perobs 4.0.0 → 4.1.0

data/test/FNV_Hash_1a_64_spec.rb

@@ -0,0 +1,59 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2019 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# This file contains tests for Array that are similar to the tests for the
+# Array implementation in MRI. The ideas of these tests were replicated in
+# this code.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+
+require 'perobs'
+
+describe PEROBS::FNV_Hash_1a_64 do
+
+  it 'should generate stable hashes for Strings' do
+    refs = [
+      [ 'foo', 15902901984413996407 ],
+      [ 'foo', 15902901984413996407 ],
+      [ 'bar', 16101355973854746 ],
+      [ 'foobar', 9625390261332436968 ],
+      [ 'PEROBS rocks your application!', 4089220442501866848 ],
+      [ 'Permission is hereby granted, free of charge, to any person ' +
+        'obtaining a copy of this software and associated documentation ' +
+        'files (the "Software"), to deal in the Software without ' +
+        'restriction, including without limitation the rights to use, ' +
+        'copy, modify, merge, publish, distribute, sublicense, and/or ' +
+        'sell copies of the Software, and to permit persons to whom the ' +
+        'Software is furnished to do so, subject to the following conditions:',
+        17637146001033534275 ]
+    ]
+
+    refs.each do |v|
+      expect(PEROBS::FNV_Hash_1a_64::digest(v[0])).to eql(v[1])
+    end
+  end
+
+end
+

data/test/FlatFileDB_spec.rb

@@ -28,6 +28,7 @@ require 'fileutils'
 require 'spec_helper'
 require 'perobs/FlatFileDB'
 require 'perobs/Store'
+require 'LegacyDBs/LegacyDB'
 
 class FlatFileDB_O < PEROBS::Object
 
@@ -47,7 +48,12 @@ describe PEROBS::FlatFileDB do
   before(:each) do
     @db_dir = generate_db_name(__FILE__)
     FileUtils.mkdir_p(@db_dir)
-    @store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    @store_options = {
+      :engine => PEROBS::FlatFileDB,
+      :log => $stderr,
+      :log_level => Logger::ERROR
+    }
+    @store = PEROBS::Store.new(@db_dir, @store_options)
   end
 
   after(:each) do
@@ -67,17 +73,13 @@ describe PEROBS::FlatFileDB do
 
   it 'should do a version upgrade' do
     # Close the store
-    @store['o'] = @store.new(FlatFileDB_O)
     @store.exit
+    src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
+    FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
 
-    # Manually downgrade the version file to version 1
-    version_file = File.join(@db_dir, 'version')
-    File.write(version_file, '1')
-
-    # Open the store again
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
-    expect(File.read(version_file).to_i).to eql(PEROBS::FlatFileDB::VERSION)
-    expect(store['o'].b).to eql(42)
+    db = LegacyDB.new(@db_dir)
+    capture_io { db.open }
+    capture_io { expect(db.check).to be true }
   end
 
   it 'should refuse a version downgrade' do
@@ -89,7 +91,7 @@ describe PEROBS::FlatFileDB do
     File.write(version_file, '1000000')
 
     # Open the store again
-    expect { PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB) }.to raise_error(PEROBS::FatalError)
+    expect { PEROBS::Store.new(@db_dir) }.to raise_error(PEROBS::FatalError)
   end
 
   it 'should recover from a lost index file' do
@@ -97,7 +99,10 @@ describe PEROBS::FlatFileDB do
     @store.exit
 
     File.delete(File.join(@db_dir, 'index.blobs'))
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    store = nil
+    capture_io do
+      store = PEROBS::Store.new(@db_dir)
+    end
     expect(store['o'].b).to eql(42)
   end
 
@@ -106,10 +111,54 @@ describe PEROBS::FlatFileDB do
     @store.exit
 
     File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
-    store.check(true)
+    store = nil
+    capture_io do
+      store = PEROBS::Store.new(@db_dir)
+    end
+    capture_io { store.check(true) }
     expect(store['o'].b).to eql(42)
   end
 
+  it 'should repair a corrupted database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}", i + 1)
+    end
+    db.close
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 10}:#{'Y' * (i + 1) * 25}", i + 10)
+    end
+    pos = db.instance_variable_get('@flat_file').find_obj_addr_by_id(10)
+    db.close
+
+    f = File.open(db_file, 'rb+')
+    f.seek(pos)
+    f.write('ZZZZZ')
+    f.close
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(5) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}")
+    end
+    expect(db.get_object(10)).to be_nil
+    1.upto(5) do |i|
+      expect(db.get_object(i + 10)).to eql("#{i + 10}:#{'Y' * (i + 1) * 25}")
+    end
+    db.close
+  end
+
 end
 

data/test/Hash_spec.rb

@@ -169,9 +169,8 @@ describe PEROBS::Hash do
   it 'should catch a leaked PEROBS::ObjectBase object' do
     @store['a'] = a = @store.new(PEROBS::Hash)
     o = @store.new(PO)
-    a['a'] = o.get_self
     PEROBS.log.open(StringIO.new)
-    expect { @store.sync }.to raise_error(PEROBS::FatalError)
+    expect { a['a'] = o.get_self }.to raise_error(PEROBS::FatalError)
     PEROBS.log.open($stderr)
   end
 

data/test/IDList_spec.rb

@@ -0,0 +1,77 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+require 'perobs/IDList'
+
+module PEROBS
+
+  describe IDList do
+
+    before(:all) do
+      @db_dir = generate_db_name('IDList')
+      FileUtils.mkdir_p(@db_dir)
+      @list = PEROBS::IDList.new(@db_dir, 'idlist', 16, 64)
+    end
+
+    after(:all) do
+      @list.erase
+      FileUtils.rm_rf(@db_dir)
+    end
+
+    it 'should not contain any values' do
+      expect(@list.to_a).to eql []
+      expect(@list.include?(0)).to be false
+      expect(@list.include?(1)).to be false
+      expect { @list.check }.to_not raise_error
+    end
+
+    it 'should store a large number of values' do
+      vals = []
+      50000.times do
+        v = rand(2 ** 64)
+        vals << v
+
+        next if @list.include?(v)
+        @list.insert(v)
+        #expect(@list.include?(v)).to be true
+        0.upto(rand(10)) do
+          v = vals[rand(vals.length)]
+          expect(@list.include?(v)).to be true
+        end
+
+        #expect { @list.check }.to_not raise_error if rand(1000) == 0
+      end
+      expect { @list.check }.to_not raise_error
+
+      vals.each do |v|
+        expect(@list.include?(v)).to be true
+      end
+    end
+
+  end
+
+end
+

data/test/LegacyDBs/LegacyDB.rb

@@ -0,0 +1,151 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+$:.unshift File.join(File.dirname(__FILE__), '..', '..', 'lib')
+
+require 'perobs'
+
+# This class creates and manages a simple DB with some toy data to check the
+# conversion routines for legacy DB formats.
+class LegacyDB
+
+  class Fragment < PEROBS::Object
+
+    attr_persist :str, :pred, :succ
+
+    def initialize(p, str, pred = nil)
+      super(p)
+      self.str = str
+      self.pred = pred
+      self.succ = nil
+    end
+
+  end
+
+  N1 = 293
+  N2 = 427
+
+  def initialize(name)
+    @name = name
+    @store = nil
+  end
+
+  def create
+    @store = PEROBS::Store.new(@name)
+    @store['fragments'] = @store.new(PEROBS::Array)
+    @store['metadata'] = @store.new(PEROBS::Hash)
+    @store['by_length'] = @store.new(PEROBS::Hash)
+
+    # Create a long string of digits.
+    number = (N1**N2).to_s
+    # Find a suitable digit that we can use a separator to split the long
+    # string into smaller strings.
+    separator = find_separator(number)
+    @store['metadata']['separator'] = separator
+    pred = nil
+    # Store all the fragments in the @store['fragments'] array.
+    number.split(separator).each do |fragment|
+      @store['fragments'] << (f = @store.new(Fragment, fragment, pred))
+      # Additionally, we create the doubly-linked list of the fragments.
+      pred.succ = f if pred
+      pred = f
+      # And we store the fragments hashed by their length.
+      length = fragment.length.to_s
+      if @store['by_length'][length].nil?
+        @store['by_length'][length] = @store.new(PEROBS::Array)
+      end
+      @store['by_length'][length] << f
+    end
+    @store.exit
+  end
+
+  def open
+    @store = PEROBS::Store.new(@name)
+  end
+
+  def check
+    # Recreate the original number from the @store['fragments'] list.
+    number = @store['fragments'].map { |f| f.str }.
+      join(@store['metadata']['separator'])
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    # Check the total number of digits based on the bash by length.
+    fragment_counter = 0
+    total_fragment_length = 0
+    @store['by_length'].each do |length, fragments|
+      fragment_counter += fragments.length
+      total_fragment_length += length.to_i * fragments.length
+    end
+    if number.length != total_fragment_length + fragment_counter - 1
+      raise RuntimeError, "Number length mismatch"
+    end
+
+    # Recreate the original number from the linked list forward traversal.
+    number = ''
+    f = @store['fragments'][0]
+    while f
+      number += @store['metadata']['separator'] unless number.empty?
+      number += f.str
+      f = f.succ
+    end
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    # Recreate the original number from the linked list backwards traversal.
+    number = ''
+    f = @store['fragments'][-1]
+    while f
+      number = @store['metadata']['separator'] + number unless number.empty?
+      number = f.str + number
+      f = f.pred
+    end
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    true
+  end
+
+  private
+
+  def find_separator(str)
+    0.upto(9) do |digit|
+      c = digit.to_s
+      return c if str[0] != c && str[-1] != c
+    end
+
+    raise RuntimeError, "Could not find separator"
+  end
+
+end
+
+#db = LegacyDB.new('test')
+#db.create
+#db.open
+#db.check
+

data/test/LegacyDBs/version_3/class_map.json

@@ -0,0 +1 @@
+{"PEROBS::Hash":0,"PEROBS::Array":1,"LegacyDB::Fragment":2}

data/test/LegacyDBs/version_3/config.json

@@ -0,0 +1 @@
+{"serializer":{"json_class":"Symbol","s":"json"}}

data/test/LegacyDBs/version_3/version

@@ -0,0 +1 @@
+3

data/test/LockFile_spec.rb

@@ -30,7 +30,11 @@ require 'perobs/LockFile'
 describe PEROBS::LockFile do
 
   before(:each) do
-    @dir = Dir.mktmpdir('LockFile')
+    PEROBS.log.open($stderr)
+    PEROBS.log.level = Logger::INFO
+    @dir = File.join(Dir.tmpdir,
+                        "#{File.basename('LockFile_spec')}.#{rand(2**32)}")
+    FileUtils.mkdir_p(@dir)
     @file = File.join(@dir, 'LockFile.lock')
   end
 
@@ -42,7 +46,6 @@ describe PEROBS::LockFile do
     capture_io do
       expect(PEROBS::LockFile.new('/foo/bar/foobar').lock).to be false
     end
-    PEROBS.log.open($stderr)
   end
 
   it 'should support taking and releasing the lock' do
@@ -59,7 +62,7 @@ describe PEROBS::LockFile do
     expect(lock.is_locked?).to be true
     lock.forced_unlock
     expect(lock.is_locked?).to be false
-    out = capture_io{ expect(lock.unlock).to be false }
+    out = capture_io{ expect(lock.unlock).to be false }.log
     expect(out).to include('There is no current lock to release')
   end
 
@@ -67,7 +70,7 @@ describe PEROBS::LockFile do
     lock1 = PEROBS::LockFile.new(@file)
     expect(lock1.lock).to be true
     lock2 = PEROBS::LockFile.new(@file)
-    out = capture_io { expect(lock2.lock).to be false }
+    out = capture_io { expect(lock2.lock).to be false }.log
     expect(out).to include('due to timeout')
     expect(lock1.unlock).to be true
     expect(lock2.lock).to be true
@@ -105,7 +108,7 @@ describe PEROBS::LockFile do
     end
     lock2 = PEROBS::LockFile.new(@file,
                                  { :max_retries => 2, :pause_secs => 0.5 })
-    out = capture_io { expect(lock2.lock).to be false }
+    out = capture_io { expect(lock2.lock).to be false }.log
     expect(out).to include('due to timeout')
     Process.wait(pid)
   end
@@ -123,7 +126,7 @@ describe PEROBS::LockFile do
     end
 
     lock2 = PEROBS::LockFile.new(@file, { :timeout_secs => 1 })
-    out = capture_io { expect(lock2.lock).to be true }
+    out = capture_io { expect(lock2.lock).to be true }.log
     expect(out).to include('Old lock file found for PID')
     expect(lock2.unlock).to be true
     Process.wait(pid)