perobs 3.0.1 → 4.3.0

data/test/FuzzyStringMatcher_spec.rb

@@ -0,0 +1,261 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+require 'perobs/Store'
+require 'perobs/FuzzyStringMatcher'
+
+module PEROBS
+
+  class WordRef < PEROBS::Object
+
+    attr_persist :word, :line
+
+    def initialize(store, word, line)
+      super(store)
+      self.word = word
+      self.line = line
+    end
+
+  end
+
+  describe FuzzyStringMatcher do
+
+    before(:all) do
+      @db_name = generate_db_name(__FILE__)
+      @store = PEROBS::Store.new(@db_name)
+      @store['fsm'] = @fsm = @store.new(FuzzyStringMatcher)
+      @store['fsm2'] = @fsm2 = @store.new(FuzzyStringMatcher, true, 2)
+    end
+
+    after(:all) do
+      @store.delete_store
+    end
+
+    it 'should have no matches for empty dict' do
+      expect(@fsm.best_matches('foobar')).to eql([])
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(0)
+      expect(stats['max_list_size']).to eql(0)
+      expect(stats['avg_list_size']).to eql(0)
+    end
+
+    it 'should learn a word' do
+      @fsm.learn('kindergarten')
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(11)
+      expect(stats['max_list_size']).to eql(1)
+      expect(stats['avg_list_size']).to eql(1.0)
+    end
+
+    it 'should clear the dictionary' do
+      @fsm.clear
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(0)
+      expect(stats['max_list_size']).to eql(0)
+      expect(stats['avg_list_size']).to eql(0)
+    end
+
+    it 'should learn some words' do
+      %w( one two three four five six seven eight nine ten
+          eleven twelve thirteen fourteen fifteen sixteen
+          seventeen eighteen nineteen twenty ).each do |w|
+        @fsm.learn(w, w)
+      end
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(65)
+      expect(stats['max_list_size']).to eql(7)
+      expect(stats['avg_list_size']).to be_within(0.001).of(1.415)
+    end
+
+    it 'should find a match' do
+      dut = {
+        [ 'one' ] => [ [ 'one', 1.0 ] ],
+        [ 'three' ] => [ [ 'three', 1.0 ] ],
+        [ 'four' ]=> [ [ 'four', 1.0 ], [ 'fourteen', 0.666 ] ],
+        [ 'four', 1.0 ]=> [ [ 'four', 1.0 ] ],
+        [ 'even' ] => [ [ 'seven', 0.666 ], [ 'eleven', 0.666 ] ],
+        [ 'teen' ] => [ ['thirteen', 0.6666666666666666],
+                      ['fourteen', 0.6666666666666666],
+                      ['fifteen', 0.6666666666666666],
+                      ['sixteen', 0.6666666666666666],
+                      ['seventeen', 0.6666666666666666],
+                      ['eighteen', 0.6666666666666666],
+                      ['nineteen', 0.6666666666666666] ],
+        [ 'aight' ] => [ [ 'eight', 0.5 ] ],
+        [ 'thirdteen' ] => [ [ 'thirteen', 0.5 ] ],
+        [ 'shirt teen', 0.3 ] => [ [ 'thirteen', 0.333 ] ]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should not find an unknown match' do
+      expect(@fsm.best_matches('foobar')).to eql([])
+    end
+
+    it 'should find a match' do
+      dut = {
+        [ 'one' ] => [ [ 'one', 1.0 ] ],
+        [ 'three' ] => [ [ 'three', 1.0 ] ],
+        [ 'four' ]=> [ [ 'four', 1.0 ], [ 'fourteen', 0.666 ] ],
+        [ 'four', 1.0 ]=> [ [ 'four', 1.0 ] ],
+        [ 'even' ] => [ [ 'seven', 0.666 ], [ 'eleven', 0.666 ] ],
+        [ 'teen' ] => [ ['thirteen', 0.6666666666666666],
+                      ['fourteen', 0.6666666666666666],
+                      ['fifteen', 0.6666666666666666],
+                      ['sixteen', 0.6666666666666666],
+                      ['seventeen', 0.6666666666666666],
+                      ['eighteen', 0.6666666666666666],
+                      ['nineteen', 0.6666666666666666] ],
+        [ 'aight' ] => [ [ 'eight', 0.5 ] ],
+        [ 'thirdteen' ] => [ [ 'thirteen', 0.5 ] ],
+        [ 'shirt teen', 0.3 ] => [ [ 'thirteen', 0.333 ] ]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should sort best to worst matches' do
+      @fsm.clear
+      %w( xbar xfoox foor bar foobar barfoo foo rab baar fool xbarx
+          foobarx xfoobarx foo_bar ).each do |w|
+        @fsm.learn(w, w)
+      end
+      dut = {
+        [ 'foo' ] => [["foo", 1.0], ["foor", 0.5], ["foobar", 0.5],
+                      ["fool", 0.5], ["foobarx", 0.5], ["foo_bar", 0.5],
+                      ["barfoo", 0.5]],
+        [ 'bar' ] => [["bar", 1.0], ["barfoo", 0.5], ["xbar", 0.5],
+                      ["foobar", 0.5], ["foo_bar", 0.5]],
+        [ 'foobar' ] => [["foobar", 1.0], ["foobarx", 0.8], ["xfoobarx", 0.6]]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should handle a larger text' do
+      text =<<-EOT
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+EOT
+
+      text.split.each do |word|
+        @fsm2.learn(word, word)
+      end
+      stats = @fsm2.stats
+      expect(stats['dictionary_size']).to eql(352)
+      expect(stats['max_list_size']).to eql(22)
+      expect(stats['avg_list_size']).to be_within(0.001).of(2.409)
+    end
+
+    it 'should find case sensitive matches' do
+      dut = {
+        [ 'SOFTWARE', 0.5, 20 ] => [ [ 'SOFTWARE', 1.0 ], [ 'SOFTWARE.', 0.888 ] ],
+        [ 'three', 0.5, 20 ] => [ [ 'the', 0.5 ], [ 'free', 0.5 ] ]
+      }
+
+      check_data_under_test(@fsm2, dut)
+    end
+
+    it 'should support references to PEROBS objects' do
+      text =<<-EOT
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+EOT
+
+      line_no = 1
+      @store['fsm'] = fsm = @store.new(FuzzyStringMatcher)
+      @store['refs'] = refs = @store.new(Array)
+      text.each_line do |line|
+        line.split.each do |word|
+          ref = @store.new(WordRef, word, line_no)
+          refs << ref
+          fsm.learn(word, ref)
+        end
+        line_no += 1
+      end
+
+      found_lines = []
+      fsm.best_matches('SOFTWARE').each do |match|
+        found_lines << match[0].line
+      end
+      expect(found_lines.sort).to eql([ 4, 5, 5, 7, 8 ])
+    end
+
+    it 'should with small search words' do
+      @fsm.clear
+      mats = 'Yukihiro Matsumoto'
+      @fsm.learn(mats)
+      expect(@fsm.best_matches('Yukihiro').first.first).to eql(mats)
+      expect(@fsm.best_matches('Mats', 0.3).first.first).to eql(mats)
+    end
+
+    def check_data_under_test(fsm, dut)
+      dut.each do |inputs, reference|
+        key = inputs[0]
+        results = fsm.best_matches(*inputs)
+
+        expect(results.length).to eql(reference.length),
+          "Wrong number of results for '#{key}': \n#{results}\n#{reference}"
+
+        reference.each do |key, rating|
+          match = results.find { |v| v[0] == key}
+          expect(match).not_to be_nil,
+            "result is missing key #{key}: #{results}"
+          expect(match[0]).to eql(key),
+            "Wrong match returned for key #{key}: #{match}"
+          expect(match[1]).to be_within(0.001).of(rating),
+            "Wrong rating returend for key #{key}: #{match}"
+        end
+      end
+    end
+
+  end
+
+end
+

data/test/Hash_spec.rb

@@ -31,10 +31,9 @@ require 'spec_helper'
 
 require 'perobs'
 
-
 class PO < PEROBS::Object
 
-  po_attr :name
+  attr_persist :name
 
   def initialize(store, name = nil)
     super(store)
@@ -68,16 +67,28 @@ describe PEROBS::Hash do
     h['po'] = po = @store.new(PO)
     po.name = 'foobar'
     h['b'] = 'B'
+    @store['po_key'] = po_key = @store.new(PO)
+    po_key.name = 'po key'
+    h[po_key] = 'PO Key'
 
     expect(h['a']).to eq('A')
     expect(h['b']).to eq('B')
-    @store.sync
+    expect(h[@store['po_key']]).to eq('PO Key')
+    @store.exit
 
     @store = PEROBS::Store.new(@db_name)
     h = @store['h']
     expect(h['a']).to eq('A')
     expect(h['b']).to eq('B')
     expect(h['po'].name).to eq('foobar')
+    po_key = @store['po_key']
+    expect(po_key.name).to eq('po key')
+    expect(h[po_key]).to eq('PO Key')
+  end
+
+  it 'should not allow hash keys that conflict with internal notations' do
+    @store['h'] = h = @store.new(PEROBS::Hash)
+    expect { h['#<PEROBS::POReference id=1234>'] = 'foo'; @store.sync }.to raise_error(ArgumentError)
   end
 
   it 'should have an each method to iterate' do
@@ -88,6 +99,7 @@ describe PEROBS::Hash do
     vs = []
     h.each { |k, v| vs << k + v }
     expect(vs.sort.join).to eq('aAbBcC')
+    @store.exit
 
     @store = PEROBS::Store.new(@db_name)
     @store['h'] = h = @store.new(PEROBS::Hash)
@@ -100,15 +112,15 @@ describe PEROBS::Hash do
   end
 
   # Utility method to create a PEROBS::Hash from a normal Hash.
-  def cph(hash = nil)
+  def cph(hash = nil, id = 'a')
     a = @store.new(PEROBS::Hash)
     a.replace(hash) unless hash.nil?
-    @store['a'] = a
+    @store[id] = a
   end
 
   def pcheck
     yield
-    @store.sync
+    @store.exit
     @store = PEROBS::Store.new(@db_name)
     yield
   end
@@ -142,20 +154,20 @@ describe PEROBS::Hash do
   end
 
   it 'should support merge!' do
-    h1 = cph({ 1 => 2, 2 => 3, 3 => 4 })
-    h2 = cph({ 2 => 'two', 4 => 'four' })
+    h1 = cph({ '1' => 2, '2' => 3, '3' => 4 }, 'h1')
+    h2 = cph({ '2' => 'two', '4' => 'four' }, 'h2')
 
-    ha = { 1 => 2, 2 => 'two', 3 => 4, 4 => 'four' }
-    hb = { 1 => 2, 2 => 3, 3 => 4, 4 => 'four' }
+    ha = { '1' => 2, '2' => 'two', '3' => 4, '4' => 'four' }
+    hb = { '1' => 2, '2' => 3, '3' => 4, '4' => 'four' }
 
     expect(h1.update(h2)).to eq(ha)
-    pcheck { expect(h1).to eq(ha) }
+    pcheck { expect(@store['h1'].to_hash).to eq(ha) }
 
-    h1 = cph({ 1 => 2, 2 => 3, 3 => 4 })
-    h2 = cph({ 2 => 'two', 4 => 'four' })
+    h1 = cph({ '1' => 2, '2' => 3, '3' => 4 }, 'h1')
+    h2 = cph({ '2' => 'two', '4' => 'four' }, 'h2')
 
     expect(h2.update(h1)).to eq(hb)
-    pcheck { expect(h2).to eq(hb) }
+    pcheck { expect(@store['h2'].to_hash).to eq(hb) }
   end
 
   it 'should support inspect' do
@@ -168,9 +180,8 @@ describe PEROBS::Hash do
   it 'should catch a leaked PEROBS::ObjectBase object' do
     @store['a'] = a = @store.new(PEROBS::Hash)
     o = @store.new(PO)
-    a['a'] = o.get_self
     PEROBS.log.open(StringIO.new)
-    expect { @store.sync }.to raise_error(PEROBS::FatalError)
+    expect { a['a'] = o.get_self }.to raise_error(PEROBS::FatalError)
     PEROBS.log.open($stderr)
   end
 

data/test/IDList_spec.rb

@@ -0,0 +1,77 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+require 'perobs/IDList'
+
+module PEROBS
+
+  describe IDList do
+
+    before(:all) do
+      @db_dir = generate_db_name('IDList')
+      FileUtils.mkdir_p(@db_dir)
+      @list = PEROBS::IDList.new(@db_dir, 'idlist', 16, 64)
+    end
+
+    after(:all) do
+      @list.erase
+      FileUtils.rm_rf(@db_dir)
+    end
+
+    it 'should not contain any values' do
+      expect(@list.to_a).to eql []
+      expect(@list.include?(0)).to be false
+      expect(@list.include?(1)).to be false
+      expect { @list.check }.to_not raise_error
+    end
+
+    it 'should store a large number of values' do
+      vals = []
+      50000.times do
+        v = rand(2 ** 64)
+        vals << v
+
+        next if @list.include?(v)
+        @list.insert(v)
+        #expect(@list.include?(v)).to be true
+        0.upto(rand(10)) do
+          v = vals[rand(vals.length)]
+          expect(@list.include?(v)).to be true
+        end
+
+        #expect { @list.check }.to_not raise_error if rand(1000) == 0
+      end
+      expect { @list.check }.to_not raise_error
+
+      vals.each do |v|
+        expect(@list.include?(v)).to be true
+      end
+    end
+
+  end
+
+end
+

data/test/LegacyDBs/LegacyDB.rb

@@ -0,0 +1,155 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+$:.unshift File.join(File.dirname(__FILE__), '..', '..', 'lib')
+
+require 'perobs'
+
+# This class creates and manages a simple DB with some toy data to check the
+# conversion routines for legacy DB formats.
+class LegacyDB
+
+  class Fragment < PEROBS::Object
+
+    attr_persist :str, :pred, :succ
+
+    def initialize(p, str, pred = nil)
+      super(p)
+      self.str = str
+      self.pred = pred
+      self.succ = nil
+    end
+
+  end
+
+  N1 = 293
+  N2 = 427
+
+  def initialize(name)
+    @name = name
+    @store = nil
+  end
+
+  def create
+    @store = PEROBS::Store.new(@name)
+    @store['fragments'] = @store.new(PEROBS::Array)
+    @store['metadata'] = @store.new(PEROBS::Hash)
+    @store['by_length'] = @store.new(PEROBS::Hash)
+
+    # Create a long string of digits.
+    number = (N1**N2).to_s
+    # Find a suitable digit that we can use a separator to split the long
+    # string into smaller strings.
+    separator = find_separator(number)
+    @store['metadata']['separator'] = separator
+    pred = nil
+    # Store all the fragments in the @store['fragments'] array.
+    number.split(separator).each do |fragment|
+      @store['fragments'] << (f = @store.new(Fragment, fragment, pred))
+      # Additionally, we create the doubly-linked list of the fragments.
+      pred.succ = f if pred
+      pred = f
+      # And we store the fragments hashed by their length.
+      length = fragment.length.to_s
+      if @store['by_length'][length].nil?
+        @store['by_length'][length] = @store.new(PEROBS::Array)
+      end
+      @store['by_length'][length] << f
+    end
+    @store.exit
+  end
+
+  def open
+    @store = PEROBS::Store.new(@name)
+  end
+
+  def check
+    # Recreate the original number from the @store['fragments'] list.
+    number = @store['fragments'].map { |f| f.str }.
+      join(@store['metadata']['separator'])
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    # Check the total number of digits based on the bash by length.
+    fragment_counter = 0
+    total_fragment_length = 0
+    @store['by_length'].each do |length, fragments|
+      fragment_counter += fragments.length
+      total_fragment_length += length.to_i * fragments.length
+    end
+    if number.length != total_fragment_length + fragment_counter - 1
+      raise RuntimeError, "Number length mismatch"
+    end
+
+    # Recreate the original number from the linked list forward traversal.
+    number = ''
+    f = @store['fragments'][0]
+    while f
+      number += @store['metadata']['separator'] unless number.empty?
+      number += f.str
+      f = f.succ
+    end
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    # Recreate the original number from the linked list backwards traversal.
+    number = ''
+    f = @store['fragments'][-1]
+    while f
+      number = @store['metadata']['separator'] + number unless number.empty?
+      number = f.str + number
+      f = f.pred
+    end
+    if number.to_i != N1 ** N2
+      raise RuntimeError, "Number mismatch\n#{number}\n#{N1 ** N2}"
+    end
+
+    true
+  end
+
+  def repair
+    @store.check(true)
+  end
+
+  private
+
+  def find_separator(str)
+    0.upto(9) do |digit|
+      c = digit.to_s
+      return c if str[0] != c && str[-1] != c
+    end
+
+    raise RuntimeError, "Could not find separator"
+  end
+
+end
+
+#db = LegacyDB.new('test')
+#db.create
+#db.open
+#db.check
+