perobs 4.1.0 → 4.2.0

data/test/FuzzyStringMatcher_spec.rb

@@ -0,0 +1,171 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+require 'perobs/Store'
+require 'perobs/FuzzyStringMatcher'
+
+module PEROBS
+
+  describe FuzzyStringMatcher do
+
+    before(:all) do
+      @db_name = generate_db_name(__FILE__)
+      @store = PEROBS::Store.new(@db_name)
+      @fsm = FuzzyStringMatcher.new(@store, 'test')
+      @fsm2 = FuzzyStringMatcher.new(@store, 'test', true, 2)
+    end
+
+    after(:all) do
+      @store.delete_store
+    end
+
+    it 'should have no matches for empty dict' do
+      expect(@fsm.best_matches('foobar')).to eql([])
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(0)
+      expect(stats['max_list_size']).to eql(0)
+      expect(stats['avg_list_size']).to eql(0)
+    end
+
+    it 'should learn a word' do
+      @fsm.learn('kindergarten')
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(11)
+      expect(stats['max_list_size']).to eql(1)
+      expect(stats['avg_list_size']).to eql(1.0)
+    end
+
+    it 'should clear the dictionary' do
+      @fsm.clear
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(0)
+      expect(stats['max_list_size']).to eql(0)
+      expect(stats['avg_list_size']).to eql(0)
+    end
+
+    it 'should learn some words' do
+      %w( one two three four five six seven eight nine ten
+          eleven twelve thirteen fourteen fifteen sixteen
+          seventeen eighteen nineteen twenty ).each do |w|
+        @fsm.learn(w, w)
+      end
+      expect(stats = @fsm.stats).not_to be_nil
+      expect(stats['dictionary_size']).to eql(65)
+      expect(stats['max_list_size']).to eql(7)
+      expect(stats['avg_list_size']).to be_within(0.001).of(1.415)
+    end
+
+    it 'should find a match' do
+      dut = {
+        [ 'one' ] => [ [ 'one', 1.0 ] ],
+        [ 'three' ] => [ [ 'three', 1.0 ] ],
+        [ 'four' ]=> [ [ 'four', 1.0 ], [ 'fourteen', 0.666 ] ],
+        [ 'four', 1.0 ]=> [ [ 'four', 1.0 ] ],
+        [ 'even' ] => [ [ 'seven', 0.666 ], [ 'eleven', 0.666 ] ],
+        [ 'teen' ] => [ ['thirteen', 0.6666666666666666],
+                      ['fourteen', 0.6666666666666666],
+                      ['fifteen', 0.6666666666666666],
+                      ['sixteen', 0.6666666666666666],
+                      ['seventeen', 0.6666666666666666],
+                      ['eighteen', 0.6666666666666666],
+                      ['nineteen', 0.6666666666666666] ],
+        [ 'aight' ] => [ [ 'eight', 0.5 ] ],
+        [ 'thirdteen' ] => [ [ 'thirteen', 0.5 ] ],
+        [ 'shirt teen', 0.3 ] => [ [ 'thirteen', 0.333 ] ]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should not find an unknown match' do
+      expect(@fsm.best_matches('foobar')).to eql([])
+    end
+
+    it 'should handle a larger text' do
+      text =<<-EOT
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+EOT
+
+      text.split.each do |word|
+        @fsm2.learn(word, word)
+      end
+      stats = @fsm2.stats
+      expect(stats['dictionary_size']).to eql(363)
+      expect(stats['max_list_size']).to eql(22)
+      expect(stats['avg_list_size']).to be_within(0.001).of(2.366)
+    end
+
+    it 'should find case sensitive matches' do
+      dut = {
+        [ 'SOFTWARE', 0.5, 20 ] => [ [ 'SOFTWARE', 1.0 ], [ 'SOFTWARE.', 0.888 ] ],
+        [ 'three', 0.5, 20 ] => [ [ 'the', 0.5 ], [ 'free', 0.5 ] ]
+      }
+
+      check_data_under_test(@fsm2, dut)
+    end
+
+    def check_data_under_test(fsm, dut)
+      dut.each do |inputs, reference|
+        key = inputs[0]
+        results = fsm.best_matches(*inputs)
+
+        expect(results.length).to eql(reference.length),
+          "Wrong number of results for '#{key}': \n#{results}\n#{reference}"
+
+        reference.each do |key, rating|
+          match = results.find { |v| v[0] == key}
+          expect(match).not_to be_nil,
+            "result is missing key #{key}: #{results}"
+          expect(match[0]).to eql(key),
+            "Wrong match returned for key #{key}: #{match}"
+          expect(match[1]).to be_within(0.001).of(rating),
+            "Wrong rating returend for key #{key}: #{match}"
+        end
+      end
+    end
+
+  end
+
+end
+

data/test/LegacyDBs/LegacyDB.rb

@@ -131,6 +131,10 @@ class LegacyDB
     true
   end
 
+  def repair
+    @store.check(true)
+  end
+
   private
 
   def find_separator(str)

data/test/SpaceManager_spec.rb

@@ -0,0 +1,176 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+
+require 'spec_helper'
+require 'perobs/SpaceManager'
+require 'perobs/ProgressMeter'
+
+describe PEROBS::SpaceManager do
+
+  before(:all) do
+    @db_dir = generate_db_name('SpaceManager')
+    FileUtils.mkdir_p(@db_dir)
+    @m = PEROBS::SpaceManager.new(@db_dir, PEROBS::ProgressMeter.new, 5)
+    PEROBS.log.level = Logger::ERROR
+    PEROBS.log.open($stderr)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@db_dir)
+  end
+
+  it 'should open the space manager' do
+    @m.open
+    expect(@m.to_a).to eql([])
+  end
+
+  it 'should support adding spaces' do
+    @m.add_space(80, 8)
+    expect(@m.has_space?(80, 8)).to be true
+    expect(@m.to_a).to eql([[80, 8]])
+    expect(@m.check).to be true
+    @m.add_space(40, 4)
+    expect(@m.has_space?(40, 4)).to be true
+    expect(@m.to_a).to eql([[40, 4], [80, 8]])
+    @m.add_space(20, 2)
+    expect(@m.has_space?(20, 2)).to be true
+    expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8]])
+    @m.add_space(160, 16)
+    expect(@m.has_space?(160, 16)).to be true
+    expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16]])
+    @m.add_space(320, 32)
+    expect(@m.has_space?(320, 32)).to be true
+    expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16], [320, 32]])
+    @m.add_space(100, 10)
+    expect(@m.has_space?(100, 10)).to be true
+    expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [100, 10], [160, 16], [320, 32]])
+    @m.add_space(81, 8)
+    expect(@m.has_space?(81, 8)).to be true
+    expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
+    expect(@m.check).to be true
+  end
+
+  it 'should keep values over an close/open' do
+    @m.add_space(1, 15)
+    expect(@m.check).to be true
+    @m.close
+    @m.open
+    expect(@m.check).to be true
+    expect(@m.to_a).to eql([[1, 15], [20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
+  end
+
+  it 'should support clearing all spaces' do
+    @m.clear
+    expect(@m.to_a).to eql([])
+    @m.add_space(1, 1)
+    @m.add_space(2, 2)
+    @m.clear
+    expect(@m.to_a).to eql([])
+  end
+
+  it 'should return exactly matching spaces' do
+    @m.clear
+    add_sizes([ 10, 5, 15, 10 ])
+    expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10]])
+    expect(@m.get_space(10)).to eql([3, 10])
+    expect(@m.get_space(10)).to eql([0, 10])
+    expect(@m.to_a).to eql([[1, 5], [2, 15]])
+    expect(@m.added_spaces).to eql(4)
+    expect(@m.recycled_spaces).to eql(2)
+    expect(@m.failed_requests).to eql(0)
+
+    @m.clear
+    add_sizes([ 10, 5, 15, 10, 10 ])
+    expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10], [4, 10]])
+    expect(@m.get_space(10)).to eql([4, 10])
+    expect(@m.get_space(10)).to eql([3, 10])
+    expect(@m.get_space(10)).to eql([0, 10])
+    expect(@m.to_a).to eql([[1, 5], [2, 15]])
+    expect(@m.added_spaces).to eql(5)
+    expect(@m.recycled_spaces).to eql(3)
+    expect(@m.failed_requests).to eql(0)
+  end
+
+  it "should return nil if no space can be found" do
+    expect(@m.get_space(42)).to be nil
+    expect(@m.get_space(9)).to be nil
+    expect(@m.get_space(11)).to be nil
+    expect(@m.recycled_spaces).to eql(3)
+    expect(@m.failed_requests).to eql(3)
+  end
+
+  it 'should support a real-world traffic pattern' do
+    address = 0
+    spaces = []
+    @m.clear
+    0.upto(1500) do
+      case rand(4)
+      when 0
+        # Insert new values
+        rand(9).times do
+          size = 20 + rand(80)
+          @m.add_space(address, size)
+          spaces << [ address, size ]
+          address += size
+        end
+      when 1
+        # Remove some values
+        rand(7).times do
+          size = rand(110)
+          if (space = @m.get_space(size))
+            expect(spaces.include?(space)).to be true
+            spaces.delete(space)
+          end
+        end
+      when 2
+        if rand(10) == 0
+          expect(@m.check).to be true
+          spaces.each do |address, size|
+            expect(@m.has_space?(address, size)).to be true
+          end
+          @m.to_a.each do |address, size|
+            expect(spaces.include?([ address, size ])).to be true
+          end
+        end
+      when 3
+        if rand(200) == 0
+          expect(@m.check).to be true
+          @m.close
+          @m.open
+          expect(@m.check).to be true
+        end
+      end
+    end
+  end
+
+  def add_sizes(sizes)
+    sizes.each_with_index do |size, i|
+      @m.add_space(i, size)
+    end
+  end
+
+end

data/test/Store_spec.rb

@@ -271,7 +271,6 @@ describe PEROBS::Store do
     capture_io { expect(store.gc).to eq(0) }
     p0 = p1 = p2 = nil
     capture_io { store.exit }
-    GC.start
     store = PEROBS::Store.new(@db_file)
     expect(store['person0']._id).to eq(id0)
     expect(store['person0'].related._id).to eq(id1)
@@ -279,9 +278,8 @@ describe PEROBS::Store do
 
     store['person0'].related = nil
     capture_io { expect(store.gc).to eq(2) }
-    GC.start
-    expect(store.object_by_id(id1)).to be_nil
-    expect(store.object_by_id(id2)).to be_nil
+    stats = store.statistics
+    expect(stats.swept_objects).to eql(2)
     capture_io { store.exit }
 
     store = PEROBS::Store.new(@db_file)
@@ -494,7 +492,6 @@ describe PEROBS::Store do
     # We have the root hash and the Person object.
     expect(store.statistics[:in_memory_objects]).to eq(2)
     store.sync
-    GC.start
     # Now the Person should be gone from memory.
     # Ruby 2.3 and later has changed the GC so that this does not work
     # reliably anymore. The GC seems to operate lazyly.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-08 00:00:00.000000000 Z
+date: 2020-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.1'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.1'
+        version: 12.3.3
 description: Library to provide a persistent object store
 email:
 - chris@linux.com
@@ -87,6 +87,7 @@ files:
 - lib/perobs/FlatFile.rb
 - lib/perobs/FlatFileBlobHeader.rb
 - lib/perobs/FlatFileDB.rb
+- lib/perobs/FuzzyStringMatcher.rb
 - lib/perobs/Handle.rb
 - lib/perobs/Hash.rb
 - lib/perobs/IDList.rb
@@ -101,6 +102,7 @@ files:
 - lib/perobs/PersistentObjectCacheLine.rb
 - lib/perobs/ProgressMeter.rb
 - lib/perobs/RobustFile.rb
+- lib/perobs/SpaceManager.rb
 - lib/perobs/SpaceTree.rb
 - lib/perobs/SpaceTreeNode.rb
 - lib/perobs/SpaceTreeNodeLink.rb
@@ -123,6 +125,7 @@ files:
 - test/EquiBlobsFile_spec.rb
 - test/FNV_Hash_1a_64_spec.rb
 - test/FlatFileDB_spec.rb
+- test/FuzzyStringMatcher_spec.rb
 - test/Hash_spec.rb
 - test/IDList_spec.rb
 - test/LegacyDBs/LegacyDB.rb
@@ -134,6 +137,7 @@ files:
 - test/LegacyDBs/version_3/version
 - test/LockFile_spec.rb
 - test/Object_spec.rb
+- test/SpaceManager_spec.rb
 - test/SpaceTree_spec.rb
 - test/StackFile_spec.rb
 - test/Store_spec.rb
@@ -151,7 +155,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -159,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Persistent Ruby Object Store
@@ -175,6 +179,7 @@ test_files:
 - test/EquiBlobsFile_spec.rb
 - test/FNV_Hash_1a_64_spec.rb
 - test/FlatFileDB_spec.rb
+- test/FuzzyStringMatcher_spec.rb
 - test/Hash_spec.rb
 - test/IDList_spec.rb
 - test/LegacyDBs/LegacyDB.rb
@@ -186,6 +191,7 @@ test_files:
 - test/LegacyDBs/version_3/version
 - test/LockFile_spec.rb
 - test/Object_spec.rb
+- test/SpaceManager_spec.rb
 - test/SpaceTree_spec.rb
 - test/StackFile_spec.rb
 - test/Store_spec.rb