perobs 4.2.0 → 4.5.0

data/lib/perobs/Store.rb

@@ -2,7 +2,7 @@
 #
 # = Store.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016, 2017, 2018, 2019
+# Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
 # by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
@@ -27,6 +27,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'set'
+require 'monitor'
 
 require 'perobs/Log'
 require 'perobs/Handle'
@@ -46,7 +47,7 @@ require 'perobs/ConsoleProgressMeter'
 # PErsistent Ruby OBject Store
 module PEROBS
 
-  Statistics = Struct.new(:in_memory_objects, :root_objects, :zombie_objects,
+  Statistics = Struct.new(:in_memory_objects, :root_objects,
                           :marked_objects, :swept_objects,
                           :created_objects, :collected_objects)
 
@@ -160,9 +161,6 @@ module PEROBS
       # List of PEROBS objects that are currently available as Ruby objects
       # hashed by their ID.
       @in_memory_objects = {}
-      # List of objects that were destroyed already but were still found in
-      # the in_memory_objects list. _collect has not yet been called for them.
-      @zombie_objects = {}
 
       # This objects keeps some counters of interest.
       @stats = Statistics.new
@@ -173,6 +171,9 @@ module PEROBS
       # objects in memory.
       @cache = Cache.new(options[:cache_bits] || 16)
 
+      # Lock to serialize access to the Store and all stored data.
+      @lock = Monitor.new
+
       # The named (global) objects IDs hashed by their name
       unless options[:no_root_objects]
         unless (@root_objects = object_by_id(0))
@@ -243,8 +244,8 @@ module PEROBS
         end
       end
 
-      @db = @class_map = @in_memory_objects = @zombie_objects =
-        @stats = @cache = @root_objects = nil
+      @db = @class_map = @in_memory_objects = @stats = @cache =
+        @root_objects = nil
     end
 
     # You need to call this method to create new PEROBS objects that belong to
@@ -259,11 +260,13 @@ module PEROBS
         PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
       end
 
-      obj = _construct_po(klass, _new_id, *args)
-      # Mark the new object as modified so it gets pushed into the database.
-      @cache.cache_write(obj)
-      # Return a POXReference proxy for the newly created object.
-      obj.myself
+      @lock.synchronize do
+        obj = _construct_po(klass, _new_id, *args)
+        # Mark the new object as modified so it gets pushed into the database.
+        @cache.cache_write(obj)
+        # Return a POXReference proxy for the newly created object.
+        obj.myself
+      end
     end
 
     # For library internal use only!
@@ -280,9 +283,11 @@ module PEROBS
     # method was called. This is an alternative to exit() that additionaly
     # deletes the entire database.
     def delete_store
-      @db.delete_database
-      @db = @class_map = @in_memory_objects = @zombie_objects =
-        @stats = @cache = @root_objects = nil
+      @lock.synchronize do
+        @db.delete_database
+        @db = @class_map = @in_memory_objects = @stats = @cache =
+          @root_objects = nil
+      end
     end
 
     # Store the provided object under the given name. Use this to make the
@@ -294,25 +299,27 @@ module PEROBS
     # @param obj [PEROBS::Object] The object to store
     # @return [PEROBS::Object] The stored object.
     def []=(name, obj)
-      # If the passed object is nil, we delete the entry if it exists.
-      if obj.nil?
-        @root_objects.delete(name)
-        return nil
-      end
+      @lock.synchronize do
+        # If the passed object is nil, we delete the entry if it exists.
+        if obj.nil?
+          @root_objects.delete(name)
+          return nil
+        end
 
-      # We only allow derivatives of PEROBS::Object to be stored in the
-      # store.
-      unless obj.is_a?(ObjectBase)
-        PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
-          "is of class #{obj.class}"
-      end
+        # We only allow derivatives of PEROBS::Object to be stored in the
+        # store.
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
+            "is of class #{obj.class}"
+        end
 
-      unless obj.store == self
-        PEROBS.log.fatal 'The object does not belong to this store.'
-      end
+        unless obj.store == self
+          PEROBS.log.fatal 'The object does not belong to this store.'
+        end
 
-      # Store the name and mark the name list as modified.
-      @root_objects[name] = obj._id
+        # Store the name and mark the name list as modified.
+        @root_objects[name] = obj._id
+      end
 
       obj
     end
@@ -322,28 +329,34 @@ module PEROBS
     #        returned.
     # @return The requested object or nil if it doesn't exist.
     def [](name)
-      # Return nil if there is no object with that name.
-      return nil unless (id = @root_objects[name])
+      @lock.synchronize do
+        # Return nil if there is no object with that name.
+        return nil unless (id = @root_objects[name])
 
-      POXReference.new(self, id)
+        POXReference.new(self, id)
+      end
     end
 
     # Return a list with all the names of the root objects.
     # @return [Array of Symbols]
     def names
-      @root_objects.keys
+      @lock.synchronize do
+        @root_objects.keys
+      end
     end
 
     # Flush out all modified objects to disk and shrink the in-memory list if
     # needed.
     def sync
-      if @cache.in_transaction?
-        @cache.abort_transaction
+      @lock.synchronize do
+        if @cache.in_transaction?
+          @cache.abort_transaction
+          @cache.flush
+          PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
+            Kernel.caller.join("\n")
+        end
         @cache.flush
-        PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
-          Kernel.caller.join("\n")
       end
-      @cache.flush
     end
 
     # Return the number of object stored in the store. CAVEAT: This method
@@ -353,7 +366,9 @@ module PEROBS
     def size
       # We don't include the Hash that stores the root objects into the object
       # count.
-      @db.item_counter - 1
+      @lock.synchronize do
+        @db.item_counter - 1
+      end
     end
 
     # Discard all objects that are not somehow connected to the root objects
@@ -362,51 +377,20 @@ module PEROBS
     # method periodically.
     # @return [Integer] The number of collected objects
     def gc
-      sync
-      mark
-      sweep
+      @lock.synchronize do
+        sync
+        mark
+        sweep
+      end
     end
 
     # Return the object with the provided ID. This method is not part of the
     # public API and should never be called by outside users. It's purely
     # intended for internal use.
     def object_by_id(id)
-      if (ruby_object_id = @in_memory_objects[id])
-        # We have the object in memory so we can just return it.
-        begin
-          object = ObjectSpace._id2ref(ruby_object_id)
-          # Let's make sure the object is really the object we are looking
-          # for. The GC might have recycled it already and the Ruby object ID
-          # could now be used for another object.
-          if object.is_a?(ObjectBase) && object._id == id
-            return object
-          end
-        rescue RangeError => e
-          # Due to a race condition the object can still be in the
-          # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again. The _collect() call
-          # will happen much later, potentially after we have registered a new
-          # object with the same ID.
-          @zombie_objects[id] = @in_memory_objects.delete(id)
-        end
+      @lock.synchronize do
+        object_by_id_internal(id)
       end
-
-      if (obj = @cache.object_by_id(id))
-        PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
-      end
-
-      # We don't have the object in memory. Let's find it in the storage.
-      if @db.include?(id)
-        # Great, object found. Read it into memory and return it.
-        obj = ObjectBase::read(self, id)
-        # Add the object to the in-memory storage list.
-        @cache.cache_read(obj)
-
-        return obj
-      end
-
-      # The requested object does not exist. Return nil.
-      nil
     end
 
     # This method can be used to check the database and optionally repair it.
@@ -471,38 +455,54 @@ module PEROBS
     # beginning of the transaction. The exception is passed on to the
     # enclosing scope, so you probably want to handle it accordingly.
     def transaction
-      @cache.begin_transaction
+      transaction_not_started = true
+      while transaction_not_started do
+        begin
+          @lock.synchronize do
+            @cache.begin_transaction
+            # If we get to this point, the transaction was successfully
+            # started. We can exit the loop.
+            transaction_not_started = false
+          end
+        rescue TransactionInOtherThread
+          # sleep up to 50ms
+          sleep(rand(50) / 1000.0)
+        end
+      end
+
       begin
         yield if block_given?
       rescue => e
-        @cache.abort_transaction
+        @lock.synchronize { @cache.abort_transaction }
         raise e
       end
-      @cache.end_transaction
+      @lock.synchronize { @cache.end_transaction }
     end
 
     # Calls the given block once for each object, passing that object as a
     # parameter.
     def each
-      @db.clear_marks
-      # Start with the object 0 and the indexes of the root objects. Push them
-      # onto the work stack.
-      stack = [ 0 ] + @root_objects.values
-      while !stack.empty?
-        # Get an object index from the stack.
-        id = stack.pop
-        next if @db.is_marked?(id)
-
-        unless (obj = object_by_id(id))
-          PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
-            "not found."
-        end
-        # Mark the object so it will never be pushed to the stack again.
-        @db.mark(id)
-        yield(obj.myself) if block_given?
-        # Push the IDs of all unmarked referenced objects onto the stack
-        obj._referenced_object_ids.each do |r_id|
-          stack << r_id unless @db.is_marked?(r_id)
+      @lock.synchronize do
+        @db.clear_marks
+        # Start with the object 0 and the indexes of the root objects. Push them
+        # onto the work stack.
+        stack = [ 0 ] + @root_objects.values
+        while !stack.empty?
+          # Get an object index from the stack.
+          id = stack.pop
+          next if @db.is_marked?(id)
+
+          unless (obj = object_by_id_internal(id))
+            PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
+              "not found."
+          end
+          # Mark the object so it will never be pushed to the stack again.
+          @db.mark(id)
+          yield(obj.myself) if block_given?
+          # Push the IDs of all unmarked referenced objects onto the stack
+          obj._referenced_object_ids.each do |r_id|
+            stack << r_id unless @db.is_marked?(r_id)
+          end
         end
       end
     end
@@ -510,7 +510,7 @@ module PEROBS
     # Rename classes of objects stored in the data base.
     # @param rename_map [Hash] Hash that maps the old name to the new name
     def rename_classes(rename_map)
-      @class_map.rename(rename_map)
+      @lock.synchronize { @class_map.rename(rename_map) }
     end
 
     # Internal method. Don't use this outside of this library!
@@ -518,14 +518,16 @@ module PEROBS
     # random numbers between 0 and 2**64 - 1.
     # @return [Integer]
     def _new_id
-      begin
-        # Generate a random number. It's recommended to not store more than
-        # 2**62 objects in the same store.
-        id = rand(2**64)
-        # Ensure that we don't have already another object with this ID.
-      end while @in_memory_objects.include?(id) || @db.include?(id)
+      @lock.synchronize do
+        begin
+          # Generate a random number. It's recommended to not store more than
+          # 2**62 objects in the same store.
+          id = rand(2**64)
+          # Ensure that we don't have already another object with this ID.
+        end while @in_memory_objects.include?(id) || @db.include?(id)
 
-      id
+        id
+      end
     end
 
     # Internal method. Don't use this outside of this library!
@@ -536,16 +538,18 @@ module PEROBS
     # @param obj [BasicObject] Object to register
     # @param id [Integer] object ID
     def _register_in_memory(obj, id)
-      unless obj.is_a?(ObjectBase)
-        PEROBS.log.fatal "You can only register ObjectBase objects"
-      end
-      if @in_memory_objects.include?(id)
-        PEROBS.log.fatal "The Store::_in_memory_objects list already " +
-          "contains an object for ID #{id}"
-      end
+      @lock.synchronize do
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal "You can only register ObjectBase objects"
+        end
+        if @in_memory_objects.include?(id)
+          PEROBS.log.fatal "The Store::_in_memory_objects list already " +
+            "contains an object for ID #{id}"
+        end
 
-      @in_memory_objects[id] = obj.object_id
-      @stats[:created_objects] += 1
+        @in_memory_objects[id] = obj.object_id
+        @stats[:created_objects] += 1
+      end
     end
 
     # Remove the object from the in-memory list. This is an internal method
@@ -553,26 +557,73 @@ module PEROBS
     # finalizer, so many restrictions apply!
     # @param id [Integer] Object ID of object to remove from the list
     def _collect(id, ruby_object_id)
-      if @in_memory_objects[id] == ruby_object_id
+      # This method should only be called from the Ruby garbage collector.
+      # Therefor no locking is needed or even possible. The GC can kick in at
+      # any time and we could be anywhere in the code. So there is a small
+      # risk for a race here, but it should not have any serious consequences.
+      if @in_memory_objects && @in_memory_objects[id] == ruby_object_id
         @in_memory_objects.delete(id)
         @stats[:collected_objects] += 1
-      elsif @zombie_objects[id] == ruby_object_id
-        @zombie_objects.delete(id)
-        @stats[:collected_objects] += 1
       end
     end
 
     # This method returns a Hash with some statistics about this store.
     def statistics
-      @stats.in_memory_objects = @in_memory_objects.length
-      @stats.root_objects = @root_objects.length
-      @stats.zombie_objects = @zombie_objects.length
+      @lock.synchronize do
+        @stats.in_memory_objects = @in_memory_objects.length
+        @stats.root_objects = @root_objects.length
+      end
 
       @stats
     end
 
     private
 
+    def object_by_id_internal(id)
+      if (ruby_object_id = @in_memory_objects[id])
+        # We have the object in memory so we can just return it.
+        begin
+          object = ObjectSpace._id2ref(ruby_object_id)
+          # Let's make sure the object is really the object we are looking
+          # for. The GC might have recycled it already and the Ruby object ID
+          # could now be used for another object.
+          if object.is_a?(ObjectBase) && object._id == id
+            return object
+          end
+        rescue RangeError => e
+          # Due to a race condition the object can still be in the
+          # @in_memory_objects list but has been collected already by the Ruby
+          # GC. The _collect() call has not been completed yet. We now have to
+          # wait until this has been done. I think the GC lock will prevent a
+          # race on @in_memory_objects.
+          GC.start
+          while @in_memory_objects.include?(id)
+            sleep 0.01
+          end
+        end
+      end
+
+      # This is just a safety check. It has never triggered, so we can disable
+      # it for now.
+      #if (obj = @cache.object_by_id(id))
+      #  PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
+      #    "cache but not in_memory"
+      #end
+
+      # We don't have the object in memory. Let's find it in the storage.
+      if @db.include?(id)
+        # Great, object found. Read it into memory and return it.
+        obj = ObjectBase::read(self, id)
+        # Add the object to the in-memory storage list.
+        @cache.cache_read(obj)
+
+        return obj
+      end
+
+      # The requested object does not exist. Return nil.
+      nil
+    end
+
     # Mark phase of a mark-and-sweep garbage collector. It will mark all
     # objects that are reachable from the root objects.
     def mark
@@ -599,6 +650,7 @@ module PEROBS
       @stats.swept_objects = @db.delete_unmarked_objects do |id|
         @cache.evict(id)
       end
+      @db.clear_marks
       GC.start
       PEROBS.log.debug "#{@stats.swept_objects} objects collected"
       @stats.swept_objects

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "4.2.0"
+  VERSION = "4.5.0"
 end

data/lib/perobs.rb

@@ -28,3 +28,4 @@
 require "perobs/version"
 require 'perobs/Store'
 require 'perobs/ConsoleProgressMeter'
+require 'perobs/FuzzyStringMatcher'

data/perobs.gemspec

@@ -20,5 +20,5 @@ GEM_SPEC = Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'yard', '~>0.9.12'
-  spec.add_development_dependency 'rake', '~> 12.3.3'
+  spec.add_development_dependency 'rake', '~> 13.0.3'
 end

data/test/FlatFileDB_spec.rb

@@ -265,5 +265,35 @@ describe PEROBS::FlatFileDB do
     db.close
   end
 
+  it 'should handle duplicate entries for the same ID in database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    # This appends the entry 2 again
+    blob2 = File.read(db_file, 319 - 199, 199)
+    File.write(db_file, blob2, File.size(db_file))
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::WARN })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(5) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    db.close
+  end
+
 end
 

data/test/FuzzyStringMatcher_spec.rb

@@ -29,13 +29,25 @@ require 'perobs/FuzzyStringMatcher'
 
 module PEROBS
 
+  class WordRef < PEROBS::Object
+
+    attr_persist :word, :line
+
+    def initialize(store, word, line)
+      super(store)
+      self.word = word
+      self.line = line
+    end
+
+  end
+
   describe FuzzyStringMatcher do
 
     before(:all) do
       @db_name = generate_db_name(__FILE__)
       @store = PEROBS::Store.new(@db_name)
-      @fsm = FuzzyStringMatcher.new(@store, 'test')
-      @fsm2 = FuzzyStringMatcher.new(@store, 'test', true, 2)
+      @store['fsm'] = @fsm = @store.new(FuzzyStringMatcher)
+      @store['fsm2'] = @fsm2 = @store.new(FuzzyStringMatcher, true, 2)
     end
 
     after(:all) do
@@ -103,6 +115,44 @@ module PEROBS
       expect(@fsm.best_matches('foobar')).to eql([])
     end
 
+    it 'should find a match' do
+      dut = {
+        [ 'one' ] => [ [ 'one', 1.0 ] ],
+        [ 'three' ] => [ [ 'three', 1.0 ] ],
+        [ 'four' ]=> [ [ 'four', 1.0 ], [ 'fourteen', 0.666 ] ],
+        [ 'four', 1.0 ]=> [ [ 'four', 1.0 ] ],
+        [ 'even' ] => [ [ 'seven', 0.666 ], [ 'eleven', 0.666 ] ],
+        [ 'teen' ] => [ ['thirteen', 0.6666666666666666],
+                      ['fourteen', 0.6666666666666666],
+                      ['fifteen', 0.6666666666666666],
+                      ['sixteen', 0.6666666666666666],
+                      ['seventeen', 0.6666666666666666],
+                      ['eighteen', 0.6666666666666666],
+                      ['nineteen', 0.6666666666666666] ],
+        [ 'aight' ] => [ [ 'eight', 0.5 ] ],
+        [ 'thirdteen' ] => [ [ 'thirteen', 0.5 ] ],
+        [ 'shirt teen', 0.3 ] => [ [ 'thirteen', 0.333 ] ]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should sort best to worst matches' do
+      @fsm.clear
+      %w( xbar xfoox foor bar foobar barfoo foo rab baar fool xbarx
+          foobarx xfoobarx foo_bar ).each do |w|
+        @fsm.learn(w, w)
+      end
+      dut = {
+        [ 'foo' ] => [["foo", 1.0], ["foor", 0.5], ["foobar", 0.5],
+                      ["fool", 0.5], ["foobarx", 0.5], ["foo_bar", 0.5],
+                      ["barfoo", 0.5]],
+        [ 'bar' ] => [["bar", 1.0], ["barfoo", 0.5], ["xbar", 0.5],
+                      ["foobar", 0.5], ["foo_bar", 0.5]],
+        [ 'foobar' ] => [["foobar", 1.0], ["foobarx", 0.8], ["xfoobarx", 0.6]]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
     it 'should handle a larger text' do
       text =<<-EOT
 MIT License
@@ -131,9 +181,9 @@ EOT
         @fsm2.learn(word, word)
       end
       stats = @fsm2.stats
-      expect(stats['dictionary_size']).to eql(363)
+      expect(stats['dictionary_size']).to eql(352)
       expect(stats['max_list_size']).to eql(22)
-      expect(stats['avg_list_size']).to be_within(0.001).of(2.366)
+      expect(stats['avg_list_size']).to be_within(0.001).of(2.409)
     end
 
     it 'should find case sensitive matches' do
@@ -145,6 +195,46 @@ EOT
       check_data_under_test(@fsm2, dut)
     end
 
+    it 'should support references to PEROBS objects' do
+      text =<<-EOT
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+EOT
+
+      line_no = 1
+      @store['fsm'] = fsm = @store.new(FuzzyStringMatcher)
+      @store['refs'] = refs = @store.new(Array)
+      text.each_line do |line|
+        line.split.each do |word|
+          ref = @store.new(WordRef, word, line_no)
+          refs << ref
+          fsm.learn(word, ref)
+        end
+        line_no += 1
+      end
+
+      found_lines = []
+      fsm.best_matches('SOFTWARE').each do |match|
+        found_lines << match[0].line
+      end
+      expect(found_lines.sort).to eql([ 4, 5, 5, 7, 8 ])
+    end
+
+    it 'should with small search words' do
+      @fsm.clear
+      mats = 'Yukihiro Matsumoto'
+      @fsm.learn(mats)
+      expect(@fsm.best_matches('Yukihiro').first.first).to eql(mats)
+      expect(@fsm.best_matches('Mats', 0.3).first.first).to eql(mats)
+    end
+
     def check_data_under_test(fsm, dut)
       dut.each do |inputs, reference|
         key = inputs[0]