perobs 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7790ee42586bb2b8fca115f93ed277c4a8057f7a7027b356baea7b066da953e5
-  data.tar.gz: 110e0710a84ef544a4874cf868ec1662dfc900077d1894b24f53bcdaeaeeed34
+  metadata.gz: a61fc945e0ef9f5ed6558080931d2acae42cc0401f375275684e4ee32fefe4f7
+  data.tar.gz: 4d864fdc0791aa78d8c180b4686ee825cd25e209284fca1966f144813c063280
 SHA512:
-  metadata.gz: d95d845c7e8bd183f53b60369415bde86cd766c224bcbc2c52c870c60542a786f359a63eecc4e8829055cee6a1674bc952277749da183432b4b3abae7536efbb
-  data.tar.gz: 5fa1712fb01118d955d86396aec87b319c75856f6602ccf1a19f475a3dc64dc65a540e35271e6f2c6c7ee6a10c5cb03da30d43945a0dc264c7b0e48f575269d1
+  metadata.gz: f3834a9caae693d82837fb9f75141cb35e85f1a2c1439d1bb898f8578d9ae082f46deb0233d2b8a02d6ae6b0bf66862098b47ff571ff3d9a6b874fadaef6d23a
+  data.tar.gz: 883f1b5e553fae2be0039aa090d89bf6eb44ec1d0dc31488aeaa727ec8bc2844c9b722568cd7dde0b33c507121afd11e720154d4ff27e66aa3ac3812d5603954

data/README.md

@@ -108,7 +108,7 @@ class Person < PEROBS::Object
     attr_init(:father) do { @store.new(Person, 'Dad') }
   end
 
-  def merry(spouse)
+  def marry(spouse)
     self.spouse = spouse
     self.status = :married
   end

data/lib/perobs.rb

@@ -28,3 +28,4 @@
 require "perobs/version"
 require 'perobs/Store'
 require 'perobs/ConsoleProgressMeter'
+require 'perobs/FuzzyStringMatcher'

data/lib/perobs/BTree.rb

@@ -70,7 +70,7 @@ module PEROBS
       @nodes.register_custom_data('first_leaf')
       @nodes.register_custom_data('last_leaf')
       @nodes.register_custom_data('btree_size')
-      @node_cache = PersistentObjectCache.new(2**16, -1, BTreeNode, self)
+      @node_cache = PersistentObjectCache.new(2**13, 2**13, BTreeNode, self)
       @root = @first_leaf = @last_leaf = nil
       @size = 0
 
@@ -190,7 +190,7 @@ module PEROBS
         "Number of leave nodes: #{stats.leave_nodes}; " +
         "Number of leaves: #{stats.leaves}"
 
-      !stats.nil?
+      true
     end
 
     # Register a new node as root node of the tree.

data/lib/perobs/BTreeNode.rb

@@ -59,7 +59,7 @@ module PEROBS
     #        if not
     def initialize(tree, node_address = nil, parent = nil, is_leaf = true,
                    prev_sibling = nil, next_sibling = nil,
-                   keys = [], values = [], children = [])
+                   keys = nil, values = nil, children = nil)
       @tree = tree
       if node_address == 0
         PEROBS.log.fatal "Node address may not be 0"
@@ -68,13 +68,13 @@ module PEROBS
       @parent = link(parent)
       @prev_sibling = link(prev_sibling)
       @next_sibling = link(next_sibling)
-      @keys = keys
+      @keys = keys || []
       if (@is_leaf = is_leaf)
-        @values = values
-        @children = []
+        @values = values || []
+        @children = nil
       else
-        @children = children
-        @values = []
+        @children = children || []
+        @values = nil
       end
     end
 
@@ -585,11 +585,11 @@ module PEROBS
     end
 
     def trim(idx)
-      @keys = @keys[0..idx - 1]
+      @keys.slice!(idx, @keys.length - idx)
       if @is_leaf
-        @values = @values[0..idx - 1]
+        @values.slice!(idx, @values.length - idx)
       else
-        @children = @children[0..idx]
+        @children.slice!(idx + 1, @children.length - idx - 1)
       end
       @tree.node_cache.insert(self)
     end
@@ -654,13 +654,18 @@ module PEROBS
     # @yield [key, value]
     # @return [nil or Hash] nil in case of errors or a hash with some
     #         statistical information about the tree
-    def check
+    def check(&block)
       stats = Stats.new(nil, 0, 0, 0)
 
       traverse do |node, position, stack|
         if position == 0
           stats.nodes_count += 1
           if node.parent
+            unless node.parent.is_a?(BTreeNodeLink)
+              node.error "parent is a #{node.parent.class} instead of a " +
+                "BTreeNodeLink"
+              return nil
+            end
             # After a split the nodes will only have half the maximum keys.
             # For branch nodes one of the split nodes will have even 1 key
             # less as this will become the branch key in a parent node.
@@ -695,6 +700,16 @@ module PEROBS
             else
               stats.branch_depth = node.tree_level
             end
+            if node.prev_sibling && !node.prev_sibling.is_a?(BTreeNodeLink)
+              node.error "prev_sibling is a #{node.prev_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
+            if node.next_sibling && !node.next_sibling.is_a?(BTreeNodeLink)
+              node.error "next_sibling is a #{node.next_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
             if node.prev_sibling.nil? && @tree.first_leaf != node
               node.error "Leaf node #{node.node_address} has no previous " +
                 "sibling but is not the first leaf of the tree"
@@ -708,9 +723,9 @@ module PEROBS
             unless node.keys.size == node.values.size
               node.error "Key count (#{node.keys.size}) and value " +
                 "count (#{node.values.size}) don't match"
-                return nil
+              return nil
             end
-            unless node.children.empty?
+            unless node.children.nil?
               node.error "@children must be nil for a leaf node"
               return nil
             end
@@ -718,14 +733,14 @@ module PEROBS
             stats.leave_nodes += 1
             stats.leaves += node.keys.length
           else
-            unless node.values.empty?
+            unless node.values.nil?
               node.error "@values must be nil for a branch node"
               return nil
             end
             unless node.children.size == node.keys.size + 1
               node.error "Key count (#{node.keys.size}) must be one " +
                 "less than children count (#{node.children.size})"
-                return nil
+              return nil
             end
             node.children.each_with_index do |child, i|
               unless child.is_a?(BTreeNodeLink)
@@ -789,7 +804,9 @@ module PEROBS
           else
             if block_given?
               # If a block was given, call this block with the key and value.
-              return nil unless yield(node.keys[index], node.values[index])
+              unless yield(node.keys[index], node.values[index])
+                return nil
+              end
             end
           end
         end

data/lib/perobs/FlatFile.rb

@@ -293,7 +293,7 @@ module PEROBS
       header = FlatFileBlobHeader.read(@f, addr, id)
       if header.id != id
         PEROBS.log.fatal "Database index corrupted: Index for object " +
-          "#{id} points to object with ID #{header.id}"
+          "#{id} points to object with ID #{header.id} at address #{addr}"
       end
 
       buf = nil
@@ -302,7 +302,8 @@ module PEROBS
         @f.seek(addr + FlatFileBlobHeader::LENGTH)
         buf = @f.read(header.length)
       rescue IOError => e
-        PEROBS.log.fatal "Cannot read blob for ID #{id}: #{e.message}"
+        PEROBS.log.fatal "Cannot read blob for ID #{id} at address #{addr}: " +
+          e.message
       end
 
       # Uncompress the data if the compression bit is set in the flags byte.
@@ -311,12 +312,13 @@ module PEROBS
           buf = Zlib.inflate(buf)
         rescue Zlib::BufError, Zlib::DataError
           PEROBS.log.fatal "Corrupted compressed block with ID " +
-            "#{header.id} found."
+            "#{id} found at address #{addr}."
         end
       end
 
       if checksum(buf) != header.crc
-        PEROBS.log.fatal "Checksum failure while reading blob ID #{id}"
+        PEROBS.log.fatal "Checksum failure while reading blob ID #{id} " +
+          "at address #{addr}"
       end
 
       buf
@@ -339,7 +341,7 @@ module PEROBS
       if @marks
         @marks.clear
       else
-        @marks = IDList.new(@db_dir, 'marks', 8)
+        @marks = IDList.new(@db_dir, 'marks', item_counter)
       end
     end
 
@@ -452,16 +454,14 @@ module PEROBS
       regenerate_index_and_spaces
     end
 
-    # Check (and repair) the FlatFile.
-    # @param repair [Boolean] True if errors should be fixed.
+    # Check the FlatFile.
     # @return [Integer] Number of errors found
-    def check(repair = false)
+    def check()
       errors = 0
       return errors unless @f
 
       t = Time.now
-      PEROBS.log.info "Checking FlatFile database" +
-        "#{repair ? ' in repair mode' : ''}..."
+      PEROBS.log.info "Checking FlatFile database..."
 
       # First check the database blob file. Each entry should be readable and
       # correct and all IDs must be unique. We use a shadow index to keep
@@ -483,7 +483,6 @@ module PEROBS
               if buf.bytesize != header.length
                 PEROBS.log.error "Premature end of file in blob with ID " +
                   "#{header.id}."
-                discard_damaged_blob(header) if repair
                 errors += 1
                 next
               end
@@ -496,7 +495,6 @@ module PEROBS
                 rescue Zlib::BufError, Zlib::DataError
                   PEROBS.log.error "Corrupted compressed block with ID " +
                     "#{header.id} found."
-                  discard_damaged_blob(header) if repair
                   errors += 1
                   next
                 end
@@ -505,7 +503,6 @@ module PEROBS
               if header.crc && checksum(buf) != header.crc
                 PEROBS.log.error "Checksum failure while checking blob " +
                   "with ID #{header.id}"
-                discard_damaged_blob(header) if repair
                 errors += 1
                 next
               end
@@ -521,22 +518,6 @@ module PEROBS
               errors += 1
               previous_header = FlatFileBlobHeader.read(@f, previous_address,
                                                         header.id)
-              if repair
-                # We have two blobs with the same ID and we must discard one of
-                # them.
-                if header.is_outdated?
-                  discard_damaged_blob(header)
-                elsif previous_header.is_outdated?
-                  discard_damaged_blob(previous_header)
-                else
-                  PEROBS.log.error "None of the blobs with same ID have " +
-                    "the outdated flag set. Deleting the smaller one."
-                  errors += 1
-                  discard_damaged_blob(header.length < previous_header.length ?
-                                       header : previous_header)
-                end
-                next
-              end
             else
               # ID is unique so far. Add it to the shadow index.
               new_index.insert(header.id, header.addr)
@@ -553,12 +534,6 @@ module PEROBS
           PEROBS.log.error "#{@f.size - end_of_last_healthy_blob} corrupted " +
             'bytes found at the end of FlatFile.'
           corrupted_blobs += 1
-          if repair
-            PEROBS.log.error "Truncating FlatFile to " +
-              "#{end_of_last_healthy_blob} bytes by discarding " +
-              "#{@f.size - end_of_last_healthy_blob} bytes"
-            @f.truncate(end_of_last_healthy_blob)
-          end
         end
 
         errors += corrupted_blobs
@@ -568,17 +543,19 @@ module PEROBS
       new_index.close
       new_index.erase
 
-      if repair && corrupted_blobs > 0
-        erase_index_files
-        defragmentize
-        regenerate_index_and_spaces
-      elsif corrupted_blobs == 0
+      if corrupted_blobs == 0
         # Now we check the index data. It must be correct and the entries must
         # match the blob file. All entries in the index must be in the blob file
         # and vise versa.
         begin
           index_ok = @index.check do |id, address|
-            has_id_at?(id, address)
+            unless has_id_at?(id, address)
+              PEROBS.log.error "Index contains an entry for " +
+                "ID #{id} at address #{address} that is not in FlatFile"
+              false
+            else
+              true
+            end
           end
           x_check_errs = 0
           space_check_ok = true
@@ -586,16 +563,13 @@ module PEROBS
             (x_check_errs = cross_check_entries) == 0
             errors += 1 unless index_ok && space_check_ok
             errors += x_check_errs
-            regenerate_index_and_spaces if repair
           end
         rescue PEROBS::FatalError
           errors += 1
-          regenerate_index_and_spaces if repair
         end
       end
 
-      sync if repair
-      PEROBS.log.info "check_db completed in #{Time.now - t} seconds. " +
+      PEROBS.log.info "FlatFile check completed in #{Time.now - t} seconds. " +
         "#{errors} errors found."
 
       errors
@@ -687,17 +661,7 @@ module PEROBS
                                                         header.id)
               # We have two blobs with the same ID and we must discard one of
               # them.
-              if header.is_outdated?
-                discard_damaged_blob(header)
-              elsif previous_header.is_outdated?
-                discard_damaged_blob(previous_header)
-              else
-                PEROBS.log.error "None of the blobs with same ID have " +
-                  "the outdated flag set. Deleting the smaller one."
-                errors += 1
-                discard_damaged_blob(header.length < previous_header.length ?
-                                     header : previous_header)
-              end
+              discard_duplicate_blobs(header, previous_header)
             else
               # ID is unique so far. Add it to the shadow index.
               @index.insert(header.id, header.addr)
@@ -927,6 +891,23 @@ module PEROBS
       header.clear_flags
     end
 
+    def discard_duplicate_blobs(header, previous_header)
+      if header.is_outdated?
+        discard_damaged_blob(header)
+      elsif previous_header.is_outdated?
+        discard_damaged_blob(previous_header)
+      else
+        smaller, larger = header.length < previous_header.length ?
+          [ header, previous_header ] : [ previous_header, header ]
+        PEROBS.log.error "None of the blobs with same ID have " +
+          "the outdated flag set. Deleting the smaller one " +
+          "at address #{smaller.addr}"
+        discard_damaged_blob(smaller)
+        @space_list.add_space(smaller.addr, smaller.length)
+        @index.insert(larger.id, larger.addr)
+      end
+    end
+
     def open_index_files(abort_on_missing_files = false)
       begin
         @index.open(abort_on_missing_files)

data/lib/perobs/FuzzyStringMatcher.rb

@@ -26,40 +26,42 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'perobs/Log'
-require 'perobs/ObjectBase'
+require 'perobs/Object'
 
 module PEROBS
 
   # The fuzzy string matcher can be used to perform a fuzzy string search
   # against a known set of strings. The dictionary of known strings does not
-  # store the actual strings but references to arbitrary objects. These could
-  # be the string, but can be something else related to the learned strings.
-  # To use this class a list of strings with their references must be learned.
-  # Once the dictionary has been established, fuzzy matches can be done.
-  class FuzzyStringMatcher
+  # store the actual strings but references to String or PEROBS objects.
+  # Once the dictionary has been established, fuzzy matches can be done. Since
+  # the actual input strings are not directly stored, you cannot remove or
+  # modified already stored strings. To remove strings, you have to clear the
+  # matcher and add the strings again that you want to keep.
+  class FuzzyStringMatcher < PEROBS::Object
+
+    attr_persist :case_sensitive, :n, :dict
 
     # Create a new FuzzyStringMatcher.
-    # @param store [PEROBS::Store] place to store the dictionary
-    # @param name [String] Unique name of the string matcher
+    # @param p [PEROBS::Store] place to store the dictionary
     # @param case_sensitive [Boolean] True if case matters for matching
     # @param n [Integer] Determines what kind of n-gramm is used to store the
     #        references in the dictionary. It also determines the minimum word
-    #        length that can be used for fuzzy matches.
-    def initialize(store, name, case_sensitive = false, n = 4)
-      @store = store
-      @dict_name = "FuzzyStringMatcher::#{name}"
+    #        length that can be used for fuzzy matches. Values between 2 and
+    #        10 are supported. The default is 4.
+    def initialize(p, case_sensitive = false, n = 4)
+      super(p)
       if n < 2 || n > 10
         raise ArgumentError, 'n must be between 2 and 10'
       end
-      @case_sensitive = case_sensitive
-      @n = n
+      self.case_sensitive = case_sensitive
+      self.n = n
 
-      clear unless (@dict = @store[@dict_name])
+      clear unless @dict
     end
 
     # Wipe the dictionary.
     def clear
-      @store[@dict_name] = @dict = @store.new(BigHash)
+      self.dict = @store.new(BigHash)
     end
 
     # Add a string with its reference to the dictionary.
@@ -79,11 +81,8 @@ module PEROBS
           @dict[n_gramm] = ng_list = @store.new(Hash)
         end
 
-        if ng_list.include?(reference)
-          ng_list[reference] += 1
-        else
-          ng_list[reference] = 0
-        end
+        # We use the Hash as a Set. The value doesn't matter.
+        ng_list[reference] = true unless ng_list.include?(reference)
       end
 
       nil
@@ -109,22 +108,12 @@ module PEROBS
 
       matches = {}
 
-      # This will be the best possible score for a perfect match.
-      best_possible_score = 0
       each_n_gramm(string) do |n_gramm|
-        best_possible_score += 1
         if (ng_list = @dict[n_gramm])
-          ng_list.each do |reference, count|
+          ng_list.each do |reference, dummy|
             if matches.include?(reference)
               matches[reference] += 1
             else
-              # We use internally a 10 times larger list so that we don't
-              # throw away good matches too early. If the max_count value is
-              # chosen too small there is a risk of not finding the best
-              # matches!
-              if matches.size > 10 * max_count
-                matches = discard_worst_match(matches)
-              end
               matches[reference] = 1
             end
           end
@@ -133,19 +122,23 @@ module PEROBS
 
       return [] if matches.empty?
 
-      # Sort in the order of occurance count downwards.
-      match_list = matches.to_a.sort do |a, b|
-        b[1] <=> a[1]
-      end
+      match_list = matches.to_a
 
       # Set occurance counters to scores relative to the best possible score.
+      # This will be the best possible score for a perfect match.
+      best_possible_score = string.length - @n + 1
       match_list.map! { |a, b| [ a, b.to_f / best_possible_score ] }
 
-      # Delete all matches that occured less than half as often than the
-      # top match.
+      # Delete all matches that don't have the required minimum match score.
       match_list.delete_if { |a| a[1] < min_score }
 
-      match_list[0..max_count]
+      # Sort the list best to worst match
+      match_list.sort! do |a, b|
+        b[1] <=> a[1]
+      end
+
+      # Return the top max_count matches.
+      match_list[0..max_count - 1]
     end
 
     # Returns some internal stats about the dictionary.
@@ -176,16 +169,6 @@ module PEROBS
       end
     end
 
-    def discard_worst_match(matches)
-      # Sort in the order of occurance count downwards.
-      match_list = matches.to_a.sort do |a, b|
-        b[1] <=> a[1]
-      end
-      # Discard the lowest half of the matches
-      match_list = match_list[0..match_list.length / 2]
-      match_list.to_h
-    end
-
   end
 
 end

data/lib/perobs/Hash.rb

@@ -124,9 +124,9 @@ module PEROBS
 
     # Proxy for assignment method.
     def []=(key, value)
-      unless key.is_a?(String)
-        raise ArgumentError, "PEROBS::Hash[] key must be a String but is a " +
-          "#{key.class}"
+      unless key.is_a?(String) || key.respond_to?(:is_poxreference?)
+        raise ArgumentError, "PEROBS::Hash[] key must be a String or " +
+          "a PEROBS object but is a #{key.class}"
       end
       _check_assignment_value(value)
       @store.cache.cache_write(self)
@@ -143,18 +143,33 @@ module PEROBS
     # is referencing.
     # @return [Array of Integer] IDs of referenced objects
     def _referenced_object_ids
-      @data.each_value.select { |v| v && v.respond_to?(:is_poxreference?) }.
-        map { |o| o.id }
+      ids = []
+      @data.each do |k, v|
+        if k && k.respond_to?(:is_poxreference?)
+          ids << k.id
+        end
+        if v && v.respond_to?(:is_poxreference?)
+          ids << v.id
+        end
+      end
+
+      ids
     end
 
     # This method should only be used during store repair operations. It will
     # delete all referenced to the given object ID.
     # @param id [Integer] targeted object ID
     def _delete_reference_to_id(id)
+      original_length = @data.length
+
       @data.delete_if do |k, v|
-        v && v.respond_to?(:is_poxreference?) && v.id == id
+        (k && k.respond_to?(:is_poxreference?) && k.id == id) ||
+        (v && v.respond_to?(:is_poxreference?) && v.id == id)
+      end
+
+      if @data.length != original_length
+        @store.cache.cache_write(self)
       end
-      @store.cache.cache_write(self)
     end
 
     # Restore the persistent data from a single data structure.
@@ -163,8 +178,18 @@ module PEROBS
     # @private
     def _deserialize(data)
       @data = {}
-      data.each { |k, v| @data[k] = v.is_a?(POReference) ?
-                                    POXReference.new(@store, v.id) : v }
+
+      data.each do |k, v|
+        # References to other PEROBS Objects are marshalled with our own
+        # format. If we detect such a marshalled String we convert it into a
+        # POXReference object.
+        if (match = /^#<PEROBS::POReference id=([0-9]+)>$/.match(k))
+          k = POXReference.new(@store, match[1].to_i)
+        end
+        dv = v.is_a?(POReference) ? POXReference.new(@store, v.id) : v
+        @data[k] = dv
+      end
+
       @data
     end
 
@@ -185,26 +210,46 @@ module PEROBS
       data = {}
 
       @data.each do |k, v|
-        if v.respond_to?(:is_poxreference?)
-          data[k] = POReference.new(v.id)
-        else
-          # Outside of the PEROBS library all PEROBS::ObjectBase derived
-          # objects should not be used directly. The library only exposes them
-          # via POXReference proxy objects.
-          if v.is_a?(ObjectBase)
-            PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
-              "It is stored in a PEROBS::Hash with key #{k.inspect}. " +
-              'Have you used self() instead of myself() to ' +
-              "get the reference of this PEROBS object?\n" +
-              v.inspect
-          end
-          data[k] = v
+        if k.respond_to?(:is_poxreference?)
+          # JSON only supports Strings as hash keys. Since JSON is the default
+          # internal storage format in the database, we have to marshall
+          # PEROBS::Object references ourselves.
+          k = "#<PEROBS::POReference id=#{k.id}>"
+        elsif k[0..24] == '#<PEROBS::POReference id='
+          # This could obviously result in conflicts with 'normal' String hash
+          # keys. This is extremely unlikely, but we better catch this case
+          # before it causes hard to debug trouble.
+          raise ArgumentError, "Hash key #{k} conflicts with PEROBS " +
+            "internal representation of marshalled hash keys!"
         end
+        data[k] = serialize_helper(v)
       end
 
       data
     end
 
+    def serialize_helper(v)
+      if v.respond_to?(:is_poxreference?)
+        # References to other PEROBS objects (POXReference) are stored as
+        # POReference in the database.
+        return POReference.new(v.id)
+      else
+        # Outside of the PEROBS library all PEROBS::ObjectBase derived
+        # objects should not be used directly. The library only exposes them
+        # via POXReference proxy objects.
+        if v.is_a?(ObjectBase)
+          PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
+            "It is stored in a PEROBS::Hash. " +
+            'Have you used self() instead of myself() to ' +
+            "get the reference of this PEROBS object?\n" +
+            v.inspect
+        end
+
+        # All other objects are serialized by their native methods.
+        return v
+      end
+    end
+
   end
 
 end

data/lib/perobs/IDListPageFile.rb

@@ -54,7 +54,8 @@ module PEROBS
       @file_name = File.join(dir, name + '.cache')
       @page_size = page_size
       open
-      @pages = PersistentObjectCache.new(max_in_memory, -1, IDListPage, self)
+      @pages = PersistentObjectCache.new(max_in_memory, max_in_memory,
+                                         IDListPage, self)
       @page_counter = 0
     end
 

data/lib/perobs/ObjectBase.rb

@@ -102,6 +102,13 @@ module PEROBS
       end
     end
 
+    # To allow POXReference objects to be used as Hash keys we need to
+    # implement this function. Conveniently, we can just use the PEROBS object
+    # ID since that is unique.
+    def hash
+      @id
+    end
+
     # Shortcut to access the _id() method of the referenced object.
     def _id
       @id

data/lib/perobs/SpaceTree.rb

@@ -54,7 +54,7 @@ module PEROBS
 
       # Benchmark runs showed a cache size of 128 to be a good compromise
       # between read and write performance trade-offs and memory consumption.
-      @cache = PersistentObjectCache.new(256, -1, SpaceTreeNode, self)
+      @cache = PersistentObjectCache.new(256, 256, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.

data/lib/perobs/Store.rb

@@ -599,6 +599,7 @@ module PEROBS
       @stats.swept_objects = @db.delete_unmarked_objects do |id|
         @cache.evict(id)
       end
+      @db.clear_marks
       GC.start
       PEROBS.log.debug "#{@stats.swept_objects} objects collected"
       @stats.swept_objects

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "4.2.0"
+  VERSION = "4.3.0"
 end

data/test/FlatFileDB_spec.rb

@@ -265,5 +265,35 @@ describe PEROBS::FlatFileDB do
     db.close
   end
 
+  it 'should handle duplicate entries for the same ID in database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    # This appends the entry 2 again
+    blob2 = File.read(db_file, 319 - 199, 199)
+    File.write(db_file, blob2, File.size(db_file))
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::WARN })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(5) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    db.close
+  end
+
 end
 

data/test/FuzzyStringMatcher_spec.rb

@@ -29,13 +29,25 @@ require 'perobs/FuzzyStringMatcher'
 
 module PEROBS
 
+  class WordRef < PEROBS::Object
+
+    attr_persist :word, :line
+
+    def initialize(store, word, line)
+      super(store)
+      self.word = word
+      self.line = line
+    end
+
+  end
+
   describe FuzzyStringMatcher do
 
     before(:all) do
       @db_name = generate_db_name(__FILE__)
       @store = PEROBS::Store.new(@db_name)
-      @fsm = FuzzyStringMatcher.new(@store, 'test')
-      @fsm2 = FuzzyStringMatcher.new(@store, 'test', true, 2)
+      @store['fsm'] = @fsm = @store.new(FuzzyStringMatcher)
+      @store['fsm2'] = @fsm2 = @store.new(FuzzyStringMatcher, true, 2)
     end
 
     after(:all) do
@@ -103,6 +115,44 @@ module PEROBS
       expect(@fsm.best_matches('foobar')).to eql([])
     end
 
+    it 'should find a match' do
+      dut = {
+        [ 'one' ] => [ [ 'one', 1.0 ] ],
+        [ 'three' ] => [ [ 'three', 1.0 ] ],
+        [ 'four' ]=> [ [ 'four', 1.0 ], [ 'fourteen', 0.666 ] ],
+        [ 'four', 1.0 ]=> [ [ 'four', 1.0 ] ],
+        [ 'even' ] => [ [ 'seven', 0.666 ], [ 'eleven', 0.666 ] ],
+        [ 'teen' ] => [ ['thirteen', 0.6666666666666666],
+                      ['fourteen', 0.6666666666666666],
+                      ['fifteen', 0.6666666666666666],
+                      ['sixteen', 0.6666666666666666],
+                      ['seventeen', 0.6666666666666666],
+                      ['eighteen', 0.6666666666666666],
+                      ['nineteen', 0.6666666666666666] ],
+        [ 'aight' ] => [ [ 'eight', 0.5 ] ],
+        [ 'thirdteen' ] => [ [ 'thirteen', 0.5 ] ],
+        [ 'shirt teen', 0.3 ] => [ [ 'thirteen', 0.333 ] ]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
+    it 'should sort best to worst matches' do
+      @fsm.clear
+      %w( xbar xfoox foor bar foobar barfoo foo rab baar fool xbarx
+          foobarx xfoobarx foo_bar ).each do |w|
+        @fsm.learn(w, w)
+      end
+      dut = {
+        [ 'foo' ] => [["foo", 1.0], ["foor", 0.5], ["foobar", 0.5],
+                      ["fool", 0.5], ["foobarx", 0.5], ["foo_bar", 0.5],
+                      ["barfoo", 0.5]],
+        [ 'bar' ] => [["bar", 1.0], ["barfoo", 0.5], ["xbar", 0.5],
+                      ["foobar", 0.5], ["foo_bar", 0.5]],
+        [ 'foobar' ] => [["foobar", 1.0], ["foobarx", 0.8], ["xfoobarx", 0.6]]
+      }
+      check_data_under_test(@fsm, dut)
+    end
+
     it 'should handle a larger text' do
       text =<<-EOT
 MIT License
@@ -131,9 +181,9 @@ EOT
         @fsm2.learn(word, word)
       end
       stats = @fsm2.stats
-      expect(stats['dictionary_size']).to eql(363)
+      expect(stats['dictionary_size']).to eql(352)
       expect(stats['max_list_size']).to eql(22)
-      expect(stats['avg_list_size']).to be_within(0.001).of(2.366)
+      expect(stats['avg_list_size']).to be_within(0.001).of(2.409)
     end
 
     it 'should find case sensitive matches' do
@@ -145,6 +195,46 @@ EOT
       check_data_under_test(@fsm2, dut)
     end
 
+    it 'should support references to PEROBS objects' do
+      text =<<-EOT
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+EOT
+
+      line_no = 1
+      @store['fsm'] = fsm = @store.new(FuzzyStringMatcher)
+      @store['refs'] = refs = @store.new(Array)
+      text.each_line do |line|
+        line.split.each do |word|
+          ref = @store.new(WordRef, word, line_no)
+          refs << ref
+          fsm.learn(word, ref)
+        end
+        line_no += 1
+      end
+
+      found_lines = []
+      fsm.best_matches('SOFTWARE').each do |match|
+        found_lines << match[0].line
+      end
+      expect(found_lines.sort).to eql([ 4, 5, 5, 7, 8 ])
+    end
+
+    it 'should with small search words' do
+      @fsm.clear
+      mats = 'Yukihiro Matsumoto'
+      @fsm.learn(mats)
+      expect(@fsm.best_matches('Yukihiro').first.first).to eql(mats)
+      expect(@fsm.best_matches('Mats', 0.3).first.first).to eql(mats)
+    end
+
     def check_data_under_test(fsm, dut)
       dut.each do |inputs, reference|
         key = inputs[0]

data/test/Hash_spec.rb

@@ -31,7 +31,6 @@ require 'spec_helper'
 
 require 'perobs'
 
-
 class PO < PEROBS::Object
 
   attr_persist :name
@@ -68,9 +67,13 @@ describe PEROBS::Hash do
     h['po'] = po = @store.new(PO)
     po.name = 'foobar'
     h['b'] = 'B'
+    @store['po_key'] = po_key = @store.new(PO)
+    po_key.name = 'po key'
+    h[po_key] = 'PO Key'
 
     expect(h['a']).to eq('A')
     expect(h['b']).to eq('B')
+    expect(h[@store['po_key']]).to eq('PO Key')
     @store.exit
 
     @store = PEROBS::Store.new(@db_name)
@@ -78,6 +81,14 @@ describe PEROBS::Hash do
     expect(h['a']).to eq('A')
     expect(h['b']).to eq('B')
     expect(h['po'].name).to eq('foobar')
+    po_key = @store['po_key']
+    expect(po_key.name).to eq('po key')
+    expect(h[po_key]).to eq('PO Key')
+  end
+
+  it 'should not allow hash keys that conflict with internal notations' do
+    @store['h'] = h = @store.new(PEROBS::Hash)
+    expect { h['#<PEROBS::POReference id=1234>'] = 'foo'; @store.sync }.to raise_error(ArgumentError)
   end
 
   it 'should have an each method to iterate' do

data/test/Store_spec.rb

@@ -251,6 +251,20 @@ describe PEROBS::Store do
     end
     expect(i).to eq(6)
 
+    capture_io { store.gc }
+    capture_io { expect { store.check }.to_not raise_error }
+    capture_io { store.exit }
+
+    store = PEROBS::Store.new(@db_file)
+    capture_io { expect { store.check }.to_not raise_error }
+
+    person = store['person1']
+    i = 0
+    while (person = person.related) do
+      i += 1
+    end
+    expect(i).to eq(6)
+
     capture_io { store.gc }
     capture_io { expect { store.check }.to_not raise_error }
     expect { store.delete_store }.to_not raise_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors:
 - Chris Schlaeger
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-02 00:00:00.000000000 Z
+date: 2021-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -147,7 +147,7 @@ homepage: https://github.com/scrapper/perobs
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -162,9 +162,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Persistent Ruby Object Store
 test_files: