perobs 3.0.1 → 4.3.0

data/lib/perobs/FuzzyStringMatcher.rb

@@ -0,0 +1,175 @@
+# encoding: UTF-8
+#
+# = FuzzyStringMatcher.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+require 'perobs/Object'
+
+module PEROBS
+
+  # The fuzzy string matcher can be used to perform a fuzzy string search
+  # against a known set of strings. The dictionary of known strings does not
+  # store the actual strings but references to String or PEROBS objects.
+  # Once the dictionary has been established, fuzzy matches can be done. Since
+  # the actual input strings are not directly stored, you cannot remove or
+  # modified already stored strings. To remove strings, you have to clear the
+  # matcher and add the strings again that you want to keep.
+  class FuzzyStringMatcher < PEROBS::Object
+
+    attr_persist :case_sensitive, :n, :dict
+
+    # Create a new FuzzyStringMatcher.
+    # @param p [PEROBS::Store] place to store the dictionary
+    # @param case_sensitive [Boolean] True if case matters for matching
+    # @param n [Integer] Determines what kind of n-gramm is used to store the
+    #        references in the dictionary. It also determines the minimum word
+    #        length that can be used for fuzzy matches. Values between 2 and
+    #        10 are supported. The default is 4.
+    def initialize(p, case_sensitive = false, n = 4)
+      super(p)
+      if n < 2 || n > 10
+        raise ArgumentError, 'n must be between 2 and 10'
+      end
+      self.case_sensitive = case_sensitive
+      self.n = n
+
+      clear unless @dict
+    end
+
+    # Wipe the dictionary.
+    def clear
+      self.dict = @store.new(BigHash)
+    end
+
+    # Add a string with its reference to the dictionary.
+    # @param string [String] The string to store
+    # @param reference [Object] Any object that is associated with the string
+    def learn(string, reference = string)
+      reference = string if reference.nil?
+
+      unless @case_sensitive
+        string = string.downcase
+      end
+      # Enclose string in 'start of text' and 'end of text' ASCII values.
+      string = "\002" + string + "\003"
+
+      each_n_gramm(string) do |n_gramm|
+        unless (ng_list = @dict[n_gramm])
+          @dict[n_gramm] = ng_list = @store.new(Hash)
+        end
+
+        # We use the Hash as a Set. The value doesn't matter.
+        ng_list[reference] = true unless ng_list.include?(reference)
+      end
+
+      nil
+    end
+
+    # Find the references who's string best matches the given string.
+    # @param string [String] string to search for
+    # @param min_score [Float] Value 0.01 and 1.0 that specifies how strict
+    #        the matching should be done. The larger the value the more closer
+    #        the given string needs to be.
+    # @param max_count [Integer] The maximum number of matches that should be
+    #        returned.
+    # @return [Array] The result is an Array of Arrays. The nested Arrays only
+    #         have 2 entries. The reference and a Float value between 0 and
+    #         1.0 that describes how good the match is. The matches are sorted
+    #         in descending order by the match score.
+    def best_matches(string, min_score = 0.5, max_count = 100)
+      unless @case_sensitive
+        string = string.downcase
+      end
+      # Enclose string in 'start of text' and 'end of text' ASCII values.
+      string = "\002" + string + "\003"
+
+      matches = {}
+
+      each_n_gramm(string) do |n_gramm|
+        if (ng_list = @dict[n_gramm])
+          ng_list.each do |reference, dummy|
+            if matches.include?(reference)
+              matches[reference] += 1
+            else
+              matches[reference] = 1
+            end
+          end
+        end
+      end
+
+      return [] if matches.empty?
+
+      match_list = matches.to_a
+
+      # Set occurance counters to scores relative to the best possible score.
+      # This will be the best possible score for a perfect match.
+      best_possible_score = string.length - @n + 1
+      match_list.map! { |a, b| [ a, b.to_f / best_possible_score ] }
+
+      # Delete all matches that don't have the required minimum match score.
+      match_list.delete_if { |a| a[1] < min_score }
+
+      # Sort the list best to worst match
+      match_list.sort! do |a, b|
+        b[1] <=> a[1]
+      end
+
+      # Return the top max_count matches.
+      match_list[0..max_count - 1]
+    end
+
+    # Returns some internal stats about the dictionary.
+    def stats
+      s = {}
+      s['dictionary_size'] = @dict.size
+      max = total = 0
+      @dict.each do |n_gramm, ng_list|
+        size = ng_list.length
+        max = size if size > max
+        total += size
+      end
+      s['max_list_size'] = max
+      s['avg_list_size'] = total > 0 ? total.to_f / s['dictionary_size'] : 0
+
+      s
+    end
+
+    private
+
+    def each_n_gramm(string, &block)
+      return if string.length < @n
+
+      0.upto(string.length - @n) do |i|
+        n_gramm = string[i, @n]
+
+        yield(n_gramm)
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/Hash.rb

@@ -2,7 +2,7 @@
 #
 # = Hash.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -37,20 +37,36 @@ module PEROBS
   # The implementation is largely a proxy around the standard Hash class. But
   # all mutating methods must be re-implemented to convert PEROBS::Objects to
   # POXReference objects and to register the object as modified with the
-  # cache.
+  # cache. However, it is not designed for large data sets as it always reads
+  # and writes the full data set for every access (unless it is cached). For
+  # data sets that could have more than a few hundred entries BigHash is the
+  # recommended alternative.
   #
   # We explicitely don't support Hash::store() as it conflicts with
   # ObjectBase::store() method to access the store.
   class Hash < ObjectBase
 
+    # These methods do not mutate the Hash. They only perform read
+    # operations and return a new PEROBS::Hash object.
+    ([
+      :invert, :merge, :reject, :select
+    ] + Enumerable.instance_methods).uniq.each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the read operation with the cache.
+        @store.cache.cache_read(self)
+        @store.new(PEROBS::Hash, @data.send(method_sym, *args, &block))
+      end
+    end
+
     # These methods do not mutate the Hash. They only perform read
     # operations.
     ([
       :==, :[], :assoc, :compare_by_identity, :compare_by_identity?, :default,
       :default_proc, :each, :each_key, :each_pair, :each_value, :empty?,
       :eql?, :fetch, :flatten, :has_key?, :has_value?, :hash, :include?,
-      :invert, :key, :key?, :keys, :length, :member?, :merge,
-      :pretty_print, :pretty_print_cycle, :rassoc, :reject, :select, :size,
+      :key, :key?, :keys, :length, :member?,
+      :pretty_print, :pretty_print_cycle, :rassoc, :size,
       :to_a, :to_h, :to_hash, :to_s, :value?, :values, :values_at
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
       # Create a wrapper method that passes the call to @data.
@@ -61,11 +77,22 @@ module PEROBS
       end
     end
 
-    # These methods mutate the Hash.
+    # These methods mutate the Hash and return self
+    [
+      :clear, :keep_if, :merge!, :rehash, :reject!, :replace, :select!, :update
+    ].each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the write operation with the cache.
+        @store.cache.cache_write(self)
+        @data.send(method_sym, *args, &block)
+        myself
+      end
+    end
+
+    # These methods mutate the Hash and return basic Ruby type objects.
     [
-      :[]=, :clear, :default=, :default_proc=, :delete, :delete_if,
-      :initialize_copy, :keep_if, :merge!, :rehash, :reject!, :replace,
-      :select!, :shift, :update
+      :delete, :delete_if, :shift
     ].each do |method_sym|
       # Create a wrapper method that passes the call to @data.
       define_method(method_sym) do |*args, &block|
@@ -79,33 +106,70 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param default [Any] The default value that is returned when no value is
-    #        stored for a specific key.
-    def initialize(p, default = nil)
+    # @param default [Object] The default value that is returned when no value
+    #        is stored for a specific key. The default must be of the
+    #        supported type.
+    def initialize(p, default = nil, &block)
       super(p)
-      @default = nil
-      @data = {}
+      _check_assignment_value(default)
+      if block_given?
+        @data = ::Hash.new(&block)
+      else
+        @data = ::Hash.new(default)
+      end
 
       # Ensure that the newly created object will be pushed into the database.
       @store.cache.cache_write(self)
     end
 
+    # Proxy for assignment method.
+    def []=(key, value)
+      unless key.is_a?(String) || key.respond_to?(:is_poxreference?)
+        raise ArgumentError, "PEROBS::Hash[] key must be a String or " +
+          "a PEROBS object but is a #{key.class}"
+      end
+      _check_assignment_value(value)
+      @store.cache.cache_write(self)
+      @data[key] = value
+    end
+
+    # Proxy for default= method.
+    def default=(value)
+      _check_assignment_value(value)
+      @data.default=(value)
+    end
+
     # Return a list of all object IDs of all persistend objects that this Hash
     # is referencing.
-    # @return [Array of Fixnum or Bignum] IDs of referenced objects
+    # @return [Array of Integer] IDs of referenced objects
     def _referenced_object_ids
-      @data.each_value.select { |v| v && v.respond_to?(:is_poxreference?) }.
-        map { |o| o.id }
+      ids = []
+      @data.each do |k, v|
+        if k && k.respond_to?(:is_poxreference?)
+          ids << k.id
+        end
+        if v && v.respond_to?(:is_poxreference?)
+          ids << v.id
+        end
+      end
+
+      ids
     end
 
     # This method should only be used during store repair operations. It will
     # delete all referenced to the given object ID.
-    # @param id [Fixnum/Bignum] targeted object ID
+    # @param id [Integer] targeted object ID
     def _delete_reference_to_id(id)
+      original_length = @data.length
+
       @data.delete_if do |k, v|
-        v && v.respond_to?(:is_poxreference?) && v.id == id
+        (k && k.respond_to?(:is_poxreference?) && k.id == id) ||
+        (v && v.respond_to?(:is_poxreference?) && v.id == id)
+      end
+
+      if @data.length != original_length
+        @store.cache.cache_write(self)
       end
-      @store.cache.cache_write(self)
     end
 
     # Restore the persistent data from a single data structure.
@@ -114,8 +178,18 @@ module PEROBS
     # @private
     def _deserialize(data)
       @data = {}
-      data.each { |k, v| @data[k] = v.is_a?(POReference) ?
-                                    POXReference.new(@store, v.id) : v }
+
+      data.each do |k, v|
+        # References to other PEROBS Objects are marshalled with our own
+        # format. If we detect such a marshalled String we convert it into a
+        # POXReference object.
+        if (match = /^#<PEROBS::POReference id=([0-9]+)>$/.match(k))
+          k = POXReference.new(@store, match[1].to_i)
+        end
+        dv = v.is_a?(POReference) ? POXReference.new(@store, v.id) : v
+        @data[k] = dv
+      end
+
       @data
     end
 
@@ -136,26 +210,46 @@ module PEROBS
       data = {}
 
       @data.each do |k, v|
-        if v.respond_to?(:is_poxreference?)
-          data[k] = POReference.new(v.id)
-        else
-          # Outside of the PEROBS library all PEROBS::ObjectBase derived
-          # objects should not be used directly. The library only exposes them
-          # via POXReference proxy objects.
-          if v.is_a?(ObjectBase)
-            PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
-              "It is stored in a PEROBS::Hash with key #{k.inspect}. " +
-              'Have you used self() instead of myself() to ' +
-              "get the reference of this PEROBS object?\n" +
-              v.inspect
-          end
-          data[k] = v
+        if k.respond_to?(:is_poxreference?)
+          # JSON only supports Strings as hash keys. Since JSON is the default
+          # internal storage format in the database, we have to marshall
+          # PEROBS::Object references ourselves.
+          k = "#<PEROBS::POReference id=#{k.id}>"
+        elsif k[0..24] == '#<PEROBS::POReference id='
+          # This could obviously result in conflicts with 'normal' String hash
+          # keys. This is extremely unlikely, but we better catch this case
+          # before it causes hard to debug trouble.
+          raise ArgumentError, "Hash key #{k} conflicts with PEROBS " +
+            "internal representation of marshalled hash keys!"
         end
+        data[k] = serialize_helper(v)
       end
 
       data
     end
 
+    def serialize_helper(v)
+      if v.respond_to?(:is_poxreference?)
+        # References to other PEROBS objects (POXReference) are stored as
+        # POReference in the database.
+        return POReference.new(v.id)
+      else
+        # Outside of the PEROBS library all PEROBS::ObjectBase derived
+        # objects should not be used directly. The library only exposes them
+        # via POXReference proxy objects.
+        if v.is_a?(ObjectBase)
+          PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
+            "It is stored in a PEROBS::Hash. " +
+            'Have you used self() instead of myself() to ' +
+            "get the reference of this PEROBS object?\n" +
+            v.inspect
+        end
+
+        # All other objects are serialized by their native methods.
+        return v
+      end
+    end
+
   end
 
 end

data/lib/perobs/IDList.rb

@@ -0,0 +1,144 @@
+# encoding: UTF-8
+#
+# = IDList.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/IDListPageFile'
+require 'perobs/IDListPageRecord'
+
+module PEROBS
+
+  # This class stores a list of 64 bit values. Values can be added to the list
+  # and the presence of a certain value can be checked. It can hold up to 2^64
+  # values. It tries to keep values in memory but can store them in a file if
+  # needed. A threshold for the in-memory values can be set in the
+  # constructor. The stored values are grouped in pages. Each page can hold up
+  # to page_size entries.
+  class IDList
+
+    # Create a new IDList object. The data that can't be kept in memory will
+    # be stored in the specified directory under the given name.
+    # @param dir [String] Path of the directory
+    # @param name [String] Name of the file
+    # @param max_in_memory [Integer] Specifies the maximum number of values
+    #        that will be kept in memory. If the list is larger, values will
+    #        be cached in the specified file.
+    # @param page_size [Integer] The number of values per page. The default
+    #        value is 32 which was found the best performing config in  tests.
+    def initialize(dir, name, max_in_memory, page_size = 32)
+      # The page_file manages the pages that store the values.
+      @page_file = IDListPageFile.new(self, dir, name,
+                                      max_in_memory, page_size)
+      clear
+    end
+
+    # Insert a new value into the list.
+    # @param id [Integer] The value to add
+    def insert(id)
+      # Find the index of the page that should hold ID.
+      index = @page_records.bsearch_index { |pr| pr.max_id >= id }
+      # Get the corresponding IDListPageRecord object.
+      page = @page_records[index]
+
+      # In case the page is already full we'll have to create a new page.
+      # There is no guarantee that a split will yield an page with space as we
+      # split by ID range, not by distributing the values evenly across the
+      # two pages.
+      while page.is_full?
+        new_page = page.split
+        # Store the newly created page into the page_records list.
+        @page_records.insert(index + 1, new_page)
+        if id >= new_page.min_id
+          # We need to insert the ID into the newly created page. Adjust index
+          # and page reference accordingly.
+          index += 1
+          page = new_page
+        end
+      end
+
+      # Insert the ID into the page.
+      page.insert(id)
+    end
+
+    # Check if a given value is already stored in the list.
+    # @param id [Integer] The value to check for
+    def include?(id)
+      @page_records.bsearch { |pr| pr.max_id >= id }.include?(id)
+    end
+
+    # Clear the list and empty the filesystem cache file.
+    def clear
+      @page_file.clear
+      @page_records = [ IDListPageRecord.new(@page_file, 0, 2 ** 64) ]
+    end
+
+    # Erase the list including the filesystem cache file. The IDList is no
+    # longer usable after this call but the cache file is removed from the
+    # filesystem.
+    def erase
+      @page_file.erase
+      @page_records = nil
+    end
+
+    # Perform some consistency checks on the internal data structures. Raises
+    # a RuntimeError in case a problem is found.
+    def check
+      last_max = -1
+      unless (min_id = @page_records.first.min_id) == 0
+        raise RuntimeError, "min_id of first record (#{min_id}) " +
+          "must be 0."
+      end
+
+      @page_records.each do |pr|
+        unless pr.min_id == last_max + 1
+          raise RuntimeError, "max_id of previous record (#{last_max}) " +
+            "must be exactly 1 smaller than current record (#{pr.min_id})."
+        end
+        last_max = pr.max_id
+        pr.check
+      end
+
+      unless last_max == 2 ** 64
+        raise RuntimeError, "max_id of last records " +
+          "(#{@page_records.last.max_id}) must be #{2 ** 64})."
+      end
+    end
+
+    def to_a
+      a = []
+      @page_records.each { |pr| a += pr.values }
+      a
+    end
+
+    # Print a human readable form of the tree that stores the list. This is
+    # only meant for debugging purposes and does not scale for larger trees.
+    def to_s
+      "\n" + @root.to_s
+    end
+
+  end
+
+end
+