perobs 4.0.0 → 4.4.0

data/lib/perobs/Hash.rb

@@ -2,7 +2,7 @@
 #
 # = Hash.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -37,20 +37,36 @@ module PEROBS
   # The implementation is largely a proxy around the standard Hash class. But
   # all mutating methods must be re-implemented to convert PEROBS::Objects to
   # POXReference objects and to register the object as modified with the
-  # cache.
+  # cache. However, it is not designed for large data sets as it always reads
+  # and writes the full data set for every access (unless it is cached). For
+  # data sets that could have more than a few hundred entries BigHash is the
+  # recommended alternative.
   #
   # We explicitely don't support Hash::store() as it conflicts with
   # ObjectBase::store() method to access the store.
   class Hash < ObjectBase
 
+    # These methods do not mutate the Hash. They only perform read
+    # operations and return a new PEROBS::Hash object.
+    ([
+      :invert, :merge, :reject, :select
+    ] + Enumerable.instance_methods).uniq.each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the read operation with the cache.
+        @store.cache.cache_read(self)
+        @store.new(PEROBS::Hash, @data.send(method_sym, *args, &block))
+      end
+    end
+
     # These methods do not mutate the Hash. They only perform read
     # operations.
     ([
       :==, :[], :assoc, :compare_by_identity, :compare_by_identity?, :default,
       :default_proc, :each, :each_key, :each_pair, :each_value, :empty?,
       :eql?, :fetch, :flatten, :has_key?, :has_value?, :hash, :include?,
-      :invert, :key, :key?, :keys, :length, :member?, :merge,
-      :pretty_print, :pretty_print_cycle, :rassoc, :reject, :select, :size,
+      :key, :key?, :keys, :length, :member?,
+      :pretty_print, :pretty_print_cycle, :rassoc, :size,
       :to_a, :to_h, :to_hash, :to_s, :value?, :values, :values_at
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
       # Create a wrapper method that passes the call to @data.
@@ -61,11 +77,22 @@ module PEROBS
       end
     end
 
-    # These methods mutate the Hash.
+    # These methods mutate the Hash and return self
+    [
+      :clear, :keep_if, :merge!, :rehash, :reject!, :replace, :select!, :update
+    ].each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the write operation with the cache.
+        @store.cache.cache_write(self)
+        @data.send(method_sym, *args, &block)
+        myself
+      end
+    end
+
+    # These methods mutate the Hash and return basic Ruby type objects.
     [
-      :[]=, :clear, :default=, :default_proc=, :delete, :delete_if,
-      :initialize_copy, :keep_if, :merge!, :rehash, :reject!, :replace,
-      :select!, :shift, :update
+      :delete, :delete_if, :shift
     ].each do |method_sym|
       # Create a wrapper method that passes the call to @data.
       define_method(method_sym) do |*args, &block|
@@ -79,33 +106,70 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param default [Any] The default value that is returned when no value is
-    #        stored for a specific key.
-    def initialize(p, default = nil)
+    # @param default [Object] The default value that is returned when no value
+    #        is stored for a specific key. The default must be of the
+    #        supported type.
+    def initialize(p, default = nil, &block)
       super(p)
-      @default = nil
-      @data = {}
+      _check_assignment_value(default)
+      if block_given?
+        @data = ::Hash.new(&block)
+      else
+        @data = ::Hash.new(default)
+      end
 
       # Ensure that the newly created object will be pushed into the database.
       @store.cache.cache_write(self)
     end
 
+    # Proxy for assignment method.
+    def []=(key, value)
+      unless key.is_a?(String) || key.respond_to?(:is_poxreference?)
+        raise ArgumentError, "PEROBS::Hash[] key must be a String or " +
+          "a PEROBS object but is a #{key.class}"
+      end
+      _check_assignment_value(value)
+      @store.cache.cache_write(self)
+      @data[key] = value
+    end
+
+    # Proxy for default= method.
+    def default=(value)
+      _check_assignment_value(value)
+      @data.default=(value)
+    end
+
     # Return a list of all object IDs of all persistend objects that this Hash
     # is referencing.
     # @return [Array of Integer] IDs of referenced objects
     def _referenced_object_ids
-      @data.each_value.select { |v| v && v.respond_to?(:is_poxreference?) }.
-        map { |o| o.id }
+      ids = []
+      @data.each do |k, v|
+        if k && k.respond_to?(:is_poxreference?)
+          ids << k.id
+        end
+        if v && v.respond_to?(:is_poxreference?)
+          ids << v.id
+        end
+      end
+
+      ids
     end
 
     # This method should only be used during store repair operations. It will
     # delete all referenced to the given object ID.
     # @param id [Integer] targeted object ID
     def _delete_reference_to_id(id)
+      original_length = @data.length
+
       @data.delete_if do |k, v|
-        v && v.respond_to?(:is_poxreference?) && v.id == id
+        (k && k.respond_to?(:is_poxreference?) && k.id == id) ||
+        (v && v.respond_to?(:is_poxreference?) && v.id == id)
+      end
+
+      if @data.length != original_length
+        @store.cache.cache_write(self)
       end
-      @store.cache.cache_write(self)
     end
 
     # Restore the persistent data from a single data structure.
@@ -114,8 +178,18 @@ module PEROBS
     # @private
     def _deserialize(data)
       @data = {}
-      data.each { |k, v| @data[k] = v.is_a?(POReference) ?
-                                    POXReference.new(@store, v.id) : v }
+
+      data.each do |k, v|
+        # References to other PEROBS Objects are marshalled with our own
+        # format. If we detect such a marshalled String we convert it into a
+        # POXReference object.
+        if (match = /^#<PEROBS::POReference id=([0-9]+)>$/.match(k))
+          k = POXReference.new(@store, match[1].to_i)
+        end
+        dv = v.is_a?(POReference) ? POXReference.new(@store, v.id) : v
+        @data[k] = dv
+      end
+
       @data
     end
 
@@ -136,26 +210,46 @@ module PEROBS
       data = {}
 
       @data.each do |k, v|
-        if v.respond_to?(:is_poxreference?)
-          data[k] = POReference.new(v.id)
-        else
-          # Outside of the PEROBS library all PEROBS::ObjectBase derived
-          # objects should not be used directly. The library only exposes them
-          # via POXReference proxy objects.
-          if v.is_a?(ObjectBase)
-            PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
-              "It is stored in a PEROBS::Hash with key #{k.inspect}. " +
-              'Have you used self() instead of myself() to ' +
-              "get the reference of this PEROBS object?\n" +
-              v.inspect
-          end
-          data[k] = v
+        if k.respond_to?(:is_poxreference?)
+          # JSON only supports Strings as hash keys. Since JSON is the default
+          # internal storage format in the database, we have to marshall
+          # PEROBS::Object references ourselves.
+          k = "#<PEROBS::POReference id=#{k.id}>"
+        elsif k[0..24] == '#<PEROBS::POReference id='
+          # This could obviously result in conflicts with 'normal' String hash
+          # keys. This is extremely unlikely, but we better catch this case
+          # before it causes hard to debug trouble.
+          raise ArgumentError, "Hash key #{k} conflicts with PEROBS " +
+            "internal representation of marshalled hash keys!"
         end
+        data[k] = serialize_helper(v)
       end
 
       data
     end
 
+    def serialize_helper(v)
+      if v.respond_to?(:is_poxreference?)
+        # References to other PEROBS objects (POXReference) are stored as
+        # POReference in the database.
+        return POReference.new(v.id)
+      else
+        # Outside of the PEROBS library all PEROBS::ObjectBase derived
+        # objects should not be used directly. The library only exposes them
+        # via POXReference proxy objects.
+        if v.is_a?(ObjectBase)
+          PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
+            "It is stored in a PEROBS::Hash. " +
+            'Have you used self() instead of myself() to ' +
+            "get the reference of this PEROBS object?\n" +
+            v.inspect
+        end
+
+        # All other objects are serialized by their native methods.
+        return v
+      end
+    end
+
   end
 
 end

data/lib/perobs/IDList.rb

@@ -0,0 +1,144 @@
+# encoding: UTF-8
+#
+# = IDList.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/IDListPageFile'
+require 'perobs/IDListPageRecord'
+
+module PEROBS
+
+  # This class stores a list of 64 bit values. Values can be added to the list
+  # and the presence of a certain value can be checked. It can hold up to 2^64
+  # values. It tries to keep values in memory but can store them in a file if
+  # needed. A threshold for the in-memory values can be set in the
+  # constructor. The stored values are grouped in pages. Each page can hold up
+  # to page_size entries.
+  class IDList
+
+    # Create a new IDList object. The data that can't be kept in memory will
+    # be stored in the specified directory under the given name.
+    # @param dir [String] Path of the directory
+    # @param name [String] Name of the file
+    # @param max_in_memory [Integer] Specifies the maximum number of values
+    #        that will be kept in memory. If the list is larger, values will
+    #        be cached in the specified file.
+    # @param page_size [Integer] The number of values per page. The default
+    #        value is 32 which was found the best performing config in  tests.
+    def initialize(dir, name, max_in_memory, page_size = 32)
+      # The page_file manages the pages that store the values.
+      @page_file = IDListPageFile.new(self, dir, name,
+                                      max_in_memory, page_size)
+      clear
+    end
+
+    # Insert a new value into the list.
+    # @param id [Integer] The value to add
+    def insert(id)
+      # Find the index of the page that should hold ID.
+      index = @page_records.bsearch_index { |pr| pr.max_id >= id }
+      # Get the corresponding IDListPageRecord object.
+      page = @page_records[index]
+
+      # In case the page is already full we'll have to create a new page.
+      # There is no guarantee that a split will yield an page with space as we
+      # split by ID range, not by distributing the values evenly across the
+      # two pages.
+      while page.is_full?
+        new_page = page.split
+        # Store the newly created page into the page_records list.
+        @page_records.insert(index + 1, new_page)
+        if id >= new_page.min_id
+          # We need to insert the ID into the newly created page. Adjust index
+          # and page reference accordingly.
+          index += 1
+          page = new_page
+        end
+      end
+
+      # Insert the ID into the page.
+      page.insert(id)
+    end
+
+    # Check if a given value is already stored in the list.
+    # @param id [Integer] The value to check for
+    def include?(id)
+      @page_records.bsearch { |pr| pr.max_id >= id }.include?(id)
+    end
+
+    # Clear the list and empty the filesystem cache file.
+    def clear
+      @page_file.clear
+      @page_records = [ IDListPageRecord.new(@page_file, 0, 2 ** 64) ]
+    end
+
+    # Erase the list including the filesystem cache file. The IDList is no
+    # longer usable after this call but the cache file is removed from the
+    # filesystem.
+    def erase
+      @page_file.erase
+      @page_records = nil
+    end
+
+    # Perform some consistency checks on the internal data structures. Raises
+    # a RuntimeError in case a problem is found.
+    def check
+      last_max = -1
+      unless (min_id = @page_records.first.min_id) == 0
+        raise RuntimeError, "min_id of first record (#{min_id}) " +
+          "must be 0."
+      end
+
+      @page_records.each do |pr|
+        unless pr.min_id == last_max + 1
+          raise RuntimeError, "max_id of previous record (#{last_max}) " +
+            "must be exactly 1 smaller than current record (#{pr.min_id})."
+        end
+        last_max = pr.max_id
+        pr.check
+      end
+
+      unless last_max == 2 ** 64
+        raise RuntimeError, "max_id of last records " +
+          "(#{@page_records.last.max_id}) must be #{2 ** 64})."
+      end
+    end
+
+    def to_a
+      a = []
+      @page_records.each { |pr| a += pr.values }
+      a
+    end
+
+    # Print a human readable form of the tree that stores the list. This is
+    # only meant for debugging purposes and does not scale for larger trees.
+    def to_s
+      "\n" + @root.to_s
+    end
+
+  end
+
+end
+

data/lib/perobs/IDListPage.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+#
+# = IDListPage.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  class IDListPage
+
+    attr_reader :uid, :values
+    attr_accessor :record
+
+    def initialize(page_file, record, uid, values = [])
+      @page_file = page_file
+      @record = record
+      @uid = uid
+      @values = values
+      @record.page_entries = @values.length
+    end
+
+    def IDListPage::load(page_file, uid, ref)
+      page_file.load(uid, ref)
+    end
+
+    def is_full?
+      @values.length >= @page_file.page_size
+    end
+
+    def length
+      @values.length
+    end
+
+    def save
+      @page_file.save_page(self)
+    end
+
+    def insert(id)
+      if is_full?
+        raise ArgumentError, "IDListPage is already full"
+      end
+      index = @values.bsearch_index { |v| v >= id } || @values.length
+
+      # If the value isn't stored already, insert it.
+      if @values[index] != id
+        @values.insert(index, id)
+        @record.page_entries = @values.length
+        @page_file.mark_page_as_modified(self)
+      end
+    end
+
+    def include?(id)
+      !(v = @values.bsearch { |v| v >= id }).nil? && v == id
+    end
+
+    def delete(max_id)
+      a = []
+      @values.delete_if { |v| v > max_id ? a << v : false }
+
+      unless a.empty?
+        @record.page_entries = @values.length
+        @page_file.mark_page_as_modified(self)
+      end
+
+      a
+    end
+
+    def check
+      last_value = nil
+      @values.each_with_index do |v, i|
+        if last_value && last_value >= v
+          raise RuntimeError, "The values #{last_value} and #{v} must be " +
+            "strictly ascending: #{@values.inspect}"
+        end
+        last_value = v
+      end
+    end
+
+    def to_s
+      "[ #{@values.join(', ')} ]"
+    end
+
+  end
+
+end
+

data/lib/perobs/IDListPageFile.rb

@@ -0,0 +1,180 @@
+# encoding: UTF-8
+#
+# = IDListPageFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/IDListPage'
+require 'perobs/IDListPageRecord'
+require 'perobs/Log'
+require 'perobs/PersistentObjectCache'
+
+module PEROBS
+
+  # The IDListPageFile class provides filesystem based cache for the
+  # IDListPage objects. The IDListRecord objects only hold the index of the
+  # page in this cache. This allows the pages to be garbage collected and
+  # swapped to the file. If accessed, the pages will be swaped in again. While
+  # this process is similar to the demand paging of the OS it has absolutely
+  # nothing to do with it.
+  class IDListPageFile
+
+    attr_reader :page_size, :pages
+
+    # Create a new IDListPageFile object that uses the given file in the given
+    # directory as cache file.
+    # @param list [IDList] The IDList object that caches pages here
+    # @param dir [String] An existing directory
+    # @param name [String] A file name (without path)
+    # @param max_in_memory [Integer] Maximum number of pages to keep in memory
+    # @param page_size [Integer] The number of values in each page
+    def initialize(list, dir, name, max_in_memory, page_size)
+      @list = list
+      @file_name = File.join(dir, name + '.cache')
+      @page_size = page_size
+      open
+      @pages = PersistentObjectCache.new(max_in_memory, max_in_memory,
+                                         IDListPage, self)
+      @page_counter = 0
+    end
+
+    # Load the IDListPage from the cache file.
+    # @param page_idx [Integer] The page index in the page file
+    # @param record [IDListPageRecord] the corresponding IDListPageRecord
+    # @return [IDListPage] The loaded values
+    def load(page_idx, record)
+      # The IDListPageRecord will tell us the actual number of values stored
+      # in this page.
+      values = []
+      unless (entries = record.page_entries) == 0
+        begin
+          @f.seek(page_idx * @page_size * 8)
+          values = @f.read(entries * 8).unpack("Q#{entries}")
+        rescue IOError => e
+          PEROBS.log.fatal "Cannot read cache file #{@file_name}: #{e.message}"
+        end
+      end
+
+      # Create the IDListPage object with the given values.
+      p = IDListPage.new(self, record, page_idx, values)
+      @pages.insert(p, false)
+
+      p
+    end
+
+    # Return the number of registered pages.
+    def page_count
+      @page_counter
+    end
+
+    # Create a new IDListPage and register it.
+    # @param record [IDListPageRecord] The corresponding record.
+    # @param values [Array of Integer] The values stored in the page
+    # @return [IDListPage]
+    def new_page(record, values = [])
+      idx = @page_counter
+      @page_counter += 1
+      mark_page_as_modified(IDListPage.new(self, record, idx, values))
+      idx
+    end
+
+    # Return the IDListPage object with the given index.
+    # @param record [IDListPageRecord] the corresponding IDListPageRecord
+    # @return [IDListPage] The page corresponding to the index.
+    def page(record)
+      p = @pages.get(record.page_idx, record) || load(record.page_idx, record)
+      unless p.uid == record.page_idx
+        raise RuntimeError, "Page reference mismatch. Record " +
+          "#{record.page_idx} points to page #{p.uid}"
+      end
+
+      p
+    end
+
+    # Mark a page as modified. This means it has to be written into the cache
+    # before it is removed from memory.
+    # @param p [IDListPage] page reference
+    def mark_page_as_modified(p)
+      @pages.insert(p)
+      @pages.flush
+    end
+
+    # Clear all pages, erase the cache and re-open it again.
+    def clear
+      @pages.clear
+      @page_counter = 0
+      begin
+        @f.truncate(0)
+      rescue IOError => e
+        raise RuntimeError, "Cannote truncate cache file #{@file_name}: " +
+          e.message
+      end
+    end
+
+    # Discard all pages and erase the cache file.
+    def erase
+      @pages.clear
+      @page_counter = 0
+      close
+    end
+
+    # Save the given IDListPage into the cache file.
+    # @param p [IDListPage] page to store
+    def save_page(p)
+      if p.record.page_entries != p.values.length
+        raise RuntimeError, "page_entries mismatch for node #{p.uid}"
+      end
+      begin
+        @f.seek(p.uid * @page_size * 8)
+        @f.write(p.values.pack('Q*'))
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write cache file #{@file_name}: #{e.message}"
+      end
+    end
+
+    private
+
+    def open
+      begin
+        # Create a new file by writing a new header.
+        @f = File.open(@file_name, 'wb+')
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot open cache file #{@file_name}: #{e.message}"
+      end
+    end
+
+    def close
+      begin
+        @f.close
+        File.delete(@file_name) if File.exist?(@file_name)
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot erase cache file #{@file_name}: #{e.message}"
+      end
+      @f = nil
+    end
+
+  end
+
+end
+