perobs 4.0.0 → 4.4.0

data/lib/perobs/IDListPageRecord.rb

@@ -0,0 +1,142 @@
+# encoding: UTF-8
+#
+# = IDListPageRecord.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  # The IDListPageRecord class models the elements of the IDList. Each page
+  # holds up to a certain number of IDs that can be cached into a file if
+  # needed. Each page holds IDs within a given interval. The cache is managed
+  # by the IDListPageFile object.
+  class IDListPageRecord
+
+    attr_reader :min_id, :max_id, :page_idx
+    attr_accessor :page_entries
+
+    # Create a new IDListPageRecord object.
+    # @param page_file [IDListPageFile] The page file that manages the cache.
+    # @param min_id [Integer] The smallest ID that can be stored in this page
+    # @param max_id [Integer] the largest ID that can be stored in this page
+    # @param values [Array] An array of IDs to be stored in this page
+    def initialize(page_file, min_id, max_id, values = [])
+      @page_file = page_file
+      @min_id = min_id
+      @max_id = max_id
+      @page_entries = 0
+      @page_idx = @page_file.new_page(self, values)
+    end
+
+    # Check if the given ID is included in this page.
+    # @param id [Integer]
+    # @return [True of False] Return true if found, false otherwise.
+    def include?(id)
+      return false if id < @min_id || @max_id < id
+
+      page.include?(id)
+    end
+
+    # Check if the page is full and can't store any more IDs.
+    # @return [True or False]
+    def is_full?
+      page.is_full?
+    end
+
+    # Insert an ID into the page.
+    # @param id [Integer] The ID to store
+    def insert(id)
+      unless @min_id <= id && id <= @max_id
+        raise ArgumentError, "IDs for this page must be between #{@min_id} " +
+          "and #{@max_id}. #{id} is outside this range."
+      end
+
+      page.insert(id)
+    end
+
+    # Split the current page. This split is done by splitting the ID range in
+    # half. This page will keep the first half, the newly created page will
+    # get the second half. This may not actually yield an empty page as all
+    # values could remain with one of the pages. In this case further splits
+    # need to be issued by the caller.
+    # @return [IDListPageRecord] A new IDListPageRecord object.
+    def split
+      # Determine the new max_id for the old page.
+      max_id = @min_id + (@max_id - @min_id) / 2
+      # Create a new page that stores the upper half of the ID range. Remove
+      # all IDs from this page that now belong into the new page and transfer
+      # them.
+      new_page_record = IDListPageRecord.new(@page_file, max_id + 1, @max_id,
+                                             page.delete(max_id))
+      # Adjust the max_id of the current page.
+      @max_id = max_id
+
+      new_page_record
+    end
+
+    def values
+      page.values
+    end
+
+    def <=>(pr)
+      @min_id <=> pr.min_id
+    end
+
+    def check
+      unless @min_id < @max_id
+        raise RuntimeError, "min_id must be smaller than max_id"
+      end
+
+      p = page
+      values = p.values
+      unless @page_entries == values.length
+        raise RuntimeError, "Mismatch between node page_entries " +
+          "(#{@page_entries}) and number of values (#{p.values.length})"
+      end
+
+      values.each do |v|
+        if v < @min_id
+          raise RuntimeError, "Page value #{v} is smaller than min_id " +
+            "#{@min_id}"
+        end
+        if v > @max_id
+          raise RuntimeError, "Page value #{v} is larger than max_id #{@max_id}"
+        end
+      end
+
+      p.check
+    end
+
+    private
+
+    def page
+      # The leaf pages reference the IDListPage objects only by their index.
+      # This method will convert the index into a reference to the actual
+      # object. These references should be very short-lived as a life
+      # reference prevents the page object from being collected.
+      @page_file.page(self)
+    end
+  end
+
+end

data/lib/perobs/Object.rb

@@ -2,7 +2,7 @@
 #
 # = Object.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -212,17 +212,7 @@ module PEROBS
     end
 
     def _set(attr, val)
-      if val.respond_to?(:is_poxreference?)
-        # References to other PEROBS::Objects must be handled somewhat
-        # special.
-        if @store != val.store
-          PEROBS.log.fatal 'The referenced object is not part of this store'
-        end
-      elsif val.is_a?(ObjectBase)
-        PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
-          'Have you used self() instead of myself() to get the reference ' +
-          'of the PEROBS object that you are trying to assign here?'
-      end
+      _check_assignment_value(val)
       instance_variable_set(('@' + attr.to_s).to_sym, val)
       # Let the store know that we have a modified object. If we restored the
       # object from the DB, we don't mark it as modified.
@@ -236,13 +226,26 @@ module PEROBS
     end
 
     def _all_attributes
+      # Collect all persistent attributes from this class and all
+      # super classes into a single Array.
+      attributes = []
+      klass = self.class
+      while klass && klass.respond_to?(:attributes)
+        if (attrs = klass.attributes)
+          attributes += attrs
+        end
+        klass = klass.superclass
+      end
+
       # PEROBS objects that don't have persistent attributes declared don't
       # really make sense.
-      unless self.class.attributes
+      if attributes.empty?
         PEROBS.log.fatal "No persistent attributes have been declared for " +
-          "class #{self.class}. Use 'po_attr' to declare them."
+          "class #{self.class} or any parent class. Use 'attr_persist' " +
+          "to declare them."
       end
-      self.class.attributes
+
+      attributes
     end
 
   end

data/lib/perobs/ObjectBase.rb

@@ -86,6 +86,10 @@ module PEROBS
       _referenced_object == obj
     end
 
+    def eql?(obj)
+      _referenced_object._id == obj._id
+    end
+
     # BasicObject provides a equal?() method that prevents method_missing from
     # being called. So we have to pass the call manually to the referenced
     # object.
@@ -98,6 +102,13 @@ module PEROBS
       end
     end
 
+    # To allow POXReference objects to be used as Hash keys we need to
+    # implement this function. Conveniently, we can just use the PEROBS object
+    # ID since that is unique.
+    def hash
+      @id
+    end
+
     # Shortcut to access the _id() method of the referenced object.
     def _id
       @id
@@ -114,6 +125,20 @@ module PEROBS
   # common to all classes of persistent objects.
   class ObjectBase
 
+    # This is a list of the native Ruby classes that are supported for
+    # instance variable assignements in addition to other PEROBS objects.
+    if RUBY_VERSION < '2.2'
+      NATIVE_CLASSES = [
+        NilClass, Integer, Bignum, Fixnum, Float, String, Time,
+        TrueClass, FalseClass
+      ]
+    else
+      NATIVE_CLASSES = [
+        NilClass, Integer, Float, String, Time,
+        TrueClass, FalseClass
+      ]
+    end
+
     attr_reader :_id, :store, :myself
 
     # New PEROBS objects must always be created by calling # Store.new().
@@ -192,6 +217,25 @@ module PEROBS
       @store.db.put_object(db_obj, @_id)
     end
 
+    #
+    def _check_assignment_value(val)
+      if val.respond_to?(:is_poxreference?)
+        # References to other PEROBS::Objects must be handled somewhat
+        # special.
+        if @store != val.store
+          PEROBS.log.fatal 'The referenced object is not part of this store'
+        end
+      elsif val.is_a?(ObjectBase)
+        PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
+          'Have you used self() instead of myself() to get the reference ' +
+          'of the PEROBS object that you are trying to assign here?'
+      elsif !NATIVE_CLASSES.include?(val.class)
+        PEROBS.log.fatal "Assigning objects of class #{val.class} is not " +
+          "supported. Only PEROBS objects or one of the following classes " +
+          "are supported: #{NATIVE_CLASSES.join(', ')}"
+      end
+    end
+
     # Read an raw object with the specified ID from the backing store and
     # instantiate a new object of the specific type.
     def ObjectBase.read(store, id)
@@ -213,15 +257,12 @@ module PEROBS
     def _restore(level)
       # Find the most recently stored state of this object. This could be on
       # any previous stash level or in the regular object DB. If the object
-      # was created during the transaction, there is not previous state to
+      # was created during the transaction, there is no previous state to
       # restore to.
       data = nil
       if @_stash_map
         (level - 1).downto(0) do |lvl|
-          if @_stash_map[lvl]
-            data = @_stash_map[lvl]
-            break
-          end
+          break if (data = @_stash_map[lvl])
         end
       end
       if data

data/lib/perobs/PersistentObjectCache.rb

@@ -31,29 +31,32 @@ module PEROBS
 
   class PersistentObjectCache
 
-    FLUSH_WATERMARK = 500
-
     # This cache class manages the presence of objects that primarily live in
     # a backing store but temporarily exist in memory as well. To work with
     # these objects, direct references must be only very short lived. Indirect
     # references can be done via a unique ID that the object must provide. Due
     # to the indirect references the Ruby garbage collector can collect these
-    # objects and the cache is notified via a finalizer that the objects must
-    # provide. The finalize must call the _collect() method. To reduce the
-    # read and write latencies of the backing store this class keeps a subset
-    # of the object in memory which prevents them from being collected. All
-    # references to the objects must be resolved via the get() method to
-    # prevent duplicate instances in memory of the same object.
-    # @param size [Integer] Maximum number of objects to be cached at a time
+    # objects. To reduce the read and write latencies of the backing store
+    # this class keeps a subset of the objects in memory which prevents them
+    # from being collected. All references to the objects must be resolved via
+    # the get() method to prevent duplicate instances in memory of the same
+    # in-store object. The cache uses a least-recently-used (LRU) scheme to
+    # cache objects.
+    # @param size [Integer] Minimum number of objects to be cached at a time
+    # @param flush_delay [Integer] Determines how often non-forced flushes are
+    #        ignored in a row before the flush is really done. If flush_delay
+    #        is smaller than 0 non-forced flushed will always be ignored.
     # @param klass [Class] The class of the objects to be cached. Objects must
     #        provide a uid() method that returns a unique ID for every object.
     # @param collection [] The object collection the objects belong to. It
     #        must provide a ::load method.
-    def initialize(size, klass, collection)
+    def initialize(size, flush_delay, klass, collection)
       @size = size
       @klass = klass
       @collection = collection
-      @flush_counter = FLUSH_WATERMARK
+      @flush_delay = @flush_counter = flush_delay
+      @flush_times = 0
+
       clear
     end
 
@@ -61,65 +64,46 @@ module PEROBS
     # @param object [Object] Object to cache
     # @param modified [Boolean] True if the object was modified, false otherwise
     def insert(object, modified = true)
-      # Store the object via its Ruby object ID instead of a direct reference.
-      # This allows the object to be collected by the garbage collector.
-      @in_memory_objects[object.uid] = object.object_id
+      unless object.is_a?(@klass)
+        raise ArgumentError, "You can insert only #{@klass} objects in this " +
+          "cache. You have tried to insert a #{object.class} instead."
+      end
+
+      if modified
+        @modified_entries[object.uid] = object
+      else
+        @unmodified_entries[object.uid % @size] = object
+      end
 
-      @lines[object.uid % @size].insert(object, modified)
+      nil
     end
 
     # Retrieve a object reference from the cache.
     # @param uid [Integer] uid of the object to retrieve.
-    def get(uid)
-      if (entry = @lines[uid % @size].get(uid))
-        return entry.obj
+    # @param ref [Object] optional reference to be used by the load method
+    def get(uid, ref = nil)
+      # First check if it's a modified object.
+      if (object = @modified_entries[uid])
+        return object
       end
 
-      if (ruby_object_id = @in_memory_objects[uid])
-        # We have the object in memory so we can just return it.
-        begin
-          object = ObjectSpace._id2ref(ruby_object_id)
-          # Let's make sure the object is really the object we are looking
-          # for. The GC might have recycled it already and the Ruby object ID
-          # could now be used for another object.
-          if object.is_a?(@klass) && object.uid == uid
-            # Let's put the object in the cache. We might need it soon again.
-            insert(object, false)
-            return object
-          end
-        rescue RangeError
-          # Due to a race condition the object can still be in the
-          # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again. In this case the
-          # _collect() call will happen much later, potentially after we have
-          # registered a new object with the same ID.
-          @in_memory_objects.delete(uid)
-        end
+      # Then check the unmodified object list.
+      if (object = @unmodified_entries[uid % @size]) && object.uid == uid
+        return object
       end
 
-      @klass::load(@collection, uid)
+      # If we don't have it in memory we need to load it.
+      @klass::load(@collection, uid, ref)
     end
 
     # Remove a object from the cache.
     # @param uid [Integer] unique ID of object to remove.
     def delete(uid)
-      # The object is likely still in memory, but we really don't want to
-      # access it anymore.
-      @in_memory_objects.delete(uid)
-
-      @lines[uid % @size].delete(uid)
-    end
+      @modified_entries.delete(uid)
 
-    # Remove a object from the in-memory list. This is an internal method
-    # and should never be called from user code. It will be called from a
-    # finalizer, so many restrictions apply!
-    # @param uid [Integer] Object address of the object to remove from
-    #        the list
-    # @param ruby_object_id [Integer] The Ruby object ID of the collected
-    #        object
-    def _collect(address, ruby_object_id)
-      if @in_memory_objects[id] == ruby_object_id
-        @in_memory_objects.delete(address)
+      index = uid % @size
+      if (object = @unmodified_entries[index]) && object.uid == uid
+        @unmodified_entries[index] = nil
       end
     end
 
@@ -127,24 +111,29 @@ module PEROBS
     # all modified objects will be written.
     # @param now [Boolean]
     def flush(now = false)
-      if now || (@flush_counter -= 1) <= 0
-        @lines.each { |line| line.flush(now) }
-        @flush_counter = FLUSH_WATERMARK
+      if now || (@flush_delay >= 0 && (@flush_counter -= 1) <= 0)
+        @modified_entries.each do |id, object|
+          object.save
+          # Add the object to the unmodified object cache. We might still need
+          # it again soon.
+          @unmodified_entries[object.uid % @size] = object
+        end
+        @modified_entries = ::Hash.new
+        @flush_counter = @flush_delay
       end
+      @flush_times += 1
     end
 
     # Remove all entries from the cache.
     def clear
-      # A hash that stores all objects by the Ruby object ID that are
-      # currently in memory. Objects are added via insert() and will be
-      # removed via delete() or _collect() called from a Object
-      # finalizer. It only stores the object Ruby object ID hashed by their
-      # address in the file.  This enables them from being collected by the
-      # Ruby garbage collector.
-      @in_memory_objects = {}
-      # This is the actual cache. The Array stores objects as Entry objects to
-      # also store the modified/not-modified state.
-      @lines = ::Array.new(@size) { |i| PersistentObjectCacheLine.new }
+      # This Array stores all unmodified entries. It has a fixed size and uses
+      # a % operation to compute the index from the object ID.
+      @unmodified_entries = ::Array.new(@size)
+
+      # This Hash stores all modified entries. It can grow and shrink as
+      # needed. A flush operation writes all modified objects into the backing
+      # store.
+      @modified_entries = ::Hash.new
     end
 
   end

data/lib/perobs/PersistentObjectCacheLine.rb

@@ -34,30 +34,42 @@ module PEROBS
     class Entry < Struct.new(:obj, :modified)
     end
 
-    WATERMARK = 4
+    # This defines the minimum size of the cache line. If it is too large, the
+    # time to find an entry will grow too much. If it is too small the number
+    # of cache lines will be too large and create more store overhead. By
+    # running benchmarks it turned out that 8 is a pretty good compromise.
+    WATERMARK = 8
 
     def initialize
       @entries = []
     end
 
     def insert(object, modified)
-      @entries.each do |e|
-        if e.obj.uid == object.uid
-          e.modified = true if modified
-          return
-        end
+      if (index = @entries.find_index{ |e| e.obj.uid == object.uid })
+        # We have found and removed an existing entry for this particular
+        # object. If the modified flag is set, ensure that the entry has it
+        # set as well.
+        entry = @entries.delete_at(index)
+        entry.modified = true if modified && !entry.modified
+      else
+        # There is no existing entry for this object. Create a new one.
+        entry = Entry.new(object, modified)
       end
 
-      # Insert the new entry at the beginning of the line.
-      @entries.unshift(Entry.new(object, modified))
+      # Insert the entry at the beginning of the line.
+      @entries.unshift(entry)
     end
 
     def get(uid)
-      @entries.each do |e|
-        return e if e.obj.uid == uid
+      if (index = @entries.find_index{ |e| e.obj.uid == uid })
+        if index > 0
+          # Move the entry to the front.
+          @entries.unshift(@entries.delete_at(index))
+        end
+        @entries.first
+      else
+        nil
       end
-
-      nil
     end
 
     # Delete the entry that matches the given UID

data/lib/perobs/ProgressMeter.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+#
+# = ProgressMeter.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+
+module PEROBS
+
+  # This is the base class for all ProgressMeter classes. It only logs into
+  # the PEROBS log. You need to create a derived class that overloads
+  # print_bar() and print_time() to provide more fancy outputs.
+  class ProgressMeter
+
+    def initialize
+      @name = nil
+      @max_value = nil
+      @current_value = nil
+      @start_time = nil
+      @end_time = nil
+    end
+
+    def start(name, max_value)
+      @name = name
+      unless max_value >= 0
+        raise ArgumentError, "Maximum value (#{max_value}) must be larger " +
+          "or equal to 0"
+      end
+      @max_value = max_value
+      @current_value = 0
+      @start_time = Time.now
+      @end_time = nil
+      print_bar
+
+      if block_given?
+        yield(self)
+        done
+      end
+    end
+
+    def update(value)
+      return unless (value_i = value.to_i) > @current_value
+
+      @current_value = value_i
+      print_bar
+    end
+
+    def done
+      @end_time = Time.now
+      print_time
+      PEROBS.log.info "#{@name} completed in " +
+        secsToHMS(@end_time - @start_time)
+    end
+
+    private
+
+    def print_bar
+    end
+
+    def print_time
+    end
+
+    def secsToHMS(secs)
+      secs = secs.to_i
+      s = secs % 60
+      mins = secs / 60
+      m = mins % 60
+      h = mins / 60
+      "#{h}:#{'%02d' % m}:#{'%02d' % s}"
+    end
+
+  end
+
+end
+