perobs 2.3.1 → 2.4.0

data/lib/perobs/IndexTreeNode.rb

@@ -0,0 +1,296 @@
+# encoding: UTF-8
+#
+# = IndexTreeNode.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+
+module PEROBS
+
+  # The IndexTreeNode is the building block of the IndexTree. Each node can
+  # hold up to 16 entries. An entry is described by type bits and can be empty
+  # (0), an reference into the object ID file (1) or a reference to another
+  # IndexTreeNode for the next nibble (2). Each level of the tree is
+  # associated with an specific nibble of the ID. The nibble is used to
+  # identify the entry within the node. IndexTreeNode objects are in-memory
+  # represenations of the nodes in the IndexTree file.
+  class IndexTreeNode
+
+    attr_reader :address
+
+    ENTRIES = 16
+    ENTRY_BYTES = 8
+    TYPE_BYTES = 4
+    NODE_BYTES = TYPE_BYTES + ENTRIES * ENTRY_BYTES
+
+    # Create a new IndexTreeNode.
+    # @param tree [IndexTree] The tree this node belongs to
+    # @param nibble_idx [Fixnum] the level of the node in the tree (root
+    #        being 0)
+    # @param address [Integer] The address of this node in the blob file
+    def initialize(tree, nibble_idx, address = nil)
+      @tree = tree
+      if nibble_idx >= 16
+        # We are processing 64 bit numbers, so we have at most 16 nibbles.
+        PEROBS.log.fatal 'nibble must be 0 - 15'
+      end
+      @nibble_idx = nibble_idx
+      if (@address = address).nil? || !read_node
+        # Create a new node if none with this address exists already.
+        @entry_types = 0
+        @entries = ::Array.new(ENTRIES, 0)
+        @address = @tree.nodes.free_address
+        write_node
+      end
+    end
+
+    # Store a value for the given ID. Existing values will be overwritten.
+    # @param id [Integer] ID (or key)
+    # @param value [Integer] value
+    def put_value(id, value)
+      index = calc_index(id)
+      case get_entry_type(index)
+      when 0
+        # The entry is still empty. Store the id and value and set the entry
+        # to holding a value (1).
+        set_entry_type(index, 1)
+        @entries[index] = address = @tree.ids.free_address
+        store_id_and_value(address, id, value)
+        write_node
+      when 1
+        existing_value = @entries[index]
+        existing_id, existing_address = get_id_and_address(existing_value)
+        if id == existing_id
+          if value != existing_address
+            # The entry already holds another value.
+            store_id_and_value(@entries[index], id, value)
+          end
+        else
+          # The entry already holds a value. We need to create a new node and
+          # store the existing value and the new value in it.
+          # First get the exiting value of the entry and the corresponding ID.
+          # Create a new node.
+          node = @tree.get_node(@nibble_idx + 1)
+          # The entry of the current node is now a reference to the new node.
+          set_entry_type(index, 2)
+          @entries[index] = node.address
+          # Store the existing value and the new value with their IDs.
+          node.set_entry(existing_id, existing_value)
+          node.put_value(id, value)
+        end
+        write_node
+      when 2
+        # The entry is a reference to another node.
+        node = @tree.get_node(@nibble_idx + 1, @entries[index])
+        node.put_value(id, value)
+      else
+        PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
+      end
+    end
+
+    # Retrieve the value for the given ID.
+    # @param id [Integer] ID (or key)
+    # @return [Integer] value or nil
+    def get_value(id)
+      index = calc_index(id)
+      case get_entry_type(index)
+      when 0
+        # There is no entry for this ID.
+        return nil
+      when 1
+        # There is a value stored for the ID part that we have seen so far. We
+        # still need to compare the requested ID with the full ID to determine
+        # a match.
+        stored_id, address = get_id_and_address(@entries[index])
+        if id == stored_id
+          # We have a match. Return the value.
+          return address
+        else
+          # Just a partial match of the least significant nibbles.
+          return nil
+        end
+      when 2
+        # The entry is a reference to another node. Just follow it and look at
+        # the next nibble.
+        return @tree.get_node(@nibble_idx + 1, @entries[index]).
+          get_value(id)
+      else
+        PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
+      end
+    end
+
+    # Delete the entry for the given ID.
+    # @param id [Integer] ID or key
+    # @return [Boolean] True if a key was found and deleted, otherwise false.
+    def delete_value(id)
+      index = calc_index(id)
+      case get_entry_type(index)
+      when 0
+        # There is no entry for this ID.
+        return false
+      when 1
+        # We have a value. Check that the ID matches and delete the value.
+        stored_id, address = get_id_and_address(@entries[index])
+        if id == stored_id
+          @tree.ids.delete_blob(@entries[index])
+          @entries[index] = 0
+          set_entry_type(index, 0)
+          write_node
+          return true
+        else
+          # Just a partial ID match.
+          return false
+        end
+      when 2
+        # The entry is a reference to another node.
+        node = @tree.get_node(@nibble_idx + 1, @entries[index])
+        result = node.delete_value(id)
+        if node.empty?
+          # If the sub-node is empty after the delete we delete the whole
+          # sub-node.
+          @tree.delete_node(@nibble_idx + 1, @entries[index])
+          # Eliminate the reference to the sub-node and update this node in
+          # the file.
+          set_entry_type(index, 0)
+          write_node
+        end
+        return result
+      else
+        PEROBS.node.fatal "Illegal node type #{get_entry_type(index)}"
+      end
+    end
+
+    # Recursively check this node and all sub nodes. Compare the found
+    # ID/address pairs with the corresponding entry in the given FlatFile.
+    # @param flat_file [FlatFile]
+    # @return [Boolean] true if no errors were found, false otherwise
+    def check(flat_file)
+      ENTRIES.times do |index|
+        case get_entry_type(index)
+        when 0
+          # Empty entry, nothing to do here.
+        when 1
+          # There is a value stored for the ID part that we have seen so far.
+          # We still need to compare the requested ID with the full ID to
+          # determine a match.
+          id, address = get_id_and_address(@entries[index])
+          unless flat_file.has_id_at?(id, address)
+            PEROBS.log.error "The entry for ID #{id} in the index was not " +
+              "found in the FlatFile at address #{address}"
+            return false
+          end
+        when 2
+          # The entry is a reference to another node. Just follow it and look
+          # at the next nibble.
+          unless @tree.get_node(@nibble_idx + 1, @entries[index]).
+                 check(flat_file)
+            return false
+          end
+        else
+          return false
+        end
+      end
+
+      true
+    end
+
+    # Convert the node and all sub-nodes to human readable format.
+    def inspect
+      str = "{\n"
+      0.upto(15) do |i|
+        case get_entry_type(i)
+        when 0
+          # Don't show empty entries.
+        when 1
+          id, address = get_id_and_address(@entries[i])
+          str += "  #{id} => #{address},\n"
+        when 2
+          str += "  " + @tree.get_node(@nibble_idx + 1, @entries[i]).
+            inspect.gsub(/\n/, "\n  ")
+        end
+      end
+      str + "}\n"
+    end
+
+    # Utility method to set the value of an existing node entry.
+    # @param id [Integer] ID or key
+    # @param value [Integer] value to set. Note that this value must be an
+    # address from the ids list.
+    def set_entry(id, value)
+      index = calc_index(id)
+      set_entry_type(index, 1)
+      @entries[index] = value
+    end
+
+    # Check if the node is empty.
+    # @return [Boolean] True if all entries are empty.
+    def empty?
+      @entry_types == 0
+    end
+
+    private
+
+    def calc_index(id)
+      (id >> (4 * @nibble_idx)) & 0xF
+    end
+
+    def read_node
+      return false unless (bytes = @tree.nodes.retrieve_blob(@address))
+      @entry_types = bytes[0, TYPE_BYTES].unpack('L')[0]
+      @entries = bytes[TYPE_BYTES, ENTRIES * ENTRY_BYTES].unpack('Q16')
+      true
+    end
+
+    def write_node
+      bytes = ([ @entry_types ] + @entries).pack('LQ16')
+      @tree.nodes.store_blob(@address, bytes)
+    end
+
+    def set_entry_type(index, type)
+      if index < 0 || index > 15
+        PEROBS.log.fatal "Index must be between 0 and 15"
+      end
+      @entry_types = ((@entry_types & ~(0x3 << 2 * index)) |
+                     ((type & 0x3) << 2 * index)) & 0xFFFFFFFF
+    end
+
+    def get_entry_type(index)
+      if index < 0 || index > 15
+        PEROBS.log.fatal "Index must be between 0 and 15"
+      end
+      (@entry_types >> 2 * index) & 0x3
+    end
+
+    def get_id_and_address(id_address)
+      @tree.ids.retrieve_blob(id_address).unpack('QQ')
+    end
+
+    def store_id_and_value(address, id, value)
+      @tree.ids.store_blob(address, [ id, value ].pack('QQ'))
+    end
+
+  end
+
+end

data/lib/perobs/Log.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+#
+# = Log.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+#
+
+require 'monitor'
+require 'logger'
+require 'singleton'
+
+module PEROBS
+
+  # This is the Exception type that will be thrown for all unrecoverable
+  # library internal (program logic) errors.
+  class FatalError < StandardError ; end
+
+  # This is the Exception type that will be thrown for all program errors that
+  # are caused by user error rather than program logic errors.
+  class UsageError < StandardError ; end
+
+  # The ILogger class is a singleton that provides a common logging mechanism
+  # to all objects. It exposes essentially the same interface as the Logger
+  # class, just as a singleton and extends fatal to raise an FatalError
+  # exception.
+  class ILogger < Monitor
+
+    include Singleton
+
+    # Default options to create a logger. Keep 4 log files, each 1MB max.
+    @@options = [ 4, 2**20 ]
+    @@level = Logger::INFO
+    @@formatter = proc do |severity, time, progname, msg|
+      "#{time} #{severity} #{msg}\n"
+    end
+    @@logger = nil
+
+    # Set log level.
+    # @param l [Logger::WARN, Logger:INFO, etc]
+    def level=(l)
+      @@level = l
+    end
+
+    # Set Logger formatter.
+    # @param f [Proc]
+    def formatter=(f)
+      @@formatter = f
+    end
+
+    # Set Logger options
+    # @param o [Array] Optional parameters for Logger.new().
+    def options=(o)
+      @@options = o
+    end
+
+    # Redirect all log messages to the given IO.
+    # @param io [IO] Output file descriptor
+    def open(io)
+      begin
+        @@logger = Logger.new(io, *@@options)
+      rescue IOError => e
+        @@logger = Logger.new($stderr)
+        $stderr.puts "Cannot open log file: #{e.message}"
+      end
+      @@logger.level = @@level
+      @@logger.formatter = @@formatter
+    end
+
+    # Pass all calls to unknown methods to the @@logger object.
+    def method_missing(method, *args, &block)
+      @@logger.send(method, *args, &block)
+    end
+
+    # Make it properly introspectable.
+    def respond_to?(method, include_private = false)
+      @@logger.respond_to?(method)
+    end
+
+    # Print an error message via the Logger and raise a Fit4Ruby::Error.
+    # This method should be used to abort the program in case of program logic
+    # errors.
+    def fatal(msg, &block)
+      @@logger.fatal(msg, &block)
+      raise FatalError, msg
+    end
+
+  end
+
+  class << self
+
+    ILogger.instance.open($stderr)
+
+    # Convenience method to we can use PEROBS::log instead of
+    # PEROBS::ILogger.instance.
+    def log
+      ILogger.instance
+    end
+
+  end
+
+end
+

data/lib/perobs/Object.rb

@@ -27,6 +27,7 @@
 
 require 'time'
 
+require 'perobs/Log'
 require 'perobs/ObjectBase'
 
 module PEROBS
@@ -53,7 +54,7 @@ module PEROBS
       def po_attr(*attributes)
         attributes.each do |attr_name|
           unless attr_name.is_a?(Symbol)
-            raise ArgumentError, "name must be a symbol but is a " +
+            PEROBS.log.fatal "name must be a symbol but is a " +
               "#{attr_name.class}"
           end
 
@@ -115,8 +116,8 @@ module PEROBS
         end
         return true
       else
-        raise ArgumentError, "'#{attr}' is not a defined persistent " +
-                             "attribute of class #{self.class}"
+        PEROBS.log.fatal "'#{attr}' is not a defined persistent " +
+          "attribute of class #{self.class}"
       end
 
       false
@@ -177,7 +178,7 @@ module PEROBS
     # Textual dump for debugging purposes
     # @return [String]
     def inspect
-      "#{to_s}:#{@_id}\n{\n" +
+      "<#{self.class}:#{@_id}>\n{\n" +
       _all_attributes.map do |attr|
         ivar = ('@' + attr.to_s).to_sym
         if (value = instance_variable_get(ivar)).respond_to?(:is_poxreference?)
@@ -209,13 +210,12 @@ module PEROBS
         # References to other PEROBS::Objects must be handled somewhat
         # special.
         if @store != val.store
-          raise ArgumentError, 'The referenced object is not part of this store'
+          PEROBS.log.fatal 'The referenced object is not part of this store'
         end
       elsif val.is_a?(ObjectBase)
-        raise ArgumentError, 'A PEROBS::ObjectBase object escaped! ' +
-                             'Have you used self() instead of myself() to' +
-                             'get the reference of the PEROBS object that ' +
-                             'you are trying to assign here?'
+        PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
+          'Have you used self() instead of myself() to get the reference ' +
+          'of the PEROBS object that you are trying to assign here?'
       end
       instance_variable_set(('@' + attr.to_s).to_sym, val)
       # Let the store know that we have a modified object. If we restored the
@@ -233,8 +233,7 @@ module PEROBS
       # PEROBS objects that don't have persistent attributes declared don't
       # really make sense.
       unless self.class.attributes
-        raise StandardError
-          "No persistent attributes have been declared for " +
+        PEROBS.log.fatal "No persistent attributes have been declared for " +
           "class #{self.class}. Use 'po_attr' to declare them."
       end
       self.class.attributes

data/lib/perobs/ObjectBase.rb

@@ -25,6 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/Log'
 require 'perobs/ClassMap'
 
 module PEROBS
@@ -47,13 +48,11 @@ module PEROBS
     # Proxy all calls to unknown methods to the referenced object.
     def method_missing(method_sym, *args, &block)
       unless (obj = _referenced_object)
-        ::Kernel.raise ::RuntimeError,
-          "Internal consistency error. No object with ID #{@id} found in " +
-          'the store.'
+        ::PEROBS.log.fatal "Internal consistency error. No object with ID " +
+          "#{@id} found in the store."
       end
       if obj.respond_to?(:is_poxreference?)
-        ::Kernel.raise ::RuntimeError,
-          "POXReference that references a POXReference found."
+        ::PEROBS.log.fatal "POXReference that references a POXReference found."
       end
       obj.send(method_sym, *args, &block)
     end
@@ -149,6 +148,20 @@ module PEROBS
       proc { store._collect(id) }
     end
 
+    # Library internal method to transfer the Object to a new store.
+    # @param store [Store] New store
+    def _transfer(store)
+      @store = store
+      # Remove the previously defined finalizer as it is attached to the old
+      # store.
+      ObjectSpace.undefine_finalizer(self)
+      # Register the object as in-memory object with the new store.
+      @store._register_in_memory(self, @_id)
+      # Register the finalizer for the new store.
+      ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
+      @myself = POXReference.new(@store, @_id)
+    end
+
     # This method can be overloaded by derived classes to do some massaging on
     # the data after it has been restored from the database. This could either
     # be some sanity check or code to migrate the object from one version to

data/lib/perobs/StackFile.rb

@@ -0,0 +1,137 @@
+# encoding: UTF-8
+#
+# = StackFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+
+module PEROBS
+
+  # This class implements a file based stack. All entries must have the same
+  # size.
+  class StackFile
+
+    # Create a new stack file in the given directory with the given file name.
+    # @param dir [String] Directory
+    # @param name [String] File name
+    # @param entry_bytes [Fixnum] Number of bytes each entry must have
+    def initialize(dir, name, entry_bytes)
+      @file_name = File.join(dir, name + '.stack')
+      @entry_bytes = entry_bytes
+      @f = nil
+    end
+
+    # Open the stack file.
+    def open
+      begin
+        if File.exist?(@file_name)
+          @f = File.open(@file_name, 'rb+')
+        else
+          @f = File.open(@file_name, 'wb+')
+        end
+      rescue => e
+        PEROBS.log.fatal "Cannot open stack file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Close the stack file. This method must be called before the program is
+    # terminated to avoid data loss.
+    def close
+      begin
+        @f.flush
+        @f.close
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot close stack file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Flush out unwritten data to file.
+    def sync
+      begin
+        @f.flush
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot sync stack file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Push the given bytes onto the stack file.
+    # @param bytes [String] Bytes to write.
+    def push(bytes)
+      if bytes.length != @entry_bytes
+        PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
+          "long. This entry is #{bytes.length} bytes long."
+      end
+      begin
+        @f.seek(0, IO::SEEK_END)
+        @f.write(bytes)
+      rescue => e
+        PEROBS.log.fatal "Cannot push to stack file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Pop the last entry from the stack file.
+    # @return [String or nil] Popped entry or nil if stack is already empty.
+    def pop
+      begin
+        return nil if @f.size == 0
+
+        @f.seek(-@entry_bytes, IO::SEEK_END)
+        bytes = @f.read(@entry_bytes)
+        @f.truncate(@f.size - @entry_bytes)
+        @f.flush
+      rescue => e
+        PEROBS.log.fatal "Cannot pop from stack file #{@file_name}: " +
+          e.message
+      end
+
+      bytes
+    end
+
+    # Remove all entries from the stack.
+    def clear
+      @f.truncate(0)
+      @f.flush
+    end
+
+    # Iterate over all entries in the stack and call the given block for the
+    # bytes.
+    def each
+      @f.seek(0)
+      while !@f.eof
+        yield(@f.read(@entry_bytes))
+      end
+    end
+
+    # Return the content of the stack as an Array.
+    # @return [Array]
+    def to_ary
+      a = []
+      each { |bytes| a << bytes }
+      a
+    end
+
+  end
+
+end