perobs 0.0.1

data/lib/perobs/HashedBlocksDB.rb

@@ -0,0 +1,153 @@
+# encoding: UTF-8
+#
+# = HashedBlocksDB.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+require 'json'
+require 'json/add/core'
+require 'json/add/struct'
+require 'yaml'
+require 'fileutils'
+
+require 'perobs/DataBase'
+require 'perobs/BlockDB'
+
+module PEROBS
+
+  # This class provides a filesytem based database store for objects.
+  class HashedBlocksDB < DataBase
+
+    @@Extensions = {
+      :marshal => '.mshl',
+      :json => '.json',
+      :yaml => '.yml'
+    }
+
+    # Create a new FileSystemDB object. This will create a DB with the given
+    # name. A database will live in a directory of that name.
+    # @param db_name [String] name of the DB directory
+    # @param options [Hash] options to customize the behavior. Currently only
+    #        the following options are supported:
+    #        :serializer  : Can be :marshal, :json, :yaml
+    #        :dir_nibbles : The number of nibbles to use for directory names.
+    #                       Meaningful values are 1, 2, and 3. The larger the
+    #                       number the more back-end files are used. Each
+    #                       nibble provides 16 times more directories.
+    #        :block_size  : The size of the blocks inside the storage files in
+    #                       bytes. This should roughly correspond to the size
+    #                       of the smallest serialized objects you want to
+    #                       store in quantities. It also should be an fraction
+    #                       of 4096, the native storage system block size.
+    def initialize(db_name, options = {})
+      super(options[:serializer] || :json)
+      @db_dir = db_name
+      @dir_nibbles = options[:dir_nibbles] || 2
+      @block_size = options[:block_size] || 256
+
+      # Create the database directory if it doesn't exist yet.
+      ensure_dir_exists(@db_dir)
+    end
+
+    # Return true if the object with given ID exists
+    # @param id [Fixnum or Bignum]
+    def include?(id)
+      !BlockDB.new(directory(id), @block_size).find(id).nil?
+    end
+
+    # Store the given object into the cluster files.
+    # @param obj [Hash] Object as defined by PEROBS::ObjectBase
+    def put_object(obj, id)
+      BlockDB.new(directory(id), @block_size).write_object(id, serialize(obj))
+    end
+
+    # Load the given object from the filesystem.
+    # @param id [Fixnum or Bignum] object ID
+    # @return [Hash] Object as defined by PEROBS::ObjectBase
+    def get_object(id)
+      deserialize(BlockDB.new(directory(id), @block_size).read_object(id))
+    end
+
+    # This method must be called to initiate the marking process.
+    def clear_marks
+      Dir.glob(File.join(@db_dir, '*')) do |dir|
+        BlockDB.new(dir, @block_size).clear_marks
+      end
+    end
+
+    # Permanently delete all objects that have not been marked. Those are
+    # orphaned and are no longer referenced by any actively used object.
+    def delete_unmarked_objects
+      Dir.glob(File.join(@db_dir, '*')) do |dir|
+        BlockDB.new(dir, @block_size).delete_unmarked_entries
+      end
+    end
+
+    # Mark an object.
+    # @param id [Fixnum or Bignum] ID of the object to mark
+    def mark(id)
+      BlockDB.new(directory(id), @block_size).mark(id)
+    end
+
+    # Check if the object is marked.
+    # @param id [Fixnum or Bignum] ID of the object to check
+    def is_marked?(id)
+      BlockDB.new(directory(id), @block_size).is_marked?(id)
+    end
+
+    # Check if the stored object is syntactically correct.
+    # @param id [Fixnum/Bignum] Object ID
+    # @param repair [TrueClass/FalseClass] True if an repair attempt should be
+    #        made.
+    # @return [TrueClass/FalseClass] True if the object is OK, otherwise
+    #         false.
+    def check(id, repair)
+      begin
+        get_object(id)
+      rescue => e
+        $stderr.puts "Cannot read object with ID #{id}: #{e.message}"
+        return false
+      end
+
+      true
+    end
+
+    private
+
+    # Determine the file name to store the object. The object ID determines
+    # the directory and file name inside the store.
+    # @param id [Fixnum or Bignum] ID of the object
+    def directory(id)
+      hex_id = "%016X" % id
+      dir = hex_id[0..(@dir_nibbles - 1)]
+      ensure_dir_exists(dir_name = File.join(@db_dir, dir))
+
+      dir_name
+    end
+
+  end
+
+end
+

data/lib/perobs/Object.rb

@@ -0,0 +1,189 @@
+# encoding: UTF-8
+#
+# = Object.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+
+require 'perobs/ObjectBase'
+
+module PEROBS
+
+  # The PEROBS::Object class is the base class for user-defined objects to be
+  # stored in the Store. It provides all the plumbing to define the class
+  # attributes and to transparently load and store the instances of the class
+  # in the database. You can use instance variables like normal instance
+  # variables unless they refer to other PEROBS objects. In these cases you
+  # must use the accessor methods for these instance variables. You must use
+  # accessor methods for any read and write operation to instance variables
+  # that hold or should hold PEROBS objects.
+  class Object < ObjectBase
+
+    # Modify the Metaclass of PEROBS::Object to add the attribute method and
+    # instance variables to store the default values of the attributes.
+    class << self
+
+      attr_reader :attributes
+
+      # This method can be used to define instance variable for
+      # PEROBS::Object derived classes.
+      # @param attributes [Symbol] Name of the instance variable
+      def po_attr(*attributes)
+        attributes.each do |attr_name|
+          unless attr_name.is_a?(Symbol)
+            raise ArgumentError, "name must be a symbol but is a " +
+              "#{attr_name.class}"
+          end
+
+          # Create the attribute reader method with name of attr_name.
+          define_method(attr_name.to_s) do
+            _get(attr_name)
+          end
+          # Create the attribute writer method with name of attr_name.
+          define_method(attr_name.to_s + '=') do |val|
+            _set(attr_name, val)
+          end
+
+          # Store a list of the attribute names
+          @attributes ||= []
+          @attributes << attr_name unless @attributes.include?(attr_name)
+        end
+      end
+
+    end
+
+    attr_reader :attributes
+
+    # Create a new PEROBS::Object object.
+    def initialize(store)
+      super
+    end
+
+    # Initialize the specified attribute _attr_ with the value _val_ unless
+    # the attribute has been initialized already. Use this method in the class
+    # constructor to avoid overwriting values that have been set when the
+    # object was reconstructed from the store.
+    # @param attr [Symbol] Name of the attribute
+    # @param val [Any] Value to be set
+    # @return [true|false] True if the value was initialized, otherwise false.
+    def init_attr(attr, val)
+      if self.class.attributes.include?(attr)
+        _set(attr, val)
+        return true
+      end
+
+      false
+    end
+
+    # Return a list of all object IDs that the attributes of this instance are
+    # referencing.
+    # @return [Array of Fixnum or Bignum] IDs of referenced objects
+    def _referenced_object_ids
+      ids = []
+      self.class.attributes.each do |attr|
+        value = instance_variable_get(('@' + attr.to_s).to_sym)
+        ids << value.id if value && value.is_a?(POReference)
+      end
+      ids
+    end
+
+    # This method should only be used during store repair operations. It will
+    # delete all referenced to the given object ID.
+    # @param id [Fixnum/Bignum] targeted object ID
+    def _delete_reference_to_id(id)
+      self.class.attributes.each do |attr|
+        ivar = ('@' + attr.to_s).to_sym
+        value = instance_variable_get(ivar)
+        if value && value.is_a?(POReference) && value.id == id
+          instance_variable_set(ivar, nil)
+        end
+      end
+    end
+
+    # Restore the persistent data from a single data structure.
+    # This is a library internal method. Do not use outside of this library.
+    # @param data [Hash] attribute values hashed by their name
+    # @private
+    def _deserialize(data)
+      # Initialize all attributes with the provided values.
+      data.each do |attr_name, value|
+        instance_variable_set(('@' + attr_name).to_sym, value)
+      end
+    end
+
+    private
+
+    # Return a single data structure that holds all persistent data for this
+    # class.
+    def _serialize
+      attributes = {}
+      self.class.attributes.each do |attr|
+        ivar = ('@' + attr.to_s).to_sym
+        if (value = instance_variable_get(ivar)).is_a?(ObjectBase)
+          raise ArgumentError, "The instance variable #{ivar} contains a " +
+                               "reference to a PEROBS::ObjectBase object! " +
+                               "This is not allowed. You must use the " +
+                               "accessor method to assign a reference to " +
+                               "another PEROBS object."
+        end
+        attributes[attr.to_s] = value
+      end
+      attributes
+    end
+
+    def _set(attr, val)
+      ivar = ('@' + attr.to_s).to_sym
+      if val.is_a?(ObjectBase)
+        # References to other PEROBS::Objects must be handled somewhat
+        # special.
+        if @store != val.store
+          raise ArgumentError, 'The referenced object is not part of this store'
+        end
+        # To release the object from the Ruby object list later, we store the
+        # PEROBS::Store ID of the referenced object instead of the actual
+        # reference.
+        instance_variable_set(ivar, POReference.new(val._id))
+      else
+        instance_variable_set(ivar, val)
+      end
+      # Let the store know that we have a modified object.
+      @store.cache.cache_write(self)
+
+      val
+    end
+
+    def _get(attr)
+      value = instance_variable_get(('@' + attr.to_s).to_sym)
+      if value.is_a?(POReference)
+        @store.object_by_id(value.id)
+      else
+        value
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/ObjectBase.rb

@@ -0,0 +1,159 @@
+# encoding: UTF-8
+#
+# = ObjectBase.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  # This class is used to replace a direct reference to another Ruby object by
+  # the Store ID. This makes object disposable by the Ruby garbage collector
+  # since it's no longer referenced once it has been evicted from the
+  # PEROBS::Store cache.
+  class POReference < Struct.new(:id)
+  end
+
+  # Base class for all persistent objects. It provides the functionality
+  # common to all classes of persistent objects.
+  class ObjectBase
+
+    attr_reader :_id, :store
+
+    # Create a new PEROBS::ObjectBase object.
+    def initialize(store)
+      @store = store
+      @_id = @store.db.new_id
+      @_stash_map = nil
+
+      # Let the store know that we have a modified object.
+      @store.cache.cache_write(self)
+    end
+
+    # Two objects are considered equal if their object IDs are the same.
+    def ==(obj)
+      obj && @_id == obj._id
+    end
+
+    # Write the object into the backing store database.
+    def _sync
+      # Reset the stash map to ensure that it's reset before the next
+      # transaction is being started.
+      @_stash_map = nil
+
+      db_obj = {
+        'class' => self.class.to_s,
+        'data' => _serialize
+      }
+      @store.db.put_object(db_obj, @_id)
+    end
+
+    # Read an raw object with the specified ID from the backing store and
+    # instantiate a new object of the specific type.
+    def ObjectBase.read(store, id)
+      # Read the object from database.
+      db_obj = store.db.get_object(id)
+
+      # Call the constructor of the specified class.
+      obj = Object.const_get(db_obj['class']).new(store)
+      # The object gets created with a new ID by default. We need to restore
+      # the old one.
+      obj._change_id(id)
+      obj._deserialize(db_obj['data'])
+
+      obj
+    end
+
+    # Restore the object state from the storage back-end.
+    # @param level [Fixnum] the transaction nesting level
+    def _restore(level)
+      # Find the most recently stored state of this object. This could be on
+      # any previous stash level or in the regular object DB. If the object
+      # was created during the transaction, there is not previous state to
+      # restore to.
+      id = nil
+      if @_stash_map
+        (level - 1).downto(0) do |lvl|
+          if @_stash_map[lvl]
+            id = @_stash_map[lvl]
+            break
+          end
+        end
+      end
+      unless id
+        if @store.db.include?(@_id)
+          id = @_id
+        end
+      end
+      if id
+        db_obj = store.db.get_object(id)
+        _deserialize(db_obj['data'])
+      end
+    end
+
+    # Save the object state for this transaction level to the storage
+    # back-end. The object gets a new ID that is stored in @_stash_map to map
+    # the stash ID back to the original data.
+    def _stash(level)
+      db_obj = {
+        'class' => self.class.to_s,
+        'data' => _serialize
+      }
+      @_stash_map = [] unless @_stash_map
+      # Get a new ID to store this version of the object.
+      @_stash_map[level] = stash_id = @store.db.new_id
+      @store.db.put_object(db_obj, stash_id)
+    end
+
+    # Library internal method. Do not use outside of this library.
+    # @private
+    def _change_id(id)
+      # Unregister the object with the old ID from the write cache to prevent
+      # cache corruption. The objects are index by ID in the cache.
+      store.cache.unwrite(self)
+      @_id = id
+    end
+
+    private
+
+    def _dereferenced(v)
+      v.is_a?(POReference) ? @store.object_by_id(v.id) : v
+    end
+
+    def _referenced(obj)
+      if obj.is_a?(ObjectBase)
+        # The obj is a reference to another persistent object. Store the ID
+        # of that object in a POReference object.
+        if @store != obj.store
+          raise ArgumentError, 'The referenced object is not part of this store'
+        end
+        POReference.new(obj._id)
+      else
+        obj
+      end
+    end
+
+  end
+
+end
+