perobs 0.0.1

data/lib/perobs/Cache.rb

@@ -0,0 +1,201 @@
+# encoding: UTF-8
+#
+# = Cache.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Store'
+
+module PEROBS
+
+  # The Cache provides two functions for the PEROBS Store. It keeps some
+  # amount of objects in memory to substantially reduce read access latencies.
+  # It # also stores a list of objects that haven't been synced to the
+  # permanent store yet to accelerate object writes.
+  class Cache
+
+    # Create a new Cache object.
+    # @param bits [Fixnum] Number of bits for the cache index. This parameter
+    #        heavilty affects the performance and memory consumption of the
+    #        cache.
+    def initialize(bits = 16)
+      @bits = bits
+      # This mask is used to access the _bits_ least significant bits of the
+      # object ID.
+      @mask = 2 ** bits - 1
+      # Initialize the read and write cache
+      reset
+    end
+
+    # Add an PEROBS::Object to the read cache.
+    # @param obj [PEROBS::ObjectBase]
+    def cache_read(obj)
+      @reads[index(obj)] = obj
+    end
+
+    # Add a PEROBS::Object to the write cache.
+    # @param obj [PEROBS::ObjectBase]
+    def cache_write(obj)
+      if @transaction_stack.empty?
+        idx = index(obj)
+        if (old_obj = @writes[idx]) && old_obj._id != obj._id
+          # There is another old object using this cache slot. Before we can
+          # re-use the slot, we need to sync it to the permanent storage.
+          old_obj._sync
+        end
+        @writes[idx] = obj
+      else
+        # When a transaction is active, we don't have a write cache. The read
+        # cache is used to speed up access to recently used objects.
+        cache_read(obj)
+        # Push the reference of the modified object into the write buffer for
+        # this transaction level.
+        unless @transaction_stack.last.include?(obj)
+          @transaction_stack.last << obj
+        end
+      end
+    end
+
+    # Remove an object from the write cache. This will prevent a modified
+    # object from being written to the back-end store.
+    def unwrite(obj)
+      if @transaction_stack.empty?
+        idx = index(obj)
+        if (old_obj = @writes[idx]).nil? || old_obj._id != obj._id
+          raise RuntimeError, "Object to unwrite is not in cache"
+        end
+        @writes[idx] = nil
+      else
+        unless @transaction_stack.last.include?(obj)
+          raise RuntimeError, 'unwrite failed'
+        end
+        @transaction_stack.last.delete(obj)
+      end
+    end
+
+    # Return the PEROBS::Object with the specified ID or nil if not found.
+    # @param id [Fixnum or Bignum] ID of the cached PEROBS::ObjectBase
+    def object_by_id(id)
+      idx = id & @mask
+      # The index is just a hash. We still need to check if the object IDs are
+      # actually the same before we can return the object.
+      if (obj = @writes[idx]) && obj._id == id
+        # The object was in the write cache.
+        return obj
+      elsif (obj = @reads[idx]) && obj._id == id
+        # The object was in the read cache.
+        return obj
+      end
+
+      nil
+    end
+
+    # Flush all pending writes to the persistant storage back-end.
+    def flush
+      @writes.each { |w| w._sync if w }
+      @writes = ::Array.new(2 ** @bits)
+    end
+
+    # Returns true if the Cache is currently handling a transaction, false
+    # otherwise.
+    # @return [true/false]
+    def in_transaction?
+      !@transaction_stack.empty?
+    end
+
+    # Tell the cache to start a new transaction. If no other transaction is
+    # active, the write cached is flushed before the transaction is started.
+    def begin_transaction
+      if @transaction_stack.empty?
+        # This is the top-level transaction. Flush the write buffer to save
+        # the current state of all objects.
+        flush
+      else
+        @transaction_stack.last.each do |o|
+          o._stash(@transaction_stack.length - 1)
+        end
+      end
+      # Push a transaction buffer onto the transaction stack. This buffer will
+      # hold a reference to all objects modified during this transaction.
+      @transaction_stack.push(::Array.new)
+    end
+
+    # Tell the cache to end the currently active transaction. All write
+    # operations of the current transaction will be synced to the storage
+    # back-end.
+    def end_transaction
+      case @transaction_stack.length
+      when 0
+        raise RuntimeError, 'No ongoing transaction to end'
+      when 1
+        # All transactions completed successfully. Write all modified objects
+        # into the backend storage.
+        @transaction_stack.pop.each { |o| o._sync }
+      else
+        # A nested transaction completed successfully. We add the list of
+        # modified objects to the list of the enclosing transaction.
+        transactions = @transaction_stack.pop
+        # Merge the two lists
+        @transaction_stack.push(@transaction_stack.pop + transactions)
+        # Ensure that each object is only included once in the list.
+        @transaction_stack.last.uniq!
+      end
+    end
+
+    # Tell the cache to abort the currently active transaction. All modified
+    # objects will be restored from the storage back-end to their state before
+    # the transaction started.
+    def abort_transaction
+      if @transaction_stack.empty?
+        raise RuntimeError, 'No ongoing transaction to abort'
+      end
+      @transaction_stack.pop.each { |o| o._restore(@transaction_stack.length) }
+    end
+
+    # Clear all cached entries. You must call flush before calling this
+    # method. Otherwise unwritten objects will be lost.
+    def reset
+      # The read and write caches are Arrays. We use the _bits_ least
+      # significant bits of the PEROBS::ObjectBase ID to select the index in
+      # the read or write cache Arrays.
+      @reads = ::Array.new(2 ** @bits)
+      @writes = ::Array.new(2 ** @bits)
+      @transaction_stack = []
+    end
+
+    # Don't include the cache buffers in output of other objects that
+    # reference Cache.
+    def inspect
+    end
+
+    private
+
+    def index(obj)
+      obj._id & @mask
+    end
+
+  end
+
+end
+

data/lib/perobs/DataBase.rb

@@ -0,0 +1,115 @@
+# encoding: UTF-8
+#
+# = DataBase.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+require 'json'
+require 'json/add/core'
+require 'json/add/struct'
+require 'yaml'
+require 'fileutils'
+
+require 'perobs/ObjectBase'
+
+module PEROBS
+
+  # Base class for all storage back-ends.
+  class DataBase
+
+    def initialize(serializer = :json)
+      @serializer = serializer
+    end
+
+    # Serialize the given object using the object serializer.
+    # @param obj [ObjectBase] Object to serialize
+    # @return [String] Serialized version
+    def serialize(obj)
+      begin
+        case @serializer
+        when :marshal
+          Marshal.dump(obj)
+        when :json
+          obj.to_json
+        when :yaml
+          YAML.dump(obj)
+        end
+      rescue => e
+        raise RuntimeError,
+              "Cannot serialize object as #{@serializer}: #{e.message}"
+      end
+    end
+
+    # De-serialize the given String into a Ruby object.
+    # @param raw [String]
+    # @return [Hash] Deserialized version
+    def deserialize(raw)
+      begin
+        case @serializer
+        when :marshal
+          Marshal.load(raw)
+        when :json
+          JSON.parse(raw, :create_additions => true)
+        when :yaml
+          YAML.load(raw)
+        end
+      rescue => e
+        raise RuntimeError,
+              "Cannot de-serialize object with #{@serializer} parser: " +
+              e.message
+      end
+    end
+
+    # Generate a new unique ID. It uses random numbers between 0 and 2**64 -
+    # 1. Deriving classes can overwrite this method. They must implement a
+    # include? method.
+    # @return [Fixnum or Bignum]
+    def new_id
+      begin
+        # Generate a random number. It's recommended to not store more than
+        # 2**62 objects in the same store.
+        id = rand(2**64)
+        # Ensure that we don't have already another object with this ID.
+      end while include?(id)
+
+      id
+    end
+
+    private
+
+    # Ensure that we have a directory to store the DB items.
+    def ensure_dir_exists(dir)
+      unless Dir.exists?(dir)
+        begin
+          Dir.mkdir(dir)
+        rescue IOError => e
+          raise IOError, "Cannote create DB directory '#{dir}': #{e.message}"
+        end
+      end
+    end
+
+  end
+
+end

data/lib/perobs/FileSystemDB.rb

@@ -0,0 +1,171 @@
+# encoding: UTF-8
+#
+# = FileSystemDB.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+require 'json'
+require 'json/add/core'
+require 'json/add/struct'
+require 'yaml'
+require 'fileutils'
+
+require 'perobs/DataBase'
+require 'perobs/ObjectBase'
+
+module PEROBS
+
+  # This class provides a filesytem based database store for objects.
+  class FileSystemDB < DataBase
+
+    @@Extensions = {
+      :marshal => '.mshl',
+      :json => '.json',
+      :yaml => '.yml'
+    }
+
+    # Create a new FileSystemDB object. This will create a DB with the given
+    # name. A database will live in a directory of that name.
+    # @param db_name [String] name of the DB directory
+    # @param options [Hash] options to customize the behavior. Currently only
+    #        the following option is supported:
+    #        :serializer : Can be :marshal, :json, :yaml
+    def initialize(db_name, options = {})
+      super(options[:serializer] || :json)
+      @db_dir = db_name
+
+      # Create the database directory if it doesn't exist yet.
+      ensure_dir_exists(@db_dir)
+    end
+
+    # Return true if the object with given ID exists
+    # @param id [Fixnum or Bignum]
+    def include?(id)
+      File.exists?(object_file_name(id))
+    end
+
+    # Store the given object into the filesystem.
+    # @param obj [Hash] Object as defined by PEROBS::ObjectBase
+    def put_object(obj, id)
+      File.write(object_file_name(id), serialize(obj))
+    end
+
+    # Load the given object from the filesystem.
+    # @param id [Fixnum or Bignum] object ID
+    # @return [Hash] Object as defined by PEROBS::ObjectBase
+    def get_object(id)
+      begin
+        raw = File.read(file_name = object_file_name(id))
+      rescue => e
+        raise RuntimeError, "Error in #{file_name}: #{e.message}"
+      end
+      deserialize(raw)
+    end
+
+    # This method must be called to initiate the marking process.
+    def clear_marks
+      @mark_start = Time.now
+      # The filesystem stores access times with second granularity. We need to
+      # wait 1 sec. to ensure that all marks are noticeable.
+      sleep(1)
+    end
+
+    # Permanently delete all objects that have not been marked. Those are
+    # orphaned and are no longer referenced by any actively used object.
+    def delete_unmarked_objects
+      Dir.glob(File.join(@db_dir, '*')) do |dir|
+        next unless Dir.exists?(dir)
+
+        Dir.glob(File.join(dir, '*')) do |file|
+          if File.atime(file) <= @mark_start
+            File.delete(file)
+          end
+        end
+      end
+    end
+
+    # Mark an object.
+    # @param id [Fixnum or Bignum] ID of the object to mark
+    def mark(id)
+      FileUtils.touch(object_file_name(id))
+    end
+
+    # Check if the object is marked.
+    # @param id [Fixnum or Bignum] ID of the object to check
+    def is_marked?(id)
+      File.atime(object_file_name(id)) > @mark_start
+    end
+
+    # Check if the stored object is syntactically correct.
+    # @param id [Fixnum/Bignum] Object ID
+    # @param repair [TrueClass/FalseClass] True if an repair attempt should be
+    #        made.
+    # @return [TrueClass/FalseClass] True if the object is OK, otherwise
+    #         false.
+    def check(id, repair)
+      file_name = object_file_name(id)
+      unless File.exists?(file_name)
+        $stderr.puts "Object file for ID #{id} does not exist"
+        return false
+      end
+
+      begin
+        get_object(id)
+      rescue => e
+        $stderr.puts "Cannot read object file #{file_name}: #{e.message}"
+        return false
+      end
+
+      true
+    end
+
+    private
+
+    # Ensure that we have a directory to store the DB items.
+    def ensure_dir_exists(dir)
+      unless Dir.exists?(dir)
+        begin
+          Dir.mkdir(dir)
+        rescue IOError => e
+          raise IOError, "Cannote create DB directory '#{dir}': #{e.message}"
+        end
+      end
+    end
+
+    # Determine the file name to store the object. The object ID determines
+    # the directory and file name inside the store.
+    # @param id [Fixnum or Bignum] ID of the object
+    def object_file_name(id)
+      hex_id = "%016X" % id
+      dir = hex_id[0..1]
+      ensure_dir_exists(File.join(@db_dir, dir))
+
+      File.join(@db_dir, dir, hex_id + @@Extensions[@serializer])
+    end
+
+  end
+
+end
+

data/lib/perobs/Hash.rb

@@ -0,0 +1,175 @@
+# encoding: UTF-8
+#
+# = Hash.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/ObjectBase'
+
+module PEROBS
+
+  # A Hash that is transparently persisted in the back-end storage. It is very
+  # similar to the Ruby built-in Hash class but has some additional
+  # limitations. The hash key must always be a String.
+  class Hash < ObjectBase
+
+    # Create a new PersistentHash object.
+    # @param store [Store] The Store this hash is stored in
+    # @param default [Any] The default value that is returned when no value is
+    #        stored for a specific key.
+    def initialize(store, default = nil)
+      super(store)
+      @default = nil
+      @data = {}
+    end
+
+    # Retrieves the value object corresponding to the
+    # key object. If not found, returns the default value.
+    def [](key)
+      #unless key.is_a?(String)
+      #  raise ArgumentError, 'The Hash key must be of type String'
+      #end
+      _dereferenced(@data.include?(key) ? @data[key] : @default)
+    end
+
+    # Associates the value given by value with the key given by key.
+    # @param key [String] The key
+    # @param value [Any] The value to store
+    def []=(key, value)
+      #unless key.is_a?(String)
+      #  raise ArgumentError, 'The Hash key must be of type String'
+      #end
+      @data[key] = _referenced(value)
+      @store.cache.cache_write(self)
+
+      value
+    end
+
+    # Equivalent to Hash::clear
+    def clear
+      @store.cache.cache_write(self)
+      @data.clear
+    end
+
+    # Equivalent to Hash::delete
+    def delete(key)
+      @store.cache.cache_write(self)
+      @data.delete(key)
+    end
+
+    # Equivalent to Hash::delete_if
+    def delete_if
+      @store.cache.cache_write(self)
+      @data.delete_if do |k, v|
+        yield(k, _dereferenced(v))
+      end
+    end
+
+    # Equivalent to Hash::each
+    def each
+      @data.each do |k, v|
+        yield(k, _dereferenced(v))
+      end
+    end
+
+    # Equivalent to Hash::each_key
+    def each_key
+      @data.each_key { |k| yield(k) }
+    end
+
+    # Equivalent to Hash::each_value
+    def each_value
+      @data.each_value do |v|
+        yield(_dereferenced(v))
+      end
+    end
+
+    # Equivalent to Hash::empty?
+    def emtpy?
+      @data.empty?
+    end
+
+    # Equivalent to Hash::has_key?
+    def has_key?(key)
+      @data.has_key?(key)
+    end
+    alias include? has_key?
+    alias key? has_key?
+    alias member? has_key?
+
+    # Equivalent to Hash::keys
+    def keys
+      @data.keys
+    end
+
+    # Equivalent to Hash::length
+    def length
+      @data.length
+    end
+    alias size length
+
+    # Equivalent to Hash::map
+    def map
+      @data.map do |k, v|
+        yield(k, _dereferenced(v))
+      end
+    end
+
+    # Equivalent to Hash::values
+    def values
+      @data.values.map { |v| _dereferenced(v) }
+    end
+
+    # Return a list of all object IDs of all persistend objects that this Hash
+    # is referencing.
+    # @return [Array of Fixnum or Bignum] IDs of referenced objects
+    def _referenced_object_ids
+      @data.each_value.select { |v| v && v.is_a?(POReference) }.map { |o| o.id }
+    end
+
+    # This method should only be used during store repair operations. It will
+    # delete all referenced to the given object ID.
+    # @param id [Fixnum/Bignum] targeted object ID
+    def _delete_reference_to_id(id)
+      @data.delete_if { |k, v| v && v.is_a?(POReference) && v.id == id }
+    end
+
+    # Restore the persistent data from a single data structure.
+    # This is a library internal method. Do not use outside of this library.
+    # @param data [Hash] the actual Hash object
+    # @private
+    def _deserialize(data)
+      @data = data
+    end
+
+    private
+
+    def _serialize
+      @data
+    end
+
+  end
+
+end
+