perobs 2.3.1 → 2.4.0

data/lib/perobs/FlatFileDB.rb

@@ -0,0 +1,249 @@
+# encoding: UTF-8
+#
+# = FlatFileDB.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+require 'zlib'
+
+require 'perobs/Log'
+require 'perobs/DataBase'
+require 'perobs/FlatFile'
+
+module PEROBS
+
+  # The FlatFileDB is a storage backend that uses a single flat file to store
+  # the value blobs.
+  class FlatFileDB < DataBase
+
+    # This version number increases whenever the on-disk format changes in a
+    # way that requires conversion actions after an update.
+    VERSION = 1
+
+    attr_reader :max_blob_size
+
+    # Create a new FlatFileDB object.
+    # @param db_name [String] name of the DB directory
+    # @param options [Hash] options to customize the behavior. Currently only
+    #        the following options are supported:
+    #        :serializer    : Can be :marshal, :json, :yaml
+    def initialize(db_name, options = {})
+      super(options[:serializer] || :json)
+
+      @db_dir = db_name
+      # Create the database directory if it doesn't exist yet.
+      ensure_dir_exists(@db_dir)
+      PEROBS.log.open(File.join(@db_dir, 'log'))
+      check_version
+
+      # Read the existing DB config.
+      @config = get_hash('config')
+      check_option('serializer')
+
+      put_hash('config', @config)
+    end
+
+    # Open the FlatFileDB for transactions.
+    def open
+      @flat_file = FlatFile.new(@db_dir)
+      @flat_file.open
+      PEROBS.log.info "FlatFile opened"
+    end
+
+    # Close the FlatFileDB.
+    def close
+      @flat_file.close
+      @flat_file = nil
+      PEROBS.log.info "FlatFile closed"
+    end
+
+    # Delete the entire database. The database is no longer usable after this
+    # method was called.
+    def delete_database
+      FileUtils.rm_rf(@db_dir)
+    end
+
+    def FlatFileDB::delete_db(db_name)
+      FileUtils.rm_rf(db_name)
+    end
+
+    # Return true if the object with given ID exists
+    # @param id [Fixnum or Bignum]
+    def include?(id)
+      !@flat_file.find_obj_addr_by_id(id).nil?
+    end
+
+    # Store a simple Hash as a JSON encoded file into the DB directory.
+    # @param name [String] Name of the hash. Will be used as file name.
+    # @param hash [Hash] A Hash that maps String objects to strings or
+    # numbers.
+    def put_hash(name, hash)
+      file_name = File.join(@db_dir, name + '.json')
+      begin
+        File.write(file_name, hash.to_json)
+      rescue => e
+        PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
+      end
+    end
+
+    # Load the Hash with the given name.
+    # @param name [String] Name of the hash.
+    # @return [Hash] A Hash that maps String objects to strings or numbers.
+    def get_hash(name)
+      file_name = File.join(@db_dir, name + '.json')
+      return ::Hash.new unless File.exist?(file_name)
+
+      begin
+        json = File.read(file_name)
+      rescue => e
+        PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
+      end
+      JSON.parse(json, :create_additions => true)
+    end
+
+    # Store the given object into the cluster files.
+    # @param obj [Hash] Object as defined by PEROBS::ObjectBase
+    def put_object(obj, id)
+      @flat_file.delete_obj_by_id(id)
+      @flat_file.write_obj_by_id(id, serialize(obj))
+    end
+
+    # Load the given object from the filesystem.
+    # @param id [Fixnum or Bignum] object ID
+    # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
+    #         not exist
+    def get_object(id)
+      if (raw_obj = @flat_file.read_obj_by_id(id))
+        return deserialize(raw_obj)
+      else
+        nil
+      end
+    end
+
+    # This method must be called to initiate the marking process.
+    def clear_marks
+      t = Time.now
+      PEROBS.log.info "Clearing all marks"
+      @flat_file.clear_all_marks
+      PEROBS.log.info "All marks cleared in #{Time.now - t} seconds"
+    end
+
+    # Permanently delete all objects that have not been marked. Those are
+    # orphaned and are no longer referenced by any actively used object.
+    # @return [Array] List of IDs that have been removed from the DB.
+    def delete_unmarked_objects
+      t = Time.now
+      PEROBS.log.info "Deleting unmarked objects"
+      retval = @flat_file.delete_unmarked_objects
+      PEROBS.log.info "Unmarked objects deleted in #{Time.now - t} seconds"
+      retval
+    end
+
+    # Mark an object.
+    # @param id [Fixnum or Bignum] ID of the object to mark
+    def mark(id)
+      @flat_file.mark_obj_by_id(id)
+    end
+
+    # Check if the object is marked.
+    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param ignore_errors [Boolean] If set to true no errors will be raised
+    #        for non-existing objects.
+    def is_marked?(id, ignore_errors = false)
+      @flat_file.is_marked_by_id?(id)
+    end
+
+    # Basic consistency check.
+    # @param repair [TrueClass/FalseClass] True if found errors should be
+    #        repaired.
+    def check_db(repair = false)
+      t = Time.now
+      PEROBS.log.info "check_db started"
+      @flat_file.check(repair)
+      PEROBS.log.info "check_db completed in #{Time.now - t} seconds"
+    end
+
+    # Check if the stored object is syntactically correct.
+    # @param id [Fixnum/Bignum] Object ID
+    # @param repair [TrueClass/FalseClass] True if an repair attempt should be
+    #        made.
+    # @return [TrueClass/FalseClass] True if the object is OK, otherwise
+    #         false.
+    def check(id, repair)
+      begin
+        get_object(id)
+      rescue => e
+        PEROBS.log.warn "Cannot read object with ID #{id}: #{e.message}"
+        return false
+      end
+
+      true
+    end
+
+    # Store the given serialized object into the cluster files. This method is
+    # for internal use only!
+    # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
+    # @param id [Fixnum or Bignum] Object ID
+    def put_raw_object(raw, id)
+      @flat_file.delete_obj_(id)
+      @flat_file.write_obj_by_id(id, raw)
+    end
+
+    private
+
+    def check_version
+      version_file = File.join(@db_dir, 'version')
+      version = VERSION
+
+      if File.exist?(version_file)
+        begin
+          version = File.read(version_file).to_i
+        rescue => e
+          PEROBS.log.fatal "Cannot read version number file " +
+                           "'#{version_file}': " + e.message
+        end
+      else
+        write_version_file(version_file)
+      end
+
+      if version > VERSION
+        PEROBS.log.fatal "Cannot downgrade the FlatFile database from " +
+                         "version #{version} to version #{VERSION}"
+      end
+    end
+
+    def write_version_file(version_file)
+      begin
+        File.write(version_file, "#{VERSION}")
+      rescue => e
+        PEROBS.log.fatal "Cannot write version number file " +
+                         "'#{version_file}': " + e.message
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/FreeSpaceManager.rb

@@ -0,0 +1,204 @@
+# encoding: UTF-8
+#
+# = FreeSpaceManager.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+require 'perobs/StackFile'
+
+module PEROBS
+
+  # The FreeSpaceManager keeps a list of the free spaces in the FlatFile. Each
+  # space is stored with address and size. The data is persisted in the file
+  # system. Internally the free spaces are stored in different pools. Each
+  # pool holds spaces that are at least of a given size and not as big as the
+  # next pool up. Pool entry minimum sizes increase by a factor of 2 from
+  # pool to pool.
+  class FreeSpaceManager
+
+    # Create a new FreeSpaceManager object in the specified directory.
+    # @param dir [String] directory path
+    def initialize(dir)
+      @dir = dir
+      @pools = []
+    end
+
+    # Open the pool files.
+    def open
+      Dir.glob(File.join(@dir, 'free_list_*.stack')).each do |file|
+        basename = File.basename(file)
+        # Cut out the pool index from the file name.
+        index = basename[10..-7].to_i
+        @pools[index] = StackFile.new(@dir, basename[0..-7], 2 * 8)
+      end
+    end
+
+    # Close all pool files.
+    def close
+      @pools = []
+    end
+
+    # Add a new space with a given address and size.
+    # @param address [Integer] Starting address of the space
+    # @param size [Integer] size of the space in bytes
+    def add_space(address, size)
+      if size <= 0
+        PEROBS.log.fatal "Size (#{size}) must be larger than 0."
+      end
+      pool_index = msb(size)
+      new_pool(pool_index) unless @pools[pool_index]
+      push_pool(pool_index, [ address, size ].pack('QQ'))
+    end
+
+    # Get a space that has at least the requested size.
+    # @param size [Integer] Required size in bytes
+    # @return [Array] Touple with address and actual size of the space.
+    def get_space(size)
+      if size <= 0
+        PEROBS.log.fatal "Size (#{size}) must be larger than 0."
+      end
+      # When we search for a free space we need to search the pool that
+      # corresponds to (size - 1) * 2. It is the pool that has the spaces that
+      # are at least as big as size.
+      pool_index = size == 1 ? 0 : msb(size - 1) + 1
+      unless @pools[pool_index]
+        return nil
+      else
+        return nil unless (entry = pop_pool(pool_index))
+        sp_address, sp_size = entry.unpack('QQ')
+        if sp_size < size
+          PEROBS.log.fatal "Space at address #{sp_address} is too small. " +
+            "Must be at least #{size} bytes but is only #{sp_size} bytes."
+        end
+        [ sp_address, sp_size ]
+      end
+    end
+
+    # Clear all pools and forget any registered spaces.
+    def clear
+      @pools.each do |pool|
+        if pool
+          pool.open
+          pool.clear
+          pool.close
+        end
+      end
+      close
+    end
+
+    # Check if there is a space in the free space lists that matches the
+    # address and the size.
+    # @param [Integer] address Address of the space
+    # @param [Integer] size Length of the space in bytes
+    # @return [Boolean] True if space is found, false otherwise
+    def has_space?(address, size)
+      unless (pool = @pools[msb(size)])
+        return false
+      end
+
+      pool.open
+      pool.each do |entry|
+        sp_address, sp_size = entry.unpack('QQ')
+        if address == sp_address
+          if size != sp_size
+            PEROBS.log.fatal "FreeSpaceManager has space with different " +
+              "size"
+          end
+          pool.close
+          return true
+        end
+      end
+
+      pool.close
+      false
+    end
+
+    def check(flat_file)
+      @pools.each do |pool|
+        next unless pool
+
+        pool.open
+        pool.each do |entry|
+          address, size = entry.unpack('QQ')
+          unless flat_file.has_space?(address, size)
+            PEROBS.log.error "FreeSpaceManager has space that isn't " +
+              "available in the FlatFile."
+            return false
+          end
+        end
+        pool.close
+      end
+
+      true
+    end
+
+    def inspect
+      '[' + @pools.map do |p|
+        if p
+          p.open
+          r = p.to_ary.map { |bs| bs.unpack('QQ')}.inspect
+          p.close
+          r
+        else
+          'nil'
+        end
+      end.join(', ') + ']'
+    end
+
+    private
+
+    def new_pool(index)
+      # The file name pattern for the pool files.
+      filename = "free_list_#{index}"
+      @pools[index] = sf = StackFile.new(@dir, filename, 2 * 8)
+    end
+
+    def push_pool(index, value)
+      pool = @pools[index]
+      pool.open
+      pool.push(value)
+      pool.close
+    end
+
+    def pop_pool(index)
+      pool = @pools[index]
+      pool.open
+      value = pool.pop
+      pool.close
+
+      value
+    end
+
+    def msb(i)
+      unless i > 0
+        PEROBS.log.fatal "i must be larger than 0"
+      end
+      i.to_s(2).length - 1
+    end
+
+  end
+
+end
+

data/lib/perobs/Hash.rb

@@ -2,7 +2,7 @@
 #
 # = Hash.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -25,6 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/Log'
 require 'perobs/ObjectBase'
 
 module PEROBS
@@ -48,7 +49,7 @@ module PEROBS
       :==, :[], :assoc, :compare_by_identity, :compare_by_identity?, :default,
       :default_proc, :each, :each_key, :each_pair, :each_value, :empty?,
       :eql?, :fetch, :flatten, :has_key?, :has_value?, :hash, :include?,
-      :inspect, :invert, :key, :key?, :keys, :length, :member?, :merge,
+      :invert, :key, :key?, :keys, :length, :member?, :merge,
       :pretty_print, :pretty_print_cycle, :rassoc, :reject, :select, :size,
       :to_a, :to_h, :to_hash, :to_s, :value?, :values, :values_at
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
@@ -104,6 +105,7 @@ module PEROBS
       @data.delete_if do |k, v|
         v && v.respond_to?(:is_poxreference?) && v.id == id
       end
+      @store.cache.cache_write(self)
     end
 
     # Restore the persistent data from a single data structure.
@@ -117,6 +119,17 @@ module PEROBS
       @data
     end
 
+    # Textual dump for debugging purposes
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{@_id}>\n{\n" +
+      @data.map do |k, v|
+        "  #{k.inspect} => " + (v.respond_to?(:is_poxreference?) ?
+                "<PEROBS::ObjectBase:#{v._id}>" : v.inspect)
+      end.join(",\n") +
+      "\n}\n"
+    end
+
     private
 
     def _serialize
@@ -130,9 +143,9 @@ module PEROBS
           # objects should not be used directly. The library only exposes them
           # via POXReference proxy objects.
           if v.is_a?(ObjectBase)
-            raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
+            PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
               "It is stored in a PEROBS::Hash with key #{k.inspect}. " +
-              'Have you used self() instead of myself() to' +
+              'Have you used self() instead of myself() to ' +
               "get the reference of this PEROBS object?\n" +
               v.inspect
           end

data/lib/perobs/IndexTree.rb

@@ -0,0 +1,164 @@
+# encoding: UTF-8
+#
+# = IndexTree.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/FixedSizeBlobFile'
+require 'perobs/IndexTreeNode'
+
+module PEROBS
+
+  # The IndexTree maps the object ID to the address in the FlatFile. The
+  # search in the tree is much faster than the linear search in the FlatFile.
+  class IndexTree
+
+    # Determines how many levels of the IndexTree will be kept in memory to
+    # accerlerate the access. A number of 7 will keep up to 21845 entries in
+    # the cache but will accelerate the access to the FlatFile address.
+    MAX_CACHED_LEVEL = 7
+
+    attr_reader :nodes, :ids
+
+    def initialize(db_dir)
+      # Directory path used to store the files.
+      @db_dir = db_dir
+
+      # This FixedSizeBlobFile contains the nodes of the IndexTree.
+      @nodes = FixedSizeBlobFile.new(db_dir, 'database_index',
+                                     IndexTreeNode::NODE_BYTES)
+
+      # The node sequence usually only reveals a partial match with the
+      # requested ID. So, the leaves of the tree point to the object_id_index
+      # file which contains the full object ID and the address of the
+      # corresponding object in the FlatFile.
+      @ids = FixedSizeBlobFile.new(db_dir, 'object_id_index', 2 * 8)
+
+      # The first MAX_CACHED_LEVEL levels of nodes will be cached in memory to
+      # improve access times.
+      @node_cache = {}
+    end
+
+    # Open the tree files.
+    def open
+      @nodes.open
+      @ids.open
+      @root = IndexTreeNode.new(self, 0, 0)
+    end
+
+    # Close the tree files.
+    def close
+      @ids.close
+      @nodes.close
+    end
+
+    # Flush out all unwritten data
+    def sync
+      @ids.sync
+      @nodes.sync
+    end
+
+    # Delete all data from the tree.
+    def clear
+      @nodes.clear
+      @ids.clear
+      @node_cache = {}
+      @root = IndexTreeNode.new(self, 0, 0)
+    end
+
+    # Return an IndexTreeNode object that corresponds to the given address.
+    # @param nibble [Fixnum] Index of the nibble the node should correspond to
+    # @param address [Integer] Address of the node in @nodes or nil
+    def get_node(nibble, address = nil)
+      # Generate a mask for the least significant bits up to and including the
+      # nibble.
+      mask = (2 ** ((1 + nibble) * 4)) - 1
+      if address && (node = @node_cache[address & mask])
+        # We have an address and have found the node in the node cache.
+        return node
+      else
+        # We don't have a IndexTreeNode object yet for this node. Create it
+        # with the data from the 'database_index' file.
+        node = IndexTreeNode.new(self, nibble, address)
+        # Add the node to the node cache if it's up to MAX_CACHED_LEVEL levels
+        # down from the root.
+        @node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
+        return node
+      end
+    end
+
+    # Delete a node from the tree that corresponds to the address.
+    # @param nibble [Fixnum] The corresponding nibble for the node
+    # @param address [Integer] The address of the node in @nodes
+    def delete_node(nibble, address)
+      # First delete the node from the node cache.
+      mask = (2 ** ((1 + nibble) * 4)) - 1
+      @node_cache.delete(address & mask)
+      # Then delete it from the 'database_index' file.
+      @nodes.delete_blob(address)
+    end
+
+    # Store a ID/value touple into the tree. The value can later be retrieved
+    # by the ID again. IDs are always unique in the tree. If the ID already
+    # exists in the tree, the value will be overwritten.
+    # @param id [Integer] ID or key
+    # @param value [Integer] value to store
+    def put_value(id, value)
+      #MAX_CACHED_LEVEL.downto(0) do |i|
+      #  mask = (2 ** ((1 + i) * 4)) - 1
+      #  if (node = @node_cache[value & mask])
+      #    return node.put_value(id, value)
+      #  end
+      #end
+      @root.put_value(id, value)
+    end
+
+    # Retrieve the value that was stored with the given ID.
+    # @param id [Integer] ID of the value to retrieve
+    # @return [Fixnum] value
+    def get_value(id)
+      @root.get_value(id)
+    end
+
+    # Delete the value with the given ID.
+    # @param [Integer] id
+    def delete_value(id)
+      @root.delete_value(id)
+    end
+
+    # Check if the index is OK and matches the flat_file data.
+    def check(flat_file)
+      @root.check(flat_file)
+    end
+
+    # Convert the tree into a human readable form.
+    # @return [String]
+    def inspect
+      @root.inspect
+    end
+
+  end
+
+end
+