perobs 0.0.1 → 1.0.0

data/lib/perobs/BlockDB.rb

@@ -1,242 +0,0 @@
-# encoding: UTF-8
-#
-# = BlockDB.rb -- Persistent Ruby Object Store
-#
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
-#
-# MIT License
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-require 'json'
-require 'json/add/core'
-require 'json/add/struct'
-
-module PEROBS
-
-  # This class manages the usage of the data blocks in the corresponding
-  # HashedBlocks object.
-  class BlockDB
-
-    # Create a new BlockDB object.
-    def initialize(dir, block_size)
-      @dir = dir
-      @block_size = block_size
-
-      @index_file_name = File.join(dir, 'index.json')
-      @block_file_name = File.join(dir, 'data')
-      read_index
-    end
-
-    # Write the given bytes with the given ID into the DB.
-    # @param id [Fixnum or Bignum] ID
-    # @param raw [String] sequence of bytes
-    def write_object(id, raw)
-      bytes = raw.bytesize
-      start_address = reserve_blocks(id, bytes)
-      if write_to_block_file(raw, start_address) != bytes
-        raise RuntimeError, 'Object length does not match written bytes'
-      end
-      write_index
-    end
-
-    # Read the entry for the given ID and return it as bytes.
-    # @param id [Fixnum or Bignum] ID
-    # @return [String] sequence of bytes
-    def read_object(id)
-      read_from_block_file(*find(id))
-    end
-
-
-    # Find the data for the object with given id.
-    # @param id [Fixnum or Bignum] Object ID
-    # @return [Array] Returns an Array with two Fixnum entries. The first is
-    #         the number of bytes and the second is the starting offset in the
-    #         block storage file.
-    def find(id)
-      @entries.each do |entry|
-        if entry['id'] == id
-          return [ entry['bytes'], entry['first_block'] * @block_size ]
-        end
-      end
-
-      nil
-    end
-
-    # Write a string of bytes into the file at the given address.
-    # @param raw [String] bytes to write
-    # @param address [Fixnum] offset in the file
-    # @return [Fixnum] number of bytes written
-    def write_to_block_file(raw, address)
-      begin
-        File.write(@block_file_name, raw, address)
-      rescue => e
-        raise IOError,
-              "Cannot write block file #{@block_file_name}: #{e.message}"
-      end
-    end
-
-    # Read _bytes_ bytes from the file starting at offset _address_.
-    # @param bytes [Fixnum] number of bytes to read
-    # @param address [Fixnum] offset in the file
-    def read_from_block_file(bytes, address)
-      begin
-        File.read(@block_file_name, bytes, address)
-      rescue => e
-        raise IOError,
-              "Cannot read block file #{@block_file_name}: #{e.message}"
-      end
-    end
-
-    # Clear the mark on all entries in the index.
-    def clear_marks
-      @entries.each { |e| e['marked'] = false}
-      write_index
-    end
-
-    # Set a mark on the entry with the given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
-    def mark(id)
-      found = false
-      @entries.each do |entry|
-        if entry['id'] == id
-          entry['marked'] = true
-          found = true
-          break
-        end
-      end
-
-      unless found
-        raise ArgumentError, "Cannot find an entry for ID #{id} to mark"
-      end
-
-      write_index
-    end
-
-    # Check if the entry for a given ID is marked.
-    # @param id [Fixnum or Bignum] ID of the entry
-    # @return [TrueClass or FalseClass] true if marked, false otherwise
-    def is_marked?(id)
-      @entries.each do |entry|
-        return entry['marked'] if entry['id'] == id
-      end
-
-      raise ArgumentError, "Cannot find an entry for ID #{id} to check"
-    end
-
-    # Remove all entries from the index that have not been marked.
-    def delete_unmarked_entries
-      @entries.delete_if { |e| e['marked'] == false }
-      write_index
-    end
-
-    private
-
-    # Reserve the blocks needed for the specified number of bytes with the
-    # given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
-    # @param bytes [Fixnum] number of bytes for this entry
-    # @return [Fixnum] the start address of the reserved block
-    def reserve_blocks(id, bytes)
-      # size of the entry in blocks
-      blocks = size_in_blocks(bytes)
-      # index of first block after the last seen entry
-      end_of_last_entry = 0
-      # block index of best fit segment
-      best_fit_start = nil
-      # best fir segment size in blocks
-      best_fit_blocks = nil
-      # If there is already an entry for an object with the _id_, we mark it
-      # for deletion.
-      entry_to_delete = nil
-
-      @entries.each do |entry|
-        if entry['id'] == id
-          # We've found an old entry for this ID.
-          if entry['blocks'] >= blocks
-            # The old entry still fits. Let's just reuse it.
-            entry['bytes'] = bytes
-            entry['blocks'] = blocks
-            return entry['first_block'] * @block_size
-          end
-          # It does not fit. Ignore the entry and mark it for deletion.
-          entry_to_delete = entry
-          next
-        end
-
-        gap = entry['first_block'] - end_of_last_entry
-        if gap >= blocks &&
-          (best_fit_blocks.nil? || gap < best_fit_blocks)
-          # We've found a segment that fits the requested bytes and fits
-          # better than any previous find.
-          best_fit_start = end_of_last_entry
-          best_fit_blocks = gap
-        end
-        end_of_last_entry = entry['first_block'] + entry['blocks']
-      end
-
-      # Delete the old entry if requested.
-      @entries.delete(entry_to_delete) if entry_to_delete
-
-      # Create a new entry and insert it.
-      entry = {
-        'id' => id,
-        'bytes' => bytes,
-        'first_block' => best_fit_start || end_of_last_entry,
-        'blocks' => blocks,
-        'marked' => false
-      }
-      @entries << entry
-      @entries.sort! { |e1, e2| e1['first_block'] <=> e2['first_block'] }
-
-      entry['first_block'] * @block_size
-    end
-
-    def read_index
-      if File.exists?(@index_file_name)
-        begin
-          @entries = JSON.parse(File.read(@index_file_name))
-        rescue => e
-          raise RuntimeError,
-                "BlockDB file #{@index_file_name} corrupted: #{e.message}"
-        end
-      else
-        @entries = []
-      end
-    end
-
-    def write_index
-      begin
-        File.write(@index_file_name, @entries.to_json)
-      rescue => e
-        raise RuntimeError,
-              "Cannot write BlockDB index file #{@index_file_name}: " +
-              e.message
-      end
-    end
-
-    def size_in_blocks(bytes)
-      bytes / @block_size + (bytes % @block_size != 0 ? 1 : 0)
-    end
-
-  end
-
-end
-

data/lib/perobs/FileSystemDB.rb

@@ -1,171 +0,0 @@
-# encoding: UTF-8
-#
-# = FileSystemDB.rb -- Persistent Ruby Object Store
-#
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
-#
-# MIT License
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-require 'time'
-require 'json'
-require 'json/add/core'
-require 'json/add/struct'
-require 'yaml'
-require 'fileutils'
-
-require 'perobs/DataBase'
-require 'perobs/ObjectBase'
-
-module PEROBS
-
-  # This class provides a filesytem based database store for objects.
-  class FileSystemDB < DataBase
-
-    @@Extensions = {
-      :marshal => '.mshl',
-      :json => '.json',
-      :yaml => '.yml'
-    }
-
-    # Create a new FileSystemDB object. This will create a DB with the given
-    # name. A database will live in a directory of that name.
-    # @param db_name [String] name of the DB directory
-    # @param options [Hash] options to customize the behavior. Currently only
-    #        the following option is supported:
-    #        :serializer : Can be :marshal, :json, :yaml
-    def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
-      @db_dir = db_name
-
-      # Create the database directory if it doesn't exist yet.
-      ensure_dir_exists(@db_dir)
-    end
-
-    # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
-    def include?(id)
-      File.exists?(object_file_name(id))
-    end
-
-    # Store the given object into the filesystem.
-    # @param obj [Hash] Object as defined by PEROBS::ObjectBase
-    def put_object(obj, id)
-      File.write(object_file_name(id), serialize(obj))
-    end
-
-    # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
-    # @return [Hash] Object as defined by PEROBS::ObjectBase
-    def get_object(id)
-      begin
-        raw = File.read(file_name = object_file_name(id))
-      rescue => e
-        raise RuntimeError, "Error in #{file_name}: #{e.message}"
-      end
-      deserialize(raw)
-    end
-
-    # This method must be called to initiate the marking process.
-    def clear_marks
-      @mark_start = Time.now
-      # The filesystem stores access times with second granularity. We need to
-      # wait 1 sec. to ensure that all marks are noticeable.
-      sleep(1)
-    end
-
-    # Permanently delete all objects that have not been marked. Those are
-    # orphaned and are no longer referenced by any actively used object.
-    def delete_unmarked_objects
-      Dir.glob(File.join(@db_dir, '*')) do |dir|
-        next unless Dir.exists?(dir)
-
-        Dir.glob(File.join(dir, '*')) do |file|
-          if File.atime(file) <= @mark_start
-            File.delete(file)
-          end
-        end
-      end
-    end
-
-    # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
-    def mark(id)
-      FileUtils.touch(object_file_name(id))
-    end
-
-    # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
-    def is_marked?(id)
-      File.atime(object_file_name(id)) > @mark_start
-    end
-
-    # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
-    # @param repair [TrueClass/FalseClass] True if an repair attempt should be
-    #        made.
-    # @return [TrueClass/FalseClass] True if the object is OK, otherwise
-    #         false.
-    def check(id, repair)
-      file_name = object_file_name(id)
-      unless File.exists?(file_name)
-        $stderr.puts "Object file for ID #{id} does not exist"
-        return false
-      end
-
-      begin
-        get_object(id)
-      rescue => e
-        $stderr.puts "Cannot read object file #{file_name}: #{e.message}"
-        return false
-      end
-
-      true
-    end
-
-    private
-
-    # Ensure that we have a directory to store the DB items.
-    def ensure_dir_exists(dir)
-      unless Dir.exists?(dir)
-        begin
-          Dir.mkdir(dir)
-        rescue IOError => e
-          raise IOError, "Cannote create DB directory '#{dir}': #{e.message}"
-        end
-      end
-    end
-
-    # Determine the file name to store the object. The object ID determines
-    # the directory and file name inside the store.
-    # @param id [Fixnum or Bignum] ID of the object
-    def object_file_name(id)
-      hex_id = "%016X" % id
-      dir = hex_id[0..1]
-      ensure_dir_exists(File.join(@db_dir, dir))
-
-      File.join(@db_dir, dir, hex_id + @@Extensions[@serializer])
-    end
-
-  end
-
-end
-

data/lib/perobs/HashedBlocksDB.rb

@@ -1,153 +0,0 @@
-# encoding: UTF-8
-#
-# = HashedBlocksDB.rb -- Persistent Ruby Object Store
-#
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
-#
-# MIT License
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-require 'time'
-require 'json'
-require 'json/add/core'
-require 'json/add/struct'
-require 'yaml'
-require 'fileutils'
-
-require 'perobs/DataBase'
-require 'perobs/BlockDB'
-
-module PEROBS
-
-  # This class provides a filesytem based database store for objects.
-  class HashedBlocksDB < DataBase
-
-    @@Extensions = {
-      :marshal => '.mshl',
-      :json => '.json',
-      :yaml => '.yml'
-    }
-
-    # Create a new FileSystemDB object. This will create a DB with the given
-    # name. A database will live in a directory of that name.
-    # @param db_name [String] name of the DB directory
-    # @param options [Hash] options to customize the behavior. Currently only
-    #        the following options are supported:
-    #        :serializer  : Can be :marshal, :json, :yaml
-    #        :dir_nibbles : The number of nibbles to use for directory names.
-    #                       Meaningful values are 1, 2, and 3. The larger the
-    #                       number the more back-end files are used. Each
-    #                       nibble provides 16 times more directories.
-    #        :block_size  : The size of the blocks inside the storage files in
-    #                       bytes. This should roughly correspond to the size
-    #                       of the smallest serialized objects you want to
-    #                       store in quantities. It also should be an fraction
-    #                       of 4096, the native storage system block size.
-    def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
-      @db_dir = db_name
-      @dir_nibbles = options[:dir_nibbles] || 2
-      @block_size = options[:block_size] || 256
-
-      # Create the database directory if it doesn't exist yet.
-      ensure_dir_exists(@db_dir)
-    end
-
-    # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
-    def include?(id)
-      !BlockDB.new(directory(id), @block_size).find(id).nil?
-    end
-
-    # Store the given object into the cluster files.
-    # @param obj [Hash] Object as defined by PEROBS::ObjectBase
-    def put_object(obj, id)
-      BlockDB.new(directory(id), @block_size).write_object(id, serialize(obj))
-    end
-
-    # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
-    # @return [Hash] Object as defined by PEROBS::ObjectBase
-    def get_object(id)
-      deserialize(BlockDB.new(directory(id), @block_size).read_object(id))
-    end
-
-    # This method must be called to initiate the marking process.
-    def clear_marks
-      Dir.glob(File.join(@db_dir, '*')) do |dir|
-        BlockDB.new(dir, @block_size).clear_marks
-      end
-    end
-
-    # Permanently delete all objects that have not been marked. Those are
-    # orphaned and are no longer referenced by any actively used object.
-    def delete_unmarked_objects
-      Dir.glob(File.join(@db_dir, '*')) do |dir|
-        BlockDB.new(dir, @block_size).delete_unmarked_entries
-      end
-    end
-
-    # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
-    def mark(id)
-      BlockDB.new(directory(id), @block_size).mark(id)
-    end
-
-    # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
-    def is_marked?(id)
-      BlockDB.new(directory(id), @block_size).is_marked?(id)
-    end
-
-    # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
-    # @param repair [TrueClass/FalseClass] True if an repair attempt should be
-    #        made.
-    # @return [TrueClass/FalseClass] True if the object is OK, otherwise
-    #         false.
-    def check(id, repair)
-      begin
-        get_object(id)
-      rescue => e
-        $stderr.puts "Cannot read object with ID #{id}: #{e.message}"
-        return false
-      end
-
-      true
-    end
-
-    private
-
-    # Determine the file name to store the object. The object ID determines
-    # the directory and file name inside the store.
-    # @param id [Fixnum or Bignum] ID of the object
-    def directory(id)
-      hex_id = "%016X" % id
-      dir = hex_id[0..(@dir_nibbles - 1)]
-      ensure_dir_exists(dir_name = File.join(@db_dir, dir))
-
-      dir_name
-    end
-
-  end
-
-end
-