perobs 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2167d14af9d60510dd03e04309d7ca87a281624
-  data.tar.gz: 849c7b4c782ad6de0b7c0f49e26e91bebc86fdb0
+  metadata.gz: 2cb73944d6f6f8968d868d89e19e229dfecd5fc8
+  data.tar.gz: d9c7598cd06bd8b88678eaec6f71f494eb1a0c1e
 SHA512:
-  metadata.gz: 5469262c566ffe7dca585a4b2d4316a664b756224b321455993422f67e2e8ce3928c8edc5c4d625a87c316b06018a084c6989c3369d9636c19d0db3fd5cddda0
-  data.tar.gz: b66d1656c1a2967e098c4f8fa6c6eb4dd58c2dfc1ffa7cc6711c559dc16225b844b045c4053914a7180acc010c572687bc59378bfd14a9b1f775a726bccfe344
+  metadata.gz: befb35a3bc54b7f261889bd7b6495aaaa51c75f50cbb91fef4a657fda323bacb8080f91934062d2ae28e751ab23bc6130ec033fd802d54ae4c67a86b092442a0
+  data.tar.gz: 013e98cd4fe0fce2063ac8a204a33152f1e96c8e22786870046a66a6aa5d6d941444c3b7caaff319404d5d59a0ad2449fec5638266b883e833833a76131b90b2

data/README.md

@@ -127,7 +127,7 @@ jim.father = joe
 joe.kids << jim
 jim.mother = jane
 jane.kids << jim
-store.sync
+store.exit
 ```
 
 When you run this script, a folder named 'family' will be created. It

data/lib/perobs/Array.rb

@@ -2,7 +2,7 @@
 #
 # = Array.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -25,6 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/Log'
 require 'perobs/ObjectBase'
 
 module PEROBS
@@ -49,7 +50,7 @@ module PEROBS
       :&, :*, :+, :-, :==, :[], :<=>, :at, :abbrev, :assoc, :bsearch, :collect,
       :combination, :compact, :count, :cycle, :dclone, :drop, :drop_while,
       :each, :each_index, :empty?, :eql?, :fetch, :find_index, :first,
-      :flatten, :frozen?, :hash, :include?, :index, :inspect, :join, :last,
+      :flatten, :frozen?, :hash, :include?, :index, :join, :last,
       :length, :map, :pack, :permutation, :pretty_print, :pretty_print_cycle,
       :product, :rassoc, :reject, :repeated_combination,
       :repeated_permutation, :reverse, :reverse_each, :rindex, :rotate,
@@ -107,6 +108,7 @@ module PEROBS
       @data.delete_if do |v|
         v && v.respond_to?(:is_poxreference?) && v.id == id
       end
+      @store.cache.cache_write(self)
     end
 
     # Restore the persistent data from a single data structure.
@@ -118,6 +120,17 @@ module PEROBS
                          POXReference.new(@store, v.id) : v }
     end
 
+    # Textual dump for debugging purposes
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{@_id}>\n[\n" +
+      @data.map do |v|
+        "  " + (v.respond_to?(:is_poxreference?) ?
+                "<PEROBS::ObjectBase:#{v._id}>" : v.inspect)
+      end.join(",\n") +
+      "\n]\n"
+    end
+
     private
 
     def _serialize
@@ -129,9 +142,9 @@ module PEROBS
           # objects should not be used directly. The library only exposes them
           # via POXReference proxy objects.
           if v.is_a?(ObjectBase)
-            raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
+            PEROBS.log.fatal 'A PEROBS::ObjectBase object escaped! ' +
               "It is stored in a PEROBS::Array at index #{@data.index(v)}. " +
-              'Have you used self() instead of myself() to' +
+              'Have you used self() instead of myself() to ' +
               "get the reference of this PEROBS object?\n" +
               v.inspect
           end

data/lib/perobs/BTreeBlob.rb

@@ -27,6 +27,8 @@
 
 require 'zlib'
 
+require 'perobs/Log'
+
 module PEROBS
 
   # This class manages the usage of the data blobs in the corresponding
@@ -77,7 +79,7 @@ module PEROBS
         crc32 = Zlib.crc32(raw, 0)
         start_address = reserve_bytes(id, bytes, crc32)
         if write_to_blobs_file(raw, start_address) != bytes
-          raise RuntimeError, 'Object length does not match written bytes'
+          PEROBS.log.fatal 'Object length does not match written bytes'
         end
         write_index
       end
@@ -118,8 +120,8 @@ module PEROBS
       end
 
       unless found
-        raise ArgumentError,
-              "Cannot find an entry for ID #{'%016X' % id} #{id} to mark"
+        PEROBS.log.fatal "Cannot find an entry for ID #{'%016X' % id} " +
+          "#{id} to mark"
       end
 
       write_index
@@ -136,8 +138,7 @@ module PEROBS
       end
 
       return false if ignore_errors
-      raise ArgumentError,
-            "Cannot find an entry for ID #{'%016X' % id} to check"
+      PEROBS.log.fatal "Cannot find an entry for ID #{'%016X' % id} to check"
     end
 
     # Remove all entries from the index that have not been marked.
@@ -165,7 +166,7 @@ module PEROBS
     # @return [TrueClass/FalseClass] Always true right now
     def check(repair = false)
       # Determine size of the data blobs file.
-      data_file_size = File.exists?(@blobs_file_name) ?
+      data_file_size = File.exist?(@blobs_file_name) ?
         File.size(@blobs_file_name) : 0
 
       next_start = 0
@@ -173,7 +174,7 @@ module PEROBS
       @entries.each do |entry|
         # Entries should never overlap
         if prev_entry && next_start > entry[START]
-          raise RuntimeError,
+          PEROBS.log.fatal
                 "#{@dir}: Index entries are overlapping\n" +
                 "ID: #{'%016X' % prev_entry[ID]}  " +
                 "Start: #{prev_entry[START]}  " +
@@ -185,7 +186,7 @@ module PEROBS
 
         # Entries must fit within the data file
         if next_start > data_file_size
-          raise RuntimeError,
+          PEROBS.log.fatal
                 "#{@dir}: Entry for ID #{'%016X' % entry[ID]} " +
                 "goes beyond 'data' file " +
                 "size (#{data_file_size})\n" +
@@ -209,8 +210,8 @@ module PEROBS
       begin
         File.write(@blobs_file_name, raw, address)
       rescue => e
-        raise IOError,
-              "Cannot write blobs file #{@blobs_file_name}: #{e.message}"
+        PEROBS.log.fatal "Cannot write blobs file #{@blobs_file_name}: " +
+          e.message
       end
     end
 
@@ -221,13 +222,12 @@ module PEROBS
       begin
         raw = File.read(@blobs_file_name, entry[BYTES], entry[START])
       rescue => e
-        raise IOError,
-              "Cannot read blobs file #{@blobs_file_name}: #{e.message}"
+        PEROBS.log.fatal "Cannot read blobs file #{@blobs_file_name}: " +
+          e.message
       end
       if Zlib.crc32(raw, 0) != entry[CRC]
-        raise RuntimeError,
-              "BTreeBlob for object #{entry[ID]} has been corrupted: " +
-              "Checksum mismatch"
+        PEROBS.log.fatal "BTreeBlob for object #{entry[ID]} has been " +
+          "corrupted: Checksum mismatch"
       end
 
       raw
@@ -296,7 +296,7 @@ module PEROBS
       entry_bytes = 29
       entry_format = 'QQQCL'
       restore_crc = false
-      if File.exists?(@index_file_name)
+      if File.exist?(@index_file_name)
         begin
           File.open(@index_file_name, 'rb') do |f|
             # Since version 2.3.0, all index files start with a header.
@@ -338,8 +338,8 @@ module PEROBS
                 begin
                   raw = File.read(@blobs_file_name, e[BYTES], e[START])
                 rescue => e
-                  raise IOError,
-                    "Cannot read blobs file #{@blobs_file_name}: #{e.message}"
+                  PEROBS.log.fatal "Cannot read blobs file " +
+                    "#{@blobs_file_name}: #{e.message}"
                 end
                 e[CRC] = Zlib.crc32(raw)
               end
@@ -348,8 +348,8 @@ module PEROBS
             end
           end
         rescue => e
-          raise RuntimeError,
-                "BTreeBlob file #{@index_file_name} corrupted: #{e.message}"
+          PEROBS.log.fatal "BTreeBlob file #{@index_file_name} corrupted: " +
+            e.message
         end
       end
     end
@@ -364,9 +364,8 @@ module PEROBS
           end
         end
       rescue => e
-        raise RuntimeError,
-              "Cannot write BTreeBlob index file #{@index_file_name}: " +
-              e.message
+        PEROBS.log.fatal "Cannot write BTreeBlob index file " +
+          "#{@index_file_name}: " + e.message
       end
     end
 

data/lib/perobs/BTreeDB.rb

@@ -27,6 +27,7 @@
 
 require 'fileutils'
 
+require 'perobs/Log'
 require 'perobs/DataBase'
 require 'perobs/BTreeBlob'
 
@@ -69,15 +70,15 @@ module PEROBS
       # Check and set @dir_bits, the number of bits used for each tree level.
       @dir_bits = options[:dir_bits] || 12
       if @dir_bits < 4 || @dir_bits > 14
-        raise ArgumentError,
-              "dir_bits option (#{@dir_bits}) must be between 4 and 12"
+        PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
+          "and 12"
       end
       check_option('dir_bits')
 
       @max_blob_size = options[:max_blob_size] || 32
       if @max_blob_size < 4 || @max_blob_size > 128
-        raise ArgumentError,
-          "max_blob_size option (#{@max_blob_size}) must be between 4 and 128"
+        PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
+          "between 4 and 128"
       end
       check_option('max_blob_size')
 
@@ -115,8 +116,7 @@ module PEROBS
       begin
         File.write(file_name, hash.to_json)
       rescue => e
-        raise RuntimeError,
-              "Cannot write hash file '#{file_name}': #{e.message}"
+        PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
       end
     end
 
@@ -125,13 +125,12 @@ module PEROBS
     # @return [Hash] A Hash that maps String objects to strings or numbers.
     def get_hash(name)
       file_name = File.join(@db_dir, name + '.json')
-      return ::Hash.new unless File.exists?(file_name)
+      return ::Hash.new unless File.exist?(file_name)
 
       begin
         json = File.read(file_name)
       rescue => e
-        raise RuntimeError,
-              "Cannot read hash file '#{file_name}': #{e.message}"
+        PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
       end
       JSON.parse(json, :create_additions => true)
     end
@@ -219,8 +218,8 @@ module PEROBS
         dir_bits = id & @dir_mask
         dir_name = File.join(dir_name, @dir_format_string % dir_bits)
 
-        if Dir.exists?(dir_name)
-          if File.exists?(File.join(dir_name, 'index'))
+        if Dir.exist?(dir_name)
+          if File.exist?(File.join(dir_name, 'index'))
             # The directory is a blob directory and not a BTree node dir.
             return BTreeBlob.new(dir_name, self)
           end
@@ -260,7 +259,7 @@ module PEROBS
       # contrast to BTree node directories that only contain other
       # directories.
       index_file = File.join(dir_name, 'index')
-      File.exists?(index_file)
+      File.exist?(index_file)
     end
 
   end

data/lib/perobs/Cache.rb

@@ -25,6 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/Log'
 require 'perobs/Store'
 
 module PEROBS
@@ -55,7 +56,7 @@ module PEROBS
       # to increase performance.
       if obj.respond_to?(:is_poxreference?)
         # If this condition triggers, we have a bug in the library.
-        raise RuntimeError, "POXReference objects should never be cached"
+        PEROBS.log.fatal "POXReference objects should never be cached"
       end
       @reads[index(obj)] = obj
     end
@@ -67,7 +68,7 @@ module PEROBS
       # to increase performance.
       if obj.respond_to?(:is_poxreference?)
         # If this condition triggers, we have a bug in the library.
-        raise RuntimeError, "POXReference objects should never be cached"
+        PEROBS.log.fatal "POXReference objects should never be cached"
       end
 
       if @transaction_stack.empty?
@@ -147,7 +148,7 @@ module PEROBS
     def end_transaction
       case @transaction_stack.length
       when 0
-        raise RuntimeError, 'No ongoing transaction to end'
+        PEROBS.log.fatal 'No ongoing transaction to end'
       when 1
         # All transactions completed successfully. Write all modified objects
         # into the backend storage.
@@ -169,7 +170,7 @@ module PEROBS
     # the transaction started.
     def abort_transaction
       if @transaction_stack.empty?
-        raise RuntimeError, 'No ongoing transaction to abort'
+        PEROBS.log.fatal 'No ongoing transaction to abort'
       end
       @transaction_stack.pop.each do |id|
         @transaction_objects[id]._restore(@transaction_stack.length)

data/lib/perobs/DataBase.rb

@@ -32,6 +32,7 @@ require 'json/add/struct'
 require 'yaml'
 require 'fileutils'
 
+require 'perobs/Log'
 require 'perobs/ObjectBase'
 
 module PEROBS
@@ -39,11 +40,23 @@ module PEROBS
   # Base class for all storage back-ends.
   class DataBase
 
+    # Create a new DataBase object. This method must be overwritten by the
+    # deriving classes and then called via their constructor.
     def initialize(serializer = :json)
       @serializer = serializer
       @config = {}
     end
 
+    # A dummy open method. Deriving classes must overload them to insert their
+    # open/close semantics.
+    def open
+    end
+
+    # A dummy close method. Deriving classes must overload them to insert their
+    # open/close semantics.
+    def close
+    end
+
     # Serialize the given object using the object serializer.
     # @param obj [ObjectBase] Object to serialize
     # @return [String] Serialized version
@@ -58,8 +71,8 @@ module PEROBS
           YAML.dump(obj)
         end
       rescue => e
-        raise RuntimeError,
-              "Cannot serialize object as #{@serializer}: #{e.message}"
+        PEROBS.log.fatal "Cannot serialize object as #{@serializer}: " +
+          e.message
       end
     end
 
@@ -77,9 +90,8 @@ module PEROBS
           YAML.load(raw)
         end
       rescue => e
-        raise RuntimeError,
-              "Cannot de-serialize object with #{@serializer} parser: " +
-              e.message
+        PEROBS.log.fatal "Cannot de-serialize object with #{@serializer} " +
+          "parser: " + e.message
       end
     end
 
@@ -106,11 +118,11 @@ module PEROBS
 
     # Ensure that we have a directory to store the DB items.
     def ensure_dir_exists(dir)
-      unless Dir.exists?(dir)
+      unless Dir.exist?(dir)
         begin
           Dir.mkdir(dir)
         rescue IOError => e
-          raise IOError, "Cannote create DB directory '#{dir}': #{e.message}"
+          PEROBS.log.fatal "Cannote create DB directory '#{dir}': #{e.message}"
         end
       end
     end

data/lib/perobs/DynamoDB.rb

@@ -176,7 +176,7 @@ module PEROBS
       begin
         get_object(id)
       rescue => e
-        $stderr.puts "Cannot read object with ID #{id}: #{e.message}"
+        PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
         return false
       end
 

data/lib/perobs/FixedSizeBlobFile.rb

@@ -0,0 +1,189 @@
+# encoding: UTF-8
+#
+# = FixedSizeBlobFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+require 'perobs/StackFile'
+
+module PEROBS
+
+  # This class implements persistent storage space for fixed size data blobs.
+  # The blobs can be stored and retrieved and can be deleted again. The
+  # FixedSizeBlobFile manages the storage of the blobs and free storage
+  # spaces. The files grows and shrinks as needed. A blob is referenced by its
+  # address.
+  class FixedSizeBlobFile
+
+    # Create a new stack file in the given directory with the given file name.
+    # @param dir [String] Directory
+    # @param name [String] File name
+    # @param entry_bytes [Fixnum] Number of bytes each entry must have
+    def initialize(dir, name, entry_bytes)
+      @file_name = File.join(dir, name + '.blobs')
+      @entry_bytes = entry_bytes
+      @free_list = StackFile.new(dir, name + '-freelist', 8)
+      @f = nil
+    end
+
+    # Open the blob file.
+    def open
+      begin
+        if File.exist?(@file_name)
+          @f = File.open(@file_name, 'rb+')
+        else
+          @f = File.open(@file_name, 'wb+')
+        end
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
+      end
+      @free_list.open
+    end
+
+    # Close the blob file. This method must be called before the program is
+    # terminated to avoid data loss.
+    def close
+      @free_list.close
+      begin
+        @f.flush
+        @f.close
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Flush out all unwritten data.
+    def sync
+      @free_list.sync
+      begin
+        @f.sync
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
+      end
+    end
+
+    # Delete all data.
+    def clear
+      @f.truncate(0)
+      @f.flush
+      @free_list.clear
+    end
+
+    def empty?
+      sync
+      @f.size == 0
+    end
+
+    # Return the address of a free blob storage space. Addresses start at 0
+    # and increase linearly.
+    # @return [Fixnum] address of a free blob space
+    def free_address
+      if (bytes = @free_list.pop)
+        # Return an entry from the free list.
+        return bytes.unpack('Q')[0]
+      else
+        # There is currently no free entry. Return the address at the end of
+        # the file.
+        offset_to_address(@f.size)
+      end
+    end
+
+    # Store the given byte blob at the specified address. If the blob space is
+    # already in use the content will be overwritten.
+    # @param address [Fixnum] Address to store the blob
+    # @param bytes [String] bytes to store
+    def store_blob(address, bytes)
+      if bytes.length != @entry_bytes
+        PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
+          "long. This entry is #{bytes.length} bytes long."
+      end
+      begin
+        @f.seek(address_to_offset(address))
+        # The first byte is tha flag byte. It's set to 1 for cells that hold a
+        # blob. 0 for empty cells.
+        @f.write([ 1 ].pack('C'))
+        @f.write(bytes)
+        @f.flush
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot store blob at address #{address}: #{e.message}"
+      end
+    end
+
+    # Retrieve a blob from the given address.
+    # @param address [Fixnum] Address to store the blob
+    # @return [String] blob bytes
+    def retrieve_blob(address)
+      begin
+        if (offset = address_to_offset(address)) >= @f.size
+          return nil
+        end
+
+        @f.seek(address_to_offset(address))
+        if (@f.read(1).unpack('C')[0] != 1)
+          return nil
+        end
+        bytes = @f.read(@entry_bytes)
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot retrieve blob at adress #{address}: " +
+          e.message
+      end
+
+      bytes
+    end
+
+    # Delete the blob at the given address.
+    # @param address [Fixnum] Address of blob to delete
+    def delete_blob(address)
+      begin
+        @f.seek(address_to_offset(address))
+        if (@f.read(1).unpack('C')[0] != 1)
+          PEROBS.log.fatal "There is no blob stored at address #{address}"
+        end
+        @f.seek(address_to_offset(address))
+        @f.write([ 0 ].pack('C'))
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
+          e.message
+      end
+      # Add the address to the free list.
+      @free_list.push([ address ].pack('Q'))
+    end
+
+    private
+
+    # Translate a blob address to the actual offset in the file.
+    def address_to_offset(address)
+      address * (1 + @entry_bytes)
+    end
+
+    # Translate the file offset to the address of a blob.
+    def offset_to_address(offset)
+      offset / (1 + @entry_bytes)
+    end
+
+  end
+
+end
+