perobs 4.0.0 → 4.4.0

data/lib/perobs/Cache.rb

@@ -2,7 +2,7 @@
 #
 # = Cache.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -66,10 +66,10 @@ module PEROBS
     def cache_write(obj)
       # This is just a safety check. It can probably be disabled in the future
       # to increase performance.
-      if obj.respond_to?(:is_poxreference?)
-        # If this condition triggers, we have a bug in the library.
-        PEROBS.log.fatal "POXReference objects should never be cached"
-      end
+      #if obj.respond_to?(:is_poxreference?)
+      #  # If this condition triggers, we have a bug in the library.
+      #  PEROBS.log.fatal "POXReference objects should never be cached"
+      #end
 
       if @transaction_stack.empty?
         # We are not in transaction mode.
@@ -93,16 +93,54 @@ module PEROBS
       end
     end
 
-    # Return the PEROBS::Object with the specified ID or nil if not found.
+    # Evict the object with the given ID from the cache.
     # @param id [Integer] ID of the cached PEROBS::ObjectBase
-    def object_by_id(id)
+    # @return [True/False] True if object was stored in the cache. False
+    #         otherwise.
+    def evict(id)
+      unless @transaction_stack.empty?
+        PEROBS.log.fatal "You cannot evict entries during a transaction."
+      end
+
       idx = id & @mask
       # The index is just a hash. We still need to check if the object IDs are
       # actually the same before we can return the object.
       if (obj = @writes[idx]) && obj._id == id
-        # The object was in the write cache.
-        return obj
+        # The object is in the write cache.
+        @writes[idx] = nil
+        return true
       elsif (obj = @reads[idx]) && obj._id == id
+        # The object is in the read cache.
+        @reads[idx] = nil
+        return true
+      end
+
+      false
+    end
+
+    # Return the PEROBS::Object with the specified ID or nil if not found.
+    # @param id [Integer] ID of the cached PEROBS::ObjectBase
+    def object_by_id(id)
+      idx = id & @mask
+
+      if @transaction_stack.empty?
+        # The index is just a hash. We still need to check if the object IDs are
+        # actually the same before we can return the object.
+        if (obj = @writes[idx]) && obj._id == id
+          # The object was in the write cache.
+          return obj
+        end
+      else
+        # During transactions, the read cache is used to provide fast access
+        # to modified objects. But it does not store all modified objects
+        # since there can be hash collisions. So we also have to check all
+        # transaction objects first.
+        if (obj = @transaction_objects[id])
+          return obj
+        end
+      end
+
+      if (obj = @reads[idx]) && obj._id == id
         # The object was in the read cache.
         return obj
       end
@@ -160,7 +198,7 @@ module PEROBS
         transactions = @transaction_stack.pop
         # Merge the two lists
         @transaction_stack.push(@transaction_stack.pop + transactions)
-        # Ensure that each object is only included once in the list.
+        # Ensure that each object ID is only included once in the list.
         @transaction_stack.last.uniq!
       end
     end

data/lib/perobs/ConsoleProgressMeter.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+#
+# = ConsoleProgressMeter.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/ProgressMeter'
+
+module PEROBS
+
+  class ConsoleProgressMeter < ProgressMeter
+
+    LINE_LENGTH = 79
+
+    private
+
+    def print_bar
+      percent = @max_value == 0 ? 100.0 :
+        (@current_value.to_f / @max_value) * 100.0
+      percent = 0.0 if percent < 0
+      percent = 100.0 if percent > 100.0
+
+      meter = "<#{percent.to_i}%>"
+
+      bar_length = LINE_LENGTH - @name.chars.length - 3 - meter.chars.length
+      left_bar = '*' * (bar_length * percent / 100.0)
+      right_bar = ' ' * (bar_length - left_bar.chars.length)
+
+      print "\r#{@name} [#{left_bar}#{meter}#{right_bar}]"
+    end
+
+    def print_time
+      s = "\r#{@name} [#{secsToHMS(@end_time - @start_time)}]"
+      puts s + (' ' * (LINE_LENGTH - s.chars.length + 1))
+    end
+
+  end
+
+end
+

data/lib/perobs/DataBase.rb

@@ -2,7 +2,7 @@
 #
 # = DataBase.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -42,8 +42,9 @@ module PEROBS
 
     # Create a new DataBase object. This method must be overwritten by the
     # deriving classes and then called via their constructor.
-    def initialize(serializer = :json)
-      @serializer = serializer
+    def initialize(options)
+      @serializer = options[:serializer] || :json
+      @progressmeter = options[:progressmeter] || ProgressMeter.new
       @config = {}
     end
 

data/lib/perobs/DynamoDB.rb

@@ -2,7 +2,8 @@
 #
 # = DynamoDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017, 2018
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -25,7 +26,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require 'aws-sdk'
+require 'aws-sdk-dynamodb'
 
 require 'perobs/DataBase'
 require 'perobs/BTreeBlob'
@@ -35,6 +36,10 @@ module PEROBS
   # This class implements an Amazon DynamoDB storage engine for PEROBS.
   class DynamoDB < DataBase
 
+    INTERNAL_ITEMS = %w( config item_counter )
+
+    attr_reader :item_counter
+
     # Create a new DynamoDB object.
     # @param db_name [String] name of the DB directory
     # @param options [Hash] options to customize the behavior. Currently only
@@ -50,7 +55,7 @@ module PEROBS
         options[:serializer] = :yaml
       end
 
-      super(options[:serializer] || :json)
+      super(options)
 
       if options.include?(:aws_id) && options.include?(:aws_key)
         Aws.config[:credentials] = Aws::Credentials.new(options[:aws_id],
@@ -62,12 +67,25 @@ module PEROBS
 
       @dynamodb = Aws::DynamoDB::Client.new
       @table_name = db_name
-      ensure_table_exists(@table_name)
+      @config = nil
+      # The number of items currently stored in the DB.
+      @item_counter = nil
+      if create_table(@table_name)
+        @config = { 'serializer' => @serializer }
+        put_hash('config', @config)
+        @item_counter = 0
+        dynamo_put_item('item_counter', @item_counter.to_s)
+      else
+        @config = get_hash('config')
+        if @config['serializer'] != @serializer
+          raise ArgumentError, "DynamoDB #{@table_name} was created with " +
+            "serializer #{@config['serializer']} but was now opened with " +
+            "serializer #{@serializer}."
+        end
+        @item_counter = dynamo_get_item('item_counter').to_i
+      end
 
       # Read the existing DB config.
-      @config = get_hash('config')
-      check_option('serializer')
-      put_hash('config', @config)
     end
 
     # Delete the entire database. The database is no longer usable after this
@@ -112,6 +130,13 @@ module PEROBS
     # Store the given object into the cluster files.
     # @param obj [Hash] Object as defined by PEROBS::ObjectBase
     def put_object(obj, id)
+      id_str = id.to_s
+      unless dynamo_get_item(id_str)
+        # The is no object with this ID yet. Increase the item counter.
+        @item_counter += 1
+        dynamo_put_item('item_counter', @item_counter.to_s)
+      end
+
       dynamo_put_item(id.to_s, serialize(obj))
     end
 
@@ -128,23 +153,23 @@ module PEROBS
       each_item do |id|
         dynamo_mark_item(id, false)
       end
-      # Mark the 'config' item so it will not get deleted.
-      dynamo_mark_item('config')
     end
 
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
-    # @return [Array] List of object IDs of the deleted objects.
+    # @return [Integer] Count of the deleted objects.
     def delete_unmarked_objects
-      deleted_ids = []
+      deleted_objects_count = 0
       each_item do |id|
         unless dynamo_is_marked?(id)
           dynamo_delete_item(id)
-          deleted_ids << id
+          deleted_objects_count += 1
+          @item_counter -= 1
         end
       end
+      dynamo_put_item('item_counter', @item_counter.to_s)
 
-      deleted_ids
+      deleted_objects_count
     end
 
     # Mark an object.
@@ -163,7 +188,17 @@ module PEROBS
     # @param repair [TrueClass/FalseClass] True if found errors should be
     #        repaired.
     def check_db(repair = false)
-      # TODO: See if we can add checks here
+      unless (item_counter = dynamo_get_item('item_counter')) &&
+             item_counter == @item_counter
+        PEROBS.log.error "@item_counter variable (#{@item_counter}) and " +
+          "item_counter table entry (#{item_counter}) don't match"
+      end
+      item_counter = 0
+      each_item { item_counter += 1 }
+      unless item_counter == @item_counter
+        PEROBS.log.error "Table contains #{item_counter} items but " +
+          "@item_counter is #{@item_counter}"
+      end
     end
 
     # Check if the stored object is syntactically correct.
@@ -185,9 +220,11 @@ module PEROBS
 
     private
 
-    def ensure_table_exists(table_name)
+    def create_table(table_name)
       begin
         @dynamodb.describe_table(:table_name => table_name)
+        # The table exists already. No need to create it.
+        return false
       rescue Aws::DynamoDB::Errors::ResourceNotFoundException
         @dynamodb.create_table(
           :table_name => table_name,
@@ -210,6 +247,8 @@ module PEROBS
         )
 
         @dynamodb.wait_until(:table_exists, table_name: table_name)
+        # The table was successfully created.
+        return true
       end
     end
 
@@ -251,6 +290,9 @@ module PEROBS
         break if resp.count <= 0
 
         resp.items.each do |item|
+          # Skip all internal items
+          next if INTERNAL_ITEMS.include?(item['Id'])
+
           yield(item['Id'])
         end
 

data/lib/perobs/EquiBlobsFile.rb

@@ -2,7 +2,8 @@
 #
 # = EquiBlobsFile.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -26,6 +27,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'perobs/Log'
+require 'perobs/ProgressMeter'
 
 module PEROBS
 
@@ -37,6 +39,11 @@ module PEROBS
   # is used to represent an undefined address or nil. The file has a 4 * 8
   # bytes long header that stores the total entry count, the total space
   # count, the offset of the first entry and the offset of the first space.
+  # The header is followed by a custom entry section. Each entry is also 8
+  # bytes long. After the custom entry section the data blobs start. Each data
+  # blob starts with a mark byte that indicates if the blob is valid data (2),
+  # a free space (0) or reseved space (1). Then it is followed by @entry_bytes
+  # number of bytes for the data blob.
   class EquiBlobsFile
 
     TOTAL_ENTRIES_OFFSET = 0
@@ -50,15 +57,21 @@ module PEROBS
     # Create a new stack file in the given directory with the given file name.
     # @param dir [String] Directory
     # @param name [String] File name
+    # @param progressmeter [ProgressMeter] Reference to a progress meter
+    #        object
     # @param entry_bytes [Integer] Number of bytes each entry must have
     # @param first_entry_default [Integer] Default address of the first blob
-    def initialize(dir, name, entry_bytes, first_entry_default = 0)
+    def initialize(dir, name, progressmeter, entry_bytes,
+                   first_entry_default = 0)
+      @name = name
       @file_name = File.join(dir, name + '.blobs')
+      @progressmeter = progressmeter
       if entry_bytes < 8
         PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
       end
       @entry_bytes = entry_bytes
       @first_entry_default = first_entry_default
+      clear_custom_data
       reset_counters
 
       # The File handle.
@@ -102,6 +115,57 @@ module PEROBS
       end
     end
 
+    # In addition to the standard offsets for the first entry and the first
+    # space any number of additional data fields can be registered. This must be
+    # done right after the object is instanciated and before the open() method
+    # is called. Each field represents a 64 bit unsigned integer.
+    # @param name [String] The label for this offset
+    # @param default_value [Integer] The default value for the offset
+    def register_custom_data(name, default_value = 0)
+      if @custom_data_labels.include?(name)
+        PEROBS.log.fatal "Custom data field #{name} has already been registered"
+      end
+
+      @custom_data_labels << name
+      @custom_data_values << default_value
+      @custom_data_defaults << default_value
+    end
+
+    # Reset (delete) all custom data labels that have been registered.
+    def clear_custom_data
+      unless @f.nil?
+        PEROBS.log.fatal "clear_custom_data should only be called when " +
+          "the file is not opened"
+      end
+
+      @custom_data_labels = []
+      @custom_data_values = []
+      @custom_data_defaults = []
+    end
+
+    # Set the registered custom data field to the given value.
+    # @param name [String] Label of the offset
+    # @param value [Integer] Value
+    def set_custom_data(name, value)
+      unless @custom_data_labels.include?(name)
+        PEROBS.log.fatal "Unknown custom data field #{name}"
+      end
+
+      @custom_data_values[@custom_data_labels.index(name)] = value
+      write_header if @f
+    end
+
+    # Get the registered custom data field value.
+    # @param name [String] Label of the offset
+    # @return [Integer] Value of the custom data field
+    def get_custom_data(name)
+      unless @custom_data_labels.include?(name)
+        PEROBS.log.fatal "Unknown custom data field #{name}"
+      end
+
+      @custom_data_values[@custom_data_labels.index(name)]
+    end
+
     # Erase the backing store. This method should only be called when the file
     # is not currently open.
     def erase
@@ -285,7 +349,7 @@ module PEROBS
 
       @first_space = offset
       @total_spaces += 1
-      @total_entries -= 1
+      @total_entries -= 1 unless marker == 1
       write_header
 
       if offset == @f.size - 1 - @entry_bytes
@@ -298,12 +362,16 @@ module PEROBS
     # Check the file for logical errors.
     # @return [Boolean] true of file has no errors, false otherwise.
     def check
+      sync
+
       return false unless check_spaces
       return false unless check_entries
 
-      if @f.size != HEADER_SIZE + (@total_entries + @total_spaces) *
-                                  (1 + @entry_bytes)
-        PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}"
+      expected_size = address_to_offset(@total_entries + @total_spaces + 1)
+      actual_size = @f.size
+      if actual_size != expected_size
+        PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
+          "Expected #{expected_size} bytes but found #{actual_size} bytes."
         return false
       end
 
@@ -326,6 +394,9 @@ module PEROBS
       @first_entry = @first_entry_default
       # The file offset of the first empty entry.
       @first_space = 0
+
+      # Copy default custom values
+      @custom_data_values = @custom_data_defaults.dup
     end
 
     def read_header
@@ -333,6 +404,12 @@ module PEROBS
         @f.seek(0)
         @total_entries, @total_spaces, @first_entry, @first_space =
           @f.read(HEADER_SIZE).unpack('QQQQ')
+        custom_labels_count = @custom_data_labels.length
+        if custom_labels_count > 0
+          @custom_data_values =
+            @f.read(custom_labels_count * 8).unpack("Q#{custom_labels_count}")
+        end
+
       rescue IOError => e
         PEROBS.log.fatal "Cannot read EquiBlobsFile header: #{e.message}"
       end
@@ -343,7 +420,13 @@ module PEROBS
       begin
         @f.seek(0)
         @f.write(header_ary.pack('QQQQ'))
+        unless @custom_data_values.empty?
+          @f.write(@custom_data_values.
+                   pack("Q#{@custom_data_values.length}"))
+        end
         @f.flush
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write EquiBlobsFile header: " + e.message
       end
     end
 
@@ -367,25 +450,31 @@ module PEROBS
         return false
       end
 
+      return true if next_offset == 0
+
       total_spaces = 0
-      begin
-        while next_offset != 0
-          # Check that the marker byte is 0
-          @f.seek(next_offset)
-          if (marker = read_char) != 0
-            PEROBS.log.error "Marker byte at address " +
-              "#{offset_to_address(next_offset)} is #{marker} instead of 0."
-            return false
-          end
-          # Read offset of next empty space
-          next_offset = read_unsigned_int
+      @progressmeter.start("Checking #{@name} spaces list",
+                           @total_spaces) do |pm|
+        begin
+          while next_offset != 0
+            # Check that the marker byte is 0
+            @f.seek(next_offset)
+            if (marker = read_char) != 0
+              PEROBS.log.error "Marker byte at address " +
+                "#{offset_to_address(next_offset)} is #{marker} instead of 0."
+              return false
+            end
+            # Read offset of next empty space
+            next_offset = read_unsigned_int
 
-          total_spaces += 1
+            total_spaces += 1
+            pm.update(total_spaces)
+          end
+        rescue IOError => e
+          PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
+            "#{@file_name}: #{e.message}"
+          return false
         end
-      rescue IOError => e
-        PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
-          "#{@file_name}: #{e.message}"
-        return false
       end
 
       unless total_spaces == @total_spaces
@@ -414,35 +503,48 @@ module PEROBS
         return false
       end
 
-      next_offset = HEADER_SIZE
+      next_offset = address_to_offset(1)
       total_entries = 0
       total_spaces = 0
-      begin
-        @f.seek(next_offset)
-        while !@f.eof
-          marker, bytes = @f.read(1 + @entry_bytes).
-            unpack("C#{1 + @entry_bytes}")
-          case marker
-          when 0
-            total_spaces += 1
-          when 1
-            PEROBS.log.error "Entry at address " +
-              "#{offset_to_address(next_offset)} in EquiBlobsFile " +
-              "#{@file_name} has reserved marker"
-            return false
-          when 2
-            total_entries += 1
-          else
-            PEROBS.log.error "Entry at address " +
-              "#{offset_to_address(next_offset)} in EquiBlobsFile " +
-              "#{@file_name} has illegal marker #{marker}"
-            return false
+      last_entry_is_space = false
+      @progressmeter.start("Checking #{@name} entries",
+                           @total_spaces + @total_entries) do |pm|
+        begin
+          @f.seek(next_offset)
+          while !@f.eof
+            marker, bytes = @f.read(1 + @entry_bytes).
+              unpack("C#{1 + @entry_bytes}")
+            case marker
+            when 0
+              total_spaces += 1
+              last_entry_is_space = true
+            when 1
+              PEROBS.log.error "Entry at address " +
+                "#{offset_to_address(next_offset)} in EquiBlobsFile " +
+                "#{@file_name} has reserved marker"
+              return false
+            when 2
+              total_entries += 1
+              last_entry_is_space = false
+            else
+              PEROBS.log.error "Entry at address " +
+                "#{offset_to_address(next_offset)} in EquiBlobsFile " +
+                "#{@file_name} has illegal marker #{marker}"
+              return false
+            end
+            next_offset += 1 + @entry_bytes
           end
-          next_offset += 1 + @entry_bytes
+
+          pm.update(total_spaces + total_entries)
+        rescue
+          PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
+            "#{@file_name}: #{e.message}"
+          return false
         end
-      rescue
-        PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
-          "#{@file_name}: #{e.message}"
+      end
+
+      if last_entry_is_space
+        PEROBS.log.error "EquiBlobsFile #{@file_name} is not properly trimmed"
         return false
       end
 
@@ -464,7 +566,7 @@ module PEROBS
 
     def trim_file
       offset = @f.size - 1 - @entry_bytes
-      while offset >= HEADER_SIZE
+      while offset >= address_to_offset(1)
         @f.seek(offset)
         begin
           if (marker = read_char) == 0
@@ -548,12 +650,15 @@ module PEROBS
 
     # Translate a blob address to the actual offset in the file.
     def address_to_offset(address)
-      HEADER_SIZE + (address - 1) * (1 + @entry_bytes)
+      # Since address 0 is illegal, we can use address - 1 as index here.
+      HEADER_SIZE + @custom_data_labels.length * 8 +
+        (address - 1) * (1 + @entry_bytes)
     end
 
     # Translate the file offset to the address of a blob.
     def offset_to_address(offset)
-      (offset - HEADER_SIZE) / (1 + @entry_bytes) + 1
+      (offset - HEADER_SIZE - @custom_data_labels.length * 8) /
+        (1 + @entry_bytes) + 1
     end
 
     def write_char(c)