perobs 4.0.0 → 4.1.0

data/lib/perobs/FlatFileBlobHeader.rb

@@ -48,12 +48,13 @@ module PEROBS
     # The 'pack()' format of the header.
     FORMAT = 'CQQL'
     # The length of the header in bytes.
-    LENGTH = 21
+    LENGTH = 25
     VALID_FLAG_BIT = 0
     COMPRESSED_FLAG_BIT = 2
     OUTDATED_FLAG_BIT = 3
 
     attr_reader :addr, :flags, :length, :id, :crc
+    attr_accessor :corruption_start
 
     # Create a new FlatFileBlobHeader with the given flags, length, id and crc.
     # @param file [File] the FlatFile that contains the header
@@ -69,50 +70,105 @@ module PEROBS
       @length = length
       @id = id
       @crc = crc
+      # This is only set if the header is preceded by a corrupted blob.
+      @corruption_start = nil
     end
 
     # Read the header from the given File.
     # @param file [File]
-    # @return FlatFileBlobHeader
-    def FlatFileBlobHeader::read(file)
-      begin
-        addr = file.pos
-        buf = file.read(LENGTH)
-      rescue IOError => e
-        PEROBS.log.error "Cannot read blob header in flat file DB: #{e.message}"
-        return nil
+    # @param addr [Integer] address in the file to start reading. If no
+    #        address is specified use the current position in the file.
+    # @param id [Integer] Optional ID that the header should have. If no id is
+    #        specified there is no check against the actual ID done.
+    # @return FlatFileBlobHeader or nil if there are no more blobs to read in
+    #         the file.
+    def FlatFileBlobHeader::read(file, addr = nil, id = nil)
+      # If an address was specified we expect the read to always succeed. If
+      # no address is specified and we can't read the header we generate an
+      # error message but it is not fatal.
+      errors_are_fatal = !addr.nil?
+
+      mode = :searching_next_header
+      addr = file.pos unless addr
+      buf = nil
+      corruption_start = nil
+
+      loop do
+        buf_with_crc = nil
+        begin
+          file.seek(addr)
+          buf_with_crc = file.read(LENGTH)
+        rescue IOError => e
+          if errors_are_fatal
+            PEROBS.log.fatal "Cannot read blob header in flat file DB at " +
+              "address #{addr}: #{e.message}"
+          else
+            PEROBS.log.error "Cannot read blob header in flat file DB: " +
+              e.message
+            return nil
+          end
+        end
+
+        # Did we read anything?
+        if buf_with_crc.nil?
+          if errors_are_fatal
+            PEROBS.log.fatal "Cannot read blob header " +
+              "#{id ? "for ID #{id} " : ''}at address #{addr}"
+          else
+            # We have reached the end of the file.
+            return nil
+          end
+        end
+
+        # Did we get the full header?
+        if buf_with_crc.length != LENGTH
+          PEROBS.log.error "Incomplete FlatFileBlobHeader: Only " +
+            "#{buf_with_crc.length} " +
+            "bytes of #{LENGTH} could be read "
+          "#{id ? "for ID #{id} " : ''}at address #{addr}"
+          return nil
+        end
+
+        # Check the CRC of the header
+        buf = buf_with_crc[0..-5]
+        crc = buf_with_crc[-4..-1].unpack('L')[0]
+
+        if (read_crc = Zlib.crc32(buf, 0)) == crc
+          # We have found a valid header.
+          if corruption_start
+            PEROBS.log.error "FlatFile corruption ends at #{addr}. " +
+              "#{addr - corruption_start} bytes skipped. Some data may " +
+              "not be recoverable."
+          end
+          break
+        else
+          if errors_are_fatal
+            PEROBS.log.fatal "FlatFile Header CRC mismatch at address " +
+              "#{addr}. Header CRC is #{'%08x' % read_crc} but should be " +
+              "#{'%08x' % crc}."
+          else
+            if corruption_start.nil?
+              PEROBS.log.error "FlatFile corruption found. The FlatFile " +
+                "Header CRC mismatch at address #{addr}. Header CRC is " +
+                "#{'%08x' % read_crc} but should be #{'%08x' % crc}. Trying " +
+                "to find the next header."
+              corruption_start = addr
+            end
+            # The blob file is corrupted. There is no valid header at the
+            # current position in the file. We now try to find the next valid
+            # header by iterating over the remainder of the file advanding one
+            # byte with each step until we hit the end of the file or find the
+            # next valid header.
+            addr += 1
+          end
+        end
       end
 
-      return nil unless buf
-
-      if buf.length != LENGTH
-        PEROBS.log.error "Incomplete FlatFileBlobHeader: Only #{buf.length} " +
-          "bytes of #{LENGTH} could be read"
-        return nil
+      header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
+      if corruption_start
+        header.corruption_start = corruption_start
       end
 
-      FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
-    end
-
-    # Read the header from the given File.
-    # @param file [File]
-    # @param addr [Integer] address in the file to start reading
-    # @param id [Integer] Optional ID that the header should have
-    # @return FlatFileBlobHeader
-    def FlatFileBlobHeader::read_at(file, addr, id = nil)
-      buf = nil
-      begin
-        file.seek(addr)
-        buf = file.read(LENGTH)
-      rescue IOError => e
-        PEROBS.log.fatal "Cannot read blob in flat file DB: #{e.message}"
-      end
-      if buf.nil? || buf.length != LENGTH
-        PEROBS.log.fatal "Cannot read blob header " +
-          "#{id ? "for ID #{id} " : ''}at address " +
-          "#{addr}"
-      end
-      header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
       if id && header.id != id
         PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
           "found. FlatFile has entry with ID #{header.id} at address " +
@@ -123,11 +179,12 @@ module PEROBS
     end
 
     # Write the header to a given File.
-    # @param file [File]
     def write
       begin
+        buf = [ @flags, @length, @id, @crc].pack(FORMAT)
+        crc = Zlib.crc32(buf, 0)
         @file.seek(@addr)
-        @file.write([ @flags, @length, @id, @crc].pack(FORMAT))
+        @file.write(buf + [ crc ].pack('L'))
       rescue IOError => e
         PEROBS.log.fatal "Cannot write blob header into flat file DB: " +
           e.message
@@ -135,11 +192,9 @@ module PEROBS
     end
 
     # Reset all the flags bit to 0. This marks the blob as invalid.
-    # @param file [File] The file handle of the blob file.
-    # @param addr [Integer] The address of the header
     def clear_flags
       @flags = 0
-      write_flags
+      write
     end
 
     # Return true if the header is for a non-empty blob.
@@ -156,7 +211,7 @@ module PEROBS
     # transaction has been completed.
     def set_outdated_flag
       set_flag(OUTDATED_FLAG_BIT)
-      write_flags
+      write
     end
 
     # Return true if the blob contains outdated data.
@@ -166,17 +221,6 @@ module PEROBS
 
     private
 
-    def write_flags
-      begin
-        @file.seek(@addr)
-        @file.write([ @flags ].pack('C'))
-        @file.flush
-      rescue IOError => e
-        PEROBS.log.fatal "Writing flags of FlatFileBlobHeader with ID #{@id} " +
-          "failed: #{e.message}"
-      end
-    end
-
     def bit_set?(n)
       mask = 1 << n
       @flags & mask == mask

data/lib/perobs/FlatFileDB.rb

@@ -2,7 +2,8 @@
 #
 # = FlatFileDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017, 2018
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -41,7 +42,7 @@ module PEROBS
 
     # This version number increases whenever the on-disk format changes in a
     # way that requires conversion actions after an update.
-    VERSION = 2
+    VERSION = 4
 
     attr_reader :max_blob_size
 
@@ -50,13 +51,17 @@ module PEROBS
     # @param options [Hash] options to customize the behavior. Currently only
     #        the following options are supported:
     #        :serializer    : Can be :marshal, :json, :yaml
+    #        :progressmeter : Reference to a ProgressMeter object
+    #        :log           : IO that should be used for logging
+    #        :log_level     : Minimum Logger level to log
     def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
+      super(options)
 
       @db_dir = db_name
       # Create the database directory if it doesn't exist yet.
       ensure_dir_exists(@db_dir)
-      PEROBS.log.open(File.join(@db_dir, 'log'))
+      PEROBS.log.level = options[:log_level] if options[:log_level]
+      PEROBS.log.open(options[:log] || File.join(@db_dir, 'log'))
       check_version_and_upgrade
 
       # Read the existing DB config.
@@ -68,7 +73,7 @@ module PEROBS
 
     # Open the FlatFileDB for transactions.
     def open
-      @flat_file = FlatFile.new(@db_dir)
+      @flat_file = FlatFile.new(@db_dir, @progressmeter)
       @flat_file.open
       PEROBS.log.info "FlatFile '#{@db_dir}' opened"
     end
@@ -143,8 +148,9 @@ module PEROBS
       end
     end
 
-    def search_object(id)
-      @flat_file.search_object(id)
+    # @return [Integer] Number of objects stored in the DB.
+    def item_counter
+      @flat_file.item_counter
     end
 
     # This method must be called to initiate the marking process.
@@ -154,7 +160,7 @@ module PEROBS
 
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
-    # @return [Array] List of IDs that have been removed from the DB.
+    # @return [Integer] Number of the removed objects from the DB.
     def delete_unmarked_objects
       @flat_file.delete_unmarked_objects
     end
@@ -226,7 +232,8 @@ module PEROBS
                            "'#{version_file}': " + e.message
         end
       else
-        # Early versions of PEROBS did not have a version file.
+        # The DB is brand new.
+        version = VERSION
         write_version_file(version_file)
       end
 
@@ -234,25 +241,40 @@ module PEROBS
         PEROBS.log.fatal "Cannot downgrade the FlatFile database from " +
                          "version #{version} to version #{VERSION}"
       end
-
-      if version == 1
-        # Version 1 had no support for data compression. Make sure all entries
-        # are compressed to save space.
-        open
-        @flat_file.refresh
-        close
+      if version < 3
+        PEROBS.log.fatal "The upgrade of this version of the PEROBS database " +
+          "is not supported by this version of PEROBS. Please try an earlier " +
+          "version of PEROBS to upgrade the database before using this version."
       end
 
-      # After a successful upgrade change the version number in the DB as
-      # well.
-      if version < VERSION
+      # Version upgrades must be done one version number at a time. If the
+      # existing DB is multiple versions older than what the current PEROBS
+      # version expects than multiple upgrade runs will be needed.
+      while version < VERSION
+        if version == 3
+          PEROBS.log.warn "Updating FlatFileDB #{@db_dir} from version 3 to " +
+            "version 4 ..."
+          # Version 4 adds checksums for blob file headers. We have to convert
+          # the blob file to include the checksums.
+          FlatFile.insert_header_checksums(@db_dir)
+          open
+          @flat_file.regenerate_index_and_spaces
+          close
+        end
+
+        # After a successful upgrade change the version number in the DB as
+        # well.
         write_version_file(version_file)
         PEROBS.log.warn "Update of FlatFileDB '#{@db_dir}' from version " +
-          "#{version} to version #{VERSION} completed"
+          "#{version} to version #{version + 1} completed"
+
+        # Update version variable to new version.
+        version += 1
       end
     end
 
     def write_version_file(version_file)
+
       begin
         RobustFile.write(version_file, VERSION)
       rescue IOError => e

data/lib/perobs/Hash.rb

@@ -2,7 +2,7 @@
 #
 # = Hash.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -37,20 +37,36 @@ module PEROBS
   # The implementation is largely a proxy around the standard Hash class. But
   # all mutating methods must be re-implemented to convert PEROBS::Objects to
   # POXReference objects and to register the object as modified with the
-  # cache.
+  # cache. However, it is not designed for large data sets as it always reads
+  # and writes the full data set for every access (unless it is cached). For
+  # data sets that could have more than a few hundred entries BigHash is the
+  # recommended alternative.
   #
   # We explicitely don't support Hash::store() as it conflicts with
   # ObjectBase::store() method to access the store.
   class Hash < ObjectBase
 
+    # These methods do not mutate the Hash. They only perform read
+    # operations and return a new PEROBS::Hash object.
+    ([
+      :invert, :merge, :reject, :select
+    ] + Enumerable.instance_methods).uniq.each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the read operation with the cache.
+        @store.cache.cache_read(self)
+        @store.new(PEROBS::Hash, @data.send(method_sym, *args, &block))
+      end
+    end
+
     # These methods do not mutate the Hash. They only perform read
     # operations.
     ([
       :==, :[], :assoc, :compare_by_identity, :compare_by_identity?, :default,
       :default_proc, :each, :each_key, :each_pair, :each_value, :empty?,
       :eql?, :fetch, :flatten, :has_key?, :has_value?, :hash, :include?,
-      :invert, :key, :key?, :keys, :length, :member?, :merge,
-      :pretty_print, :pretty_print_cycle, :rassoc, :reject, :select, :size,
+      :key, :key?, :keys, :length, :member?,
+      :pretty_print, :pretty_print_cycle, :rassoc, :size,
       :to_a, :to_h, :to_hash, :to_s, :value?, :values, :values_at
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
       # Create a wrapper method that passes the call to @data.
@@ -61,11 +77,22 @@ module PEROBS
       end
     end
 
-    # These methods mutate the Hash.
+    # These methods mutate the Hash and return self
     [
-      :[]=, :clear, :default=, :default_proc=, :delete, :delete_if,
-      :initialize_copy, :keep_if, :merge!, :rehash, :reject!, :replace,
-      :select!, :shift, :update
+      :clear, :keep_if, :merge!, :rehash, :reject!, :replace, :select!, :update
+    ].each do |method_sym|
+      # Create a wrapper method that passes the call to @data.
+      define_method(method_sym) do |*args, &block|
+        # Register the write operation with the cache.
+        @store.cache.cache_write(self)
+        @data.send(method_sym, *args, &block)
+        myself
+      end
+    end
+
+    # These methods mutate the Hash and return basic Ruby type objects.
+    [
+      :delete, :delete_if, :shift
     ].each do |method_sym|
       # Create a wrapper method that passes the call to @data.
       define_method(method_sym) do |*args, &block|
@@ -79,17 +106,35 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param default [Any] The default value that is returned when no value is
-    #        stored for a specific key.
-    def initialize(p, default = nil)
+    # @param default [Object] The default value that is returned when no value
+    #        is stored for a specific key. The default must be of the
+    #        supported type.
+    def initialize(p, default = nil, &block)
       super(p)
-      @default = nil
-      @data = {}
+      _check_assignment_value(default)
+      if block_given?
+        @data = ::Hash.new(&block)
+      else
+        @data = ::Hash.new(default)
+      end
 
       # Ensure that the newly created object will be pushed into the database.
       @store.cache.cache_write(self)
     end
 
+    # Proxy for assignment method.
+    def []=(key, value)
+      _check_assignment_value(value)
+      @store.cache.cache_write(self)
+      @data[key] = value
+    end
+
+    # Proxy for default= method.
+    def default=(value)
+      _check_assignment_value(value)
+      @data.default=(value)
+    end
+
     # Return a list of all object IDs of all persistend objects that this Hash
     # is referencing.
     # @return [Array of Integer] IDs of referenced objects