perobs 3.0.1 → 4.3.0

data/lib/perobs/FlatFileBlobHeader.rb

@@ -33,7 +33,7 @@ module PEROBS
   #
   # 1 Byte:  Flags byte.
   #          Bit 0: 0 deleted entry, 1 valid entry
-  #          Bit 1: 0 unmarked, 1 marked
+  #          Bit 1: 0 reserved, must be 0
   #          Bit 2: 0 uncompressed data, 1 compressed data
   #          Bit 3: 0 current entry, 1 outdated entry
   #          Bit 4 - 7: reserved, must be 0
@@ -48,21 +48,21 @@ module PEROBS
     # The 'pack()' format of the header.
     FORMAT = 'CQQL'
     # The length of the header in bytes.
-    LENGTH = 21
+    LENGTH = 25
     VALID_FLAG_BIT = 0
-    MARK_FLAG_BIT = 1
     COMPRESSED_FLAG_BIT = 2
     OUTDATED_FLAG_BIT = 3
 
     attr_reader :addr, :flags, :length, :id, :crc
+    attr_accessor :corruption_start
 
     # Create a new FlatFileBlobHeader with the given flags, length, id and crc.
     # @param file [File] the FlatFile that contains the header
     # @param addr [Integer] the offset address of the header in the file
-    # @param flags [Fixnum] 8 bit number, see above
-    # @param length [Fixnum] length of the header in bytes
+    # @param flags [Integer] 8 bit number, see above
+    # @param length [Integer] length of the header in bytes
     # @param id [Integer] ID of the blob entry
-    # @param crc [Fixnum] CRC32 checksum of the blob entry
+    # @param crc [Integer] CRC32 checksum of the blob entry
     def initialize(file, addr, flags, length, id, crc)
       @file = file
       @addr = addr
@@ -70,64 +70,136 @@ module PEROBS
       @length = length
       @id = id
       @crc = crc
+      # This is only set if the header is preceded by a corrupted blob.
+      @corruption_start = nil
     end
 
     # Read the header from the given File.
     # @param file [File]
-    # @return FlatFileBlobHeader
-    def FlatFileBlobHeader::read(file)
-      begin
-        addr = file.pos
-        buf = file.read(LENGTH)
-      rescue IOError => e
-        PEROBS.log.error "Cannot read blob header in flat file DB: #{e.message}"
-        return nil
+    # @param addr [Integer] address in the file to start reading. If no
+    #        address is specified use the current position in the file.
+    # @param id [Integer] Optional ID that the header should have. If no id is
+    #        specified there is no check against the actual ID done.
+    # @return FlatFileBlobHeader or nil if there are no more blobs to read in
+    #         the file.
+    def FlatFileBlobHeader::read(file, addr = nil, id = nil)
+      # If an address was specified we expect the read to always succeed. If
+      # no address is specified and we can't read the header we generate an
+      # error message but it is not fatal.
+      errors_are_fatal = !addr.nil?
+
+      mode = :searching_next_header
+      addr = file.pos unless addr
+      buf = nil
+      corruption_start = nil
+
+      loop do
+        buf_with_crc = nil
+        begin
+          file.seek(addr)
+          buf_with_crc = file.read(LENGTH)
+        rescue IOError => e
+          if errors_are_fatal
+            PEROBS.log.fatal "Cannot read blob header in flat file DB at " +
+              "address #{addr}: #{e.message}"
+          else
+            PEROBS.log.error "Cannot read blob header in flat file DB: " +
+              e.message
+            return nil
+          end
+        end
+
+        # Did we read anything?
+        if buf_with_crc.nil?
+          if errors_are_fatal
+            PEROBS.log.fatal "Cannot read blob header " +
+              "#{id ? "for ID #{id} " : ''}at address #{addr}"
+          else
+            if corruption_start
+              PEROBS.log.error "Corruption found at end of blob file at " +
+                "address #{addr}"
+            end
+            # We have reached the end of the file.
+            return nil
+          end
+        end
+
+        # Did we get the full header?
+        if buf_with_crc.length != LENGTH
+          msg = "Incomplete FlatFileBlobHeader: Only " +
+            "#{buf_with_crc.length} " +
+            "bytes of #{LENGTH} could be read "
+          "#{id ? "for ID #{id} " : ''}at address #{addr}"
+          if errors_are_fatal
+            PEROBS.log.fatal msg
+          else
+            PEROBS.log.error msg
+          end
+          return nil
+        end
+
+        # Check the CRC of the header
+        buf = buf_with_crc[0..-5]
+        crc = buf_with_crc[-4..-1].unpack('L')[0]
+
+        if (read_crc = Zlib.crc32(buf, 0)) == crc
+          # We have found a valid header.
+          if corruption_start
+            PEROBS.log.error "FlatFile corruption ends at #{addr}. " +
+              "#{addr - corruption_start} bytes skipped. Some data may " +
+              "not be recoverable."
+          end
+          break
+        else
+          if errors_are_fatal
+            PEROBS.log.fatal "FlatFile Header CRC mismatch at address " +
+              "#{addr}. Header CRC is #{'%08x' % read_crc} but should be " +
+              "#{'%08x' % crc}."
+          else
+            if corruption_start.nil?
+              if errors_are_fatal
+                PEROBS.log.fatal "FlatFile corruption found. The FlatFile " +
+                  "Header CRC mismatch at address #{addr}. Header CRC is " +
+                  "#{'%08x' % read_crc} but should be #{'%08x' % crc}."
+              else
+                PEROBS.log.error "FlatFile corruption found. The FlatFile " +
+                  "Header CRC mismatch at address #{addr}. Header CRC is " +
+                  "#{'%08x' % read_crc} but should be #{'%08x' % crc}. " +
+                  "Trying to find the next header."
+              end
+              corruption_start = addr
+            end
+            # The blob file is corrupted. There is no valid header at the
+            # current position in the file. We now try to find the next valid
+            # header by iterating over the remainder of the file advanding one
+            # byte with each step until we hit the end of the file or find the
+            # next valid header.
+            addr += 1
+          end
+        end
       end
 
-      return nil unless buf
-
-      if buf.length != LENGTH
-        PEROBS.log.error "Incomplete FlatFileBlobHeader: Only #{buf.length} " +
-          "bytes of #{LENGTH} could be read"
-        return nil
+      header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
+      if corruption_start
+        header.corruption_start = corruption_start
       end
 
-      FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
-    end
-
-    # Read the header from the given File.
-    # @param file [File]
-    # @param addr [Integer] address in the file to start reading
-    # @param id [Integer] Optional ID that the header should have
-    # @return FlatFileBlobHeader
-    def FlatFileBlobHeader::read_at(file, addr, id = nil)
-      buf = nil
-      begin
-        file.seek(addr)
-        buf = file.read(LENGTH)
-      rescue IOError => e
-        PEROBS.log.fatal "Cannot read blob in flat file DB: #{e.message}"
-      end
-      if buf.nil? || buf.length != LENGTH
-        PEROBS.log.fatal "Cannot read blob header " +
-          "#{id ? "for ID #{id} " : ''}at address " +
-          "#{addr}"
-      end
-      header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
       if id && header.id != id
         PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
-          "found for entry with ID #{id}/#{header.id}"
+          "found. FlatFile has entry with ID #{header.id} at address " +
+          "#{addr}. Index has ID #{id} for this address."
       end
 
       return header
     end
 
     # Write the header to a given File.
-    # @param file [File]
     def write
       begin
+        buf = [ @flags, @length, @id, @crc].pack(FORMAT)
+        crc = Zlib.crc32(buf, 0)
         @file.seek(@addr)
-        @file.write([ @flags, @length, @id, @crc].pack(FORMAT))
+        @file.write(buf + [ crc ].pack('L'))
       rescue IOError => e
         PEROBS.log.fatal "Cannot write blob header into flat file DB: " +
           e.message
@@ -135,17 +207,9 @@ module PEROBS
     end
 
     # Reset all the flags bit to 0. This marks the blob as invalid.
-    # @param file [File] The file handle of the blob file.
-    # @param addr [Integer] The address of the header
     def clear_flags
-      begin
-        @file.seek(@addr)
-        @file.write([ 0 ].pack('C'))
-        @file.flush
-      rescue IOError => e
-        PEROBS.log.fatal "Clearing flags of FlatFileBlobHeader with ID " +
-          "#{@id} failed: #{e.message}"
-      end
+      @flags = 0
+      write
     end
 
     # Return true if the header is for a non-empty blob.
@@ -153,23 +217,6 @@ module PEROBS
       bit_set?(VALID_FLAG_BIT)
     end
 
-    # Return true if the blob has been marked.
-    def is_marked?
-      bit_set?(MARK_FLAG_BIT)
-    end
-
-    # Set the mark bit.
-    def set_mark_flag
-      set_flag(MARK_FLAG_BIT)
-      write_flags
-    end
-
-    # Clear the mark bit.
-    def clear_mark_flag
-      clear_flag(MARK_FLAG_BIT)
-      write_flags
-    end
-
     # Return true if the blob contains compressed data.
     def is_compressed?
       bit_set?(COMPRESSED_FLAG_BIT)
@@ -179,7 +226,7 @@ module PEROBS
     # transaction has been completed.
     def set_outdated_flag
       set_flag(OUTDATED_FLAG_BIT)
-      write_flags
+      write
     end
 
     # Return true if the blob contains outdated data.
@@ -189,17 +236,6 @@ module PEROBS
 
     private
 
-    def write_flags
-      begin
-        @file.seek(@addr)
-        @file.write([ @flags ].pack('C'))
-        @file.flush
-      rescue IOError => e
-        PEROBS.log.fatal "Writing flags of FlatFileBlobHeader with ID #{@id} " +
-          "failed: #{e.message}"
-      end
-    end
-
     def bit_set?(n)
       mask = 1 << n
       @flags & mask == mask

data/lib/perobs/FlatFileDB.rb

@@ -2,7 +2,8 @@
 #
 # = FlatFileDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -41,7 +42,7 @@ module PEROBS
 
     # This version number increases whenever the on-disk format changes in a
     # way that requires conversion actions after an update.
-    VERSION = 2
+    VERSION = 4
 
     attr_reader :max_blob_size
 
@@ -50,13 +51,17 @@ module PEROBS
     # @param options [Hash] options to customize the behavior. Currently only
     #        the following options are supported:
     #        :serializer    : Can be :marshal, :json, :yaml
+    #        :progressmeter : Reference to a ProgressMeter object
+    #        :log           : IO that should be used for logging
+    #        :log_level     : Minimum Logger level to log
     def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
+      super(options)
 
       @db_dir = db_name
       # Create the database directory if it doesn't exist yet.
       ensure_dir_exists(@db_dir)
-      PEROBS.log.open(File.join(@db_dir, 'log'))
+      PEROBS.log.level = options[:log_level] if options[:log_level]
+      PEROBS.log.open(options[:log] || File.join(@db_dir, 'log'))
       check_version_and_upgrade
 
       # Read the existing DB config.
@@ -68,7 +73,7 @@ module PEROBS
 
     # Open the FlatFileDB for transactions.
     def open
-      @flat_file = FlatFile.new(@db_dir)
+      @flat_file = FlatFile.new(@db_dir, @progressmeter)
       @flat_file.open
       PEROBS.log.info "FlatFile '#{@db_dir}' opened"
     end
@@ -87,11 +92,12 @@ module PEROBS
     end
 
     def FlatFileDB::delete_db(db_name)
+      close
       FileUtils.rm_rf(db_name)
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !@flat_file.find_obj_addr_by_id(id).nil?
     end
@@ -131,7 +137,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -142,6 +148,11 @@ module PEROBS
       end
     end
 
+    # @return [Integer] Number of objects stored in the DB.
+    def item_counter
+      @flat_file.item_counter
+    end
+
     # This method must be called to initiate the marking process.
     def clear_marks
       @flat_file.clear_all_marks
@@ -149,19 +160,19 @@ module PEROBS
 
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
-    # @return [Array] List of IDs that have been removed from the DB.
-    def delete_unmarked_objects
-      @flat_file.delete_unmarked_objects
+    # @return [Integer] Number of the removed objects from the DB.
+    def delete_unmarked_objects(&block)
+      @flat_file.delete_unmarked_objects(&block)
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       @flat_file.mark_obj_by_id(id)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     def is_marked?(id, ignore_errors = false)
@@ -173,11 +184,15 @@ module PEROBS
     #        repaired.
     # @return number of errors found
     def check_db(repair = false)
-      @flat_file.check(repair)
+      if repair
+        @flat_file.repair
+      else
+        @flat_file.check
+      end
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -202,7 +217,7 @@ module PEROBS
     # Store the given serialized object into the cluster files. This method is
     # for internal use only!
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     def put_raw_object(raw, id)
       @flat_file.write_obj_by_id(id, raw)
     end
@@ -221,7 +236,8 @@ module PEROBS
                            "'#{version_file}': " + e.message
         end
       else
-        # Early versions of PEROBS did not have a version file.
+        # The DB is brand new.
+        version = VERSION
         write_version_file(version_file)
       end
 
@@ -229,25 +245,40 @@ module PEROBS
         PEROBS.log.fatal "Cannot downgrade the FlatFile database from " +
                          "version #{version} to version #{VERSION}"
       end
-
-      if version == 1
-        # Version 1 had no support for data compression. Make sure all entries
-        # are compressed to save space.
-        open
-        @flat_file.refresh
-        close
+      if version < 3
+        PEROBS.log.fatal "The upgrade of this version of the PEROBS database " +
+          "is not supported by this version of PEROBS. Please try an earlier " +
+          "version of PEROBS to upgrade the database before using this version."
       end
 
-      # After a successful upgrade change the version number in the DB as
-      # well.
-      if version < VERSION
+      # Version upgrades must be done one version number at a time. If the
+      # existing DB is multiple versions older than what the current PEROBS
+      # version expects than multiple upgrade runs will be needed.
+      while version < VERSION
+        if version == 3
+          PEROBS.log.warn "Updating FlatFileDB #{@db_dir} from version 3 to " +
+            "version 4 ..."
+          # Version 4 adds checksums for blob file headers. We have to convert
+          # the blob file to include the checksums.
+          FlatFile.insert_header_checksums(@db_dir)
+          open
+          @flat_file.regenerate_index_and_spaces
+          close
+        end
+
+        # After a successful upgrade change the version number in the DB as
+        # well.
         write_version_file(version_file)
         PEROBS.log.warn "Update of FlatFileDB '#{@db_dir}' from version " +
-          "#{version} to version #{VERSION} completed"
+          "#{version} to version #{version + 1} completed"
+
+        # Update version variable to new version.
+        version += 1
       end
     end
 
     def write_version_file(version_file)
+
       begin
         RobustFile.write(version_file, VERSION)
       rescue IOError => e