perobs 4.1.0 → 4.2.0

data/lib/perobs/BTreeNodeLink.rb

@@ -67,6 +67,8 @@ module PEROBS
       get_node.respond_to?(method)
     end
 
+    # Directly define some commonly used methods to avoid the method_missing
+    # overhead.
     def is_leaf
       get_node.is_leaf
     end
@@ -91,10 +93,18 @@ module PEROBS
       get_node.search_key_index(key)
     end
 
+    def insert(key, value)
+      get_node.insert(key, value)
+    end
+
     def insert_element(key, voc)
       get_node.insert_element(key, voc)
     end
 
+    def split_node
+      get_node.split_node
+    end
+
     # Compare this node to another node.
     # @return [Boolean] true if node address is identical, false otherwise
     def ==(node)

data/lib/perobs/BigArray.rb

@@ -123,7 +123,11 @@ module PEROBS
     # @param index [Integer] Position in the array
     # @return [Integer or nil] found value or nil
     def [](index)
-      index = validate_index_range(index)
+      begin
+        index = validate_index_range(index)
+      rescue IndexError
+        return nil
+      end
 
       return nil if index >= @entry_counter
 
@@ -190,6 +194,20 @@ module PEROBS
       @entry_counter == 0
     end
 
+    # Return the first entry of the Array.
+    def first
+      return nil unless @first_leaf
+
+      @first_leaf.values.first
+    end
+
+    # Return the last entry of the Array.
+    def last
+      return nil unless @last_leaf
+
+      @last_leaf.values.last
+    end
+
     # Iterate over all entries in the tree. Entries are always sorted by the
     # key.
     # @yield [key, value]

data/lib/perobs/BigArrayNode.rb

@@ -147,7 +147,9 @@ module PEROBS
         else
           # Descend into the right child node to add the value to.
           cidx = node.search_child_index(index)
-          index -= node.offsets[cidx]
+          if (index -= node.offsets[cidx]) < 0
+            node.fatal "Index (#{index}) became negative"
+          end
           node = node.children[cidx]
         end
       end
@@ -392,7 +394,7 @@ module PEROBS
       @offsets.each_with_index do |offset, i|
         if i > 0
           if offset < last_offset
-            error "Offset are not strictly monotoneously " +
+            error "Offsets are not strictly monotoneously " +
               "increasing: #{@offsets.inspect}"
             return false
           end
@@ -471,11 +473,12 @@ module PEROBS
         else
           begin
             if node.is_leaf?
-              if node.values[position - 1]
+              if position <= node.size
                 str += "#{node.tree_prefix}  " +
                   "#{position == node.size ? '-' : '|'} " +
                   "[ #{node.value_index(position - 1)}: " +
-                  "#{node.values[position - 1]} ]\n"
+                  "#{node.values[position - 1].nil? ?
+                  'nil' : node.values[position - 1]} ]\n"
               end
             end
           rescue => e
@@ -613,7 +616,7 @@ module PEROBS
       # Handle special case for empty offsets list.
       return 0 if @offsets.empty? || offset <= @offsets.first
 
-      (@offsets.bsearch_index { |o| o >= offset } || @offsets.length) - 1
+      (@offsets.bsearch_index { |o| o > offset } || @offsets.length) - 1
     end
 
     # @return The index of the current node in the children list of the parent
@@ -656,7 +659,7 @@ module PEROBS
 
     # This method takes care of adjusting the offsets in tree in case elements
     # were inserted or removed. All nodes that hold children after the
-    # insert/remove operation needs to be adjusted. Since child nodes get their
+    # insert/remove operation need to be adjusted. Since child nodes get their
     # offsets via their parents, only the parent node and the direct ancestor
     # followers need to be adjusted.
     # @param after_child [BigArrayNode] specifies the modified leaf node
@@ -910,7 +913,7 @@ module PEROBS
         # Root Node             +--------------------------------+
         # Offsets               | 0                           11 |
         # Children                |                            |
-        #              prepd      v                child       v
+        #              pred       v                child       v
         # Level 1    +--------------------------++--------------------------+
         # Offsets    | 0         4         7    ||   0          2         5 |
         # Children     |         |         |         |          |         |
@@ -922,8 +925,9 @@ module PEROBS
         #
         # Remove the last predecessor offset and update the child offset with
         # it
-        delta = @offsets[child_index] - pred.offsets.last
-        @offsets[child_index] = pred.offsets.pop
+        delta = pred.children.last.values_count
+        @offsets[child_index] -= delta
+        pred.offsets.pop
         # Adjust all the offsets of the child
         child.offsets.map! { |o| o += delta }
         # And prepend the 0 offset

data/lib/perobs/BigHash.rb

@@ -68,7 +68,7 @@ module PEROBS
     class Collisions < PEROBS::Array
     end
 
-    attr_persist :btree, :entry_counter
+    attr_persist :btree
 
     # Create a new BigHash object.
     # @param p [Handle] Store handle
@@ -76,7 +76,6 @@ module PEROBS
       super(p)
       restore
       self.btree = @store.new(PEROBS::BigTree)
-      self.entry_counter = 0
     end
 
     def restore
@@ -106,7 +105,6 @@ module PEROBS
               end
               index_to_insert += 1
             end
-            self.entry_counter += 1 unless overwrite
             existing_entry[index_to_insert] = entry
           elsif existing_entry.key == key
             # The existing value is for the identical key. We can safely
@@ -119,12 +117,10 @@ module PEROBS
             array_entry << existing_entry
             array_entry << entry
             @btree.insert(hashed_key, array_entry)
-            self.entry_counter += 1
           end
         else
           # No existing entry. Insert the new entry.
           @btree.insert(hashed_key, entry)
-          self.entry_counter += 1
         end
       end
     end
@@ -170,6 +166,8 @@ module PEROBS
       false
     end
 
+    alias include? has_key?
+
     # Delete and return the entry for the given key. Return nil if no matching
     # entry exists.
     # @param key [Integer or String]
@@ -183,7 +181,6 @@ module PEROBS
       if entry.is_a?(PEROBS::Array)
         entry.each_with_index do |ae, i|
           if ae.key == key
-            self.entry_counter -= 1
             return entry.delete_at(i).value
           end
         end
@@ -197,7 +194,7 @@ module PEROBS
     # Return the number of entries stored in the hash.
     # @return [Integer]
     def length
-      @entry_counter
+      @btree.entry_counter
     end
 
     alias size length
@@ -205,7 +202,7 @@ module PEROBS
     # Return true if hash is empty. False otherweise.
     # @return [TrueClass, FalseClass]
     def empty?
-      @entry_counter == 0
+      @btree.entry_counter == 0
     end
 
     # Calls the given block for each key/value pair.
@@ -213,9 +210,9 @@ module PEROBS
     def each(&block)
       @btree.each do |index, entry|
         if entry.is_a?(Collisions)
-          break unless entry.each do |c_entry|
+          break if entry.each do |c_entry|
             yield(c_entry.key, c_entry.value)
-          end
+          end.nil?
         else
           yield(entry.key, entry.value)
         end
@@ -234,20 +231,7 @@ module PEROBS
     # Check if the data structure contains any errors.
     # @return [Boolean] true if no erros were found, false otherwise
     def check
-      return false unless @btree.check
-
-      i = 0
-      each do |k, v|
-        i += 1
-      end
-
-      unless @entry_counter == i
-        PEROBS.log.error "BigHash contains #{i} values but entry counter " +
-          "is #{@entry_counter}"
-        return false
-      end
-
-      true
+      return @btree.check
     end
 
     private

data/lib/perobs/BigTree.rb

@@ -140,7 +140,7 @@ module PEROBS
     def each(&block)
       node = @first_leaf
       while node
-        node.each_element(&block)
+        break if node.each_element(&block).nil?
         node = node.next_sibling
       end
     end
@@ -166,6 +166,19 @@ module PEROBS
     # @return [Boolean] true if no erros were found, false otherwise
     def check(&block)
       @root.check(&block)
+
+      i = 0
+      each do |k, v|
+        i += 1
+      end
+
+      unless @entry_counter == i
+        PEROBS.log.error "BigTree contains #{i} values but entry counter " +
+          "is #{@entry_counter}"
+        return false
+      end
+
+      true
     end
 
     # Gather some statistics regarding the tree structure.

data/lib/perobs/BigTreeNode.rb

@@ -227,7 +227,7 @@ module PEROBS
     # Iterate over all the key/value pairs of the node.
     # @yield [key, value]
     def each_element
-      return unless is_leaf?
+      return self unless is_leaf?
 
       0.upto(@keys.length - 1) do |i|
         yield(@keys[i], @values[i])
@@ -237,7 +237,7 @@ module PEROBS
     # Iterate over all the key/value pairs of the node in reverse order.
     # @yield [key, value]
     def reverse_each_element
-      return unless is_leaf?
+      return self unless is_leaf?
 
       (@keys.length - 1).downto(0) do |i|
         yield(@keys[i], @values[i])

data/lib/perobs/Cache.rb

@@ -2,7 +2,7 @@
 #
 # = Cache.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -66,10 +66,10 @@ module PEROBS
     def cache_write(obj)
       # This is just a safety check. It can probably be disabled in the future
       # to increase performance.
-      if obj.respond_to?(:is_poxreference?)
-        # If this condition triggers, we have a bug in the library.
-        PEROBS.log.fatal "POXReference objects should never be cached"
-      end
+      #if obj.respond_to?(:is_poxreference?)
+      #  # If this condition triggers, we have a bug in the library.
+      #  PEROBS.log.fatal "POXReference objects should never be cached"
+      #end
 
       if @transaction_stack.empty?
         # We are not in transaction mode.
@@ -93,6 +93,31 @@ module PEROBS
       end
     end
 
+    # Evict the object with the given ID from the cache.
+    # @param id [Integer] ID of the cached PEROBS::ObjectBase
+    # @return [True/False] True if object was stored in the cache. False
+    #         otherwise.
+    def evict(id)
+      unless @transaction_stack.empty?
+        PEROBS.log.fatal "You cannot evict entries during a transaction."
+      end
+
+      idx = id & @mask
+      # The index is just a hash. We still need to check if the object IDs are
+      # actually the same before we can return the object.
+      if (obj = @writes[idx]) && obj._id == id
+        # The object is in the write cache.
+        @writes[idx] = nil
+        return true
+      elsif (obj = @reads[idx]) && obj._id == id
+        # The object is in the read cache.
+        @reads[idx] = nil
+        return true
+      end
+
+      false
+    end
+
     # Return the PEROBS::Object with the specified ID or nil if not found.
     # @param id [Integer] ID of the cached PEROBS::ObjectBase
     def object_by_id(id)
@@ -160,7 +185,7 @@ module PEROBS
         transactions = @transaction_stack.pop
         # Merge the two lists
         @transaction_stack.push(@transaction_stack.pop + transactions)
-        # Ensure that each object is only included once in the list.
+        # Ensure that each object ID is only included once in the list.
         @transaction_stack.last.uniq!
       end
     end

data/lib/perobs/EquiBlobsFile.rb

@@ -40,7 +40,10 @@ module PEROBS
   # bytes long header that stores the total entry count, the total space
   # count, the offset of the first entry and the offset of the first space.
   # The header is followed by a custom entry section. Each entry is also 8
-  # bytes long. After the custom entry section the data blobs start.
+  # bytes long. After the custom entry section the data blobs start. Each data
+  # blob starts with a mark byte that indicates if the blob is valid data (2),
+  # a free space (0) or reseved space (1). Then it is followed by @entry_bytes
+  # number of bytes for the data blob.
   class EquiBlobsFile
 
     TOTAL_ENTRIES_OFFSET = 0
@@ -501,6 +504,7 @@ module PEROBS
       next_offset = address_to_offset(1)
       total_entries = 0
       total_spaces = 0
+      last_entry_is_space = false
       @progressmeter.start("Checking #{@name} entries",
                            @total_spaces + @total_entries) do |pm|
         begin
@@ -511,6 +515,7 @@ module PEROBS
             case marker
             when 0
               total_spaces += 1
+              last_entry_is_space = true
             when 1
               PEROBS.log.error "Entry at address " +
                 "#{offset_to_address(next_offset)} in EquiBlobsFile " +
@@ -518,6 +523,7 @@ module PEROBS
               return false
             when 2
               total_entries += 1
+              last_entry_is_space = false
             else
               PEROBS.log.error "Entry at address " +
                 "#{offset_to_address(next_offset)} in EquiBlobsFile " +
@@ -535,6 +541,11 @@ module PEROBS
         end
       end
 
+      if last_entry_is_space
+        PEROBS.log.error "EquiBlobsFile #{@file_name} is not properly trimmed"
+        return false
+      end
+
       unless total_spaces == @total_spaces
         PEROBS.log.error "Mismatch between space counter and spaces in " +
           "EquiBlobsFile #{@file_name}. Counter: #{@total_spaces}  " +

data/lib/perobs/FlatFile.rb

@@ -2,7 +2,7 @@
 #
 # = FlatFile.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2018 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -31,6 +31,7 @@ require 'perobs/Log'
 require 'perobs/FlatFileBlobHeader'
 require 'perobs/BTree'
 require 'perobs/SpaceTree'
+require 'perobs/SpaceManager'
 require 'perobs/IDList'
 
 module PEROBS
@@ -51,7 +52,14 @@ module PEROBS
       @f = nil
       @marks = nil
       @index = BTree.new(@db_dir, 'index', INDEX_BTREE_ORDER, @progressmeter)
-      @space_list = SpaceTree.new(@db_dir, @progressmeter)
+      old_spaces_file = File.join(@db_dir, 'database_spaces.blobs')
+      if File.exist?(old_spaces_file)
+        # PEROBS version 4.1.0 and earlier used this space list format. It is
+        # deprecated now. Newly created DBs use the SpaceManager format.
+        @space_list = SpaceTree.new(@db_dir, @progressmeter)
+      else
+        @space_list = SpaceManager.new(@db_dir, @progressmeter)
+      end
     end
 
     # Open the flat file for reading and writing.
@@ -134,7 +142,7 @@ module PEROBS
     end
 
     # Delete all unmarked objects.
-    def delete_unmarked_objects
+    def delete_unmarked_objects(&block)
       # We don't update the index and the space list during this operation as
       # we defragmentize the blob file at the end. We'll end the operation
       # with an empty space list.
@@ -145,6 +153,7 @@ module PEROBS
         each_blob_header do |header|
           if header.is_valid? && !@marks.include?(header.id)
             delete_obj_by_address(header.addr, header.id)
+            yield(header.id) if block_given?
             deleted_objects_count += 1
           end
 
@@ -183,12 +192,14 @@ module PEROBS
       # performance impact of compression is not compensated by writing
       # less data to the storage.
       compressed = false
-      if raw_obj.bytesize > 256
+      raw_obj_bytesize = raw_obj.bytesize
+      if raw_obj_bytesize > 256
         raw_obj = Zlib.deflate(raw_obj)
+        raw_obj_bytesize = raw_obj.bytesize
         compressed = true
       end
 
-      addr, length = find_free_blob(raw_obj.bytesize)
+      addr, length = find_free_blob(raw_obj_bytesize)
       begin
         if length != -1
           # Just a safeguard so we don't overwrite current data.
@@ -197,8 +208,8 @@ module PEROBS
             PEROBS.log.fatal "Length in free list (#{length}) and header " +
               "(#{header.length}) for address #{addr} don't match."
           end
-          if raw_obj.bytesize > header.length
-            PEROBS.log.fatal "Object (#{raw_obj.bytesize}) is longer than " +
+          if raw_obj_bytesize > header.length
+            PEROBS.log.fatal "Object (#{raw_obj_bytesize}) is longer than " +
               "blob space (#{header.length})."
           end
           if header.is_valid?
@@ -208,19 +219,19 @@ module PEROBS
         end
         flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
         flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
-        FlatFileBlobHeader.new(@f, addr, flags, raw_obj.bytesize, id, crc).write
+        FlatFileBlobHeader.new(@f, addr, flags, raw_obj_bytesize, id, crc).write
         @f.write(raw_obj)
-        if length != -1 && raw_obj.bytesize < length
+        if length != -1 && raw_obj_bytesize < length
           # The new object was not appended and it did not completely fill the
           # free space. So we have to write a new header to mark the remaining
           # empty space.
-          unless length - raw_obj.bytesize >= FlatFileBlobHeader::LENGTH
+          unless length - raw_obj_bytesize >= FlatFileBlobHeader::LENGTH
             PEROBS.log.fatal "Not enough space to append the empty space " +
-              "header (space: #{length} bytes, object: #{raw_obj.bytesize} " +
+              "header (space: #{length} bytes, object: #{raw_obj_bytesize} " +
               "bytes)."
           end
           space_address = @f.pos
-          space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.bytesize
+          space_length = length - FlatFileBlobHeader::LENGTH - raw_obj_bytesize
           FlatFileBlobHeader.new(@f, space_address, 0, space_length,
                                  0, 0).write
           # Register the new space with the space list.
@@ -461,6 +472,7 @@ module PEROBS
       new_index.open
 
       corrupted_blobs = 0
+      end_of_last_healthy_blob = nil
       @progressmeter.start('Checking blobs file', @f.size) do |pm|
         corrupted_blobs = each_blob_header do |header|
           if header.is_valid?
@@ -529,12 +541,26 @@ module PEROBS
               # ID is unique so far. Add it to the shadow index.
               new_index.insert(header.id, header.addr)
             end
-
           end
+          end_of_last_healthy_blob = header.addr +
+            FlatFileBlobHeader::LENGTH + header.length
 
           pm.update(header.addr)
         end
 
+        if end_of_last_healthy_blob && end_of_last_healthy_blob != @f.size
+          # The blob file ends with a corrupted blob header.
+          PEROBS.log.error "#{@f.size - end_of_last_healthy_blob} corrupted " +
+            'bytes found at the end of FlatFile.'
+          corrupted_blobs += 1
+          if repair
+            PEROBS.log.error "Truncating FlatFile to " +
+              "#{end_of_last_healthy_blob} bytes by discarding " +
+              "#{@f.size - end_of_last_healthy_blob} bytes"
+            @f.truncate(end_of_last_healthy_blob)
+          end
+        end
+
         errors += corrupted_blobs
       end
 
@@ -546,7 +572,7 @@ module PEROBS
         erase_index_files
         defragmentize
         regenerate_index_and_spaces
-      else
+      elsif corrupted_blobs == 0
         # Now we check the index data. It must be correct and the entries must
         # match the blob file. All entries in the index must be in the blob file
         # and vise versa.
@@ -575,6 +601,141 @@ module PEROBS
       errors
     end
 
+    # Repair the FlatFile. In contrast to the repair functionality in the
+    # check() method this method is much faster. It simply re-creates the
+    # index and space list from the blob file.
+    # @param repair [Boolean] True if errors should be fixed.
+    # @return [Integer] Number of errors found
+    def repair
+      errors = 0
+      return errors unless @f
+
+      t = Time.now
+      PEROBS.log.info "Repairing FlatFile database"
+
+      # Erase and re-open the index and space list files. We purposely don't
+      # close the files at it would trigger needless flushing.
+      clear_index_files(true)
+
+      # Now we scan the blob file and re-index all blobs and spaces. Corrupted
+      # blobs will be skipped.
+      corrupted_blobs = 0
+      end_of_last_healthy_blob = nil
+      @progressmeter.start('Re-indexing blobs file', @f.size) do |pm|
+        corrupted_blobs = each_blob_header do |header|
+          if header.corruption_start
+            # The blob is preceeded by a corrupted area. We create a new
+            # header of a deleted blob for this area and write the new blob
+            # over it.
+            if (data_length = header.addr - header.corruption_start -
+                FlatFileBlobHeader::LENGTH) <= 0
+              PEROBS.log.error "Found a corrupted blob that is too small to " +
+                "fit a header (#{data_length}). File must be defragmented."
+            else
+              new_header = FlatFileBlobHeader.new(@f, header.corruption_start,
+                                                  0, data_length, 0, 0)
+              new_header.write
+              @space_list.add_space(header.corruption_start, data_length)
+            end
+          end
+
+          if header.is_valid?
+            # We have a non-deleted entry.
+            begin
+              @f.seek(header.addr + FlatFileBlobHeader::LENGTH)
+              buf = @f.read(header.length)
+              if buf.bytesize != header.length
+                PEROBS.log.error "Premature end of file in blob with ID " +
+                  "#{header.id}."
+                discard_damaged_blob(header)
+                errors += 1
+                next
+              end
+
+              # Uncompress the data if the compression bit is set in the mark
+              # byte.
+              if header.is_compressed?
+                begin
+                  buf = Zlib.inflate(buf)
+                rescue Zlib::BufError, Zlib::DataError
+                  PEROBS.log.error "Corrupted compressed block with ID " +
+                    "#{header.id} found."
+                  discard_damaged_blob(header)
+                  errors += 1
+                  next
+                end
+              end
+
+              if header.crc && checksum(buf) != header.crc
+                PEROBS.log.error "Checksum failure while checking blob " +
+                  "with ID #{header.id}"
+                discard_damaged_blob(header)
+                errors += 1
+                next
+              end
+            rescue IOError => e
+              PEROBS.log.fatal "Check of blob with ID #{header.id} failed: " +
+                e.message
+            end
+
+            # Check if the ID has already been found in the file.
+            if (previous_address = @index.get(header.id))
+              PEROBS.log.error "Multiple blobs for ID #{header.id} found. " +
+                "Addresses: #{previous_address}, #{header.addr}"
+              errors += 1
+              previous_header = FlatFileBlobHeader.read(@f, previous_address,
+                                                        header.id)
+              # We have two blobs with the same ID and we must discard one of
+              # them.
+              if header.is_outdated?
+                discard_damaged_blob(header)
+              elsif previous_header.is_outdated?
+                discard_damaged_blob(previous_header)
+              else
+                PEROBS.log.error "None of the blobs with same ID have " +
+                  "the outdated flag set. Deleting the smaller one."
+                errors += 1
+                discard_damaged_blob(header.length < previous_header.length ?
+                                     header : previous_header)
+              end
+            else
+              # ID is unique so far. Add it to the shadow index.
+              @index.insert(header.id, header.addr)
+            end
+
+          else
+            if header.length > 0
+              @space_list.add_space(header.addr, header.length)
+            end
+          end
+          end_of_last_healthy_blob = header.addr +
+            FlatFileBlobHeader::LENGTH + header.length
+
+          pm.update(header.addr)
+        end
+
+        if end_of_last_healthy_blob && end_of_last_healthy_blob != @f.size
+          # The blob file ends with a corrupted blob header.
+          PEROBS.log.error "#{@f.size - end_of_last_healthy_blob} corrupted " +
+            'bytes found at the end of FlatFile.'
+          corrupted_blobs += 1
+
+          PEROBS.log.error "Truncating FlatFile to " +
+            "#{end_of_last_healthy_blob} bytes by discarding " +
+            "#{@f.size - end_of_last_healthy_blob} bytes"
+          @f.truncate(end_of_last_healthy_blob)
+        end
+
+        errors += corrupted_blobs
+      end
+
+      sync
+      PEROBS.log.info "FlatFile repair completed in #{Time.now - t} seconds. " +
+        "#{errors} errors found."
+
+      errors
+    end
+
     # This method clears the index tree and the free space list and
     # regenerates them from the FlatFile.
     def regenerate_index_and_spaces
@@ -617,7 +778,11 @@ module PEROBS
     end
 
     def has_id_at?(id, address)
-      header = FlatFileBlobHeader.read(@f, address)
+      begin
+        header = FlatFileBlobHeader.read(@f, address)
+      rescue PEROBS::FatalError
+        return false
+      end
       header.is_valid? && header.id == id
     end
 
@@ -733,7 +898,7 @@ module PEROBS
               unless @space_list.has_space?(header.addr, header.length)
                 PEROBS.log.error "FlatFile has free space " +
                   "(addr: #{header.addr}, len: #{header.length}) that is " +
-                  "not in FreeSpaceManager"
+                  "not in SpaceManager"
                 errors += 1
               end
             end
@@ -767,49 +932,36 @@ module PEROBS
         @index.open(abort_on_missing_files)
         @space_list.open
       rescue FatalError
-        # Ensure that the index is really closed.
-        @index.close
-        # Erase it completely
-        @index.erase
-        # Then create it again.
-        @index.open
-
-        # Ensure that the spaces list is really closed.
-        @space_list.close
-        # Erase it completely
-        @space_list.erase
-        # Then create it again
-        @space_list.open
-
+        clear_index_files
         regenerate_index_and_spaces
       end
     end
 
-    def erase_index_files
+    def erase_index_files(dont_close_files = false)
       # Ensure that the index is really closed.
-      @index.close
+      @index.close unless dont_close_files
       # Erase it completely
       @index.erase
 
       # Ensure that the spaces list is really closed.
-      @space_list.close
+      @space_list.close unless dont_close_files
       # Erase it completely
       @space_list.erase
+
+      if @space_list.is_a?(SpaceTree)
+        # If we still use the old SpaceTree format, this is the moment to
+        # convert it to the new SpaceManager format.
+        @space_list = SpaceManager.new(@db_dir, @progressmeter)
+        PEROBS.log.warn "Converting space list from SpaceTree format " +
+          "to SpaceManager format"
+      end
     end
 
-    def clear_index_files
-      # Ensure that the index is really closed.
-      @index.close
-      # Erase it completely
-      @index.erase
-      # Then create it again.
-      @index.open
+    def clear_index_files(dont_close_files = false)
+      erase_index_files(dont_close_files)
 
-      # Ensure that the spaces list is really closed.
-      @space_list.close
-      # Erase it completely
-      @space_list.erase
-      # Then create it again
+      # Then create them again.
+      @index.open
       @space_list.open
     end