perobs 2.4.2 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7913287f5998dfad64a99f239723ecb905798fe9
-  data.tar.gz: d667fe5eab1c96b4003640c4d924d97393fdf4b3
+  metadata.gz: 9df83ee37d61319185f94bcaf64a1f48083280fd
+  data.tar.gz: 130fbd021bfe32cad6b45e6b4063f0093552d8f3
 SHA512:
-  metadata.gz: 0140b0b11d4f6f7c5f01743d5e361cf045c4533ab339ee92e32e72b4ddca7046684a6ac00445667c0a08d953946cc0fd93d6563a9b3482759e436151d0a548df
-  data.tar.gz: 5b6b9101321994f98f42c153c788bfece79183a074baf01d1e2ed377a0ff929b3308947b272bca1892807e425abfd6d2decd208f07179151fbca703d76eaa8ac
+  metadata.gz: a1e61da3e76e0bb4e965ca00c0d4b97105fe5a37f1272cc9a4b0d997d0f8a521547c22c3318ad0bc17eabf9059c54d7d616ad287f1d48743c9f126e17c66ec97
+  data.tar.gz: 0d86ab15a8ffdb18a1b8df4f27a5f2e0fc4f9619f040934ec7904b4e17bc453b383be424190951f2c2d6f1fcb309398913aba6111a360ba908a428f1629c7b67

data/lib/perobs/FlatFile.rb

@@ -28,36 +28,17 @@
 require 'zlib'
 
 require 'perobs/Log'
+require 'perobs/FlatFileBlobHeader'
 require 'perobs/IndexTree'
 require 'perobs/FreeSpaceManager'
 
 module PEROBS
 
   # The FlatFile class manages the storage file of the FlatFileDB. It contains
-  # a sequence of blobs Each blob consists of a 25 byte header and the actual
-  # blob data bytes. The header has the following structure:
-  #
-  # 1 Byte:  Mark byte.
-  #          Bit 0: 0 deleted entry, 1 valid entry
-  #          Bit 1: 0 unmarked, 1 marked
-  #          Bit 2 - 7: reserved, must be 0
-  # 8 bytes: Length of the data blob in bytes
-  # 8 bytes: ID of the value in the data blob
-  # 4 bytes: CRC32 checksum of the data blob
-  #
-  # If the bit 0 of the mark byte is 0, only the length is valid. The blob is
-  # empty. Only of bit 0 is set then entry is valid.
+  # a sequence of blobs Each blob consists of header and the actual
+  # blob data bytes.
   class FlatFile
 
-    # Utility class to hold all the data that is stored in a blob header.
-    class Header < Struct.new(:mark, :length, :id, :crc)
-    end
-
-    # The 'pack()' format of the header.
-    BLOB_HEADER_FORMAT = 'CQQL'
-    # The length of the header in bytes.
-    BLOB_HEADER_LENGTH = 21
-
     # Create a new FlatFile object for a database in the given path.
     # @param dir [String] Directory path for the data base file
     def initialize(dir)
@@ -125,13 +106,13 @@ module PEROBS
     # @param id [Integer] ID of the blob to delete
     def delete_obj_by_address(addr, id)
       @index.delete_value(id)
-      header = read_blob_header(addr, id)
+      header = FlatFileBlobHeader.read_at(@f, addr, id)
       begin
         @f.seek(addr)
         @f.write([ 0 ].pack('C'))
         @f.flush
         @space_list.add_space(addr, header.length)
-      rescue => e
+      rescue IOError => e
         PEROBS.log.fatal "Cannot erase blob for ID #{header.id}: #{e.message}"
       end
     end
@@ -142,10 +123,10 @@ module PEROBS
       t = Time.now
 
       deleted_ids = []
-      each_blob_header do |pos, mark, length, blob_id, crc|
-        if (mark & 3 == 1)
-          delete_obj_by_address(pos, blob_id)
-          deleted_ids << blob_id
+      each_blob_header do |pos, header|
+        if header.is_valid? && !header.is_marked?
+          delete_obj_by_address(pos, header.id)
+          deleted_ids << header.id
         end
       end
       defragmentize
@@ -161,11 +142,23 @@ module PEROBS
     # @param raw_obj [String] Raw object as String
     # @return [Integer] position of the written blob in the blob file
     def write_obj_by_id(id, raw_obj)
+      crc = checksum(raw_obj)
+
+      # If the raw_obj is larger then 256 characters we will compress it to
+      # safe some space in the database file. For smaller strings the
+      # performance impact of compression is not compensated by writing
+      # less data to the storage.
+      compressed = false
+      if raw_obj.length > 256
+        raw_obj = Zlib.deflate(raw_obj)
+        compressed = true
+      end
+
       addr, length = find_free_blob(raw_obj.length)
       begin
         if length != -1
           # Just a safeguard so we don't overwrite current data.
-          header = read_blob_header(addr)
+          header = FlatFileBlobHeader.read_at(@f, addr)
           if header.length != length
             PEROBS.log.fatal "Length in free list (#{length}) and header " +
               "(#{header.length}) don't match."
@@ -174,26 +167,26 @@ module PEROBS
             PEROBS.log.fatal "Object (#{raw_obj.length}) is longer than " +
               "blob space (#{header.length})."
           end
-          if header.mark != 0
-            PEROBS.log.fatal "Mark (#{header.mark}) is not 0."
+          if header.is_valid?
+            PEROBS.log.fatal "Entry (mark: #{header.mark}) is already used."
           end
         end
         @f.seek(addr)
-        @f.write([ 1, raw_obj.length, id, checksum(raw_obj)].
-                 pack(BLOB_HEADER_FORMAT))
+        FlatFileBlobHeader.new(compressed ? (1 << 2) | 1 : 1, raw_obj.length,
+                               id, crc).write(@f)
         @f.write(raw_obj)
         if length != -1 && raw_obj.length < length
           # The new object was not appended and it did not completely fill the
           # free space. So we have to write a new header to mark the remaining
           # empty space.
-          unless length - raw_obj.length >= BLOB_HEADER_LENGTH
+          unless length - raw_obj.length >= FlatFileBlobHeader::LENGTH
             PEROBS.log.fatal "Not enough space to append the empty space " +
               "header (space: #{length} bytes, object: #{raw_obj.length} " +
               "bytes)."
           end
           space_address = @f.pos
-          space_length = length - BLOB_HEADER_LENGTH - raw_obj.length
-          @f.write([ 0, space_length, 0, 0 ].pack(BLOB_HEADER_FORMAT))
+          space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.length
+          FlatFileBlobHeader.new(0, space_length, 0, 0).write(@f)
           # Register the new space with the space list.
           @space_list.add_space(space_address, space_length) if space_length > 0
         end
@@ -230,21 +223,31 @@ module PEROBS
     # @param id [Integer] ID of the data blob
     # @return [String] Raw object data
     def read_obj_by_address(addr, id)
-      header = read_blob_header(addr, id)
+      header = FlatFileBlobHeader.read_at(@f, addr, id)
       if header.id != id
         PEROBS.log.fatal "Database index corrupted: Index for object " +
           "#{id} points to object with ID #{header.id}"
       end
+
+      buf = nil
+
       begin
-        @f.seek(addr + BLOB_HEADER_LENGTH)
+        @f.seek(addr + FlatFileBlobHeader::LENGTH)
         buf = @f.read(header.length)
-        if checksum(buf) != header.crc
-          PEROBS.log.fatal "Checksum failure while reading blob ID #{id}"
-        end
-        return buf
-      rescue => e
+      rescue IOError => e
         PEROBS.log.fatal "Cannot read blob for ID #{id}: #{e.message}"
       end
+
+      # Uncompress the data if the compression bit is set in the mark byte.
+      if header.is_compressed?
+        buf = Zlib.inflate(buf)
+      end
+
+      if checksum(buf) != header.crc
+        PEROBS.log.fatal "Checksum failure while reading blob ID #{id}"
+      end
+
+      buf
     end
 
     # Mark the object with the given ID.
@@ -259,12 +262,12 @@ module PEROBS
     # @param addr [Integer] Offset in the file
     # @param id [Integer] ID of the object
     def mark_obj_by_address(addr, id)
-      header = read_blob_header(addr, id)
+      header = FlatFileBlobHeader.read_at(@f, addr, id)
       begin
         @f.seek(addr)
-        @f.write([ header.mark | 2 ].pack('C'))
+        @f.write([ header.mark | (1 << 1) ].pack('C'))
         @f.flush
-      rescue => e
+      rescue IOError => e
         PEROBS.log.fatal "Marking of FlatFile blob with ID #{id} " +
           "failed: #{e.message}"
       end
@@ -274,8 +277,8 @@ module PEROBS
     # @param id [Integer] ID of the object
     def is_marked_by_id?(id)
       if (addr = find_obj_addr_by_id(id))
-        header = read_blob_header(addr, id)
-        return (header.mark & 2) == 2
+        header = FlatFileBlobHeader.read_at(@f, addr, id)
+        return header.is_marked?
       end
 
       false
@@ -289,16 +292,16 @@ module PEROBS
       total_blob_count = 0
       marked_blob_count = 0
 
-      each_blob_header do |pos, mark, length, blob_id, crc|
+      each_blob_header do |pos, header|
         total_blob_count += 1
-        if (mark & 3 == 3)
+        if header.is_valid? && header.is_marked?
           # Clear all valid and marked blocks.
           marked_blob_count += 1
           begin
             @f.seek(pos)
-            @f.write([ mark & 0b11111101 ].pack('C'))
+            @f.write([ header.mark & 0b11111101 ].pack('C'))
             @f.flush
-          rescue => e
+          rescue IOError => e
             PEROBS.log.fatal "Unmarking of FlatFile blob with ID #{blob_id} " +
               "failed: #{e.message}"
           end
@@ -317,10 +320,10 @@ module PEROBS
       t = Time.now
       PEROBS.log.info "Defragmenting FlatFile"
       # Iterate over all entries.
-      each_blob_header do |pos, mark, length, blob_id, crc|
+      each_blob_header do |pos, header|
         # Total size of the current entry
-        entry_bytes = BLOB_HEADER_LENGTH + length
-        if (mark & 1 == 1)
+        entry_bytes = FlatFileBlobHeader::LENGTH + header.length
+        if header.is_valid?
           # We have found a valid entry.
           valid_blobs += 1
           if distance > 0
@@ -332,14 +335,14 @@ module PEROBS
               @f.seek(pos - distance)
               @f.write(buf)
               # Update the index with the new position
-              @index.put_value(blob_id, pos - distance)
+              @index.put_value(header.id, pos - distance)
               # Mark the space between the relocated current entry and the
               # next valid entry as deleted space.
-              @f.write([ 0, distance - BLOB_HEADER_LENGTH, 0, 0 ].
-                       pack(BLOB_HEADER_FORMAT))
+              FlatFileBlobHeader.new(0, distance - FlatFileBlobHeader::LENGTH,
+                                     0, 0).write(@f)
               @f.flush
-            rescue => e
-              PEROBS.log.fatal "Error while moving blob for ID #{blob_id}: " +
+            rescue IOError => e
+              PEROBS.log.fatal "Error while moving blob for ID #{header.id}: " +
                 e.message
             end
           end
@@ -361,6 +364,35 @@ module PEROBS
       sync
     end
 
+    # This method iterates over all entries in the FlatFile and removes the
+    # entry and inserts it again. This is useful to update all entries in
+    # cased the storage format has changed.
+    def refresh
+      # This iteration might look scary as we iterate over the entries while
+      # while we are rearranging them. Re-inserted items may be inserted
+      # before or at the current entry and this is fine. They also may be
+      # inserted after the current entry and will be re-read again unless they
+      # are inserted after the original file end.
+      file_size = @f.size
+      PEROBS.log.info "Refreshing the DB..."
+      t = Time.now
+      each_blob_header do |pos, header|
+        if header.is_valid?
+          buf = read_obj_by_address(pos, header.id)
+          delete_obj_by_address(pos, header.id)
+          write_obj_by_id(header.id, buf)
+        end
+
+        # Some re-inserted blobs may be inserted after the original file end.
+        # No need to process those blobs again.
+        break if pos >= file_size
+      end
+      PEROBS.log.info "DB refresh completed in #{Time.now - t} seconds"
+
+      # Reclaim the space saved by compressing entries.
+      defragmentize
+    end
+
     def check(repair = false)
       return unless @f
 
@@ -370,24 +402,28 @@ module PEROBS
 
       # First check the database blob file. Each entry should be readable and
       # correct.
-      each_blob_header do |pos, mark, length, blob_id, crc|
-        if (mark & 1 == 1)
+      each_blob_header do |pos, header|
+        if header.is_valid?
           # We have a non-deleted entry.
           begin
-            @f.seek(pos + BLOB_HEADER_LENGTH)
-            buf = @f.read(length)
-            if crc && checksum(buf) != crc
+            @f.seek(pos + FlatFileBlobHeader::LENGTH)
+            buf = @f.read(header.length)
+            # Uncompress the data if the compression bit is set in the mark
+            # byte.
+            buf = Zlib.inflate(buf) if header.is_compressed?
+
+            if header.crc && checksum(buf) != header.crc
               if repair
                 PEROBS.log.error "Checksum failure while checking blob " +
-                  "with ID #{id}. Deleting object."
-                delete_obj_by_address(pos, blob_id)
+                  "with ID #{header.id}. Deleting object."
+                delete_obj_by_address(pos, header.id)
               else
                 PEROBS.log.fatal "Checksum failure while checking blob " +
-                  "with ID #{id}"
+                  "with ID #{header.id}"
               end
             end
-          rescue => e
-            PEROBS.log.fatal "Check of blob with ID #{blob_id} failed: " +
+          rescue IOError => e
+            PEROBS.log.fatal "Check of blob with ID #{header.id} failed: " +
               e.message
           end
         end
@@ -416,32 +452,33 @@ module PEROBS
       @index.clear
       @space_list.clear
 
-      each_blob_header do |pos, mark, length, id, crc|
-        if mark == 0
-          @space_list.add_space(pos, length) if length > 0
+      each_blob_header do |pos, header|
+        if header.is_valid?
+          @index.put_value(header.id, pos)
         else
-          @index.put_value(id, pos)
+          @space_list.add_space(pos, header.length) if header.length > 0
         end
       end
     end
 
     def has_space?(address, size)
-      header = read_blob_header(address)
+      header = FlatFileBlobHeader.read_at(@f, address)
       header.length == size
     end
 
     def has_id_at?(id, address)
-      header = read_blob_header(address)
+      header = FlatFileBlobHeader.read_at(@f, address)
       header.id == id
     end
 
     def inspect
       s = '['
-      each_blob_header do |pos, mark, length, blob_id, crc|
-        s << "{ :pos => #{pos}, :mark => #{mark}, " +
-             ":length => #{length}, :id => #{blob_id}, :crc => #{crc}"
-        if mark != 0
-          s << ", :value => #{@f.read(length)}"
+      each_blob_header do |pos, header|
+        s << "{ :pos => #{pos}, :mark => #{header.mark}, " +
+             ":length => #{header.length}, :id => #{header.id}, " +
+             ":crc => #{header.crc}"
+        if header.is_valid?
+          s << ", :value => #{@f.read(header.length)}"
         end
         s << " }\n"
       end
@@ -452,26 +489,19 @@ module PEROBS
 
     private
 
-    def read_blob_header(addr, id = nil)
-      buf = nil
+    def each_blob_header(&block)
+      pos = 0
       begin
-        @f.seek(addr)
-        buf = @f.read(BLOB_HEADER_LENGTH)
-      rescue => e
+        @f.seek(0)
+        while (header = FlatFileBlobHeader.read(@f))
+          yield(pos, header)
+
+          pos += FlatFileBlobHeader::LENGTH + header.length
+          @f.seek(pos)
+        end
+      rescue IOError => e
         PEROBS.log.fatal "Cannot read blob in flat file DB: #{e.message}"
       end
-      if buf.nil? || buf.length != BLOB_HEADER_LENGTH
-        PEROBS.log.fatal "Cannot read blob header " +
-          "#{id ? "for ID #{id} " : ''}at address " +
-          "#{addr}"
-      end
-      header = Header.new(*buf.unpack(BLOB_HEADER_FORMAT))
-      if id && header.id != id
-        PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
-          "found for entry with ID #{id}/#{header.id}"
-      end
-
-      return header
     end
 
     def find_free_blob(bytes)
@@ -480,7 +510,7 @@ module PEROBS
         # We have not found any suitable space. Return the end of the file.
         return [ @f.size, -1 ]
       end
-      if size == bytes || size - BLOB_HEADER_LENGTH >= bytes
+      if size == bytes || size - FlatFileBlobHeader::LENGTH >= bytes
         return [ address, size ]
       end
 
@@ -490,7 +520,8 @@ module PEROBS
 
       # We need a space that is large enough to hold the bytes and the gap
       # header.
-      @space_list.get_space(bytes + BLOB_HEADER_LENGTH) || [ @f.size, -1 ]
+      @space_list.get_space(bytes + FlatFileBlobHeader::LENGTH) ||
+        [ @f.size, -1 ]
     end
 
     def checksum(raw_obj)
@@ -498,19 +529,20 @@ module PEROBS
     end
 
     def cross_check_entries
-      each_blob_header do |pos, mark, length, blob_id, crc|
-        if mark == 0
-          if length > 0
-            unless @space_list.has_space?(pos, length)
+      each_blob_header do |pos, header|
+        if !header.is_valid?
+          if header.length > 0
+            unless @space_list.has_space?(pos, header.length)
               PEROBS.log.error "FlatFile has free space " +
-                "(addr: #{pos}, len: #{length}) that is not in FreeSpaceManager"
+                "(addr: #{pos}, len: #{header.length}) that is not in " +
+                "FreeSpaceManager"
               return false
             end
           end
         else
-          unless @index.get_value(blob_id) == pos
+          unless @index.get_value(header.id) == pos
             PEROBS.log.error "FlatFile blob at address #{pos} is listed " +
-              "in index with address #{@index.get_value(blob_id)}"
+              "in index with address #{@index.get_value(header.id)}"
             return false
           end
         end
@@ -519,22 +551,6 @@ module PEROBS
       true
     end
 
-    def each_blob_header(&block)
-      pos = 0
-      begin
-        @f.seek(0)
-        while (buf = @f.read(BLOB_HEADER_LENGTH))
-          mark, length, id, crc = buf.unpack(BLOB_HEADER_FORMAT)
-          yield(pos, mark, length, id, crc)
-
-          pos += BLOB_HEADER_LENGTH + length
-          @f.seek(pos)
-        end
-      rescue IOError => e
-        PEROBS.log.fatal "Cannot read blob in flat file DB: #{e.message}"
-      end
-    end
-
   end
 
 end

data/lib/perobs/FlatFileBlobHeader.rb

@@ -0,0 +1,144 @@
+# encoding: UTF-8
+#
+# = FlatFileBlobHeader.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+
+module PEROBS
+
+  # The FlatFile blob header has the following structure:
+  #
+  # 1 Byte:  Mark byte.
+  #          Bit 0: 0 deleted entry, 1 valid entry
+  #          Bit 1: 0 unmarked, 1 marked
+  #          Bit 2: 0 uncompressed data, 1 compressed data
+  #          Bit 3 - 7: reserved, must be 0
+  # 8 bytes: Length of the data blob in bytes
+  # 8 bytes: ID of the value in the data blob
+  # 4 bytes: CRC32 checksum of the data blob
+  #
+  # If the bit 0 of the mark byte is 0, only the length is valid. The blob is
+  # empty. Only of bit 0 is set then entry is valid.
+  class FlatFileBlobHeader
+
+    # The 'pack()' format of the header.
+    FORMAT = 'CQQL'
+    # The length of the header in bytes.
+    LENGTH = 21
+
+    attr_reader :mark, :length, :id, :crc
+
+    # Create a new FlatFileBlobHeader with the given mark, length, id and crc.
+    # @param mark [Fixnum] 8 bit number, see above
+    # @param length [Fixnum] length of the header in bytes
+    # @param id [Integer] ID of the blob entry
+    # @param crc [Fixnum] CRC32 checksum of the blob entry
+    def initialize(mark, length, id, crc)
+      @mark = mark
+      @length = length
+      @id = id
+      @crc = crc
+    end
+
+    # Read the header from the given File.
+    # @param file [File]
+    # @return FlatFileBlobHeader
+    def FlatFileBlobHeader::read(file)
+      begin
+        buf = file.read(LENGTH)
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot read blob header in flat file DB: #{e.message}"
+      end
+
+      return nil unless buf
+
+      FlatFileBlobHeader.new(*buf.unpack(FORMAT))
+    end
+
+    # Read the header from the given File.
+    # @param file [File]
+    # @param addr [Integer] address in the file to start reading
+    # @param id [Integer] Optional ID that the header should have
+    # @return FlatFileBlobHeader
+    def FlatFileBlobHeader::read_at(file, addr, id = nil)
+      buf = nil
+      begin
+        file.seek(addr)
+        buf = file.read(LENGTH)
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot read blob in flat file DB: #{e.message}"
+      end
+      if buf.nil? || buf.length != LENGTH
+        PEROBS.log.fatal "Cannot read blob header " +
+          "#{id ? "for ID #{id} " : ''}at address " +
+          "#{addr}"
+      end
+      header = FlatFileBlobHeader.new(*buf.unpack(FORMAT))
+      if id && header.id != id
+        PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
+          "found for entry with ID #{id}/#{header.id}"
+      end
+
+      return header
+    end
+
+    # Write the header to a given File.
+    # @param file [File]
+    def write(file)
+      begin
+        file.write([ @mark, @length, @id, @crc].pack(FORMAT))
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write blob header into flat file DB: " +
+          e.message
+      end
+    end
+
+    # Return true if the header is for a non-empty blob.
+    def is_valid?
+      bit_set?(0)
+    end
+
+    # Return true if the blob has been marked.
+    def is_marked?
+      bit_set?(1)
+    end
+
+    # Return true if the blob contains compressed data.
+    def is_compressed?
+      bit_set?(2)
+    end
+
+    private
+
+    def bit_set?(n)
+      mask = 1 << n
+      @mark & mask == mask
+    end
+
+  end
+
+end
+

data/lib/perobs/FlatFileDB.rb

@@ -41,7 +41,7 @@ module PEROBS
 
     # This version number increases whenever the on-disk format changes in a
     # way that requires conversion actions after an update.
-    VERSION = 1
+    VERSION = 2
 
     attr_reader :max_blob_size
 
@@ -57,7 +57,7 @@ module PEROBS
       # Create the database directory if it doesn't exist yet.
       ensure_dir_exists(@db_dir)
       PEROBS.log.open(File.join(@db_dir, 'log'))
-      check_version
+      check_version_and_upgrade
 
       # Read the existing DB config.
       @config = get_hash('config')
@@ -202,9 +202,9 @@ module PEROBS
 
     private
 
-    def check_version
+    def check_version_and_upgrade
       version_file = File.join(@db_dir, 'version')
-      version = VERSION
+      version = 1
 
       if File.exist?(version_file)
         begin
@@ -214,6 +214,7 @@ module PEROBS
                            "'#{version_file}': " + e.message
         end
       else
+        # Early versions of PEROBS did not have a version file.
         write_version_file(version_file)
       end
 
@@ -221,6 +222,20 @@ module PEROBS
         PEROBS.log.fatal "Cannot downgrade the FlatFile database from " +
                          "version #{version} to version #{VERSION}"
       end
+
+      if version == 1
+        # Version 1 had no support for data compression. Make sure all entries
+        # are compressed to save space.
+        open
+        @flat_file.refresh
+        close
+      end
+
+      # After a successful upgrade change the version number in the DB as
+      # well.
+      if version < VERSION
+        write_version_file(version_file)
+      end
     end
 
     def write_version_file(version_file)

data/lib/perobs/IndexTreeNode.rb

@@ -190,8 +190,8 @@ module PEROBS
     # Recursively check this node and all sub nodes. Compare the found
     # ID/address pairs with the corresponding entry in the given FlatFile.
     # @param flat_file [FlatFile]
-    # @tree_level [Fixnum] Assumed level in the tree. Must correspond with
-    #             @nibble_idx
+    # @param tree_level [Fixnum] Assumed level in the tree. Must correspond
+    #        with @nibble_idx
     # @return [Boolean] true if no errors were found, false otherwise
     def check(flat_file, tree_level)
       if tree_level >= 16

data/lib/perobs/Store.rb

@@ -160,7 +160,6 @@ module PEROBS
 
     # Copy the store content into a new Store. The arguments are identical to
     # Store.new().
-    # @param data_base [String] the name of the database
     # @param options [Hash] various options to affect the operation of the
     def copy(dir, options = {})
       # Make sure all objects are persisted.

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "2.4.2"
+  VERSION = "2.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 2.5.0
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2017-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -74,6 +74,7 @@ files:
 - lib/perobs/DynamoDB.rb
 - lib/perobs/FixedSizeBlobFile.rb
 - lib/perobs/FlatFile.rb
+- lib/perobs/FlatFileBlobHeader.rb
 - lib/perobs/FlatFileDB.rb
 - lib/perobs/FreeSpaceManager.rb
 - lib/perobs/Handle.rb