perobs 3.0.1 → 3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20f164ff526b984530355e99ab38623fd2b87cea
-  data.tar.gz: befc7531c92e0dc86c23b058cbc572657f66661f
+  metadata.gz: 985e62db6b84f509a3b390ce11ac96f4215de5ea
+  data.tar.gz: f6747e42300df5c9b55c1a5bbc6adf9822165b7b
 SHA512:
-  metadata.gz: 3b18e2e16719f896d8500852f0c1529ce2a1d9816695df04be2203cd1a10da8b70fe737afa910ec1fed71855eb6534da2b0dd16f8c6428c0ea1ae06a6509b59f
-  data.tar.gz: d6de8c99f6cffcb245b5468c6f6b5f4b5572c44019f7b9926adeae126a81ad507b815bbefda24b07f4724bc0a11abc0ab3235b304d2ca7180cde3bdc628c0b1e
+  metadata.gz: 7ed1499555741de10e6810f62041ffc879a738f488f5ce09860bddae21ed12cf999a8d4303361227a3798f3f389bd6cb9584e71e809b26d0f82fa7cdc6a28544
+  data.tar.gz: 2c56287162c7ae376ecdc091e55655fd37e4985e22914a4e21d74f28eb0c3e054ca543fb7092485ea09605b8a0ad4cc33f13dc96fa19ffcdf5137921c5b145ae

data/lib/perobs/Array.rb

@@ -81,7 +81,7 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param size [Fixnum] The requested size of the Array
+    # @param size [Integer] The requested size of the Array
     # @param default [Any] The default value that is returned when no value is
     #        stored for a specific key.
     def initialize(p, size = 0, default = nil)
@@ -94,7 +94,7 @@ module PEROBS
 
     # Return a list of all object IDs of all persistend objects that this Array
     # is referencing.
-    # @return [Array of Fixnum or Bignum] IDs of referenced objects
+    # @return [Array of Integer] IDs of referenced objects
     def _referenced_object_ids
       @data.each.select do |v|
         v && v.respond_to?(:is_poxreference?)
@@ -103,7 +103,7 @@ module PEROBS
 
     # This method should only be used during store repair operations. It will
     # delete all references to the given object ID.
-    # @param id [Fixnum/Bignum] targeted object ID
+    # @param id [Integer] targeted object ID
     def _delete_reference_to_id(id)
       @data.delete_if do |v|
         v && v.respond_to?(:is_poxreference?) && v.id == id

data/lib/perobs/BTreeBlob.rb

@@ -65,7 +65,7 @@ module PEROBS
     end
 
     # Write the given bytes with the given ID into the DB.
-    # @param id [Fixnum or Bignum] ID
+    # @param id [Integer] ID
     # @param raw [String] sequence of bytes
     def write_object(id, raw)
       if @entries.length > @btreedb.max_blob_size
@@ -87,7 +87,7 @@ module PEROBS
     end
 
     # Read the entry for the given ID and return it as bytes.
-    # @param id [Fixnum or Bignum] ID
+    # @param id [Integer] ID
     # @return [String] sequence of bytes or nil if ID is unknown
     def read_object(id)
       return nil unless (index_entry = find(id))
@@ -95,7 +95,7 @@ module PEROBS
     end
 
     # Find the data for the object with given id.
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     # @return [Array] Returns an Array that represents the index entry for the
     #         given object.
     def find(id)
@@ -109,7 +109,7 @@ module PEROBS
     end
 
     # Set a mark on the entry with the given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
+    # @param id [Integer] ID of the entry
     def mark(id)
       found = false
       @entries.each do |entry|
@@ -129,7 +129,7 @@ module PEROBS
     end
 
     # Check if the entry for a given ID is marked.
-    # @param id [Fixnum or Bignum] ID of the entry
+    # @param id [Integer] ID of the entry
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     # @return [TrueClass or FalseClass] true if marked, false otherwise
@@ -205,8 +205,8 @@ module PEROBS
 
     # Write a string of bytes into the file at the given address.
     # @param raw [String] bytes to write
-    # @param address [Fixnum] offset in the file
-    # @return [Fixnum] number of bytes written
+    # @param address [Integer] offset in the file
+    # @return [Integer] number of bytes written
     def write_to_blobs_file(raw, address)
       begin
         File.write(@blobs_file_name, raw, address)
@@ -236,9 +236,9 @@ module PEROBS
 
     # Reserve the bytes needed for the specified number of bytes with the
     # given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
-    # @param bytes [Fixnum] number of bytes for this entry
-    # @return [Fixnum] the start address of the reserved blob
+    # @param id [Integer] ID of the entry
+    # @param bytes [Integer] number of bytes for this entry
+    # @return [Integer] the start address of the reserved blob
     def reserve_bytes(id, bytes, crc32)
       # index of first blob after the last seen entry
       end_of_last_entry = 0

data/lib/perobs/BTreeDB.rb

@@ -103,7 +103,7 @@ module PEROBS
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !(blob = find_blob(id)).nil? && !blob.find(id).nil?
     end
@@ -143,7 +143,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -166,13 +166,13 @@ module PEROBS
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       (blob = find_blob(id)) && blob.mark(id)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     def is_marked?(id, ignore_errors = false)
@@ -189,7 +189,7 @@ module PEROBS
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -208,7 +208,7 @@ module PEROBS
     # Store the given serialized object into the cluster files. This method is
     # for internal use only!
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     def put_raw_object(raw, id)
       find_blob(id, true).write_object(id, raw)
     end

data/lib/perobs/Cache.rb

@@ -37,7 +37,7 @@ module PEROBS
   class Cache
 
     # Create a new Cache object.
-    # @param bits [Fixnum] Number of bits for the cache index. This parameter
+    # @param bits [Integer] Number of bits for the cache index. This parameter
     #        heavilty affects the performance and memory consumption of the
     #        cache.
     def initialize(bits = 16)
@@ -94,7 +94,7 @@ module PEROBS
     end
 
     # Return the PEROBS::Object with the specified ID or nil if not found.
-    # @param id [Fixnum or Bignum] ID of the cached PEROBS::ObjectBase
+    # @param id [Integer] ID of the cached PEROBS::ObjectBase
     #def object_by_id(id)
     #  idx = id & @mask
     #  # The index is just a hash. We still need to check if the object IDs are

data/lib/perobs/ClassMap.rb

@@ -47,14 +47,14 @@ module PEROBS
 
     # Get the ID for a given class.
     # @param klass [String] Class
-    # @return [Fixnum] ID. If klass is not yet known a new ID will be
+    # @return [Integer] ID. If klass is not yet known a new ID will be
     #         allocated.
     def class_to_id(klass)
       @by_class[klass] || new_id(klass)
     end
 
     # Get the klass for a given ID.
-    # @param id [Fixnum]
+    # @param id [Integer]
     # @return [String] String version of the class
     def id_to_class(id)
       @by_id[id]

data/lib/perobs/DynamoDB.rb

@@ -85,7 +85,7 @@ module PEROBS
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !dynamo_get_item(id.to_s).nil?
     end
@@ -116,7 +116,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -148,13 +148,13 @@ module PEROBS
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       dynamo_mark_item(id.to_s, true)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     def is_marked?(id)
       dynamo_is_marked?(id.to_s)
     end
@@ -167,7 +167,7 @@ module PEROBS
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise

data/lib/perobs/EquiBlobsFile.rb

@@ -45,13 +45,13 @@ module PEROBS
     FIRST_SPACE_OFFSET = 3 * 8
     HEADER_SIZE = 4 * 8
 
-    attr_reader :total_entries, :total_spaces, :file_name
-    attr_accessor :first_entry
+    attr_reader :total_entries, :total_spaces, :file_name, :first_entry
 
     # Create a new stack file in the given directory with the given file name.
     # @param dir [String] Directory
     # @param name [String] File name
-    # @param entry_bytes [Fixnum] Number of bytes each entry must have
+    # @param entry_bytes [Integer] Number of bytes each entry must have
+    # @param first_entry_default [Integer] Default address of the first blob
     def initialize(dir, name, entry_bytes, first_entry_default = 0)
       @file_name = File.join(dir, name + '.blobs')
       if entry_bytes < 8
@@ -125,9 +125,16 @@ module PEROBS
       write_header
     end
 
+    # Change the address of the first blob.
+    # @param address [Integer] New address
+    def first_entry=(address)
+      @first_entry = address
+      write_header
+    end
+
     # Return the address of a free blob storage space. Addresses start at 0
     # and increase linearly.
-    # @return [Fixnum] address of a free blob space
+    # @return [Integer] address of a free blob space
     def free_address
       if @first_space == 0
         # There is currently no free entry. Create a new reserved entry at the
@@ -170,7 +177,7 @@ module PEROBS
 
     # Store the given byte blob at the specified address. If the blob space is
     # already in use the content will be overwritten.
-    # @param address [Fixnum] Address to store the blob
+    # @param address [Integer] Address to store the blob
     # @param bytes [String] bytes to store
     def store_blob(address, bytes)
       unless address >= 0
@@ -217,7 +224,7 @@ module PEROBS
     end
 
     # Retrieve a blob from the given address.
-    # @param address [Fixnum] Address to store the blob
+    # @param address [Integer] Address to store the blob
     # @return [String] blob bytes
     def retrieve_blob(address)
       unless address > 0
@@ -248,7 +255,7 @@ module PEROBS
     end
 
     # Delete the blob at the given address.
-    # @param address [Fixnum] Address of blob to delete
+    # @param address [Integer] Address of blob to delete
     def delete_blob(address)
       unless address >= 0
         PEROBS.log.fatal "Blob address must be larger than 0, " +

data/lib/perobs/FlatFile.rb

@@ -116,6 +116,7 @@ module PEROBS
         PEROBS.log.fatal "Cannot sync flat file database: #{e.message}"
       end
       @index.sync
+      @space_list.sync
     end
 
     # Delete the blob for the specified ID.
@@ -195,18 +196,25 @@ module PEROBS
           header = FlatFileBlobHeader.read_at(@f, addr)
           if header.length != length
             PEROBS.log.fatal "Length in free list (#{length}) and header " +
-              "(#{header.length}) don't match."
+              "(#{header.length}) for address #{addr} don't match."
           end
           if raw_obj.length > header.length
             PEROBS.log.fatal "Object (#{raw_obj.length}) is longer than " +
               "blob space (#{header.length})."
           end
           if header.is_valid?
-            PEROBS.log.fatal "Entry (flags: #{header.flags}) is already used."
+            PEROBS.log.fatal "Entry at address #{addr} with flags: " +
+              "#{header.flags} is already used for ID #{header.id}."
           end
         end
         flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
         flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
+        if old_addr && old_header.is_marked?
+          # This method might be called in the middle of an operation that
+          # uses the mark flag. We must ensure that the flag is carried over
+          # to the new header.
+          flags |= (1 << FlatFileBlobHeader::MARK_FLAG_BIT)
+        end
         FlatFileBlobHeader.new(@f, addr, flags, raw_obj.length, id, crc).write
         @f.write(raw_obj)
         if length != -1 && raw_obj.length < length
@@ -225,15 +233,19 @@ module PEROBS
           # Register the new space with the space list.
           @space_list.add_space(space_address, space_length) if space_length > 0
         end
+
+        # Once the blob has been written we can update the index as well.
+        @index.insert(id, addr)
+
         if old_addr
           # If we had an existing object stored for the ID we have to mark
           # this entry as deleted now.
           old_header.clear_flags
+          # And register the newly freed space with the space list.
+          @space_list.add_space(old_addr, old_header.length)
         else
           @f.flush
         end
-        # Once the blob has been written we can update the index as well.
-        @index.insert(id, addr)
       rescue IOError => e
         PEROBS.log.fatal "Cannot write blob for ID #{id} to FlatFileDB: " +
           e.message
@@ -549,21 +561,31 @@ module PEROBS
 
       each_blob_header do |pos, header|
         if header.is_valid?
-          @index.insert(header.id, pos)
+          if (duplicate_pos = @index.get(header.id))
+            PEROBS.log.error "FlatFile contains multiple blobs for ID " +
+              "#{header.id}. First blob is at address #{duplicate_pos}. " +
+              "Other blob found at address #{pos}."
+            @space_list.add_space(pos, header.length) if header.length > 0
+            discard_damaged_blob(header)
+          else
+            @index.insert(header.id, pos)
+          end
         else
           @space_list.add_space(pos, header.length) if header.length > 0
         end
       end
+
+      sync
     end
 
     def has_space?(address, size)
       header = FlatFileBlobHeader.read_at(@f, address)
-      header.length == size
+      !header.is_valid? && header.length == size
     end
 
     def has_id_at?(id, address)
       header = FlatFileBlobHeader.read_at(@f, address)
-      header.id == id
+      header.is_valid? && header.id == id
     end
 
     def inspect

data/lib/perobs/FlatFileBlobHeader.rb

@@ -59,10 +59,10 @@ module PEROBS
     # Create a new FlatFileBlobHeader with the given flags, length, id and crc.
     # @param file [File] the FlatFile that contains the header
     # @param addr [Integer] the offset address of the header in the file
-    # @param flags [Fixnum] 8 bit number, see above
-    # @param length [Fixnum] length of the header in bytes
+    # @param flags [Integer] 8 bit number, see above
+    # @param length [Integer] length of the header in bytes
     # @param id [Integer] ID of the blob entry
-    # @param crc [Fixnum] CRC32 checksum of the blob entry
+    # @param crc [Integer] CRC32 checksum of the blob entry
     def initialize(file, addr, flags, length, id, crc)
       @file = file
       @addr = addr
@@ -116,7 +116,8 @@ module PEROBS
       header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
       if id && header.id != id
         PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
-          "found for entry with ID #{id}/#{header.id}"
+          "found. FlatFile has entry with ID #{header.id} at address " +
+          "#{addr}. Index has ID #{id} for this address."
       end
 
       return header
@@ -138,14 +139,8 @@ module PEROBS
     # @param file [File] The file handle of the blob file.
     # @param addr [Integer] The address of the header
     def clear_flags
-      begin
-        @file.seek(@addr)
-        @file.write([ 0 ].pack('C'))
-        @file.flush
-      rescue IOError => e
-        PEROBS.log.fatal "Clearing flags of FlatFileBlobHeader with ID " +
-          "#{@id} failed: #{e.message}"
-      end
+      @flags = 0
+      write_flags
     end
 
     # Return true if the header is for a non-empty blob.

data/lib/perobs/FlatFileDB.rb

@@ -91,7 +91,7 @@ module PEROBS
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !@flat_file.find_obj_addr_by_id(id).nil?
     end
@@ -131,7 +131,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -155,13 +155,13 @@ module PEROBS
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       @flat_file.mark_obj_by_id(id)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     def is_marked?(id, ignore_errors = false)
@@ -177,7 +177,7 @@ module PEROBS
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -202,7 +202,7 @@ module PEROBS
     # Store the given serialized object into the cluster files. This method is
     # for internal use only!
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     def put_raw_object(raw, id)
       @flat_file.write_obj_by_id(id, raw)
     end