perobs 3.0.2 → 4.0.0

data/lib/perobs/BTreeNodeCache.rb

@@ -31,11 +31,12 @@ module PEROBS
 
   class BTreeNodeCache
 
-    def initialize
+    def initialize(tree)
+      @tree = tree
       clear
     end
 
-    def [](address)
+    def get(address)
       if (node = @modified_nodes[address])
         return node
       end
@@ -48,7 +49,7 @@ module PEROBS
         return node
       end
 
-      nil
+      BTreeNode::load(@tree, address)
     end
 
     def set_root(node)
@@ -58,12 +59,15 @@ module PEROBS
       @top_nodes[node.node_address] = node
     end
 
-    def insert(node)
+    def insert(node, modified = true)
       unless node
         PEROBS.log.fatal "nil cannot be cached"
       end
       node = node.get_node if node.is_a?(BTreeNodeLink)
 
+      if modified
+        @modified_nodes[node.node_address] = node
+      end
       @ephemeral_nodes[node.node_address] = node
 
       if !@top_nodes.include?(node) && node.is_top?
@@ -71,10 +75,8 @@ module PEROBS
       end
     end
 
-    def mark_as_modified(node)
-      node = node.get_node if node.is_a?(BTreeNodeLink)
-      @modified_nodes[node.node_address] = node
-      insert(node)
+    def _collect(address, ruby_object_id)
+      # Just a dummy for now
     end
 
     # Remove a node from the cache.

data/lib/perobs/BTreeNodeLink.rb

@@ -132,7 +132,7 @@ module PEROBS
     end
 
     def get_node
-      @tree.get_node(@node_address)
+      @tree.node_cache.get(@node_address)
     end
 
   end

data/lib/perobs/Cache.rb

@@ -95,20 +95,20 @@ module PEROBS
 
     # Return the PEROBS::Object with the specified ID or nil if not found.
     # @param id [Integer] ID of the cached PEROBS::ObjectBase
-    #def object_by_id(id)
-    #  idx = id & @mask
-    #  # The index is just a hash. We still need to check if the object IDs are
-    #  # actually the same before we can return the object.
-    #  if (obj = @writes[idx]) && obj._id == id
-    #    # The object was in the write cache.
-    #    return obj
-    #  elsif (obj = @reads[idx]) && obj._id == id
-    #    # The object was in the read cache.
-    #    return obj
-    #  end
-
-    #  nil
-    #end
+    def object_by_id(id)
+      idx = id & @mask
+      # The index is just a hash. We still need to check if the object IDs are
+      # actually the same before we can return the object.
+      if (obj = @writes[idx]) && obj._id == id
+        # The object was in the write cache.
+        return obj
+      elsif (obj = @reads[idx]) && obj._id == id
+        # The object was in the read cache.
+        return obj
+      end
+
+      nil
+    end
 
     # Flush all pending writes to the persistant storage back-end.
     def flush

data/lib/perobs/DynamoDB.rb

@@ -25,7 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require 'aws-sdk-core'
+require 'aws-sdk'
 
 require 'perobs/DataBase'
 require 'perobs/BTreeBlob'

data/lib/perobs/EquiBlobsFile.rb

@@ -83,6 +83,7 @@ module PEROBS
       unless @f.flock(File::LOCK_NB | File::LOCK_EX)
         PEROBS.log.fatal 'Database blob file is locked by another process'
       end
+      @f.sync = true
     end
 
     # Close the blob file. This method must be called before the program is
@@ -92,6 +93,7 @@ module PEROBS
         if @f
           @f.flush
           @f.flock(File::LOCK_UN)
+          @f.fsync
           @f.close
           @f = nil
         end
@@ -103,7 +105,7 @@ module PEROBS
     # Erase the backing store. This method should only be called when the file
     # is not currently open.
     def erase
-      PEROBS.log.fatal 'Cannot call EquiBlobsFile::erase while it is open' if @f
+      @f = nil
       File.delete(@file_name) if File.exist?(@file_name)
       reset_counters
     end
@@ -111,7 +113,10 @@ module PEROBS
     # Flush out all unwritten data.
     def sync
       begin
-        @f.flush if @f
+        if @f
+          @f.flush
+          @f.fsync
+        end
       rescue IOError => e
         PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
       end

data/lib/perobs/FlatFile.rb

@@ -48,6 +48,7 @@ module PEROBS
       @db_dir = dir
       @f = nil
       @index = BTree.new(@db_dir, 'index', INDEX_BTREE_ORDER)
+      @marks = BTree.new(@db_dir, 'marks', INDEX_BTREE_ORDER)
       @space_list = SpaceTree.new(@db_dir)
     end
 
@@ -71,6 +72,7 @@ module PEROBS
         PEROBS.log.fatal "FlatFile database '#{file_name}' is locked by " +
           "another process"
       end
+      @f.sync = true
 
       begin
         @index.open(!new_db_created)
@@ -103,6 +105,7 @@ module PEROBS
       if @f
         @f.flush
         @f.flock(File::LOCK_UN)
+        @f.fsync
         @f.close
         @f = nil
       end
@@ -112,6 +115,7 @@ module PEROBS
     def sync
       begin
         @f.flush
+        @f.fsync
       rescue IOError => e
         PEROBS.log.fatal "Cannot sync flat file database: #{e.message}"
       end
@@ -148,7 +152,7 @@ module PEROBS
 
       deleted_ids = []
       each_blob_header do |pos, header|
-        if header.is_valid? && !header.is_marked?
+        if header.is_valid? && @marks.get(header.id).nil?
           delete_obj_by_address(pos, header.id)
           deleted_ids << header.id
         end
@@ -184,12 +188,12 @@ module PEROBS
       # performance impact of compression is not compensated by writing
       # less data to the storage.
       compressed = false
-      if raw_obj.length > 256
+      if raw_obj.bytesize > 256
         raw_obj = Zlib.deflate(raw_obj)
         compressed = true
       end
 
-      addr, length = find_free_blob(raw_obj.length)
+      addr, length = find_free_blob(raw_obj.bytesize)
       begin
         if length != -1
           # Just a safeguard so we don't overwrite current data.
@@ -198,8 +202,8 @@ module PEROBS
             PEROBS.log.fatal "Length in free list (#{length}) and header " +
               "(#{header.length}) for address #{addr} don't match."
           end
-          if raw_obj.length > header.length
-            PEROBS.log.fatal "Object (#{raw_obj.length}) is longer than " +
+          if raw_obj.bytesize > header.length
+            PEROBS.log.fatal "Object (#{raw_obj.bytesize}) is longer than " +
               "blob space (#{header.length})."
           end
           if header.is_valid?
@@ -209,25 +213,19 @@ module PEROBS
         end
         flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
         flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
-        if old_addr && old_header.is_marked?
-          # This method might be called in the middle of an operation that
-          # uses the mark flag. We must ensure that the flag is carried over
-          # to the new header.
-          flags |= (1 << FlatFileBlobHeader::MARK_FLAG_BIT)
-        end
-        FlatFileBlobHeader.new(@f, addr, flags, raw_obj.length, id, crc).write
+        FlatFileBlobHeader.new(@f, addr, flags, raw_obj.bytesize, id, crc).write
         @f.write(raw_obj)
-        if length != -1 && raw_obj.length < length
+        if length != -1 && raw_obj.bytesize < length
           # The new object was not appended and it did not completely fill the
           # free space. So we have to write a new header to mark the remaining
           # empty space.
-          unless length - raw_obj.length >= FlatFileBlobHeader::LENGTH
+          unless length - raw_obj.bytesize >= FlatFileBlobHeader::LENGTH
             PEROBS.log.fatal "Not enough space to append the empty space " +
-              "header (space: #{length} bytes, object: #{raw_obj.length} " +
+              "header (space: #{length} bytes, object: #{raw_obj.bytesize} " +
               "bytes)."
           end
           space_address = @f.pos
-          space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.length
+          space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.bytesize
           FlatFileBlobHeader.new(@f, space_address, 0, space_length,
                                  0, 0).write
           # Register the new space with the space list.
@@ -272,6 +270,15 @@ module PEROBS
       nil
     end
 
+    def search_object(id)
+      each_blob_header do |pos, header|
+        return read_obj_by_address(pos, id)
+      end
+
+      nil
+    end
+
+
     # Read the object at the specified address.
     # @param addr [Integer] Offset in the flat file
     # @param id [Integer] ID of the data blob
@@ -312,47 +319,19 @@ module PEROBS
     # Mark the object with the given ID.
     # @param id [Integer] ID of the object
     def mark_obj_by_id(id)
-      if (addr = find_obj_addr_by_id(id))
-        mark_obj_by_address(addr, id)
-      end
-    end
-
-    # Mark the object at the specified address.
-    # @param addr [Integer] Offset in the file
-    # @param id [Integer] ID of the object
-    def mark_obj_by_address(addr, id)
-      FlatFileBlobHeader.read_at(@f, addr, id).set_mark_flag
+      @marks.insert(id, 0)
     end
 
     # Return true if the object with the given ID is marked, false otherwise.
     # @param id [Integer] ID of the object
     def is_marked_by_id?(id)
-      if (addr = find_obj_addr_by_id(id))
-        header = FlatFileBlobHeader.read_at(@f, addr, id)
-        return header.is_marked?
-      end
-
-      false
+      !@marks.get(id).nil?
     end
 
     # Clear alls marks.
     def clear_all_marks
-      t = Time.now
-      PEROBS.log.info "Clearing all marks..."
-
-      total_blob_count = 0
-      marked_blob_count = 0
-
-      each_blob_header do |pos, header|
-        total_blob_count += 1
-        if header.is_valid? && header.is_marked?
-          # Clear all valid and marked blocks.
-          marked_blob_count += 1
-          header.clear_mark_flag
-        end
-      end
-      PEROBS.log.info "#{marked_blob_count} marks in #{total_blob_count} " +
-        "objects cleared in #{Time.now - t} seconds"
+      @marks.erase
+      @marks.open
     end
 
     # Eliminate all the holes in the file. This is an in-place
@@ -464,7 +443,7 @@ module PEROBS
           begin
             @f.seek(pos + FlatFileBlobHeader::LENGTH)
             buf = @f.read(header.length)
-            if buf.length != header.length
+            if buf.bytesize != header.length
               PEROBS.log.error "Premature end of file in blob with ID " +
                 "#{header.id}."
               discard_damaged_blob(header) if repair

data/lib/perobs/FlatFileBlobHeader.rb

@@ -33,7 +33,7 @@ module PEROBS
   #
   # 1 Byte:  Flags byte.
   #          Bit 0: 0 deleted entry, 1 valid entry
-  #          Bit 1: 0 unmarked, 1 marked
+  #          Bit 1: 0 reserved, must be 0
   #          Bit 2: 0 uncompressed data, 1 compressed data
   #          Bit 3: 0 current entry, 1 outdated entry
   #          Bit 4 - 7: reserved, must be 0
@@ -50,7 +50,6 @@ module PEROBS
     # The length of the header in bytes.
     LENGTH = 21
     VALID_FLAG_BIT = 0
-    MARK_FLAG_BIT = 1
     COMPRESSED_FLAG_BIT = 2
     OUTDATED_FLAG_BIT = 3
 
@@ -148,23 +147,6 @@ module PEROBS
       bit_set?(VALID_FLAG_BIT)
     end
 
-    # Return true if the blob has been marked.
-    def is_marked?
-      bit_set?(MARK_FLAG_BIT)
-    end
-
-    # Set the mark bit.
-    def set_mark_flag
-      set_flag(MARK_FLAG_BIT)
-      write_flags
-    end
-
-    # Clear the mark bit.
-    def clear_mark_flag
-      clear_flag(MARK_FLAG_BIT)
-      write_flags
-    end
-
     # Return true if the blob contains compressed data.
     def is_compressed?
       bit_set?(COMPRESSED_FLAG_BIT)

data/lib/perobs/FlatFileDB.rb

@@ -87,6 +87,7 @@ module PEROBS
     end
 
     def FlatFileDB::delete_db(db_name)
+      close
       FileUtils.rm_rf(db_name)
     end
 
@@ -142,6 +143,10 @@ module PEROBS
       end
     end
 
+    def search_object(id)
+      @flat_file.search_object(id)
+    end
+
     # This method must be called to initiate the marking process.
     def clear_marks
       @flat_file.clear_all_marks

data/lib/perobs/LockFile.rb

@@ -70,12 +70,14 @@ module PEROBS
       while retries > 0
         begin
           @file = File.open(@file_name, File::RDWR | File::CREAT, 0644)
+          @file.sync = true
 
           if @file.flock(File::LOCK_EX | File::LOCK_NB)
             # We have taken the lock. Write the PID into the file and leave it
             # open.
             @file.write($$)
             @file.flush
+            @file.fsync
             @file.truncate(@file.pos)
             PEROBS.log.debug "Lock file #{@file_name} has been taken for " +
               "process #{$$}"
@@ -129,6 +131,7 @@ module PEROBS
 
       begin
         @file.flock(File::LOCK_UN)
+        @file.fsync
         @file.close
         forced_unlock
         PEROBS.log.debug "Lock file #{@file_name} for PID #{$$} has been " +

data/lib/perobs/Object.rb

@@ -49,9 +49,11 @@ module PEROBS
       attr_reader :attributes
 
       # This method can be used to define instance variable for
-      # PEROBS::Object derived classes.
+      # PEROBS::Object derived classes. Persistent attributes always have
+      # getter and setter methods defined. So it's essentially equivalent to
+      # attr_accessor but additionally declares an attribute as persistent.
       # @param attributes [Symbol] Name of the instance variable
-      def po_attr(*attributes)
+      def attr_persist(*attributes)
         attributes.each do |attr_name|
           unless attr_name.is_a?(Symbol)
             PEROBS.log.fatal "name must be a symbol but is a " +
@@ -73,6 +75,9 @@ module PEROBS
         end
       end
 
+      # This is the deprecated name for the attr_persist method
+      alias po_attr attr_persist
+
     end
 
     attr_reader :attributes
@@ -108,7 +113,7 @@ module PEROBS
     # object was saved with an earlier version of the program that did not yet
     # have the instance variable. If you want to assign another PEROBS object
     # to the variable you should use the block variant to avoid unnecessary
-    # creation of PEROBS object that later need to be collected again.
+    # creation of PEROBS objects that later need to be collected again.
     def attr_init(attr, val = nil, &block)
       if _all_attributes.include?(attr)
         unless instance_variable_defined?('@' + attr.to_s)

data/lib/perobs/ObjectBase.rb

@@ -133,7 +133,8 @@ module PEROBS
       @store = p.store
       @_id = p.id
       @store._register_in_memory(self, @_id)
-      ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
+      ObjectSpace.define_finalizer(
+        self, ObjectBase._finalize(@store, @_id, object_id))
       @_stash_map = nil
       # Allocate a proxy object for this object. User code should only operate
       # on this proxy, never on self.
@@ -144,8 +145,8 @@ module PEROBS
     # is done this way to prevent the Proc object hanging on to a reference to
     # self which would prevent the object from being collected. This internal
     # method is not intended for users to call.
-    def ObjectBase._finalize(store, id)
-      proc { store._collect(id) }
+    def ObjectBase._finalize(store, id, ruby_object_id)
+      proc { store._collect(id, ruby_object_id) }
     end
 
     # Library internal method to transfer the Object to a new store.
@@ -158,7 +159,8 @@ module PEROBS
       # Register the object as in-memory object with the new store.
       @store._register_in_memory(self, @_id)
       # Register the finalizer for the new store.
-      ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
+      ObjectSpace.define_finalizer(
+        self, ObjectBase._finalize(@store, @_id, object_id))
       @myself = POXReference.new(@store, @_id)
     end