perobs 2.4.0 → 2.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cb73944d6f6f8968d868d89e19e229dfecd5fc8
-  data.tar.gz: d9c7598cd06bd8b88678eaec6f71f494eb1a0c1e
+  metadata.gz: 3580a4c1778350e195656c42ca100c651ec18386
+  data.tar.gz: e3c964bb3f135372dd07ffc7b36c108f27990569
 SHA512:
-  metadata.gz: befb35a3bc54b7f261889bd7b6495aaaa51c75f50cbb91fef4a657fda323bacb8080f91934062d2ae28e751ab23bc6130ec033fd802d54ae4c67a86b092442a0
-  data.tar.gz: 013e98cd4fe0fce2063ac8a204a33152f1e96c8e22786870046a66a6aa5d6d941444c3b7caaff319404d5d59a0ad2449fec5638266b883e833833a76131b90b2
+  metadata.gz: 6c12c89af79a909c65346fe8ea35e2de608dbba0ce4eb85be35b56bd07a5c50858cd5773ab94ee8e6dd3c8b75c1a872ac7400a0bbc78927c715954c337a7288e
+  data.tar.gz: f93ee4dfb7552724b78f14e682a0750e53c7984910c588050e9dc92b1d2571c35ea8ff7a5fb095ef6b537a238d15da4bb7d31a6126d1ec7aeea1d89d19138dda

data/lib/perobs/BTreeBlob.rb

@@ -28,6 +28,7 @@
 require 'zlib'
 
 require 'perobs/Log'
+require 'perobs/RobustFile'
 
 module PEROBS
 
@@ -209,7 +210,7 @@ module PEROBS
     def write_to_blobs_file(raw, address)
       begin
         File.write(@blobs_file_name, raw, address)
-      rescue => e
+      rescue IOError => e
         PEROBS.log.fatal "Cannot write blobs file #{@blobs_file_name}: " +
           e.message
       end

data/lib/perobs/BTreeDB.rb

@@ -28,6 +28,7 @@
 require 'fileutils'
 
 require 'perobs/Log'
+require 'perobs/RobustFile'
 require 'perobs/DataBase'
 require 'perobs/BTreeBlob'
 
@@ -114,8 +115,8 @@ module PEROBS
     def put_hash(name, hash)
       file_name = File.join(@db_dir, name + '.json')
       begin
-        File.write(file_name, hash.to_json)
-      rescue => e
+        RobustFile.write(file_name, hash.to_json)
+      rescue IOError => e
         PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
       end
     end

data/lib/perobs/FlatFile.rb

@@ -134,6 +134,9 @@ module PEROBS
 
     # Delete all unmarked objects.
     def delete_unmarked_objects
+      PEROBS.log.info "Deleting unmarked objects..."
+      t = Time.now
+
       deleted_ids = []
       each_blob_header do |pos, mark, length, blob_id, crc|
         if (mark & 3 == 1)
@@ -143,6 +146,8 @@ module PEROBS
       end
       defragmentize
 
+      PEROBS.log.info "#{deleted_ids.length} unmarked objects deleted " +
+        "in #{Time.now - t} seconds"
       deleted_ids
     end
 
@@ -274,8 +279,16 @@ module PEROBS
 
     # Clear alls marks.
     def clear_all_marks
+      t = Time.now
+      PEROBS.log.info "Clearing all marks..."
+
+      total_blob_count = 0
+      marked_blob_count = 0
+
       each_blob_header do |pos, mark, length, blob_id, crc|
+        total_blob_count += 1
         if (mark & 1 == 1)
+          marked_blob_count += 1
           begin
             @f.seek(pos)
             @f.write([ mark & 0b11111101 ].pack('C'))
@@ -286,6 +299,8 @@ module PEROBS
           end
         end
       end
+      PEROBS.log.info "#{marked_blob_count} marks in #{total_blob_count} " +
+        "objects cleared in #{Time.now - t} seconds"
     end
 
     # Eliminate all the holes in the file. This is an in-place
@@ -339,6 +354,10 @@ module PEROBS
     def check(repair = false)
       return unless @f
 
+      t = Time.now
+      PEROBS.log.info "Checking FlatFile database" +
+        "#{repair ? ' in repair mode' : ''}..."
+
       # First check the database blob file. Each entry should be readable and
       # correct.
       each_blob_header do |pos, mark, length, blob_id, crc|
@@ -369,16 +388,15 @@ module PEROBS
       # and vise versa.
       begin
         unless @index.check(self) && @space_list.check(self) &&
-          cross_check_entries
-          return unless repair
-
-          regenerate_index_and_spaces
+               cross_check_entries
+          regenerate_index_and_spaces if repair
         end
       rescue PEROBS::FatalError
-        regenerate_index_and_spaces
+        regenerate_index_and_spaces if repair
       end
 
-      sync
+      sync if repair
+      PEROBS.log.info "check_db completed in #{Time.now - t} seconds"
     end
 
     # This method clears the index tree and the free space list and

data/lib/perobs/FlatFileDB.rb

@@ -29,6 +29,7 @@ require 'fileutils'
 require 'zlib'
 
 require 'perobs/Log'
+require 'perobs/RobustFile'
 require 'perobs/DataBase'
 require 'perobs/FlatFile'
 
@@ -102,8 +103,8 @@ module PEROBS
     def put_hash(name, hash)
       file_name = File.join(@db_dir, name + '.json')
       begin
-        File.write(file_name, hash.to_json)
-      rescue => e
+        RobustFile.write(file_name, hash.to_json)
+      rescue IOError => e
         PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
       end
     end
@@ -144,21 +145,14 @@ module PEROBS
 
     # This method must be called to initiate the marking process.
     def clear_marks
-      t = Time.now
-      PEROBS.log.info "Clearing all marks"
       @flat_file.clear_all_marks
-      PEROBS.log.info "All marks cleared in #{Time.now - t} seconds"
     end
 
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
     # @return [Array] List of IDs that have been removed from the DB.
     def delete_unmarked_objects
-      t = Time.now
-      PEROBS.log.info "Deleting unmarked objects"
-      retval = @flat_file.delete_unmarked_objects
-      PEROBS.log.info "Unmarked objects deleted in #{Time.now - t} seconds"
-      retval
+      @flat_file.delete_unmarked_objects
     end
 
     # Mark an object.
@@ -179,10 +173,7 @@ module PEROBS
     # @param repair [TrueClass/FalseClass] True if found errors should be
     #        repaired.
     def check_db(repair = false)
-      t = Time.now
-      PEROBS.log.info "check_db started"
       @flat_file.check(repair)
-      PEROBS.log.info "check_db completed in #{Time.now - t} seconds"
     end
 
     # Check if the stored object is syntactically correct.
@@ -193,13 +184,11 @@ module PEROBS
     #         false.
     def check(id, repair)
       begin
-        get_object(id)
+        return get_object(id) != nil
       rescue => e
         PEROBS.log.warn "Cannot read object with ID #{id}: #{e.message}"
         return false
       end
-
-      true
     end
 
     # Store the given serialized object into the cluster files. This method is
@@ -236,8 +225,8 @@ module PEROBS
 
     def write_version_file(version_file)
       begin
-        File.write(version_file, "#{VERSION}")
-      rescue => e
+        RobustFile.write(version_file, VERSION)
+      rescue IOError => e
         PEROBS.log.fatal "Cannot write version number file " +
                          "'#{version_file}': " + e.message
       end

data/lib/perobs/IndexTree.rb

@@ -25,6 +25,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/Log'
 require 'perobs/FixedSizeBlobFile'
 require 'perobs/IndexTreeNode'
 
@@ -91,19 +92,24 @@ module PEROBS
     # @param nibble [Fixnum] Index of the nibble the node should correspond to
     # @param address [Integer] Address of the node in @nodes or nil
     def get_node(nibble, address = nil)
+      if nibble >= 16
+        # We only support 64 bit keys, so nibble cannot be larger than 15.
+        PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
+      end
       # Generate a mask for the least significant bits up to and including the
       # nibble.
       mask = (2 ** ((1 + nibble) * 4)) - 1
-      if address && (node = @node_cache[address & mask])
-        # We have an address and have found the node in the node cache.
-        return node
-      else
+      #if address && (node = @node_cache[address & mask])
+      #  # We have an address and have found the node in the node cache.
+      #  return node
+      #else
+      begin
         # We don't have a IndexTreeNode object yet for this node. Create it
         # with the data from the 'database_index' file.
         node = IndexTreeNode.new(self, nibble, address)
         # Add the node to the node cache if it's up to MAX_CACHED_LEVEL levels
         # down from the root.
-        @node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
+        #@node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
         return node
       end
     end
@@ -112,9 +118,13 @@ module PEROBS
     # @param nibble [Fixnum] The corresponding nibble for the node
     # @param address [Integer] The address of the node in @nodes
     def delete_node(nibble, address)
+      if nibble >= 16
+        # We only support 64 bit keys, so nibble cannot be larger than 15.
+        PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
+      end
       # First delete the node from the node cache.
       mask = (2 ** ((1 + nibble) * 4)) - 1
-      @node_cache.delete(address & mask)
+      #@node_cache.delete(address & mask)
       # Then delete it from the 'database_index' file.
       @nodes.delete_blob(address)
     end
@@ -149,7 +159,7 @@ module PEROBS
 
     # Check if the index is OK and matches the flat_file data.
     def check(flat_file)
-      @root.check(flat_file)
+      @root.check(flat_file, 0)
     end
 
     # Convert the tree into a human readable form.

data/lib/perobs/IndexTreeNode.rb

@@ -185,8 +185,14 @@ module PEROBS
     # Recursively check this node and all sub nodes. Compare the found
     # ID/address pairs with the corresponding entry in the given FlatFile.
     # @param flat_file [FlatFile]
+    # @tree_level [Fixnum] Assumed level in the tree. Must correspond with
+    #             @nibble_idx
     # @return [Boolean] true if no errors were found, false otherwise
-    def check(flat_file)
+    def check(flat_file, tree_level)
+      if tree_level >= 16
+        PEROBS.log.error "IndexTreeNode level (#{tree_level}) too large"
+        return false
+      end
       ENTRIES.times do |index|
         case get_entry_type(index)
         when 0
@@ -205,7 +211,7 @@ module PEROBS
           # The entry is a reference to another node. Just follow it and look
           # at the next nibble.
           unless @tree.get_node(@nibble_idx + 1, @entries[index]).
-                 check(flat_file)
+                 check(flat_file, tree_level + 1)
             return false
           end
         else

data/lib/perobs/RobustFile.rb

@@ -0,0 +1,88 @@
+# encoding: UTF-8
+#
+# = RobustFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+
+module PEROBS
+
+  class RobustFile
+
+    # This is a more robust version of File::write. It first writes to a
+    # temporary file and them atomically renames the new file to the
+    # potentially existing old file. This should significantly increase our
+    # chances that we never end up with a corrupted file due to problems
+    # during the file write.
+    def RobustFile::write(name, string)
+      tmp_name = name + '.atomic'
+
+      # Ensure that no file with the temporary name exists.
+      if File.exist?(tmp_name)
+        PEROBS.log.warn "Found old temporary file #{tmp_name}"
+        unless File.exist?(name)
+          # If we have a file with the temporary name but no file with the
+          # actual name, rename the temporary file to the actual name. We have
+          # no idea if that temporary file is good enough, but it's better to
+          # have a file with the actual name as backup in case this file write
+          # fails.
+          begin
+            File.rename(tmp_name, name)
+          rescue IOError => e
+            raise IOError "Could not name old temporary file to #{name}: " +
+              e.message
+          end
+          Log.warn "Found old temporary file but no corresponding original file"
+        else
+          # Delete the old temporary file.
+          begin
+            File.delete(tmp_name)
+          rescue IOError => e
+            PEROBS.log.warn "Could not delete old temporary file " +
+              "#{tmp_name}: #{e.message}"
+          end
+        end
+      end
+
+      # Write the temporary file.
+      begin
+        File.write(tmp_name, string)
+      rescue IOError => e
+        raise IOError "Could not write file #{tmp_name}: #{e.message}"
+      end
+
+      # If the temporary file was written successfully we can rename it to the
+      # actual file name and atomically replace the old file if it exists.
+      begin
+        File.rename(tmp_name, name)
+      rescue IOError => e
+         raise IOError "Could not rename #{tmp_name} to #{name}: #{e.message}"
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/Store.rb

@@ -365,7 +365,8 @@ module PEROBS
       else
         PEROBS.log.debug "No errors found"
       end
-      @root_objects.delete_if { |name, id| !@db.check(id, false) }
+      # Delete all broken root objects.
+      @root_objects.delete_if { |name, id| !@db.check(id, repair) }
 
       # Ensure that any fixes are written into the DB.
       sync
@@ -515,13 +516,17 @@ module PEROBS
           end
         else
           # Remove references to bad objects.
-          if ref_obj && repair
-            PEROBS.log.warn "Fixing broken reference to object #{id} in " +
-              "object #{ref_obj._id}:\n" + ref_obj.inspect
-            ref_obj._delete_reference_to_id(id)
+          if ref_obj
+            if repair
+              PEROBS.log.warn "Eliminating broken reference to object #{id} " +
+                "in object #{ref_obj._id}:\n" + ref_obj.inspect
+              ref_obj._delete_reference_to_id(id)
+            else
+              PEROBS.log.fatal "The following object references a " +
+                "non-existing object #{id}:\n" + ref_obj.inspect
+            end
           else
-             PEROBS.log.fatal "The following object references a " +
-               "non-existing object #{id}:\n" + ref_obj.inspect
+            PEROBS.log.warn "Eliminating root object #{id}"
           end
           errors += 1
         end

data/lib/perobs/TreeDB.rb

@@ -28,6 +28,7 @@
 require 'fileutils'
 
 require 'perobs/Log'
+require 'perobs/RobustFile'
 require 'perobs/DataBase'
 require 'perobs/BTreeBlob'
 
@@ -114,8 +115,8 @@ module PEROBS
     def put_hash(name, hash)
       file_name = File.join(@db_dir, name + '.json')
       begin
-        File.write(file_name, hash.to_json)
-      rescue => e
+        RobustFile.write(file_name, hash.to_json)
+      rescue IOError => e
         PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
       end
     end

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "2.4.0"
+  VERSION = "2.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.4.1
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-08 00:00:00.000000000 Z
+date: 2017-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -83,6 +83,7 @@ files:
 - lib/perobs/Log.rb
 - lib/perobs/Object.rb
 - lib/perobs/ObjectBase.rb
+- lib/perobs/RobustFile.rb
 - lib/perobs/StackFile.rb
 - lib/perobs/Store.rb
 - lib/perobs/TreeDB.rb