perobs 2.5.0 → 3.0.0

data/lib/perobs/FlatFileDB.rb

@@ -70,14 +70,14 @@ module PEROBS
     def open
       @flat_file = FlatFile.new(@db_dir)
       @flat_file.open
-      PEROBS.log.info "FlatFile opened"
+      PEROBS.log.info "FlatFile '#{@db_dir}' opened"
     end
 
     # Close the FlatFileDB.
     def close
       @flat_file.close
       @flat_file = nil
-      PEROBS.log.info "FlatFile closed"
+      PEROBS.log.info "FlatFile '#{@db_dir}' closed"
     end
 
     # Delete the entire database. The database is no longer usable after this
@@ -127,7 +127,6 @@ module PEROBS
     # Store the given object into the cluster files.
     # @param obj [Hash] Object as defined by PEROBS::ObjectBase
     def put_object(obj, id)
-      @flat_file.delete_obj_by_id(id)
       @flat_file.write_obj_by_id(id, serialize(obj))
     end
 
@@ -172,6 +171,7 @@ module PEROBS
     # Basic consistency check.
     # @param repair [TrueClass/FalseClass] True if found errors should be
     #        repaired.
+    # @return number of errors found
     def check_db(repair = false)
       @flat_file.check(repair)
     end
@@ -185,10 +185,18 @@ module PEROBS
     def check(id, repair)
       begin
         return get_object(id) != nil
-      rescue => e
-        PEROBS.log.warn "Cannot read object with ID #{id}: #{e.message}"
-        return false
+      rescue PEROBS::FatalError => e
+        PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
+        if repair
+          begin
+            PEROBS.log.error "Discarding broken object with ID #{id}"
+            @flat_file.delete_obj_by_id(id)
+          rescue PEROBS::FatalError
+          end
+        end
       end
+
+      return false
     end
 
     # Store the given serialized object into the cluster files. This method is
@@ -196,7 +204,6 @@ module PEROBS
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
     # @param id [Fixnum or Bignum] Object ID
     def put_raw_object(raw, id)
-      @flat_file.delete_obj_(id)
       @flat_file.write_obj_by_id(id, raw)
     end
 
@@ -235,6 +242,8 @@ module PEROBS
       # well.
       if version < VERSION
         write_version_file(version_file)
+        PEROBS.log.warn "Update of FlatFileDB '#{@db_dir}' from version " +
+          "#{version} to version #{VERSION} completed"
       end
     end
 

data/lib/perobs/LockFile.rb

@@ -0,0 +1,181 @@
+# encoding: UTF-8
+#
+# = LockFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+
+module PEROBS
+
+  # This class implements a file based lock. It can only be taken by one
+  # process at a time. It support configurable lock lifetime, maximum retries
+  # and pause between retries.
+  class LockFile
+
+    # Create a new lock for the given file.
+    # @param file_name [String] file name of the lock file
+    # @param options [Hash] See case statement
+    def initialize(file_name, options = {})
+      @file_name = file_name
+      # The handle of the lock file
+      @file = nil
+      # The maximum duration after which a lock file is considered a left-over
+      # from a dead or malefunctioning process.
+      @timeout_secs = 60 * 60
+      # The maximum number of times we try to get the lock.
+      @max_retries = 5
+      # The time we wait between retries
+      @pause_secs = 1
+
+      options.each do |name, value|
+        case name
+        when :timeout_secs
+          @timeout_secs = value
+        when :max_retries
+          @max_retries = value
+        when :pause_secs
+          @pause_secs = value
+        else
+          PEROBS.log.fatal "Unknown option #{name}"
+        end
+      end
+    end
+
+    # Attempt to take the lock.
+    # @return [Boolean] true if lock was taken, false otherwise
+    def lock
+      retries = @max_retries
+      while retries > 0
+        begin
+          @file = File.open(@file_name, File::RDWR | File::CREAT, 0644)
+
+          if @file.flock(File::LOCK_EX | File::LOCK_NB)
+            # We have taken the lock. Write the PID into the file and leave it
+            # open.
+            @file.write($$)
+            @file.flush
+            @file.truncate(@file.pos)
+            PEROBS.log.debug "Lock file #{@file_name} has been taken for " +
+              "process #{$$}"
+
+            return true
+          else
+            # We did not manage to take the lock file.
+            if @file.mtime <= Time.now - @timeout_secs
+              pid = @file.read.to_i
+              PEROBS.log.info "Old lock file found for PID #{pid}. " +
+                "Removing lock."
+              if is_running?(pid)
+                send_signal('TERM', pid)
+                # Give the process 3 seconds to terminate gracefully.
+                sleep 3
+                # Then send a SIGKILL to ensure it's gone.
+                send_signal('KILL', pid) if is_running?(pid)
+              end
+              @file.close
+              File.delete(@file_name) if File.exist?(@file_name)
+            else
+              PEROBS.log.debug "Lock file #{@file_name} is taken. Trying " +
+                "to get it #{retries} more times."
+            end
+          end
+        rescue => e
+          PEROBS.log.error "Cannot take lock file #{@file_name}: #{e.message}"
+          return false
+        end
+
+        retries -= 1
+        sleep(@pause_secs)
+      end
+
+      PEROBS.log.info "Failed to get lock file #{@file_name} due to timeout"
+      false
+    end
+
+    # Check if the lock has been taken.
+    # @return [Boolean] true if taken, false otherweise.
+    def is_locked?
+      File.exist?(@file_name)
+    end
+
+    # Release the lock again.
+    def unlock
+      unless @file
+        PEROBS.log.error "There is no current lock to release"
+        return false
+      end
+
+      begin
+        @file.flock(File::LOCK_UN)
+        @file.close
+        forced_unlock
+        PEROBS.log.debug "Lock file #{@file_name} for PID #{$$} has been " +
+          "released"
+      rescue => e
+        PEROBS.log.error "Releasing of lock file #{@file_name} failed: " +
+          e.message
+        return false
+      end
+
+      true
+    end
+
+    # Erase the lock file. It's essentially a forced unlock method.
+    def forced_unlock
+      @file = nil
+      if File.exist?(@file_name)
+        begin
+          File.delete(@file_name)
+          PEROBS.log.debug "Lock file #{@file_name} has been deleted."
+        rescue IOError => e
+          PEROBS.log.error "Cannot delete lock file #{@file_name}: " +
+            e.message
+        end
+      end
+    end
+
+    private
+
+    def send_signal(name, pid)
+      begin
+        Process.kill(name, pid)
+      rescue => e
+        PEROBS.log.info "Process kill error: #{e.message}"
+      end
+    end
+
+    def is_running?(pid)
+      begin
+        Process.getpgid(pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/Object.rb

@@ -151,7 +151,7 @@ module PEROBS
     end
 
     # This method should only be used during store repair operations. It will
-    # delete all referenced to the given object ID.
+    # delete all references to the given object ID.
     # @param id [Fixnum/Bignum] targeted object ID
     def _delete_reference_to_id(id)
       _all_attributes.each do |attr|
@@ -161,6 +161,7 @@ module PEROBS
           instance_variable_set(ivar, nil)
         end
       end
+      mark_as_modified
     end
 
     # Restore the persistent data from a single data structure.

data/lib/perobs/SpaceTree.rb

@@ -0,0 +1,181 @@
+# encoding: UTF-8
+#
+# = SpaceTree.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Log'
+require 'perobs/EquiBlobsFile'
+require 'perobs/SpaceTreeNodeCache'
+require 'perobs/SpaceTreeNode'
+require 'perobs/FlatFile'
+
+module PEROBS
+
+  # The SpaceTree keeps a complete list of all empty spaces in the FlatFile.
+  # Spaces are stored with size and address. The Tree is Tenary Tree. The
+  # nodes can link to other nodes with smaller spaces, same spaces and bigger
+  # spaces. The advantage of the ternary tree is that all nodes have equal
+  # size which drastically simplifies the backing store operation.
+  class SpaceTree
+
+    attr_reader :nodes
+
+    # Manage the free spaces tree in the specified directory
+    # @param dir [String] directory path of an existing directory
+    def initialize(dir)
+      @dir = dir
+
+      # This EquiBlobsFile contains the nodes of the SpaceTree.
+      @nodes = EquiBlobsFile.new(@dir, 'database_spaces',
+                                 SpaceTreeNode::NODE_BYTES, 1)
+
+      @node_cache = SpaceTreeNodeCache.new(128)
+    end
+
+    # Open the SpaceTree file.
+    def open
+      @nodes.open
+      @node_cache.clear
+      @root = SpaceTreeNode.new(self, nil, @nodes.total_entries == 0 ?
+                                           nil : @nodes.first_entry)
+      @node_cache.insert(@root)
+    end
+
+    # Close the SpaceTree file.
+    def close
+      @nodes.close
+      @root = nil
+      @node_cache.clear
+    end
+
+    def set_root(node)
+      @root = node
+    end
+
+
+    # Erase the SpaceTree file. This method cannot be called while the file is
+    # open.
+    def erase
+      @nodes.erase
+    end
+
+    # Add a new space with a given address and size.
+    # @param address [Integer] Starting address of the space
+    # @param size [Integer] size of the space in bytes
+    def add_space(address, size)
+      if size <= 0
+        PEROBS.log.fatal "Size (#{size}) must be larger than 0."
+      end
+      @root.add_space(address, size)
+    end
+
+    # Get a space that has at least the requested size.
+    # @param size [Integer] Required size in bytes
+    # @return [Array] Touple with address and actual size of the space.
+    def get_space(size)
+      if size <= 0
+        PEROBS.log.fatal "Size (#{size}) must be larger than 0."
+      end
+
+      if (address_size = @root.find_matching_space(size))
+        # First we try to find an exact match.
+        return address_size
+      elsif (address_size = @root.find_equal_or_larger_space(size))
+        return address_size
+      else
+        return nil
+      end
+    end
+
+    # Delete the node at the given address in the SpaceTree file.
+    # @param address [Integer] address in file
+    def delete_node(address)
+      @node_cache.delete(address)
+      @nodes.delete_blob(address)
+    end
+
+    # Clear all pools and forget any registered spaces.
+    def clear
+      @nodes.clear
+      @node_cache.clear
+      @root = SpaceTreeNode.new(self)
+      @node_cache.insert(@root)
+    end
+
+    # Create a new SpaceTreeNode.
+    # @param parent [SpaceTreeNode] parent node
+    # @param blob_address [Integer] address of the free space
+    # @param size [Integer] size of the free space
+    def new_node(parent, blob_address, size)
+      node = SpaceTreeNode.new(self, parent, nil, blob_address, size)
+      @node_cache.insert(node)
+    end
+
+    # Return the SpaceTreeNode that matches the given node address. If a blob
+    # address and size are given, a new node is created instead of being read
+    # from the file.
+    # @param node_address [Integer] Address of the node in the SpaceTree file
+    # @return [SpaceTreeNode]
+    def get_node(node_address)
+      if (node = @node_cache.get(node_address))
+        return node
+      end
+
+      @node_cache.insert(SpaceTreeNode.new(self, nil, node_address))
+    end
+
+    # Check if there is a space in the free space lists that matches the
+    # address and the size.
+    # @param [Integer] address Address of the space
+    # @param [Integer] size Length of the space in bytes
+    # @return [Boolean] True if space is found, false otherwise
+    def has_space?(address, size)
+      @root.has_space?(address, size)
+    end
+
+    # Check if the index is OK and matches the flat_file data (if given).
+    # @param flat_file [FlatFile] Flat file to compare with
+    # @return True if space list matches, flase otherwise
+    def check(flat_file = nil)
+      @nodes.check
+      @root.check(flat_file)
+    end
+
+    # Complete internal tree data structure as textual tree.
+    # @return [String]
+    def to_s
+      @root.to_tree_s
+    end
+
+    # Convert the tree into an Array of [address, size] touples.
+    # @return [Array]
+    def to_a
+      @root.to_a
+    end
+
+  end
+
+end
+