perobs 2.4.1 → 2.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3580a4c1778350e195656c42ca100c651ec18386
-  data.tar.gz: e3c964bb3f135372dd07ffc7b36c108f27990569
+  metadata.gz: 7913287f5998dfad64a99f239723ecb905798fe9
+  data.tar.gz: d667fe5eab1c96b4003640c4d924d97393fdf4b3
 SHA512:
-  metadata.gz: 6c12c89af79a909c65346fe8ea35e2de608dbba0ce4eb85be35b56bd07a5c50858cd5773ab94ee8e6dd3c8b75c1a872ac7400a0bbc78927c715954c337a7288e
-  data.tar.gz: f93ee4dfb7552724b78f14e682a0750e53c7984910c588050e9dc92b1d2571c35ea8ff7a5fb095ef6b537a238d15da4bb7d31a6126d1ec7aeea1d89d19138dda
+  metadata.gz: 0140b0b11d4f6f7c5f01743d5e361cf045c4533ab339ee92e32e72b4ddca7046684a6ac00445667c0a08d953946cc0fd93d6563a9b3482759e436151d0a548df
+  data.tar.gz: 5b6b9101321994f98f42c153c788bfece79183a074baf01d1e2ed377a0ff929b3308947b272bca1892807e425abfd6d2decd208f07179151fbca703d76eaa8ac

data/lib/perobs/BTreeBlob.rb

@@ -300,6 +300,9 @@ module PEROBS
       if File.exist?(@index_file_name)
         begin
           File.open(@index_file_name, 'rb') do |f|
+            unless f.flock(File::LOCK_NB | File::LOCK_EX)
+              PEROBS.log.fatal 'BTreeDB Database is locked by another process'
+            end
             # Since version 2.3.0, all index files start with a header.
             # Earlier versions did not yet have this header. The header is 24
             # bytes long. The 2nd set of 8 bytes must be 0 to distinguish the
@@ -347,6 +350,7 @@ module PEROBS
               @entries << e
               @entries_by_id[e[ID]] = e
             end
+            f.flock(File::LOCK_UN)
           end
         rescue => e
           PEROBS.log.fatal "BTreeBlob file #{@index_file_name} corrupted: " +
@@ -358,11 +362,15 @@ module PEROBS
     def write_index
       begin
         File.open(@index_file_name, 'wb') do |f|
+          unless f.flock(File::LOCK_NB | File::LOCK_EX)
+            PEROBS.log.fatal 'BTreeDB Database is locked by another process'
+          end
           # See read_index for data format documentation.
           f.write([ PEROBS_MAGIC, 0, 1].pack('QQQ'))
           @entries.each do |entry|
             f.write(entry.pack('QQQCL'))
           end
+          f.flock(File::LOCK_UN)
         end
       rescue => e
         PEROBS.log.fatal "Cannot write BTreeBlob index file " +

data/lib/perobs/Cache.rb

@@ -124,7 +124,7 @@ module PEROBS
     end
 
     # Tell the cache to start a new transaction. If no other transaction is
-    # active, the write cached is flushed before the transaction is started.
+    # active, the write cache is flushed before the transaction is started.
     def begin_transaction
       if @transaction_stack.empty?
         # The new transaction is the top-level transaction. Flush the write

data/lib/perobs/FixedSizeBlobFile.rb

@@ -59,6 +59,9 @@ module PEROBS
       rescue IOError => e
         PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
       end
+      unless @f.flock(File::LOCK_NB | File::LOCK_EX)
+        PEROBS.log.fatal 'Database blob file is locked by another process'
+      end
       @free_list.open
     end
 
@@ -68,6 +71,7 @@ module PEROBS
       @free_list.close
       begin
         @f.flush
+        @f.flock(File::LOCK_UN)
         @f.close
       rescue IOError => e
         PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"

data/lib/perobs/FlatFile.rb

@@ -81,6 +81,9 @@ module PEROBS
         PEROBS.log.fatal "Cannot open flat file database #{file_name}: " +
           e.message
       end
+      unless @f.flock(File::LOCK_NB | File::LOCK_EX)
+        PEROBS.log.fatal 'Database is locked by another process'
+      end
       @index.open
       @space_list.open
     end
@@ -91,6 +94,7 @@ module PEROBS
       @space_list.close
       @index.close
       @f.flush
+      @f.flock(File::LOCK_UN)
       @f.close
       @f = nil
     end
@@ -287,7 +291,8 @@ module PEROBS
 
       each_blob_header do |pos, mark, length, blob_id, crc|
         total_blob_count += 1
-        if (mark & 1 == 1)
+        if (mark & 3 == 3)
+          # Clear all valid and marked blocks.
           marked_blob_count += 1
           begin
             @f.seek(pos)
@@ -307,14 +312,17 @@ module PEROBS
     # implementation. No additional space will be needed on the file system.
     def defragmentize
       distance = 0
+      deleted_blobs = 0
+      valid_blobs = 0
       t = Time.now
-      PEROBS.log.debug "Defragmenting FlatFile"
+      PEROBS.log.info "Defragmenting FlatFile"
       # Iterate over all entries.
       each_blob_header do |pos, mark, length, blob_id, crc|
         # Total size of the current entry
         entry_bytes = BLOB_HEADER_LENGTH + length
         if (mark & 1 == 1)
           # We have found a valid entry.
+          valid_blobs += 1
           if distance > 0
             begin
               # Read current entry into a buffer
@@ -336,11 +344,13 @@ module PEROBS
             end
           end
         else
+          deleted_blobs += 1
           distance += entry_bytes
         end
       end
-      PEROBS.log.debug "FlatFile defragmented in #{Time.now - t} seconds"
-      PEROBS.log.debug "#{distance} bytes or " +
+      PEROBS.log.info "FlatFile defragmented in #{Time.now - t} seconds"
+      PEROBS.log.info "#{distance / 1000} KiB/#{deleted_blobs} blobs of " +
+        "#{@f.size / 1000} KiB/#{valid_blobs} blobs or " +
         "#{'%.1f' % (distance.to_f / @f.size * 100.0)}% reclaimed"
 
       @f.flush

data/lib/perobs/IndexTree.rb

@@ -35,11 +35,6 @@ module PEROBS
   # search in the tree is much faster than the linear search in the FlatFile.
   class IndexTree
 
-    # Determines how many levels of the IndexTree will be kept in memory to
-    # accerlerate the access. A number of 7 will keep up to 21845 entries in
-    # the cache but will accelerate the access to the FlatFile address.
-    MAX_CACHED_LEVEL = 7
-
     attr_reader :nodes, :ids
 
     def initialize(db_dir)
@@ -55,10 +50,6 @@ module PEROBS
       # file which contains the full object ID and the address of the
       # corresponding object in the FlatFile.
       @ids = FixedSizeBlobFile.new(db_dir, 'object_id_index', 2 * 8)
-
-      # The first MAX_CACHED_LEVEL levels of nodes will be cached in memory to
-      # improve access times.
-      @node_cache = {}
     end
 
     # Open the tree files.
@@ -72,6 +63,7 @@ module PEROBS
     def close
       @ids.close
       @nodes.close
+      @root = nil
     end
 
     # Flush out all unwritten data
@@ -84,7 +76,6 @@ module PEROBS
     def clear
       @nodes.clear
       @ids.clear
-      @node_cache = {}
       @root = IndexTreeNode.new(self, 0, 0)
     end
 
@@ -96,22 +87,10 @@ module PEROBS
         # We only support 64 bit keys, so nibble cannot be larger than 15.
         PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
       end
-      # Generate a mask for the least significant bits up to and including the
-      # nibble.
-      mask = (2 ** ((1 + nibble) * 4)) - 1
-      #if address && (node = @node_cache[address & mask])
-      #  # We have an address and have found the node in the node cache.
-      #  return node
-      #else
-      begin
-        # We don't have a IndexTreeNode object yet for this node. Create it
-        # with the data from the 'database_index' file.
-        node = IndexTreeNode.new(self, nibble, address)
-        # Add the node to the node cache if it's up to MAX_CACHED_LEVEL levels
-        # down from the root.
-        #@node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
-        return node
-      end
+      # We don't have a IndexTreeNode object yet for this node. Create it
+      # with the data from the 'database_index' file.
+      node = IndexTreeNode.new(self, nibble, address)
+      return node
     end
 
     # Delete a node from the tree that corresponds to the address.
@@ -122,10 +101,8 @@ module PEROBS
         # We only support 64 bit keys, so nibble cannot be larger than 15.
         PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
       end
-      # First delete the node from the node cache.
-      mask = (2 ** ((1 + nibble) * 4)) - 1
-      #@node_cache.delete(address & mask)
-      # Then delete it from the 'database_index' file.
+
+      # Delete it from the 'database_index' file.
       @nodes.delete_blob(address)
     end
 
@@ -135,12 +112,6 @@ module PEROBS
     # @param id [Integer] ID or key
     # @param value [Integer] value to store
     def put_value(id, value)
-      #MAX_CACHED_LEVEL.downto(0) do |i|
-      #  mask = (2 ** ((1 + i) * 4)) - 1
-      #  if (node = @node_cache[value & mask])
-      #    return node.put_value(id, value)
-      #  end
-      #end
       @root.put_value(id, value)
     end
 

data/lib/perobs/IndexTreeNode.rb

@@ -44,6 +44,8 @@ module PEROBS
     ENTRY_BYTES = 8
     TYPE_BYTES = 4
     NODE_BYTES = TYPE_BYTES + ENTRIES * ENTRY_BYTES
+    # How many levels of the tree should be kept in memory.
+    CACHED_LEVELS = 4
 
     # Create a new IndexTreeNode.
     # @param tree [IndexTree] The tree this node belongs to
@@ -64,6 +66,9 @@ module PEROBS
         @address = @tree.nodes.free_address
         write_node
       end
+      # These are the pointers that point to the next level of IndexTreeNode
+      # elements.
+      @node_ptrs = ::Array.new(ENTRIES, nil)
     end
 
     # Store a value for the given ID. Existing values will be overwritten.
@@ -96,6 +101,7 @@ module PEROBS
           # The entry of the current node is now a reference to the new node.
           set_entry_type(index, 2)
           @entries[index] = node.address
+          @node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
           # Store the existing value and the new value with their IDs.
           node.set_entry(existing_id, existing_value)
           node.put_value(id, value)
@@ -103,8 +109,7 @@ module PEROBS
         write_node
       when 2
         # The entry is a reference to another node.
-        node = @tree.get_node(@nibble_idx + 1, @entries[index])
-        node.put_value(id, value)
+        get_node(index).put_value(id, value)
       else
         PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
       end
@@ -134,8 +139,7 @@ module PEROBS
       when 2
         # The entry is a reference to another node. Just follow it and look at
         # the next nibble.
-        return @tree.get_node(@nibble_idx + 1, @entries[index]).
-          get_value(id)
+        return get_node(index).get_value(id)
       else
         PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
       end
@@ -156,6 +160,7 @@ module PEROBS
         if id == stored_id
           @tree.ids.delete_blob(@entries[index])
           @entries[index] = 0
+          @node_ptrs[index] = nil
           set_entry_type(index, 0)
           write_node
           return true
@@ -165,7 +170,7 @@ module PEROBS
         end
       when 2
         # The entry is a reference to another node.
-        node = @tree.get_node(@nibble_idx + 1, @entries[index])
+        node = get_node(index)
         result = node.delete_value(id)
         if node.empty?
           # If the sub-node is empty after the delete we delete the whole
@@ -210,8 +215,7 @@ module PEROBS
         when 2
           # The entry is a reference to another node. Just follow it and look
           # at the next nibble.
-          unless @tree.get_node(@nibble_idx + 1, @entries[index]).
-                 check(flat_file, tree_level + 1)
+          unless get_node(index).check(flat_file, tree_level + 1)
             return false
           end
         else
@@ -233,8 +237,7 @@ module PEROBS
           id, address = get_id_and_address(@entries[i])
           str += "  #{id} => #{address},\n"
         when 2
-          str += "  " + @tree.get_node(@nibble_idx + 1, @entries[i]).
-            inspect.gsub(/\n/, "\n  ")
+          str += "  " + get_node(i).inspect.gsub(/\n/, "\n  ")
         end
       end
       str + "}\n"
@@ -262,6 +265,17 @@ module PEROBS
       (id >> (4 * @nibble_idx)) & 0xF
     end
 
+    def get_node(index)
+      unless (node = @node_ptrs[index])
+        node = @tree.get_node(@nibble_idx + 1, @entries[index])
+        # We only cache the first levels of the tree to limit the memory
+        # consumption.
+        @node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
+      end
+
+      node
+    end
+
     def read_node
       return false unless (bytes = @tree.nodes.retrieve_blob(@address))
       @entry_types = bytes[0, TYPE_BYTES].unpack('L')[0]

data/lib/perobs/StackFile.rb

@@ -54,6 +54,9 @@ module PEROBS
       rescue => e
         PEROBS.log.fatal "Cannot open stack file #{@file_name}: #{e.message}"
       end
+      unless @f.flock(File::LOCK_NB | File::LOCK_EX)
+        PEROBS.log.fatal 'Database stack file is locked by another process'
+      end
     end
 
     # Close the stack file. This method must be called before the program is
@@ -61,6 +64,7 @@ module PEROBS
     def close
       begin
         @f.flush
+        @f.flock(File::LOCK_UN)
         @f.close
       rescue IOError => e
         PEROBS.log.fatal "Cannot close stack file #{@file_name}: #{e.message}"

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "2.4.1"
+  VERSION = "2.4.2"
 end

data/test/FlatFileDB_spec.rb

@@ -52,5 +52,10 @@ describe PEROBS::FlatFileDB do
     expect(File.read(version_file).to_i).to eq(PEROBS::FlatFileDB::VERSION)
   end
 
+  it 'should fail to open the same DB twice' do
+    db2 = PEROBS::FlatFileDB.new(@db_dir)
+    expect { db2.open }.to raise_error(PEROBS::FatalError)
+  end
+
 end
 

data/test/Store_spec.rb

@@ -479,7 +479,7 @@ describe PEROBS::Store do
       when 4
         # Sync store and reload
         if rand(15) == 0
-          @store.sync
+          @store.exit
           @store = PEROBS::Store.new(@db_file, options)
         end
       when 5

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.4.2
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-26 00:00:00.000000000 Z
+date: 2017-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -144,4 +144,3 @@ test_files:
 - test/Store_spec.rb
 - test/perobs_spec.rb
 - test/spec_helper.rb
-has_rdoc: