perobs 3.0.1 → 3.0.2

data/lib/perobs/SpaceTreeNodeCache.rb

@@ -34,40 +34,113 @@ module PEROBS
     # Simple cache that can hold up to size SpaceTreeNode entries. Entries are
     # hashed with a simple node_address % size function. This keeps the
     # overhead for managing the cache extremely low yet giving an OK
-    # probability to have cache hits.
+    # probability to have cache hits. The cache also keeps track if a node is
+    # still in memory or needs to be reloaded from the file. All node accesses
+    # must always go through this cache to avoid having duplicate in-memory
+    # nodes for the same on-disk node.
     # @param size [Integer] maximum number of cache entries
-    def initialize(size)
+    def initialize(tree, size)
+      @tree = tree
       @size = size
       clear
     end
 
-    # Insert a node into the cache.
-    # @param node [SpaceTreeNode]
-    def insert(node)
-      @entries[node.node_address % @size] = node
+    # Insert an unmodified node into the cache.
+    # @param node [SpaceTreeNode] Unmodified SpaceTreeNode
+    def insert_unmodified(node)
+      @in_memory_nodes[node.node_address] = node.object_id
+      @unmodified_nodess[node.node_address % @size] = node
+    end
+
+    # Insert a modified node into the cache.
+    # @param node [SpaceTreeNode] Modified SpaceTreeNode
+    # @param address [Integer] Address of the node in the file
+    def insert_modified(node)
+      @in_memory_nodes[node.node_address] = node.object_id
+      index = node.node_address % @size
+      if (old_node = @modified_nodes[index])
+        # If the object is already in the modified object list, we don't have
+        # to do anything.
+        return if old_node.node_address == node.node_address
+        # If the new object will replace an existing entry in the cash we have
+        # to save the object first.
+        old_node.save
+      end
+
+      @modified_nodes[index] = node
     end
 
     # Retrieve a node reference from the cache.
     # @param address [Integer] address of the node to retrieve.
     def get(address)
-      node = @entries[address % @size]
+      node = @unmodified_nodess[address % @size]
       # We can have collisions. Check if the cached node really matches the
       # requested address.
-      (node && node.node_address == address) ? node : nil
+      return node if node && node.node_address == address
+
+      node = @modified_nodes[address % @size]
+      # We can have collisions. Check if the cached node really matches the
+      # requested address.
+      return node if node && node.node_address == address
+
+      if (obj_id = @in_memory_nodes[address])
+        # We have the node in memory so we can just return it.
+        begin
+          node = ObjectSpace._id2ref(obj_id)
+          unless node.node_address == address
+            raise RuntimeError, "In memory list is corrupted"
+          end
+          insert_unmodified(node)
+          return node
+        rescue RangeError
+          # Due to a race condition the object can still be in the
+          # @in_memory_nodes list but has been collected already by the Ruby
+          # GC. In that case we need to load it again.
+          @in_memory_nodes.delete(address)
+        end
+      end
+
+      SpaceTreeNode::load(@tree, address)
     end
 
     # Remove a node from the cache.
     # @param address [Integer] address of node to remove.
     def delete(address)
+      # The object is likely still in memory, but we really don't want to
+      # access it anymore.
+      @in_memory_nodes.delete(address)
+
       index = address % @size
-      if (node = @entries[index]) && node.node_address == address
-        @entries[index] = nil
+      if (node = @unmodified_nodess[index]) && node.node_address == address
+        @unmodified_nodess[index] = nil
+      end
+
+      if (node = @modified_nodes[index]) && node.node_address == address
+        @modified_nodes[index] = nil
       end
     end
 
+    # Remove a node from the in-memory list. This is an internal method
+    # and should never be called from user code. It will be called from a
+    # finalizer, so many restrictions apply!
+    # @param node_address [Integer] Node address of the node to remove from
+    #        the list
+    def _collect(address)
+      @in_memory_nodes.delete(address)
+    end
+
+    # Write all modified objects into the backing store.
+    def flush
+      @modified_nodes.each { |node| node.save if node }
+      @modified_nodes = ::Array.new(@size)
+    end
+
     # Remove all entries from the cache.
     def clear
-      @entries = ::Array.new(@size)
+      # A hash that stores all objects by ID that are currently in memory.
+      @in_memory_nodes = {}
+      @unmodified_nodess = ::Array.new(@size)
+      @modified_nodes = ::Array.new(@size)
     end
 
   end

data/lib/perobs/SpaceTreeNodeLink.rb

@@ -94,7 +94,7 @@ module PEROBS
     private
 
     def get_node
-      @tree.get_node(@node_address)
+      @tree.cache.get(@node_address)
     end
 
   end

data/lib/perobs/StackFile.rb

@@ -36,7 +36,7 @@ module PEROBS
     # Create a new stack file in the given directory with the given file name.
     # @param dir [String] Directory
     # @param name [String] File name
-    # @param entry_bytes [Fixnum] Number of bytes each entry must have
+    # @param entry_bytes [Integer] Number of bytes each entry must have
     def initialize(dir, name, entry_bytes)
       @file_name = File.join(dir, name + '.stack')
       @entry_bytes = entry_bytes

data/lib/perobs/Store.rb

@@ -26,7 +26,6 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'set'
-require 'weakref'
 
 require 'perobs/Log'
 require 'perobs/Handle'
@@ -120,7 +119,7 @@ module PEROBS
     #                      :json : About half as fast as marshal, but the
     #                      format is rock solid and portable between
     #                      languages. It only supports basic Ruby data types
-    #                      like String, Fixnum, Float, Array, Hash. This is
+    #                      like String, Integer, Float, Array, Hash. This is
     #                      the default option.
     #                      :yaml : Can also handle most Ruby data types and is
     #                      portable between Ruby versions (1.9 and later).
@@ -221,7 +220,7 @@ module PEROBS
     # For library internal use only!
     # This method will create a new PEROBS object.
     # @param klass [BasicObject] Class of the object to create
-    # @param id [Fixnum, Bignum] Requested object ID
+    # @param id [Integer] Requested object ID
     # @param args [Array] Arguments to pass to the object constructor.
     # @return [BasicObject] Newly constructed PEROBS object
     def _construct_po(klass, id, *args)
@@ -297,7 +296,7 @@ module PEROBS
     # from the back-end storage. The garbage collector is not invoked
     # automatically. Depending on your usage pattern, you need to call this
     # method periodically.
-    # @return [Fixnum] The number of collected objects
+    # @return [Integer] The number of collected objects
     def gc
       if @cache.in_transaction?
         PEROBS.log.fatal 'You cannot call gc() during a transaction'
@@ -314,11 +313,12 @@ module PEROBS
       if (obj = @in_memory_objects[id])
         # We have the object in memory so we can just return it.
         begin
-          return obj.__getobj__
-        rescue WeakRef::RefError
+          return ObjectSpace._id2ref(obj)
+        rescue RangeError
           # Due to a race condition the object can still be in the
           # @in_memory_objects list but has been collected already by the Ruby
           # GC. In that case we need to load it again.
+          @in_memory_objects.delete(id)
         end
       end
 
@@ -342,7 +342,7 @@ module PEROBS
     # unreadable object is found, the reference will simply be deleted.
     # @param repair [TrueClass/FalseClass] true if a repair attempt should be
     #        made.
-    # @return [Fixnum] The number of references to bad objects found.
+    # @return [Integer] The number of references to bad objects found.
     def check(repair = false)
       # All objects must have in-db version.
       sync
@@ -436,7 +436,7 @@ module PEROBS
     # Internal method. Don't use this outside of this library!
     # Generate a new unique ID that is not used by any other object. It uses
     # random numbers between 0 and 2**64 - 1.
-    # @return [Fixnum or Bignum]
+    # @return [Integer]
     def _new_id
       begin
         # Generate a random number. It's recommended to not store more than
@@ -454,15 +454,15 @@ module PEROBS
     # happens the object finalizer is triggered and calls _forget() to
     # remove the object from this hash again.
     # @param obj [BasicObject] Object to register
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     def _register_in_memory(obj, id)
-      @in_memory_objects[id] = WeakRef.new(obj)
+      @in_memory_objects[id] = obj.object_id
     end
 
     # Remove the object from the in-memory list. This is an internal method
     # and should never be called from user code. It will be called from a
     # finalizer, so many restrictions apply!
-    # @param id [Fixnum or Bignum] Object ID of object to remove from the list
+    # @param id [Integer] Object ID of object to remove from the list
     def _collect(id, ignore_errors = false)
       @in_memory_objects.delete(id)
     end
@@ -502,10 +502,10 @@ module PEROBS
 
     # Check the object with the given start_id and all other objects that are
     # somehow reachable from the start object.
-    # @param start_id [Fixnum or Bignum] ID of the top-level object to start
+    # @param start_id [Integer] ID of the top-level object to start
     #        with
     # @param repair [Boolean] Delete refernces to broken objects if true
-    # @return [Fixnum] The number of references to bad objects.
+    # @return [Integer] The number of references to bad objects.
     def check_object(start_id, repair)
       errors = 0
       @db.mark(start_id)
@@ -518,7 +518,7 @@ module PEROBS
         # Get the next PEROBS object to check
         ref_obj, id = todo_list.pop
 
-        if (obj = object_by_id(id)) && (obj_ok = @db.check(id, repair))
+        if (obj = object_by_id(id))
           # The object exists and is OK. Mark is as checked.
           @db.mark(id)
           # Now look at all other objects referenced by this object.

data/lib/perobs/TreeDB.rb

@@ -103,7 +103,7 @@ module PEROBS
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !(blob = find_blob(id)).nil? && !blob.find(id).nil?
     end
@@ -143,7 +143,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -166,13 +166,13 @@ module PEROBS
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       (blob = find_blob(id)) && blob.mark(id)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     def is_marked?(id, ignore_errors = false)
@@ -187,7 +187,7 @@ module PEROBS
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -206,7 +206,7 @@ module PEROBS
     # Store the given serialized object into the cluster files. This method is
     # for internal use only!
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     def put_raw_object(raw, id)
       find_blob(id, true).write_object(id, raw)
     end
@@ -226,7 +226,7 @@ module PEROBS
         Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
           # Extract the 01-part of the filename
           lsb_string = File.basename(fqfn)[0..-6]
-          # Convert the lsb_string into a Fixnum
+          # Convert the lsb_string into a Integer
           lsb = Integer('0b' + lsb_string)
           # Bit mask to match the LSBs
           mask = (2 ** lsb_string.length) - 1

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "3.0.1"
+  VERSION = "3.0.2"
 end

data/test/Array_spec.rb

@@ -169,8 +169,8 @@ describe PEROBS::Array do
 
   it 'should support collect!()' do
     a = cpa([ 1, 'cat', 1..1 ])
-    expect(a.collect! { |e| e.class }).to eq([ Fixnum, String, Range ])
-    pcheck { expect(a).to eq([ Fixnum, String, Range ]) }
+    expect(a.collect! { |e| e.class }).to eq([ Integer, String, Range ])
+    pcheck { expect(a).to eq([ Integer, String, Range ]) }
 
     a = cpa([ 1, 'cat', 1..1 ])
     expect(a.collect! { 99 }).to eq([ 99, 99, 99])
@@ -179,8 +179,8 @@ describe PEROBS::Array do
 
   it 'should support map!()' do
     a = cpa([ 1, 'cat', 1..1 ])
-    expect(a.map! { |e| e.class }).to eq([ Fixnum, String, Range ])
-    pcheck { expect(a).to eq([ Fixnum, String, Range ]) }
+    expect(a.map! { |e| e.class }).to eq([ Integer, String, Range ])
+    pcheck { expect(a).to eq([ Integer, String, Range ]) }
 
     a = cpa([ 1, 'cat', 1..1 ])
     expect(a.map! { 99 }).to eq([ 99, 99, 99])

data/test/BTreeDB_spec.rb

@@ -56,7 +56,7 @@ describe PEROBS::BTreeDB do
     expect(@db.include?(0)).to be false
     h = {
           'String' => 'What god has wrought',
-          'Fixnum' => 42,
+          'Integer' => 42,
           'Float' => 3.14,
           'True' => true,
           'False' => false,
@@ -75,7 +75,7 @@ describe PEROBS::BTreeDB do
       expect(@db.include?(0)).to be false
       h = {
         'String' => 'What god has wrought',
-        'Fixnum' => 42,
+        'Integer' => 42,
         'Float' => 3.14,
         'True' => true,
         'False' => false,

data/test/SpaceTree_spec.rb

@@ -94,12 +94,12 @@ EOT
   end
 
   it 'should find the smallest node' do
-    node = @m.instance_variable_get('@root').find_smallest_node
+    node = @m.root.find_smallest_node
     expect(node.size).to eql(2)
   end
 
   it 'should find the largest node' do
-    node = @m.instance_variable_get('@root').find_largest_node
+    node = @m.root.find_largest_node
     expect(node.size).to eql(32)
     expect(node.node_address).to eql(5)
     expect(node.parent.size).to eql(16)
@@ -119,12 +119,14 @@ EOT
     add_sizes([ 10, 5, 15, 10 ])
     expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 10], [2, 15]])
     expect(@m.get_space(10)).to eql([0, 10])
+    expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
 
     @m.clear
     add_sizes([ 10, 5, 15, 10, 10 ])
     expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 10], [3, 10], [2, 15]])
     expect(@m.get_space(10)).to eql([0, 10])
     expect(@m.get_space(10)).to eql([4, 10])
+    expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
   end
 
   it 'should delete a smaller node' do
@@ -208,14 +210,15 @@ EOT
   it 'should support a real-world traffic pattern' do
     address = 0
     spaces = []
-    0.upto(500) do
-      case rand(3)
+    @m.clear
+    0.upto(1000) do
+      case rand(4)
       when 0
         # Insert new values
         rand(9).times do
           size = 20 + rand(5000)
           @m.add_space(address, size)
-          spaces << [address, size]
+          spaces << [ address, size ]
           address += size
         end
       when 1
@@ -228,9 +231,21 @@ EOT
           end
         end
       when 2
-        expect(@m.check).to be true
-        spaces.each do |address, size|
-          expect(@m.has_space?(address, size)).to be true
+        if rand(10) == 0
+          expect(@m.check).to be true
+          spaces.each do |address, size|
+            expect(@m.has_space?(address, size)).to be true
+          end
+          @m.each do |address, size|
+            expect(spaces.include?([ address, size ])).to be true
+          end
+        end
+      when 3
+        if rand(100) == 0
+          expect(@m.check).to be true
+          @m.close
+          @m.open
+          expect(@m.check).to be true
         end
       end
     end

data/test/Store_spec.rb

@@ -422,7 +422,9 @@ describe PEROBS::Store do
     @store.sync
     GC.start
     # Now the Person should be gone from memory.
-    expect(@store.statistics[:in_memory_objects]).to eq(1)
+    # Ruby 2.3 and later has changed the GC so that this does not work
+    # reliably anymore. The GC seems to operate lazyly.
+    #expect(@store.statistics[:in_memory_objects]).to eq(1)
   end
 
   it 'should handle nested constructors' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-13 00:00:00.000000000 Z
+date: 2017-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler