perobs 4.0.0 → 4.4.0

data/lib/perobs/BigHash.rb

@@ -0,0 +1,246 @@
+# encoding: UTF-8
+#
+# = BigHash.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Object'
+require 'perobs/BigTree'
+require 'perobs/Array'
+require 'perobs/FNV_Hash_1a_64'
+
+module PEROBS
+
+  # The BigHash is similar to the Hash object in that it provides a simple
+  # hash functionality. The difference is that this class scales to much
+  # larger data sets essentially limited to the amount of space available on
+  # your backing store. The data is persisted immediately and uses
+  # transactions to ensure the data consistent. It only provides a small
+  # subset of the methods provided by the native Hash class that make sense
+  # for giant data sets.
+  class BigHash < PEROBS::Object
+
+    # Internally this class uses BigTree to store the values by the hashed
+    # key. We are using a 64 bit hash space so collisions are fairly unlikely
+    # but not impossible. Therefor we have to store the originial key with the
+    # value to ensure that we got the right value. The key and value are
+    # stored in an Entry object.
+    #
+    # In case we have a collision we need to store multiple values for the
+    # same hashed key. In that case we store the Entry objects for the same
+    # hashed key in a Collisions object instead of storing the Entry
+    # directly in the BigTree.
+    class Entry < PEROBS::Object
+
+      attr_persist :key, :value
+
+      def initialize(p, key, value)
+        super(p)
+        self.key = key
+        self.value = value
+      end
+
+    end
+
+    # Since the BigHash can also store PEROBS::Array values we need to
+    # introduce a new class so we can tell apart collisions from Arrays.
+    class Collisions < PEROBS::Array
+    end
+
+    attr_persist :btree
+
+    # Create a new BigHash object.
+    # @param p [Handle] Store handle
+    def initialize(p)
+      super(p)
+      restore
+      self.btree = @store.new(PEROBS::BigTree)
+    end
+
+    def restore
+    end
+
+    # Insert a value that is associated with the given key. If a value for
+    # this key already exists, the value will be overwritten with the newly
+    # provided value.
+    # @param key [Integer or String]
+    # @param value [Any PEROBS storable object]
+    def []=(key, value)
+      hashed_key = hash_key(key)
+      @store.transaction do
+        entry = @store.new(Entry, key, value)
+
+        if (existing_entry = @btree.get(hashed_key))
+          # There is already an existing entry for this hashed key.
+          if existing_entry.is_a?(Collisions)
+            # Find the right index to insert the new entry. If there is
+            # already an entry with the same key overwrite that entry.
+            index_to_insert = 0
+            overwrite = false
+            existing_entry.each do |ae|
+              if ae.key == key
+                overwrite = true
+                break
+              end
+              index_to_insert += 1
+            end
+            existing_entry[index_to_insert] = entry
+          elsif existing_entry.key == key
+            # The existing value is for the identical key. We can safely
+            # overwrite
+            @btree.insert(hashed_key, entry)
+          else
+            # There is a single existing entry, but for a different key. Create
+            # a new PEROBS::Array and store both entries.
+            array_entry = @store.new(Collisions)
+            array_entry << existing_entry
+            array_entry << entry
+            @btree.insert(hashed_key, array_entry)
+          end
+        else
+          # No existing entry. Insert the new entry.
+          @btree.insert(hashed_key, entry)
+        end
+      end
+    end
+
+    # Retrieve the value for the given key. If no value for the key is found
+    # nil is returned.
+    # @param key [Integer or String]
+    # @return [Any PEROBS storable object]
+    def [](key)
+      hashed_key = hash_key(key)
+      unless (entry = @btree.get(hashed_key))
+        return nil
+      end
+
+      if entry.is_a?(PEROBS::Array)
+        entry.each do |ae|
+          return ae.value if ae.key == key
+        end
+      else
+        return entry.value if entry.key == key
+      end
+
+      nil
+    end
+
+    # Check if the is a value stored for the given key.
+    # @param key [Integer or String]
+    # @return [TrueClass or FalseClass]
+    def has_key?(key)
+      hashed_key = hash_key(key)
+      unless (entry = @btree.get(hashed_key))
+        return false
+      end
+
+      if entry.is_a?(PEROBS::Array)
+        entry.each do |ae|
+          return true if ae.key == key
+        end
+      else
+        return true if entry.key == key
+      end
+
+      false
+    end
+
+    alias include? has_key?
+
+    # Delete and return the entry for the given key. Return nil if no matching
+    # entry exists.
+    # @param key [Integer or String]
+    # @return [Object] Deleted entry
+    def delete(key)
+      hashed_key = hash_key(key)
+      unless (entry = @btree.get(hashed_key))
+        return nil
+      end
+
+      if entry.is_a?(PEROBS::Array)
+        entry.each_with_index do |ae, i|
+          if ae.key == key
+            return entry.delete_at(i).value
+          end
+        end
+      else
+        return entry.value if entry.key == key
+      end
+
+      nil
+    end
+
+    # Return the number of entries stored in the hash.
+    # @return [Integer]
+    def length
+      @btree.entry_counter
+    end
+
+    alias size length
+
+    # Return true if hash is empty. False otherweise.
+    # @return [TrueClass, FalseClass]
+    def empty?
+      @btree.entry_counter == 0
+    end
+
+    # Calls the given block for each key/value pair.
+    # @yield(key, value)
+    def each(&block)
+      @btree.each do |index, entry|
+        if entry.is_a?(Collisions)
+          break if entry.each do |c_entry|
+            yield(c_entry.key, c_entry.value)
+          end.nil?
+        else
+          yield(entry.key, entry.value)
+        end
+      end
+    end
+
+    # This is mostly intended for debugging as the result can be very big.
+    # It returns an Array of keys stored in the hash.
+    # @return [Array] A list of all keys
+    def keys
+      ks = []
+      each { |k, v| ks << k }
+      ks
+    end
+
+    # Check if the data structure contains any errors.
+    # @return [Boolean] true if no erros were found, false otherwise
+    def check
+      return @btree.check
+    end
+
+    private
+
+    def hash_key(key)
+      FNV_Hash_1a_64::digest(key)
+    end
+
+  end
+
+end
+

data/lib/perobs/BigTree.rb

@@ -0,0 +1,197 @@
+# encoding: UTF-8
+#
+# = BigTree.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/Object'
+require 'perobs/BigTreeNode'
+
+module PEROBS
+
+  # The BigTree class implements a BTree as a PEROBS object. It allows to
+  # manage huge amounts of data in a reasonably efficient way. The number of
+  # entries is limited by the space on the backing store, not the main
+  # memory. Entries are addressed by a Integer key.
+  class BigTree < PEROBS::Object
+
+    class Stats < Struct.new(:leaf_nodes, :branch_nodes, :min_depth,
+                             :max_depth)
+    end
+
+    attr_persist :node_size, :root, :first_leaf, :last_leaf, :entry_counter
+
+    # Internal constructor. Use Store.new() instead.
+    # @param p [Handle]
+    # @param node_size [Integer] The size of the tree nodes. This determines
+    #        how many entries must be read/written for each operation.
+    def initialize(p, node_size = 127)
+      super(p)
+      unless node_size > 2
+        PEROBS.log.fatal "Node size (#{node_size}) must be larger than 2"
+      end
+      attr_init(:node_size, node_size)
+      clear unless instance_variable_defined?('@root')
+    end
+
+    # Remove all entries from the BigTree.
+    def clear
+      self.root = self.first_leaf = self.last_leaf =
+        @store.new(BigTreeNode, myself, true)
+      self.entry_counter = 0
+    end
+
+    # Insert a new value into the tree using the key as a unique index. If the
+    # key already exists the old value will be overwritten.
+    # @param key [Integer] Unique key
+    # @param value [Integer] value
+    def insert(key, value)
+      @store.transaction do
+        @root.insert(key, value)
+      end
+    end
+
+    # Retrieve the value associated with the given key. If no entry was found,
+    # return nil.
+    # @param key [Integer] Unique key
+    # @return [Integer or nil] found value or nil
+    def get(key)
+      @root.get(key)
+    end
+
+    # Return the node chain from the root to the leaf node storing the
+    # key/value pair.
+    # @param key [Integer] key to search for
+    # @return [Array of BigTreeNode] node list (may be empty)
+    def node_chain(key)
+      @root.node_chain(key)
+    end
+
+    # Check if there is an entry for the given key.
+    # @param key [Integer] Unique key
+    # @return [Boolean] True if key is present, false otherwise.
+    def has_key?(key)
+      @root.has_key?(key)
+    end
+
+    # Find and remove the value associated with the given key. If no entry was
+    # found, return nil, otherwise the found value.
+    # @param key [Integer] Unique key
+    # @return [Integer or nil] found value or nil
+    def remove(key)
+      removed_value = nil
+
+      @store.transaction do
+        removed_value = @root.remove(key)
+      end
+
+      removed_value
+    end
+
+    # Delete all entries for which the passed block yields true. The
+    # implementation is optimized for large bulk deletes. It rebuilds a new
+    # BTree for the elements to keep. If only few elements are deleted the
+    # overhead of rebuilding the BTree is rather high.
+    # @yield [key, value]
+    def delete_if
+      old_root = @root
+      clear
+      old_root.each do |k, v|
+        if !yield(k, v)
+          insert(k, v)
+        end
+      end
+    end
+
+    # @return [Integer] The number of entries stored in the tree.
+    def length
+      @entry_counter
+    end
+
+    # Return true if the BigTree has no stored entries.
+    def empty?
+      @entry_counter == 0
+    end
+
+    # Iterate over all entries in the tree. Entries are always sorted by the
+    # key.
+    # @yield [key, value]
+    def each(&block)
+      node = @first_leaf
+      while node
+        break if node.each_element(&block).nil?
+        node = node.next_sibling
+      end
+    end
+
+    # Iterate over all entries in the tree in reverse order. Entries are
+    # always sorted by the key.
+    # @yield [key, value]
+    def reverse_each(&block)
+      node = @last_leaf
+      while node
+        node.reverse_each_element(&block)
+        node = node.prev_sibling
+      end
+    end
+
+
+    # @return [String] Human reable form of the tree.
+    def to_s
+      @root.to_s
+    end
+
+    # Check if the tree file contains any errors.
+    # @return [Boolean] true if no erros were found, false otherwise
+    def check(&block)
+      @root.check(&block)
+
+      i = 0
+      each do |k, v|
+        i += 1
+      end
+
+      unless @entry_counter == i
+        PEROBS.log.error "BigTree contains #{i} values but entry counter " +
+          "is #{@entry_counter}"
+        return false
+      end
+
+      true
+    end
+
+    # Gather some statistics regarding the tree structure.
+    # @return [Stats] Structs with gathered data
+    def statistics
+      stats = Stats.new(0, 0, nil, nil)
+      @root.statistics(stats)
+      stats
+    end
+
+    private
+
+  end
+
+end
+