perobs 4.0.0 → 4.1.0

data/lib/perobs/IDList.rb

@@ -0,0 +1,144 @@
+# encoding: UTF-8
+#
+# = IDList.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/IDListPageFile'
+require 'perobs/IDListPageRecord'
+
+module PEROBS
+
+  # This class stores a list of 64 bit values. Values can be added to the list
+  # and the presence of a certain value can be checked. It can hold up to 2^64
+  # values. It tries to keep values in memory but can store them in a file if
+  # needed. A threshold for the in-memory values can be set in the
+  # constructor. The stored values are grouped in pages. Each page can hold up
+  # to page_size entries.
+  class IDList
+
+    # Create a new IDList object. The data that can't be kept in memory will
+    # be stored in the specified directory under the given name.
+    # @param dir [String] Path of the directory
+    # @param name [String] Name of the file
+    # @param max_in_memory [Integer] Specifies the maximum number of values
+    #        that will be kept in memory. If the list is larger, values will
+    #        be cached in the specified file.
+    # @param page_size [Integer] The number of values per page. The default
+    #        value is 32 which was found the best performing config in  tests.
+    def initialize(dir, name, max_in_memory, page_size = 32)
+      # The page_file manages the pages that store the values.
+      @page_file = IDListPageFile.new(self, dir, name,
+                                      max_in_memory, page_size)
+      clear
+    end
+
+    # Insert a new value into the list.
+    # @param id [Integer] The value to add
+    def insert(id)
+      # Find the index of the page that should hold ID.
+      index = @page_records.bsearch_index { |pr| pr.max_id >= id }
+      # Get the corresponding IDListPageRecord object.
+      page = @page_records[index]
+
+      # In case the page is already full we'll have to create a new page.
+      # There is no guarantee that a split will yield an page with space as we
+      # split by ID range, not by distributing the values evenly across the
+      # two pages.
+      while page.is_full?
+        new_page = page.split
+        # Store the newly created page into the page_records list.
+        @page_records.insert(index + 1, new_page)
+        if id >= new_page.min_id
+          # We need to insert the ID into the newly created page. Adjust index
+          # and page reference accordingly.
+          index += 1
+          page = new_page
+        end
+      end
+
+      # Insert the ID into the page.
+      page.insert(id)
+    end
+
+    # Check if a given value is already stored in the list.
+    # @param id [Integer] The value to check for
+    def include?(id)
+      @page_records.bsearch { |pr| pr.max_id >= id }.include?(id)
+    end
+
+    # Clear the list and empty the filesystem cache file.
+    def clear
+      @page_file.clear
+      @page_records = [ IDListPageRecord.new(@page_file, 0, 2 ** 64) ]
+    end
+
+    # Erase the list including the filesystem cache file. The IDList is no
+    # longer usable after this call but the cache file is removed from the
+    # filesystem.
+    def erase
+      @page_file.erase
+      @page_records = nil
+    end
+
+    # Perform some consistency checks on the internal data structures. Raises
+    # a RuntimeError in case a problem is found.
+    def check
+      last_max = -1
+      unless (min_id = @page_records.first.min_id) == 0
+        raise RuntimeError, "min_id of first record (#{min_id}) " +
+          "must be 0."
+      end
+
+      @page_records.each do |pr|
+        unless pr.min_id == last_max + 1
+          raise RuntimeError, "max_id of previous record (#{last_max}) " +
+            "must be exactly 1 smaller than current record (#{pr.min_id})."
+        end
+        last_max = pr.max_id
+        pr.check
+      end
+
+      unless last_max == 2 ** 64
+        raise RuntimeError, "max_id of last records " +
+          "(#{@page_records.last.max_id}) must be #{2 ** 64})."
+      end
+    end
+
+    def to_a
+      a = []
+      @page_records.each { |pr| a += pr.values }
+      a
+    end
+
+    # Print a human readable form of the tree that stores the list. This is
+    # only meant for debugging purposes and does not scale for larger trees.
+    def to_s
+      "\n" + @root.to_s
+    end
+
+  end
+
+end
+

data/lib/perobs/IDListPage.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+#
+# = IDListPage.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  class IDListPage
+
+    attr_reader :uid, :values
+    attr_accessor :record
+
+    def initialize(page_file, record, uid, values = [])
+      @page_file = page_file
+      @record = record
+      @uid = uid
+      @values = values
+      @record.page_entries = @values.length
+    end
+
+    def IDListPage::load(page_file, uid, ref)
+      page_file.load(uid, ref)
+    end
+
+    def is_full?
+      @values.length >= @page_file.page_size
+    end
+
+    def length
+      @values.length
+    end
+
+    def save
+      @page_file.save_page(self)
+    end
+
+    def insert(id)
+      if is_full?
+        raise ArgumentError, "IDListPage is already full"
+      end
+      index = @values.bsearch_index { |v| v >= id } || @values.length
+
+      # If the value isn't stored already, insert it.
+      if @values[index] != id
+        @values.insert(index, id)
+        @record.page_entries = @values.length
+        @page_file.mark_page_as_modified(self)
+      end
+    end
+
+    def include?(id)
+      !(v = @values.bsearch { |v| v >= id }).nil? && v == id
+    end
+
+    def delete(max_id)
+      a = []
+      @values.delete_if { |v| v > max_id ? a << v : false }
+
+      unless a.empty?
+        @record.page_entries = @values.length
+        @page_file.mark_page_as_modified(self)
+      end
+
+      a
+    end
+
+    def check
+      last_value = nil
+      @values.each_with_index do |v, i|
+        if last_value && last_value >= v
+          raise RuntimeError, "The values #{last_value} and #{v} must be " +
+            "strictly ascending: #{@values.inspect}"
+        end
+        last_value = v
+      end
+    end
+
+    def to_s
+      "[ #{@values.join(', ')} ]"
+    end
+
+  end
+
+end
+

data/lib/perobs/IDListPageFile.rb

@@ -0,0 +1,180 @@
+# encoding: UTF-8
+#
+# = IDListPageFile.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/IDListPage'
+require 'perobs/IDListPageRecord'
+require 'perobs/Log'
+require 'perobs/PersistentObjectCache'
+
+module PEROBS
+
+  # The IDListPageFile class provides filesystem based cache for the
+  # IDListPage objects. The IDListRecord objects only hold the index of the
+  # page in this cache. This allows the pages to be garbage collected and
+  # swapped to the file. If accessed, the pages will be swaped in again. While
+  # this process is similar to the demand paging of the OS it has absolutely
+  # nothing to do with it.
+  class IDListPageFile
+
+    attr_reader :page_size, :pages
+
+    # Create a new IDListPageFile object that uses the given file in the given
+    # directory as cache file.
+    # @param list [IDList] The IDList object that caches pages here
+    # @param dir [String] An existing directory
+    # @param name [String] A file name (without path)
+    # @param max_in_memory [Integer] Maximum number of pages to keep in memory
+    # @param page_size [Integer] The number of values in each page
+    def initialize(list, dir, name, max_in_memory, page_size)
+      @list = list
+      @file_name = File.join(dir, name + '.cache')
+      @page_size = page_size
+      open
+      @pages = PersistentObjectCache.new(max_in_memory, max_in_memory / 2,
+                                         IDListPage, self)
+      @page_counter = 0
+    end
+
+    # Load the IDListPage from the cache file.
+    # @param page_idx [Integer] The page index in the page file
+    # @param record [IDListPageRecord] the corresponding IDListPageRecord
+    # @return [IDListPage] The loaded values
+    def load(page_idx, record)
+      # The IDListPageRecord will tell us the actual number of values stored
+      # in this page.
+      values = []
+      unless (entries = record.page_entries) == 0
+        begin
+          @f.seek(page_idx * @page_size * 8)
+          values = @f.read(entries * 8).unpack("Q#{entries}")
+        rescue IOError => e
+          PEROBS.log.fatal "Cannot read cache file #{@file_name}: #{e.message}"
+        end
+      end
+
+      # Create the IDListPage object with the given values.
+      p = IDListPage.new(self, record, page_idx, values)
+      @pages.insert(p, false)
+
+      p
+    end
+
+    # Return the number of registered pages.
+    def page_count
+      @page_counter
+    end
+
+    # Create a new IDListPage and register it.
+    # @param record [IDListPageRecord] The corresponding record.
+    # @param values [Array of Integer] The values stored in the page
+    # @return [IDListPage]
+    def new_page(record, values = [])
+      idx = @page_counter
+      @page_counter += 1
+      mark_page_as_modified(IDListPage.new(self, record, idx, values))
+      idx
+    end
+
+    # Return the IDListPage object with the given index.
+    # @param record [IDListPageRecord] the corresponding IDListPageRecord
+    # @return [IDListPage] The page corresponding to the index.
+    def page(record)
+      p = @pages.get(record.page_idx, record) || load(record.page_idx, record)
+      unless p.uid == record.page_idx
+        raise RuntimeError, "Page reference mismatch. Record " +
+          "#{record.page_idx} points to page #{p.uid}"
+      end
+
+      p
+    end
+
+    # Mark a page as modified. This means it has to be written into the cache
+    # before it is removed from memory.
+    # @param p [IDListPage] page reference
+    def mark_page_as_modified(p)
+      @pages.insert(p)
+      @pages.flush
+    end
+
+    # Clear all pages, erase the cache and re-open it again.
+    def clear
+      @pages.clear
+      @page_counter = 0
+      begin
+        @f.truncate(0)
+      rescue IOError => e
+        raise RuntimeError, "Cannote truncate cache file #{@file_name}: " +
+          e.message
+      end
+    end
+
+    # Discard all pages and erase the cache file.
+    def erase
+      @pages.clear
+      @page_counter = 0
+      close
+    end
+
+    # Save the given IDListPage into the cache file.
+    # @param p [IDListPage] page to store
+    def save_page(p)
+      if p.record.page_entries != p.values.length
+        raise RuntimeError, "page_entries mismatch for node #{p.uid}"
+      end
+      begin
+        @f.seek(p.uid * @page_size * 8)
+        @f.write(p.values.pack('Q*'))
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write cache file #{@file_name}: #{e.message}"
+      end
+    end
+
+    private
+
+    def open
+      begin
+        # Create a new file by writing a new header.
+        @f = File.open(@file_name, 'wb+')
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot open cache file #{@file_name}: #{e.message}"
+      end
+    end
+
+    def close
+      begin
+        @f.close
+        File.delete(@file_name) if File.exist?(@file_name)
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot erase cache file #{@file_name}: #{e.message}"
+      end
+      @f = nil
+    end
+
+  end
+
+end
+

data/lib/perobs/IDListPageRecord.rb

@@ -0,0 +1,142 @@
+# encoding: UTF-8
+#
+# = IDListPageRecord.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  # The IDListPageRecord class models the elements of the IDList. Each page
+  # holds up to a certain number of IDs that can be cached into a file if
+  # needed. Each page holds IDs within a given interval. The cache is managed
+  # by the IDListPageFile object.
+  class IDListPageRecord
+
+    attr_reader :min_id, :max_id, :page_idx
+    attr_accessor :page_entries
+
+    # Create a new IDListPageRecord object.
+    # @param page_file [IDListPageFile] The page file that manages the cache.
+    # @param min_id [Integer] The smallest ID that can be stored in this page
+    # @param max_id [Integer] the largest ID that can be stored in this page
+    # @param values [Array] An array of IDs to be stored in this page
+    def initialize(page_file, min_id, max_id, values = [])
+      @page_file = page_file
+      @min_id = min_id
+      @max_id = max_id
+      @page_entries = 0
+      @page_idx = @page_file.new_page(self, values)
+    end
+
+    # Check if the given ID is included in this page.
+    # @param id [Integer]
+    # @return [True of False] Return true if found, false otherwise.
+    def include?(id)
+      return false if id < @min_id || @max_id < id
+
+      page.include?(id)
+    end
+
+    # Check if the page is full and can't store any more IDs.
+    # @return [True or False]
+    def is_full?
+      page.is_full?
+    end
+
+    # Insert an ID into the page.
+    # @param ID [Integer] The ID to store
+    def insert(id)
+      unless @min_id <= id && id <= @max_id
+        raise ArgumentError, "IDs for this page must be between #{@min_id} " +
+          "and #{@max_id}. #{id} is outside this range."
+      end
+
+      page.insert(id)
+    end
+
+    # Split the current page. This split is done by splitting the ID range in
+    # half. This page will keep the first half, the newly created page will
+    # get the second half. This may not actually yield an empty page as all
+    # values could remain with one of the pages. In this case further splits
+    # need to be issued by the caller.
+    # @return [IDListPageRecord] A new IDListPageRecord object.
+    def split
+      # Determine the new max_id for the old page.
+      max_id = @min_id + (@max_id - @min_id) / 2
+      # Create a new page that stores the upper half of the ID range. Remove
+      # all IDs from this page that now belong into the new page and transfer
+      # them.
+      new_page_record = IDListPageRecord.new(@page_file, max_id + 1, @max_id,
+                                             page.delete(max_id))
+      # Adjust the max_id of the current page.
+      @max_id = max_id
+
+      new_page_record
+    end
+
+    def values
+      page.values
+    end
+
+    def <=>(pr)
+      @min_id <=> pr.min_id
+    end
+
+    def check
+      unless @min_id < @max_id
+        raise RuntimeError, "min_id must be smaller than max_id"
+      end
+
+      p = page
+      values = p.values
+      unless @page_entries == values.length
+        raise RuntimeError, "Mismatch between node page_entries " +
+          "(#{@page_entries}) and number of values (#{p.values.length})"
+      end
+
+      values.each do |v|
+        if v < @min_id
+          raise RuntimeError, "Page value #{v} is smaller than min_id " +
+            "#{@min_id}"
+        end
+        if v > @max_id
+          raise RuntimeError, "Page value #{v} is larger than max_id #{@max_id}"
+        end
+      end
+
+      p.check
+    end
+
+    private
+
+    def page
+      # The leaf pages reference the IDListPage objects only by their index.
+      # This method will convert the index into a reference to the actual
+      # object. These references should be very short-lived as a life
+      # reference prevents the page object from being collected.
+      @page_file.page(self)
+    end
+  end
+
+end