perobs 0.0.1 → 1.0.0

data/lib/perobs/ClassMap.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+#
+# = ClassMap.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/DataBase'
+
+module PEROBS
+
+  # PEROBS will usually store objects with a relatively small number of
+  # classes. Rather than storing the class name with each object, we map the
+  # class name to a numerical ID that represents the class in the store. This
+  # class handles the mapping and can convert class names into IDs and vice
+  # versa.
+  class ClassMap
+
+    # Create a ClassMap object for a given data base.
+    # @param db [DataBase]
+    def initialize(db)
+      @db = db
+      @by_class = {}
+      @by_id = []
+      read_map
+    end
+
+    # Get the ID for a given class.
+    # @param klass [String] Class
+    # @return [Fixnum] ID. If klass is not yet known a new ID will be
+    #         allocated.
+    def class_to_id(klass)
+      @by_class[klass] || new_id(klass)
+    end
+
+    # Get the klass for a given ID.
+    # @param id [Fixnum]
+    # @return [String] String version of the class
+    def id_to_class(id)
+      @by_id[id]
+    end
+
+    # Rename a set of classes to new names.
+    # @param rename_map [Hash] Hash that maps old names to new names
+    def rename(rename_map)
+      @by_id.each.with_index do |klass, id|
+        # Some entries can be nil. Ignore them.
+        next unless klass
+
+        if (new_name = rename_map[klass])
+          # We have a rename request. Update the current @by_id entry.
+          @by_id[id] = new_name
+          # Remove the old class name from @by_class hash.
+          @by_class.delete(klass)
+          # Insert the new one with the current ID.
+          @by_class[new_name] = id
+        end
+      end
+    end
+
+    # Delete all classes unless they are contained in _classes_.
+    # @param classes [Array of String] List of the class names
+    def keep(classes)
+      @by_id.each.with_index do |klass, id|
+        unless classes.include?(klass)
+          # Delete the class from the @by_id list by setting the entry to nil.
+          @by_id[id] = nil
+          # Delete the corresponding @by_class entry as well.
+          @by_class.delete(klass)
+        end
+      end
+    end
+
+    private
+
+    def new_id(klass)
+      # Find the first 'nil' entry and return the index.
+      idx = @by_id.find_index(nil) || @by_id.length
+
+      # Insert the new class/ID touple into the hash and reverse map.
+      @by_class[klass] = idx
+      @by_id[idx] = klass
+      # Write the updated version into the data base.
+      write_map
+
+      # Return the new ID.
+      idx
+    end
+
+    def read_map
+      # Get the hash from the data base
+      @by_class = @db.get_hash('class_map')
+      # Build the reverse map from the hash.
+      @by_class.each do |klass, id|
+        @by_id[id] = klass
+      end
+    end
+
+    def write_map
+      @db.put_hash('class_map', @by_class)
+    end
+
+  end
+
+end

data/lib/perobs/DataBase.rb

@@ -41,6 +41,7 @@ module PEROBS
 
     def initialize(serializer = :json)
       @serializer = serializer
+      @config = {}
     end
 
     # Serialize the given object using the object serializer.
@@ -83,8 +84,7 @@ module PEROBS
     end
 
     # Generate a new unique ID. It uses random numbers between 0 and 2**64 -
-    # 1. Deriving classes can overwrite this method. They must implement a
-    # include? method.
+    # 1.
     # @return [Fixnum or Bignum]
     def new_id
       begin
@@ -97,6 +97,25 @@ module PEROBS
       id
     end
 
+    # Check a config option and adjust it if needed.
+    # @param name [String] Name of the config option.
+    def check_option(name)
+      value = instance_variable_get('@' + name)
+
+      if @config.include?(name)
+        # The database already existed and has a setting for this config
+        # option. If it does not match the instance variable, adjust the
+        # instance variable accordingly.
+        unless @config[name] == value
+          instance_variable_set('@' + name, @config[name])
+        end
+      else
+        # There is no such config option yet. Create it with the value of the
+        # corresponding instance variable.
+        @config[name] = value
+      end
+    end
+
     private
 
     # Ensure that we have a directory to store the DB items.

data/lib/perobs/Hash.rb

@@ -163,6 +163,14 @@ module PEROBS
       @data = data
     end
 
+    # Textual dump for debugging purposes
+    # @return [String]
+    def inspect
+      "{\n" +
+      @data.map { |k, v| "  #{k.inspect}=>#{v.inspect}" }.join(",\n") +
+      "\n}\n"
+    end
+
     private
 
     def _serialize

data/lib/perobs/Object.rb

@@ -133,6 +133,17 @@ module PEROBS
       end
     end
 
+    # Textual dump for debugging purposes
+    # @return [String]
+    def inspect
+      "{\n" +
+      self.class.attributes.map do |attr|
+        ivar = ('@' + attr.to_s).to_sym
+        "  #{attr.inspect}=>#{instance_variable_get(ivar).inspect}"
+      end.join(",\n") +
+      "\n}\n"
+    end
+
     private
 
     # Return a single data structure that holds all persistent data for this

data/lib/perobs/ObjectBase.rb

@@ -25,6 +25,8 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'perobs/ClassMap'
+
 module PEROBS
 
   # This class is used to replace a direct reference to another Ruby object by
@@ -32,6 +34,13 @@ module PEROBS
   # since it's no longer referenced once it has been evicted from the
   # PEROBS::Store cache.
   class POReference < Struct.new(:id)
+
+    # Textual dump for debugging purposes
+    # @return [String]
+    def inspect
+      "@#{id}"
+    end
+
   end
 
   # Base class for all persistent objects. It provides the functionality
@@ -62,7 +71,7 @@ module PEROBS
       @_stash_map = nil
 
       db_obj = {
-        'class' => self.class.to_s,
+        'class_id' => @store.class_map.class_to_id(self.class.to_s),
         'data' => _serialize
       }
       @store.db.put_object(db_obj, @_id)
@@ -74,8 +83,9 @@ module PEROBS
       # Read the object from database.
       db_obj = store.db.get_object(id)
 
+      klass = store.class_map.id_to_class(db_obj['class_id'])
       # Call the constructor of the specified class.
-      obj = Object.const_get(db_obj['class']).new(store)
+      obj = Object.const_get(klass).new(store)
       # The object gets created with a new ID by default. We need to restore
       # the old one.
       obj._change_id(id)

data/lib/perobs/Store.rb

@@ -26,8 +26,8 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'perobs/Cache'
-require 'perobs/FileSystemDB'
-require 'perobs/HashedBlocksDB'
+require 'perobs/ClassMap'
+require 'perobs/BTreeDB'
 require 'perobs/Object'
 require 'perobs/Hash'
 require 'perobs/Array'
@@ -49,16 +49,16 @@ module PEROBS
   # objects are all going to persistent objects again.
   class Store
 
-    attr_reader :db, :cache
+    attr_reader :db, :cache, :class_map
 
     # Create a new Store.
     # @param data_base [String] the name of the database
     # @param options [Hash] various options to affect the operation of the
     #        database. Currently the following options are supported:
     #        :engine     : The class that provides the back-end storage
-    #                      engine. By default HashedBlocksDB is used. A user
+    #                      engine. By default BTreeDB is used. A user
     #                      can provide it's own storage engine that must
-    #                      conform to the same API exposed by HashedBlocksDB.
+    #                      conform to the same API exposed by BTreeBlobsDB.
     #        :cache_bits : the number of bits used for cache indexing. The
     #                      cache will hold 2 to the power of bits number of
     #                      objects. We have separate caches for reading and
@@ -81,7 +81,10 @@ module PEROBS
     #                      Unfortunately, it is 10x slower than marshal.
     def initialize(data_base, options = {})
       # Create a backing store handler
-      @db = (options[:engine] || HashedBlocksDB).new(data_base, options)
+      @db = (options[:engine] || BTreeDB).new(data_base, options)
+      # Create a map that can translate classes to numerical IDs and vice
+      # versa.
+      @class_map = ClassMap.new(@db)
 
       # The Cache reduces read and write latencies by keeping a subset of the
       # objects in memory.
@@ -192,6 +195,9 @@ module PEROBS
     # @param repair [TrueClass/FalseClass] true if a repair attempt should be
     #        made.
     def check(repair = true)
+      # Run basic consistency checks first.
+      @db.check_db(repair)
+
       @db.clear_marks
       # A buffer to hold a working set of object IDs.
       stack = []
@@ -269,12 +275,20 @@ module PEROBS
       end
     end
 
+    # Rename classes of objects stored in the data base.
+    # @param rename_map [Hash] Hash that maps the old name to the new name
+    def rename_classes(rename_map)
+      @class_map.rename(rename_map)
+    end
+
     private
 
     # Mark phase of a mark-and-sweep garbage collector. It will mark all
     # objects that are reachable from the root objects.
     def mark
-      each
+      classes = Set.new
+      each { |obj| classes.add(obj.class) }
+      @class_map.keep(classes.map { |c| c.to_s })
     end
 
     # Sweep phase of a mark-and-sweep garbage collector. It will remove all

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "0.0.1"
+  VERSION = "1.0.0"
 end

data/spec/{FileSystemDB_spec.rb → BTreeDB_spec.rb}

@@ -28,12 +28,12 @@ $:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 require 'fileutils'
 require 'time'
 
-require 'perobs/FileSystemDB'
+require 'perobs/BTreeDB'
 
-class TempStruct < Struct.new(:first, :second, :third)
-end
+describe PEROBS::BTreeDB do
 
-describe PEROBS::FileSystemDB do
+  class UStruct < Struct.new(:first, :second, :third)
+  end
 
   before(:all) do
     FileUtils.rm_rf('fs_test')
@@ -44,12 +44,20 @@ describe PEROBS::FileSystemDB do
   end
 
   it 'should create database' do
-    @db = PEROBS::FileSystemDB.new('fs_test')
+    @db = PEROBS::BTreeDB.new('fs_test')
     Dir.exists?('fs_test').should be_true
   end
 
+  it 'should write and read a simple Hash' do
+    @db = PEROBS::BTreeDB.new('fs_test')
+    @db.get_hash('test').should == {}
+    h = { 'A' => 1, 'B' => 2, 'D' => 4 }
+    @db.put_hash('test', h)
+    @db.get_hash('test').should == h
+  end
+
   it 'should support object insertion and retrieval' do
-    @db = PEROBS::FileSystemDB.new('fs_test')
+    @db = PEROBS::BTreeDB.new('fs_test')
     @db.include?(0).should be_false
     h = {
           'String' => 'What god has wrought',
@@ -68,7 +76,7 @@ describe PEROBS::FileSystemDB do
 
   it 'should support most Ruby objects types' do
     [ :marshal, :yaml ].each do |serializer|
-      @db = PEROBS::FileSystemDB.new('fs_test', { :serializer => serializer })
+      @db = PEROBS::BTreeDB.new('fs_test', { :serializer => serializer })
       @db.include?(0).should be_false
       h = {
         'String' => 'What god has wrought',
@@ -79,17 +87,53 @@ describe PEROBS::FileSystemDB do
         'nil' => nil,
         'Array' => [ 0, 1, 2, 3 ],
         'Time' => Time.parse('2015-05-14-13:52:17'),
-        'Struct' => TempStruct.new("Where's", 'your', 'towel?')
+        'Struct' => UStruct.new("Where's", 'your', 'towel?')
       }
       @db.put_object(h, 0)
       @db.include?(0).should be_true
       @db.check(0, false).should be_true
       @db.get_object(0).should == h
+      FileUtils.rm_rf('fs_test')
+    end
+  end
+
+  it 'should put and get multiple objects in same dir' do
+    @db = PEROBS::BTreeDB.new('fs_test')
+    0.upto(10) do |i|
+      @db.put_object({ "foo #{i}" => i }, i)
     end
+    0.upto(10) do |i|
+      @db.get_object(i)["foo #{i}"].should == i
+    end
+  end
+
+  it 'should handle deleted objects propery' do
+    @db = PEROBS::BTreeDB.new('fs_test')
+    @db.put_object({ 'a' => 'a' * 257 }, 0)
+    @db.put_object({ 'b' => 'b' * 513 }, 1)
+    @db.put_object({ 'c' => 'c' * 129 }, 2)
+    @db.put_object({ 'd' => 'd' * 1025 }, 3)
+    # Delete some objects
+    @db.clear_marks
+    @db.mark(0)
+    @db.mark(2)
+    @db.delete_unmarked_objects
+
+    @db.put_object({ 'A' => 'a' * 257 }, 4)
+    @db.put_object({ 'B' => 'b' * 513 }, 5)
+    @db.put_object({ 'C' => 'c' * 129 }, 6)
+    @db.put_object({ 'D' => 'd' * 1025 }, 7)
+
+    @db.get_object(0).should == { 'a' => 'a' * 257 }
+    @db.get_object(2).should == { 'c' => 'c' * 129 }
+    @db.get_object(4).should == { 'A' => 'a' * 257 }
+    @db.get_object(5).should == { 'B' => 'b' * 513 }
+    @db.get_object(6).should == { 'C' => 'c' * 129 }
+    @db.get_object(7).should == { 'D' => 'd' * 1025 }
   end
 
   it 'should mark objects and detect markings' do
-    @db = PEROBS::FileSystemDB.new('fs_test')
+    @db = PEROBS::BTreeDB.new('fs_test')
     h = { 'a' => 'z' }
     @db.put_object(h, 1)
     @db.put_object(h, 2)

data/spec/ClassMap_spec.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'perobs/ClassMap'
+require 'perobs/BTreeDB'
+
+describe PEROBS::ClassMap do
+
+  before(:all) do
+    FileUtils.rm_rf('fs_test')
+    @db = PEROBS::BTreeDB.new('fs_test')
+    @map = PEROBS::ClassMap.new(@db)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf('fs_test')
+  end
+
+  it 'should return nil for an unknown ID' do
+    @map.id_to_class(0).should be_nil
+  end
+
+  it 'should add a class' do
+    @map.class_to_id('Foo').should == 0
+  end
+
+  it 'should find the class again' do
+    @map.id_to_class(0).should == 'Foo'
+  end
+
+  it 'should still return nil for an unknown ID' do
+    @map.id_to_class(1).should be_nil
+  end
+
+  it 'should forget classes not in keep list' do
+    @map.class_to_id('Bar').should == 1
+    @map.class_to_id('Foobar').should == 2
+    @map.keep([ 'Bar' ])
+    @map.id_to_class(0).should be_nil
+    @map.id_to_class(1).should == 'Bar'
+    @map.id_to_class(2).should be_nil
+    @map.class_to_id('Foo1').should == 0
+    @map.class_to_id('Foo2').should == 2
+  end
+
+end

data/spec/Store_spec.rb

@@ -45,6 +45,19 @@ class Person < PEROBS::Object
 
 end
 
+class PersonN < PEROBS::Object
+
+  po_attr :name, :zip, :bmi, :married, :related, :relatives
+
+  def initialize(store)
+    super
+    init_attr(:name, '')
+    init_attr(:bmi, 22.2)
+    init_attr(:married, false)
+  end
+
+end
+
 describe PEROBS::Store do
 
   before(:all) do
@@ -96,7 +109,7 @@ describe PEROBS::Store do
 
       @store.sync
 
-      @store = PEROBS::Store.new('test_db', { :serializer => serializer })
+      @store = PEROBS::Store.new('test_db')
       john = @store['john']
       john.name.should == 'John'
       john.zip.should == 4060
@@ -127,6 +140,34 @@ describe PEROBS::Store do
     end
   end
 
+  it 'should support renaming of classes' do
+    @store = PEROBS::Store.new('test_db')
+    @store['john'] = john = Person.new(@store)
+    john.name = 'John'
+    john.zip = 4060
+    john.bmi = 25.5
+    @store['jane'] = jane = Person.new(@store)
+    jane.name = 'Jane'
+    jane.related = john
+    jane.married = true
+    jane.relatives = 'test'
+
+    @store.sync
+
+    @store = PEROBS::Store.new('test_db')
+    @store.rename_classes({ 'Person' => 'PersonN' })
+    john = @store['john']
+    john.name.should == 'John'
+    john.zip.should == 4060
+    john.bmi.should == 25.5
+    john.married.should be_false
+    john.related.should be_nil
+    jane = @store['jane']
+    jane.name.should == 'Jane'
+    jane.related.should == john
+    jane.married.should be_true
+  end
+
   it 'should detect modification to non-working objects' do
     @store = PEROBS::Store.new('test_db', :cache_bits => 3)
     0.upto(20) do |i|
@@ -346,7 +387,7 @@ describe PEROBS::Store do
   end
 
   it 'should survive a real world usage test' do
-    options = { :engine => PEROBS::HashedBlocksDB, :dir_nibbles => 1 }
+    options = { :engine => PEROBS::BTreeDB, :dir_bits => 4 }
     @store = PEROBS::Store.new('test_db', options)
     ref = {}
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
+date: 2015-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,19 +66,20 @@ files:
 - Rakefile
 - lib/perobs.rb
 - lib/perobs/Array.rb
-- lib/perobs/BlockDB.rb
+- lib/perobs/BTreeBlob.rb
+- lib/perobs/BTreeDB.rb
 - lib/perobs/Cache.rb
+- lib/perobs/ClassMap.rb
 - lib/perobs/DataBase.rb
-- lib/perobs/FileSystemDB.rb
 - lib/perobs/Hash.rb
-- lib/perobs/HashedBlocksDB.rb
 - lib/perobs/Object.rb
 - lib/perobs/ObjectBase.rb
 - lib/perobs/Store.rb
 - lib/perobs/version.rb
 - perobs.gemspec
 - spec/Array_spec.rb
-- spec/FileSystemDB_spec.rb
+- spec/BTreeDB_spec.rb
+- spec/ClassMap_spec.rb
 - spec/Hash_spec.rb
 - spec/Object_spec.rb
 - spec/Store_spec.rb
@@ -113,7 +114,8 @@ specification_version: 4
 summary: Persistent Ruby Object Store
 test_files:
 - spec/Array_spec.rb
-- spec/FileSystemDB_spec.rb
+- spec/BTreeDB_spec.rb
+- spec/ClassMap_spec.rb
 - spec/Hash_spec.rb
 - spec/Object_spec.rb
 - spec/Store_spec.rb