rod 0.6.3 → 0.7.0

data/.travis.yml

@@ -1 +1,3 @@
 rvm: 1.9.2
+before_script:
+  - "apt-get install libdb4.6-dev"

data/README.rdoc

@@ -2,6 +2,12 @@
 
 * http://github.com/apohllo/rod
 
+== WARNING
+
+The 0.7.x branch is a development branch -- incompatibilities between library
+releases might be introduced (both in API and data schema).
+You are advised to use the latest release of 0.6.x branch.
+
 == DESCRIPTION
 
 ROD (Ruby Object Database) is library which aims at providing
@@ -13,11 +19,11 @@ fast access for data, which rarely changes.
 * Ruby-to-C on-the-fly translation based on mmap and RubyInline
 * optimized for (reading) speed
 * weak reference collections for easy memory reclaims
-* segmented indices for short start-up times
+* Berkeley DB hash index for the best index performance
 * compatibility check of library version
 * compatibility check of data model
 * autogeneration of model (based on the database metadata)
-* automatic model migrations (addition/removal of properties so far)
+* automatic model migrations (limitied to addition/removal of properties and indexes)
 * full update of the database (removal of objects not available yet)
 * databases interlinking (via direct links or inverted indices)
 
@@ -25,7 +31,6 @@ fast access for data, which rarely changes.
 
 * tested mostly on 64-bit systems
 * doesn't work on Windows
-* some space is wasted when database is re-opended in read/write mode
 
 == SYNOPSIS:
 
@@ -52,13 +57,28 @@ number of disk reads was designed. The Ruby interface facilitates it's usage.
 * english
 * ActiveModel
 * bsearch
+* Berkeley DB
 
 == INSTALL
 
-Grab from rubygems:
+1. Install Berkeley DB
+
+  http://www.oracle.com/technetwork/database/berkeleydb/downloads/index.html
+
+2. Install rod gem from rubygems:
 
   gem install rod
 
+== TROUBLESHOOTING
+
+If you get the following error during library usage:
+
+  error: db.h: No such file or directory
+
+then you don't have Berkeley DB installed or its header fiels are not available
+on the default path. Make sure that the library is installed and the headers
+are available.
+
 == BASIC USAGE:
 
  class MyDatabase < Rod::Database
@@ -70,7 +90,7 @@ Grab from rubygems:
 
  class User < Model
    field :name, :string
-   field :surname, :string, :index => :flat
+   field :surname, :string, :index => :hash
    field :age, :integer
    has_one :account
    has_many :files
@@ -78,12 +98,12 @@ Grab from rubygems:
 
  class Account < Model
    field :email, :string
-   field :login, :string, :index => :flat
+   field :login, :string, :index => :hash
    field :password, :string
  end
 
  class File < Model
-   field :title, :string, :index => :flat
+   field :title, :string, :index => :hash
    field :data, :string
  end
 

data/changelog.txt

@@ -1,3 +1,9 @@
+0.7.0
+- #142 migrate indices during model migration
+- #134 Berkeley DB Hash for indexing properties
+- #137 move #special_class? to AbstractDatabase
+- #138 move development_mode accessor to AbstractDatabase
+- #136 use (DB) #opened? instead of @handler.nil?
 0.6.3
 - #94 indexing of unstored object in plural assoc
 - #128 incompatible schema message is more specific

data/contributors.txt

@@ -1,2 +1,3 @@
 People who contributed to Ruby Object Database:
-* qadro
+* Piotr Gurgul
+* Marcin Sieniek

data/features/hash_indexing.feature

@@ -0,0 +1,172 @@
+Feature: Access to objects with hash indices.
+    ROD allows for accessing objects via fields with hash indices,
+    which are useful for indices with millions of keys.
+    These are split accross multiple files for faster load-time.
+  Background:
+    Given the library works in development mode
+
+  Scenario: indexing with hash index
+      Rod should allow to access objects via values of their fields,
+      for which indices were built.
+    Given the class space is cleared
+    And the model is connected with the default database
+    And a class Caveman has a name field of type string with hash index
+    And a class Caveman has an age field of type integer with hash index
+    And a class Caveman has an identifier field of type ulong with hash index
+    And a class Caveman has a height field of type float with hash index
+    When database is created
+    And I create a Caveman
+    And his name is 'Fred'
+    And his age is '25'
+    And his identifier is '111122223333'
+    And his height is '1.86'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Barney'
+    And his age is '26'
+    And his identifier is '111122224444'
+    And his height is '1.67'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Wilma'
+    And his age is '25'
+    And his identifier is '111122225555'
+    And his height is '1.67'
+    And I store her in the database
+    And I reopen database for reading
+    Then there should be 3 Caveman(s)
+    And there should be 1 Caveman with 'Fred' name
+    And there should be 1 Caveman with 'Wilma' name
+    And there should be 1 Caveman with 'Barney' name
+    And there should be 2 Caveman(s) with '25' age
+    And there should be 1 Caveman with '26' age
+    And there should be 1 Caveman with '111122223333' identifier
+    And there should be 1 Caveman with '111122224444' identifier
+    And there should be 1 Caveman with '111122225555' identifier
+    And there should be 2 Caveman(s) with '1.67' height
+    And there should be 1 Caveman with '1.86' height
+
+    # Test re-creation of the database
+    When database is created
+    And I create a Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Barney'
+    And I store him in the database
+    And I create another Caveman
+    And her name is 'Wilma'
+    And I store her in the database
+    And I reopen database for reading
+    Then there should be 3 Caveman(s)
+    And there should be 1 Caveman with 'Fred' name
+    And there should be 1 Caveman with 'Wilma' name
+    And there should be 1 Caveman with 'Barney' name
+
+  Scenario: extending the DB when hash index is used
+      Rod should allow to extend the DB when the hash index is used.
+      The index should be properly updated.
+    Given the class space is cleared
+    And the model is connected with the default database
+    And a class Caveman has a name field of type string with hash index
+    When database is created
+    And I create a Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Barney'
+    And I store him in the database
+    And I reopen database
+    And I create another Caveman
+    And her name is 'Wilma'
+    And I store her in the database
+    And I create another Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I reopen database for reading
+    Then there should be 4 Caveman(s)
+    And there should be 1 Caveman with 'Wilma' name
+    And there should be 2 Caveman(s) with 'Fred' name
+    And there should be 1 Caveman with 'Barney' name
+
+  Scenario: indexing of fields with different DBs for the same model with hash index
+    The contents of indices should be fulshed when the database is reopened.
+    Given the class space is cleared
+    And the model is connected with the default database
+    And a class Caveman has a name field of type string with hash index
+    When database is created
+    And I create a Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Fred'
+    And I store him in the database
+    And I reopen database for reading
+    And I access the Caveman name index
+    And database is created in location2
+    And I create a Caveman
+    And his name is 'Wilma'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Wilma'
+    And I store him in the database
+    And I create another Caveman
+    And his name is 'Wilma'
+    And I store him in the database
+    And I reopen database for reading in location2
+    Then there should be 3 Caveman(s)
+    And there should be 3 Caveman(s) with 'Wilma' name
+    And there should be 0 Caveman(s) with 'Fred' name
+
+  Scenario: indexing of particular values with hash index
+    Given the class space is cleared
+    And the model is connected with the default database
+    And a class Caveman has a name field of type string with hash index
+    And a class Caveman has a surname field of type string with hash index
+    And a class Caveman has a login field of type string with hash index
+    And a class Caveman has an age field of type integer with hash index
+    When database is created
+    And I create and store the following Caveman(s):
+      | name | surname | login | age  |
+      | John | Smith   | john  | 12   |
+      | Lara | Croft   | lara  | 23   |
+      | Adam | Parker  | adam  | 12   |
+      | Adam |         | noob1 | 33   |
+      |      |         | noob2 | -1   |
+      |      | Adam    | noob1 | 33   |
+    And I reopen database for reading
+    Then there should be 6 Caveman(s)
+    And there should be 1 Caveman with 'John' name
+    And there should be 2 Caveman(s) with 'Adam' name
+    And there should be 2 Caveman(s) with '12' age
+    And there should be 1 Caveman with '-1' age
+    And there should be 2 Caveman(s) with '' name
+    And there should be 2 Caveman(s) with '' surname
+
+  Scenario: multiple object with indexed fields with hash index
+      The database should properly store thausands of objects with some indexed fields.
+    Given the class space is cleared
+    And the model is connected with the default database
+    And a class User has a name field of type string with hash index
+    And a class User has a surname field of type string with hash index
+    And a class User has an age field of type integer
+    When database is created
+    And I create a User
+    And his name is 'John'
+    And his surname is 'Smith'
+    And his age is '21'
+    And I store him in the database 1000 times
+    And I create a User
+    And her name is 'Lara'
+    And her surname is 'Croft'
+    And her age is '23'
+    And I store her in the database 1000 times
+    And I reopen database for reading
+    Then there should be 2000 User(s)
+    Then there should be 1000 User(s) with 'John' name
+    Then there should be 1000 User(s) with 'Smith' surname
+    Then there should be 1000 User(s) with 'Lara' name
+    Then there should be 1000 User(s) with 'Croft' surname

data/features/steps/model.rb

@@ -93,8 +93,8 @@ Given /^a class (\w+) inherits from ([\w:]+)$/ do |name1,name2|
 end
 
 Given /^a class (\w+) has an? (\w+) field of type (\w+)( with (\w+) index)?$/ do |class_name,field,type,index,index_type|
-  index_type = index_type == "flat" ? :flat : :segmented
   if index
+    index_type = index_type.to_sym
     get_class(class_name).send(:field,field.to_sym,type.to_sym,:index => index_type)
   else
     get_class(class_name).send(:field,field.to_sym,type.to_sym)
@@ -110,7 +110,7 @@ Given /^a class (\w+) has one (\w+ )?(\w+)( with (\w+) index)?$/ do |class_name,
     end
   end
   unless index.nil?
-    index_type = (index_type == "flat" ? :flat : :segmented)
+    index_type = index_type.to_sym
     options[:index] = index_type
   end
   get_class(class_name).send(:has_one,assoc.to_sym,options)
@@ -125,7 +125,7 @@ Given /^a class (\w+) has many (\w+ )?(\w+)( with (\w+) index)?$/ do |class_name
     end
   end
   unless index.nil?
-    index_type = (index_type == "flat" ? :flat : :segmented)
+    index_type = index_type.to_sym
     options[:index] = index_type
   end
   get_class(class_name).send(:has_many,assoc.to_sym,options)

data/lib/rod/abstract_database.rb

@@ -21,12 +21,27 @@ module Rod
     # The path which the database instance is located on.
     attr_reader :path
 
+    # This flag indicates, if Database and Model works in development
+    # mode, i.e. the dynamically loaded library has a unique, different id each time
+    # the rod library is used.
+    @@rod_development_mode = false
+
     # Initializes the classes linked with this database and the handler.
     def initialize
       @classes ||= self.special_classes
       @handler = nil
     end
 
+    # Writer of the +rod_development_mode+ flag.
+    def self.development_mode=(value)
+      @@rod_development_mode = value
+    end
+
+    # Reader of the +rod_development_mode+ flag.
+    def self.development_mode
+      @@rod_development_mode
+    end
+
     #########################################################################
     # Public API
     #########################################################################
@@ -51,7 +66,7 @@ module Rod
     #
     # WARNING: all files in the DB directory are removed during DB creation!
     def create_database(path)
-      raise DatabaseError.new("Database already opened.") unless @handler.nil?
+      raise DatabaseError.new("Database already opened.") if opened?
       @readonly = false
       @path = canonicalize_path(path)
       if File.exist?(@path)
@@ -87,25 +102,14 @@ module Rod
     #   the classes from the database metadata. If module given, the classes
     #   are generated withing the module.
     def open_database(path,options={:readonly => true})
-      raise DatabaseError.new("Database already opened.") unless @handler.nil?
+      raise DatabaseError.new("Database already opened.") if opened?
       options = convert_options(options)
       @readonly = options[:readonly]
       @path = canonicalize_path(path)
-      @metadata = {}
-      File.open(@path + DATABASE_FILE) do |input|
-        @metadata = YAML::load(input)
-      end
-      unless valid_version?(@metadata["Rod"][:version])
-        raise IncompatibleVersion.new("Incompatible versions - library #{VERSION} vs. " +
-                                      "file #{metatdata["Rod"][:version]}")
-      end
+      @metadata = load_metadata
       if options[:generate]
         module_instance = (options[:generate] == true ? Object : options[:generate])
         generate_classes(module_instance)
-      elsif options[:migrate]
-        create_legacy_classes
-        FileUtils.cp(@path + DATABASE_FILE, @path + DATABASE_FILE + LEGACY_DATA_SUFFIX)
-        remove_files(self.inline_library)
       end
       self.classes.each do |klass|
         klass.send(:build_structure)
@@ -137,13 +141,6 @@ module Rod
           raise DatabaseError.new("Size of data file of #{klass} is invalid: #{file_size}")
         end
         set_page_count(klass,file_size / _page_size)
-        if options[:migrate]
-          next unless klass.name =~ LEGACY_RE
-          new_class = klass.name.sub(LEGACY_RE,"").constantize
-          set_count(new_class,meta[:count])
-          pages = (meta[:count] * new_class.struct_size / _page_size.to_f).ceil
-          set_page_count(new_class,pages)
-        end
       end
       if metadata_copy.size > 0
         @handler = nil
@@ -151,34 +148,57 @@ module Rod
                                 metadata_copy.keys.join("\n - "))
       end
       _open(@handler)
-      if options[:migrate]
-        empty_data = "\0" * _page_size
-        self.classes.each do |klass|
-          next unless klass.to_s =~ LEGACY_RE
-          new_class = klass.name.sub(LEGACY_RE,"").constantize
-          old_metadata = klass.metadata
-          old_metadata.merge!({:superclass => old_metadata[:superclass].sub(LEGACY_RE,"")})
-          unless new_class.compatible?(old_metadata)
-            File.open(new_class.path_for_data(@path),"w") do |out|
-              send("_#{new_class.struct_name}_page_count",@handler).
-                times{|i| out.print(empty_data)}
-            end
-            klass.migrate
-            current_file_name = klass.path_for_data(@path)
-            legacy_file_name = current_file_name + LEGACY_DATA_SUFFIX
-            new_file_name = new_class.path_for_data(@path)
-            FileUtils.mv(current_file_name,legacy_file_name)
-            FileUtils.mv(new_file_name,current_file_name)
-          end
-          @classes.delete(klass)
-          new_class.model_path = nil
+    end
+
+    # Migrates the database, which is located at +path+. The
+    # old version of the DB is placed at +path+/backup.
+    def migrate_database(path)
+      raise DatabaseError.new("Database already opened.") if opened?
+      @readonly = false
+      @path = canonicalize_path(path)
+      @metadata = load_metadata
+      create_legacy_classes
+      FileUtils.mkdir_p(@path + BACKUP_PREFIX)
+      Dir.glob(@path + "*").each do |file|
+        # Don't move the directory itself and speciall classes data.
+        unless file.to_s == @path + BACKUP_PREFIX[0..-2] ||
+          special_classes.map{|c| c.path_for_data(@path)}.include?(file.to_s)
+          puts "Moving #{file} to #{@path + BACKUP_PREFIX}" if $ROD_DEBUG
+          FileUtils.mv(file,@path + BACKUP_PREFIX)
         end
-        close_database(false,true)
-        options.delete(:migrate)
-        readonly = options.delete(:old_readonly)
-        options[:readonly] = readonly
-        open_database(path,options)
       end
+      remove_files(self.inline_library)
+      self.classes.each do |klass|
+        klass.send(:build_structure)
+      end
+      generate_c_code(@path, self.classes)
+      @handler = _init_handler(@path)
+      self.classes.each do |klass|
+        next unless special_class?(klass) or legacy_class?(klass)
+        meta = @metadata[klass.name]
+        set_count(klass,meta[:count])
+        file_size = File.new(klass.path_for_data(@path)).size
+        unless file_size % _page_size == 0
+          raise DatabaseError.new("Size of data file of #{klass} is invalid: #{file_size}")
+        end
+        set_page_count(klass,file_size / _page_size)
+        next unless legacy_class?(klass)
+        new_class = klass.name.sub(LEGACY_RE,"").constantize
+        set_count(new_class,meta[:count])
+        pages = (meta[:count] * new_class.struct_size / _page_size.to_f).ceil
+        set_page_count(new_class,pages)
+      end
+      _open(@handler)
+      self.classes.each do |klass|
+        next unless legacy_class?(klass)
+        klass.migrate
+        @classes.delete(klass)
+      end
+      path_with_date = @path + BACKUP_PREFIX[0..-2] + "_" +
+        Time.new.strftime("%Y_%m_%d_%H_%M_%S") + "/"
+      puts "Moving #{@path + BACKUP_PREFIX} to #{path_with_date}" if $ROD_DEBUG
+      FileUtils.mv(@path + BACKUP_PREFIX,path_with_date)
+      close_database
     end
 
     # Closes the database.
@@ -189,7 +209,7 @@ module Rod
     #
     # If the +skip_indeces+ flat is set to true, the indices are not written.
     def close_database(purge_classes=false,skip_indices=false)
-      raise DatabaseError.new("Database not opened.") if @handler.nil?
+      raise DatabaseError.new("Database not opened.") unless opened?
 
       unless readonly_data?
         unless referenced_objects.select{|k, v| not v.empty?}.size == 0
@@ -228,6 +248,18 @@ module Rod
       @referenced_objects ||= {}
     end
 
+    # Returns true if the class is one of speciall classes
+    # (JoinElement, PolymorphicJoinElement, StringElement).
+    def special_class?(klass)
+      self.special_classes.include?(klass)
+    end
+
+    # Returns true if the +klass+ is a legacy class, i.e.
+    # a class generated during migration used to access the legacy
+    # data.
+    def legacy_class?(klass)
+      klass.name =~ LEGACY_RE
+    end
 
     # Adds the +klass+ to the set of classes linked with this database.
     def add_class(klass)
@@ -354,10 +386,11 @@ module Rod
       end
     end
 
+
     # Prints the layout of the pages in memory and other
     # internal data of the model.
     def print_layout
-      raise DatabaseError.new("Database not opened.") if @handler.nil?
+      raise DatabaseError.new("Database not opened.") unless opened?
       _print_layout(@handler)
     end
 
@@ -368,6 +401,21 @@ module Rod
 
     protected
 
+    # Returns the metadata loaded from the database's metadata file.
+    # Raises exception if the version of library and database are
+    # not compatible.
+    def load_metadata
+      metadata = {}
+      File.open(@path + DATABASE_FILE) do |input|
+        metadata = YAML::load(input)
+      end
+      unless valid_version?(metadata["Rod"][:version])
+        raise IncompatibleVersion.new("Incompatible versions - library #{VERSION} vs. " +
+                                      "file #{metatdata["Rod"][:version]}")
+      end
+      metadata
+    end
+
     # Checks if the version of the library is valid.
     # Consult https://github.com/apohllo/rod/wiki for versioning scheme.
     def valid_version?(version)
@@ -404,10 +452,6 @@ module Rod
         result[:readonly] = options
       when Hash
         result = options
-        if options[:migrate]
-          result[:old_readonly] = options[:readonly]
-          result[:readonly] = false
-        end
       else
         raise RodException.new("Invalid options for open_database: #{options}!")
       end
@@ -475,8 +519,7 @@ module Rod
     end
 
     # During migration it creats the classes which are used to read
-    # the legacy data. It also changes the path for the
-    # actual classes not to conflict with paths of legacy data.
+    # the legacy data.
     def create_legacy_classes
       legacy_module = nil
       begin
@@ -485,11 +528,11 @@ module Rod
         legacy_module = Module.new
         Object.const_set(LEGACY_MODULE,legacy_module)
       end
+      generate_classes(legacy_module)
       self.classes.each do |klass|
-        next if special_class?(klass)
-        klass.model_path = Model.struct_name_for(klass.name) + NEW_DATA_SUFFIX
+        next unless legacy_class?(klass)
+        klass.model_path = BACKUP_PREFIX + klass.model_path
       end
-      generate_classes(legacy_module)
     end
 
 

data/lib/rod/constants.rb

@@ -1,5 +1,5 @@
 module Rod
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 
   # The name of file containing the data base.
   DATABASE_FILE = "database.yml"
@@ -35,5 +35,6 @@ module Rod
   NEW_DATA_SUFFIX = ".new"
   LEGACY_MODULE = "Legacy"
   LEGACY_RE = /^#{LEGACY_MODULE}::/
+  BACKUP_PREFIX = "backup/"
 
 end

data/lib/rod/database.rb

@@ -12,21 +12,6 @@ module Rod
   # models simultaneously. This is due to the way RubyInline creates and
   # names (after the name of the class) the C code.
   class Database < AbstractDatabase
-    # This flag indicates, if Database and Model works in development
-    # mode, i.e. the dynamically loaded library has a unique, different id each time
-    # the rod library is used.
-    @@rod_development_mode = false
-
-    # Writer of the +rod_development_mode+ flag.
-    def self.development_mode=(value)
-      @@rod_development_mode = value
-    end
-
-    # Reader of the +rod_development_mode+ flag.
-    def self.development_mode
-      @@rod_development_mode
-    end
-
     protected
 
     ## Helper methods printing some generated code ##
@@ -145,12 +130,6 @@ module Rod
       str.margin
     end
 
-    # Returns true if the class is one of speciall classes
-    # (JoinElement, PolymorphicJoinElement, StringElement).
-    def special_class?(klass)
-      self.special_classes.include?(klass)
-    end
-
     #########################################################################
     # Implementations of abstract methods
     #########################################################################
@@ -167,6 +146,15 @@ module Rod
       @inline_library
     end
 
+    # Allocates the space for the +klass+ in the data file.
+    def allocate_space(klass)
+      empty_data = "\0" * _page_size
+      File.open(klass.path_for_data(@path),"w") do |out|
+        send("_#{klass.struct_name}_page_count",@handler).
+          times{|i| out.print(empty_data)}
+      end
+    end
+
 
     # Generates the code C responsible for management of the database.
     def generate_c_code(path, classes)
@@ -194,7 +182,6 @@ module Rod
 
           builder.prefix(self.class.rod_exception)
 
-
           #########################################
           # Model struct
           #########################################

data/lib/rod/exception.rb

@@ -41,6 +41,10 @@ module Rod
     end
   end
 
+  # This exception is raised, when the key is not present in the database.
+  class KeyMissing < DatabaseError
+  end
+
   # This exception is raised if argument for some Rod API call
   # (such as field, has_one, has_many) is invalid.
   class InvalidArgument < RodException

data/lib/rod/index/base.rb

@@ -87,6 +87,8 @@ module Rod
             FlatIndex.new(path,klass,options)
           when :segmented
             SegmentedIndex.new(path,klass,options)
+          when :hash
+            HashIndex.new(path,klass,options)
           else
             raise RodException.new("Invalid index type #{type}")
           end

data/lib/rod/index/flat_index.rb

@@ -18,6 +18,7 @@ module Rod
       def save
         File.open(@path,"w") do |out|
           proxy_index = {}
+          #load_index unless loaded?
           @index.each{|k,col| proxy_index[k] = [col.offset,col.size]}
           out.puts(Marshal.dump(proxy_index))
         end

data/lib/rod/index/hash_index.rb

@@ -0,0 +1,266 @@
+# encoding: utf-8
+require 'rod/index/base'
+
+module Rod
+  module Index
+    # This implementation of index is based on the
+    # Berkeley DB Hash access method.
+    class HashIndex < Base
+      # Wrapper class for the database struct.
+      class Handle
+      end
+
+      # Initializes the index with +path+ and +class+.
+      # Options are not (yet) used.
+      def initialize(path,klass,options={})
+        @path = path + ".db"
+        @klass = klass
+        _open(@path,:create => true)
+        @index = {}
+      end
+
+      # Stores the index on disk.
+      def save
+        @index.each do |key,collection|
+          key = Marshal.dump(key)
+          _put(key,collection.offset,collection.size)
+        end
+        _close()
+      end
+
+      # Clears the contents of the index.
+      def destroy
+        _close()
+        _open(@path,:truncate => true)
+      end
+
+      # Simple iterator.
+      def each
+        if block_given?
+          @index.each do |key,value|
+            yield key,value
+          end
+          _each do |key,value|
+            key = Marshal.load(key)
+            unless @index[key]
+              yield key,self[key]
+            end
+          end
+        else
+          enum_for(:each)
+        end
+      end
+
+      protected
+      def get(key)
+        return @index[key] if @index.has_key?(key)
+        begin
+          value = _get(Marshal.dump(key))
+        rescue Rod::KeyMissing => ex
+          value = nil
+        end
+        @index[key] = value
+      end
+
+      def set(key,value)
+        @index[key] = value
+      end
+
+
+      def self.rod_exception
+        str =<<-END
+        |VALUE rodException(){
+        |  VALUE klass = rb_const_get(rb_cObject, rb_intern("Rod"));
+        |  klass = rb_const_get(klass, rb_intern("DatabaseError"));
+        |  return klass;
+        |}
+        END
+        str.margin
+      end
+
+      def self.entry_struct
+        str =<<-END
+        |typedef struct rod_entry {
+        |  unsigned long offset;
+        |  unsigned long size;
+        |} rod_entry_struct;
+        END
+        str.margin
+      end
+
+      def self.convert_key
+        str =<<-END
+        |DBT _convert_key(VALUE key){
+        |  long int_key;
+        |  double float_key;
+        |  DBT db_key;
+        |
+        |  memset(&db_key, 0, sizeof(DBT));
+        |  if(rb_obj_is_kind_of(key,rb_cInteger)){
+        |    int_key = NUM2LONG(key);
+        |    db_key.data = &int_key;
+        |    db_key.size = sizeof(long);
+        |  } else if(rb_obj_is_kind_of(key,rb_cFloat)){
+        |    float_key = NUM2DBL(key);
+        |    db_key.data = &float_key;
+        |    db_key.size = sizeof(double);
+        |  } else {
+        |    db_key.data = RSTRING_PTR(key);
+        |    db_key.size = RSTRING_LEN(key);
+        |  }
+        |  // is it legal?
+        |  return db_key;
+        |}
+        END
+        str.margin
+      end
+
+      def self.key_missing_exception
+        str =<<-END
+        |VALUE keyMissingException(){
+        |  VALUE klass = rb_const_get(rb_cObject, rb_intern("Rod"));
+        |  klass = rb_const_get(klass, rb_intern("KeyMissing"));
+        |  return klass;
+        |}
+        END
+        str.margin
+      end
+
+      self.inline(:C) do |builder|
+        builder.include '<db.h>'
+        builder.include '<stdio.h>'
+        builder.add_compile_flags '-ldb-4.8'
+        builder.prefix(self.entry_struct)
+        builder.prefix(self.rod_exception)
+        builder.prefix(self.key_missing_exception)
+        builder.prefix(self.convert_key)
+
+
+        str =<<-END
+        |void _open(const char * path, VALUE options){
+        |  DB * db_pointer;
+        |  u_int32_t flags;
+        |  int return_value;
+        |  VALUE handleClass;
+        |  VALUE handle;
+        |  VALUE mod;
+        |  db_pointer = ALLOC(DB);
+        |  return_value = db_create(&db_pointer,NULL,0);
+        |  if(return_value != 0){
+        |    rb_raise(rodException(),"%s",db_strerror(return_value));
+        |  }
+        |
+        |  flags = 0;
+        |  if(rb_hash_aref(options,ID2SYM(rb_intern("create"))) == Qtrue){
+        |    flags |= DB_CREATE;
+        |  }
+        |  if(rb_hash_aref(options,ID2SYM(rb_intern("truncate"))) == Qtrue){
+        |    flags |= DB_TRUNCATE;
+        |  }
+        |
+        |  return_value = db_pointer->open(db_pointer,NULL,path,
+        |    NULL,DB_HASH,flags,0);
+        |  if(return_value != 0){
+        |    rb_raise(rodException(),"%s",db_strerror(return_value));
+        |  }
+        |  mod = rb_const_get(rb_cObject, rb_intern("Rod"));
+        |  mod = rb_const_get(mod, rb_intern("Index"));
+        |  mod = rb_const_get(mod, rb_intern("HashIndex"));
+        |  handleClass = rb_const_get(mod, rb_intern("Handle"));
+        |  // TODO the handle memory should be made free
+        |  handle = Data_Wrap_Struct(handleClass,0,0,db_pointer);
+        |  rb_iv_set(self,"@handle",handle);
+        |}
+        END
+        builder.c(str.margin)
+
+        str =<<-END
+        |void _close(){
+        |  VALUE handle;
+        |  DB *db_pointer;
+        |  handle = rb_iv_get(self,"@handle");
+        |  Data_Get_Struct(handle,DB,db_pointer);
+        |  if(db_pointer != NULL){
+        |    db_pointer->close(db_pointer,0);
+        |    rb_iv_set(self,"@handle",Qnil);
+        |  } else {
+        |    rb_raise(rodException(),"DB handle is NULL\\n");
+        |  }
+        |}
+        END
+        builder.c(str.margin)
+
+        str =<<-END
+        |void _each(){
+        |
+        |}
+        END
+        builder.c(str.margin)
+
+        str =<<-END
+        |VALUE _get(VALUE key){
+        |  VALUE handle;
+        |  DB *db_pointer;
+        |  DBT db_key, db_value;
+        |  rod_entry_struct entry;
+        |  VALUE result;
+        |  int return_value;
+        |
+        |  handle = rb_iv_get(self,"@handle");
+        |  Data_Get_Struct(handle,DB,db_pointer);
+        |  if(db_pointer != NULL){
+        |    memset(&db_value, 0, sizeof(DBT));
+        |    db_key = _convert_key(key);
+        |    db_value.data = &entry;
+        |    db_value.ulen = sizeof(rod_entry_struct);
+        |    db_value.flags = DB_DBT_USERMEM;
+        |    return_value = db_pointer->get(db_pointer, NULL, &db_key, &db_value, 0);
+        |    if(return_value == DB_NOTFOUND){
+        |      rb_raise(keyMissingException(),"%s",db_strerror(return_value));
+        |    } else if(return_value != 0){
+        |      rb_raise(rodException(),"%s",db_strerror(return_value));
+        |    } else {
+        |      result = rb_ary_new();
+        |      rb_ary_push(result,ULONG2NUM(entry.offset));
+        |      rb_ary_push(result,ULONG2NUM(entry.size));
+        |      return result;
+        |    }
+        |  } else {
+        |    rb_raise(rodException(),"DB handle is NULL\\n");
+        |  }
+        |  return Qnil;
+        |}
+        END
+        builder.c(str.margin)
+
+        str =<<-END
+        |void _put(VALUE key,unsigned long offset,unsigned long size){
+        |  VALUE handle;
+        |  DB *db_pointer;
+        |  DBT db_key, db_value;
+        |  rod_entry_struct entry;
+        |  int return_value;
+        |
+        |  handle = rb_iv_get(self,"@handle");
+        |  Data_Get_Struct(handle,DB,db_pointer);
+        |  memset(&db_value, 0, sizeof(DBT));
+        |  entry.offset = offset;
+        |  entry.size = size;
+        |  db_key = _convert_key(key);
+        |  db_value.data = &entry;
+        |  db_value.size = sizeof(rod_entry_struct);
+        |  if(db_pointer != NULL){
+        |    return_value = db_pointer->put(db_pointer, NULL, &db_key, &db_value, 0);
+        |    if(return_value != 0){
+        |      rb_raise(keyMissingException(),"%s",db_strerror(return_value));
+        |    }
+        |  } else {
+        |    rb_raise(rodException(),"DB handle is NULL\\n");
+        |  }
+        |}
+        END
+        builder.c(str.margin)
+      end
+    end
+  end
+end

data/lib/rod/index/segmented_index.rb

@@ -36,6 +36,7 @@ module Rod
       # Destroys the index (removes it from the disk completely).
       def destroy
         remove_files(@path + "*")
+        @buckets = {}
       end
 
       def each

data/lib/rod/model.rb

@@ -253,6 +253,20 @@ module Rod
       self.add_to_database
     end
 
+    # Rebuild the index for given +property+. If the property
+    # doesn't have an index, an exception is raised.
+    def self.rebuild_index(property)
+      if properties[property][:index].nil?
+        raise RodException.new("Property '#{property}' doesn't have an index!")
+      end
+      index_for(property,properties[property]).destroy
+      instance_variable_set("@#{property}_index",nil)
+      index = index_for(property,properties[property])
+      self.each do |object|
+        index[object.send(property)] << object
+      end
+    end
+
     #########################################################################
     # 'Private' instance methods
     #########################################################################
@@ -455,24 +469,39 @@ module Rod
 
     # Migrates the class to the new model, i.e. it copies all the
     # values of properties that both belong to the class in the old
-    # and the new model.
+    # and the new model; it initializes new properties with default
+    # values and migrates the indices to different implementations.
     def self.migrate
+      # check if the migration is needed
+      old_metadata = self.metadata
+      old_metadata.merge!({:superclass => old_metadata[:superclass].sub(LEGACY_RE,"")})
       new_class = self.name.sub(LEGACY_RE,"").constantize
-      old_object = self.new
-      new_object = new_class.new
+      return if new_class.compatible?(old_metadata)
+      database.send(:allocate_space,new_class)
+
       puts "Migrating #{new_class}" if $ROD_DEBUG
+      # Check for incompatible properties.
       self.properties.each do |name,options|
         next unless new_class.properties.keys.include?(name)
-        print "-  #{name}... " if $ROD_DEBUG
-        if options.map{|k,v| [k,v.to_s.sub(LEGACY_RE,"")]} !=
-          new_class.properties[name].map{|k,v| [k,v.to_s]}
+        difference = options_difference(options,new_class.properties[name])
+        difference.delete(:index)
+        # Check if there are some options which we cannot migrate at the
+        # moment.
+        unless difference.empty?
           raise IncompatibleVersion.
             new("Incompatible definition of property '#{name}'\n" +
-                "Definition is different in the old and "+
+                "Definition of '#{name}' is different in the old and "+
                 "the new schema for '#{new_class}':\n" +
-                "  #{options} \n" +
-                "  #{new_class.properties[name]}")
+                "  #{difference}")
         end
+      end
+      # Migrate the objects.
+      # initialize prototype objects
+      old_object = self.new
+      new_object = new_class.new
+      self.properties.each do |name,options|
+        next unless new_class.properties.keys.include?(name)
+        print "-  #{name}... " if $ROD_DEBUG
         if self.field?(name)
           if self.string_field?(options[:type])
             self.count.times do |position|
@@ -508,6 +537,43 @@ module Rod
         end
         puts "done" if $ROD_DEBUG
       end
+      # Migrate the indices.
+      new_class.indexed_properties.each do |name,options|
+        # Migrate to new options.
+        old_index_type = self.properties[name] && self.properties[name][:index]
+        if old_index_type.nil?
+          print "-  building index #{options[:index]} for '#{name}'... " if $ROD_DEBUG
+          new_class.rebuild_index(name)
+          puts "done" if $ROD_DEBUG
+        else
+          # TODO if index is the same, its file should be copied
+          print "-  copying index #{options[:index]} for '#{name}'... " if $ROD_DEBUG
+          new_index = new_class.index_for(name,options)
+          old_index = index_for(name,self.properties[name])
+          new_index.copy(old_index)
+          puts "done" if $ROD_DEBUG
+        end
+      end
+    end
+
+    # Returns the difference between +options1+ and +options2+.
+    def self.options_difference(options1,options2)
+      old_options = {}
+      options1.each{|k,v| old_options[k] = v.to_s.sub(LEGACY_RE,"")}
+      new_options = {}
+      options2.each{|k,v| new_options[k] = v.to_s}
+      differences = {}
+      old_options.each do |option,value|
+        if new_options[option] != value
+          differences[option] = [value,new_options[option]]
+        end
+      end
+      new_options.each do |option,value|
+        if old_options[option] != value && !differences.has_key?(option)
+          differences[option] = [old_options[option],value]
+        end
+      end
+      differences
     end
 
     protected
@@ -1100,4 +1166,3 @@ module Rod
     end
   end
 end
-

data/lib/rod.rb

@@ -27,4 +27,5 @@ require 'rod/string_element'
 require 'rod/string_ex'
 require 'rod/index/base'
 require 'rod/index/flat_index'
+require 'rod/index/hash_index'
 require 'rod/index/segmented_index'

data/tests/migration_create.rb

@@ -4,7 +4,7 @@ require File.join(".",File.dirname(__FILE__),"migration_model1")
 
 Rod::Database.development_mode = true
 
-
+FileUtils.rm_rf("tmp/migration")
 Database.instance.create_database("tmp/migration")
 
 count = (ARGV[0] || 10).to_i
@@ -19,12 +19,15 @@ count.times do |index|
                         :nick => "j#{index}")
   account.store
   user1 = User.new(:name => "John#{index}",
-                  :surname => "Smith#{index}",
-                  :account => account,
-                  :mother => users[index-1],
-                  :father => users[index-2],
-                  :friends => [users[index-3],users[index-4]],
-                  :files => [files[index],files[index + 1],files[index + 2]])
+                   :surname => "Smith#{index}",
+                   :city => "City#{index}",
+                   :street => "Street#{index}",
+                   :number => index,
+                   :account => account,
+                   :mother => users[index-1],
+                   :father => users[index-2],
+                   :friends => [users[index-3],users[index-4]],
+                   :files => [files[index],files[index + 1],files[index + 2]])
   user1.store
 
   account = Account.new(:login => "amanda#{index}",
@@ -32,6 +35,9 @@ count.times do |index|
   account.store
   user2 = User.new(:name => "Amanda#{index}",
                    :surname => "Amanda#{index}",
+                   :city => "Bigcity#{index}",
+                   :street => "Small street#{index}",
+                   :number => index,
                    :account => account,
                    :mother => users[index-1],
                    :father => users[index-2],

data/tests/migration_migrate.rb

@@ -6,8 +6,9 @@ require 'rspec/expectations'
 #$ROD_DEBUG = true
 Rod::Database.development_mode = true
 
-Database.instance.open_database("tmp/migration", :migrate => true,
-                               :readonly => false)
+Database.instance.migrate_database("tmp/migration")
+Database.instance.open_database("tmp/migration", :readonly => false)
+Dir.glob("tmp/migration/#{Rod::BACKUP_PREFIX[0..-2]}*").to_a.size.should == 1
 
 count = (ARGV[0] || 10).to_i
 count.times do |index|
@@ -19,6 +20,7 @@ count.times do |index|
   file.store
   user = User[index*2]
   user.age = index
+  user.city = "Small town#{index}"
   user.file = file
   user.accounts << account1
   user.store
@@ -35,5 +37,3 @@ count.times do |index|
 end
 
 Database.instance.close_database
-
-test(?f,"tmp/migration/#{Rod::DATABASE_FILE}#{Rod::LEGACY_DATA_SUFFIX}").should == true

data/tests/migration_model1.rb

@@ -10,6 +10,9 @@ end
 class User < Model
   field :name, :string, :index => :flat
   field :surname, :string
+  field :city, :string, :index => :flat
+  field :street, :string, :index => :flat
+  field :number, :integer, :index => :flat
   has_one :account, :index => :flat
   has_one :mother, :class_name => "User"
   has_one :father, :class_name => "User"

data/tests/migration_model2.rb

@@ -14,6 +14,15 @@ class User < Model
   # removed
   # field :surname, :string
 
+  # changed: index flat -> segmented
+  field :city, :string, :index => :segmented
+
+  # changed: index flat -> hash
+  field :street, :string, :index => :hash
+
+  # changed: index flat -> nil
+  field :number, :integer
+
   # added
   field :age, :integer
 
@@ -40,8 +49,8 @@ class User < Model
 end
 
 class Account < Model
-  # present
-  field :login, :string
+  # changed: index added
+  field :login, :string, :index => :flat
 
   # removed
   # field :nick, :string

data/tests/migration_verify.rb

@@ -12,7 +12,15 @@ count.times do |index|
   user1 = User[index*2]
   user1.should_not == nil
   user = User.find_by_name("John#{index}")
-  user1.should == user
+  user.should == user1
+  users = User.find_all_by_city("City#{index}")
+  users.size.should == 0
+  users = User.find_all_by_city("Small town#{index}")
+  users.size.should == 1
+  users[0].should == user1
+  users = User.find_all_by_street("Street#{index}")
+  users.size.should == 1
+  users[0].should == user1
   user1.name.should == "John#{index}"
   user1.age.should == index
   user1.account.should_not == nil
@@ -31,7 +39,11 @@ count.times do |index|
   user2 = User[index*2+1]
   user2.should_not == nil
   user = User.find_by_name("Amanda#{index}")
-  user2.should == user
+  user.should == user2
+  user = User.find_by_city("Bigcity#{index}")
+  #user.should == user2
+  user = User.find_by_street("Small street#{index}")
+  user.should == user2
   user2.name.should == "Amanda#{index}"
   user2.age.should == index * 2
   user2.account.should_not == nil

metadata

@@ -2,7 +2,7 @@
 name: rod
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors: 
 - Aleksander Pohl
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-09 00:00:00 Z
+date: 2011-09-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: RubyInline
@@ -143,6 +143,7 @@ files:
 - features/collection.feature
 - features/collection_proxy.feature
 - features/flat_indexing.feature
+- features/hash_indexing.feature
 - features/inheritence.feature
 - features/muliple_db.feature
 - features/persistence.feature
@@ -165,6 +166,7 @@ files:
 - lib/rod/exception.rb
 - lib/rod/index/base.rb
 - lib/rod/index/flat_index.rb
+- lib/rod/index/hash_index.rb
 - lib/rod/index/segmented_index.rb
 - lib/rod/join_element.rb
 - lib/rod/model.rb
@@ -237,6 +239,7 @@ test_files:
 - features/collection.feature
 - features/collection_proxy.feature
 - features/flat_indexing.feature
+- features/hash_indexing.feature
 - features/inheritence.feature
 - features/muliple_db.feature
 - features/persistence.feature