rod 0.6.0

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://www.rubygems.org"
+gemspec

data/README

@@ -0,0 +1,144 @@
+= ROD -- Ruby Object Database
+
+* http://github.com/apohllo/rod
+
+== DESCRIPTION
+
+ROD (Ruby Object Database) is library which aims at providing
+fast access for data, which rarely changes.
+
+== FEATURES/PROBLEMS:
+
+* nice Ruby interface which mimicks Active Record
+* Ruby-to-C on-the-fly translation based on mmap
+* optimized for speed
+* weak reference collections for easy memory reclaims
+* segmented indices for short start-up time
+
+* doesn't work on Windows
+
+== SYNOPSIS:
+
+ROD is designed for storing and accessing data which rarely changes.
+It is an opposite of RDBMS as the data is not normalized.
+It is an opposite of in-memory databases, since it is designed to cover
+out of core data sets (10 GB and more).
+
+The primary reason for designing it was to create storage facility for
+natural language dictionaries and corpora. The data in a fully fledged dictionary
+is interconnected in many ways, thus the relational model (joins) introduces
+unacceptable performance hit. The size of corpora forces them to be kept
+on disks. The in-memory data bases are unacceptable for larg corpora and
+would require the data to be kept mostly in the operational memory,
+which is not needed, while accessing dictionaries (in most cases only a friction
+of the data is needed). That's why a storage facility which minimizes the
+number of disk reads was designed. The Ruby interface facilitates it's
+usage.
+
+== REQUIREMENTS:
+
+* RubyInline
+* english
+* ActiveModel
+* weak_hash
+
+== INSTALL
+
+Grab from rubygems:
+
+  gem install rod
+
+== BASIC USAGE:
+
+ class MyDatabase < Rod::Database
+ end
+
+ class Model < Rod::Model
+   database_class MyDatabase
+ end
+
+ class User < Model
+   field :name, :string
+   field :surname, :string, :index => true
+   field :age, :integer
+   has_one :account
+   has_many :files
+ end
+
+ class Account < Model
+   field :email, :string
+   field :login, :string, :index => true
+   field :password, :string
+ end
+
+ class File < Model
+   field :title, :string, :index => true
+   field :data, :string
+ end
+
+ MyDatabase.create_database("data")
+ user = User.new
+ user.name = 'Fred'
+ user.surname = 'Smith'
+ user.age = 22
+ account = Account.new
+ account.email = "fred@smith.org"
+ account.login = "fred"
+ account.password = "password"
+ file1 = File.new
+ file1.title = "Lady Gaga video"
+ file2.data = "0012220001..."
+ file2 = File.new
+ file2.title = "Pink Floyd video"
+ file2.data = "0012220001..."
+ user.account = account
+ user.files << file1
+ user.files << file2
+ user.store
+ account.store
+ file1.store
+ file2.store
+ MyDatabase.close_database
+
+ MyDatabase.open_database("data")
+ User.each do |user|
+   puts "Name: #{user.name} surname: #{user.surname}"
+   puts "login: #{user.account.login} e-mail: #{user.account.email}"
+   user.files.each do |file|
+     puts "File: #{file.title}
+   end
+ end
+
+ User[0]                           # gives first user
+ User.find_by_surname("Smith")     # gives Fred
+ User.find_all_by_surname("Smith") # gives [Fred]
+ File[0].user                      # won't work - the data is not normalized
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008-2010 Aleksander Pohl
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+== FEEDBACK
+
+* mailto:apohllo@o2.pl

data/Rakefile

@@ -0,0 +1,58 @@
+$:.unshift "lib"
+require 'rod/constants'
+
+task :default => [:install]
+
+$gem_name = "rod"
+
+desc "Build the gem"
+task :build => :all_tests do
+  sh "gem build #$gem_name.gemspec"
+  FileUtils.mkdir("pkg") unless File.exist?("pkg")
+  sh "mv '#$gem_name-#{Rod::VERSION}.gem' pkg"
+end
+
+desc "Install the library at local machnie"
+task :install => :build do 
+  sh "sudo gem install pkg/#$gem_name-#{Rod::VERSION}.gem"
+end
+
+desc "Uninstall the library from local machnie"
+task :uninstall do
+  sh "sudo gem uninstall #$gem_name"
+end
+
+task :all_tests => [:test,:spec,:regression_test] do
+end
+
+desc "Run performence tests"
+task :perf do
+  sh "ruby tests/eff1_test.rb"
+  sh "ruby tests/eff2_test.rb"
+  sh "ruby tests/full_runs.rb"
+end
+
+desc "Run tests and specs"
+task :test do
+  sh "ruby tests/save_struct.rb"
+  sh "ruby tests/load_struct.rb"
+  sh "ruby tests/unit/model.rb"
+  sh "ruby tests/unit/model_tests.rb"
+  sh "ruby tests/unit/abstract_database.rb"
+end
+
+# Should be removed some time -- specs should cover all these cases
+task :regression_test do
+  sh "ruby tests/read_on_create.rb"
+  sh "ruby tests/check_strings.rb"
+end
+
+task :spec do
+  sh "bundle exec cucumber features/*"
+end
+
+desc "Clean"
+task :clean do
+  sh "rm #$gem_name*.gem" 
+end
+

data/changelog.txt

@@ -0,0 +1,99 @@
+0.6.0
+- #64 index for associations
+- Update legacy test to new API
+- #26 check version of Rod library agains file version
+- #1 initialization form hash (ActiveRecord style)
+- #68 the indexed objects are stored in a collection proxy and are laizly fetched
+- #9 has_many is lazy
+  no referenced element representation is created, utill the element is accessed
+- #69 fix inheritence and polymorphic associations
+- Refactoring (API change): change all internal calls to pass rod_id instead of position
+- #46 Store only rod_id and class_id for referenced objects
+- #66 after storing an object, clear it singular and plural associations to allow later garbage collection
+- #57 better implementation of struct_name
+- #61 excessive Symbol#to_s calls removal
+- #61 excessive String#to_sym calls removal
+- #58 compact implementation of update_plural
+- #41 structs associated with the object is not wrapped into another object
+- Tests for #56
+- #55 associated object is fetched from DB, even though it is present
+- Change Willma to Wilma in tests
+- #52 load bucket for segmented index during DB creation
+- Don't delete old buckets when the DB is closed
+- Fix: load bucket when the segmented index is created
+- #59 Change WeakHash to SimpleWeakHash
+- #54 don't relay on Ruby#hash in implementation of segmented index
+- #53 Remove segmented index files when the DB is created
+0.5.5
+- segmented index #31
+- don't cache objects when they are stored in the DB #19
+- #33 pre-allocate larger data segments
+- #42 storage of index in regular file
+- #23 change all runtime exceptions to RodExceptions
+- Chagne page_size variable to page_size call
+- #4 append of database (experimental)
+- Refactor index read and write #42
+- Add guards for read-only data
+- #39 remove page count from metadata
+- #38 invalid size for munmap when closing DB
+- #35 - meta-data is stored in yaml
+0.5.4
+- default implementation of to_s
+- DB data is stored in a DB, not a single file #36 #37
+- removal of legacy tests
+- index flushing when the DB is created
+0.5.3
+- implementation of field serialization
+- refactoring of field storage
+- implementation of enumerator and enumerable
+0.5.2
+- polymorphic associations
+- fix C code: change INT2NUM to UINT2NUM for unsigned values
+- make index scope error for model more descriptive
+- features for nil associations
+- store version of the library in Ruby code
+- make Rakefile more useful
+- rod.rb explicitly enumerates the required files
+0.5.1
+- force ActiveSupport::Dependencis to use regular require
+- fix initialization of models with indices
+- add more info for basic feature
+0.5.0
+- simultaneous usage of many databases
+- refactoring of service and model
+- remove page from string C definition
+- remove index packign/unpacking
+- inheritence of attributes and associations
+- features for all basic functions
+0.4.4
+- minor fix: count objects while storing
+- remove separate zero-string tests
+- Fred feature uses model step definitions
+- model step definitions refactoring
+- remove Ruby inline generated files during tests
+0.4.3
+- some test changed into features specification
+- default implementation of == for Model
+- scope check for []
+- allow for purging subclass information when closing database
+- cache clearing
+- unsued C methods are generated only in development mode
+- updated dependencies
+0.4.2
+- clear Gemfile
+- make clear statements about dev. dependencies
+0.4.1
+- allow to skip validation
+- tests updated for Ruby 1.9.2
+- Gemfile added
+0.4.0
+- page offset removed from string information (merged with string)
+- cache clearing turned off
+0.3.1
+- build_structure is called when the DB is created/opened
+- change message when not all objects are stored: only the number of objects
+- when the DB is closed only the number of not stored objects is reported
+0.3.0
+- data is stored in separated files during creation
+0.2.0
+- uses ActiveModel for validation

data/lib/rod.rb

@@ -0,0 +1,21 @@
+require 'inline'
+require 'english/inflect'
+require 'simple_weak_hash'
+require 'active_model'
+require 'active_support/dependencies'
+
+# XXX This should be done in a different way, since a library should not
+# impose on a user of another library specific way of using it.
+# See #21
+ActiveSupport::Dependencies.mechanism = :require
+
+require 'rod/abstract_database'
+require 'rod/constants'
+require 'rod/database'
+require 'rod/exception'
+require 'rod/join_element'
+require 'rod/collection_proxy'
+require 'rod/model'
+require 'rod/string_element'
+require 'rod/string_ex'
+require 'rod/segmented_index'

data/lib/rod/abstract_database.rb

@@ -0,0 +1,369 @@
+require 'singleton'
+require 'yaml'
+require 'rod/segmented_index'
+
+module Rod
+  # This class implements the database abstraction, i.e. it
+  # is a mediator between some model (a set of classes) and
+  # the generated C code, implementing the data storage functionality.
+  class AbstractDatabase
+    # This class is a singleton, since in a given time instant there
+    # is only one database (one file/set of files) storing data of
+    # a given model (set of classes).
+    include Singleton
+
+    # Initializes the classes linked with this database and the handler.
+    def initialize
+      @classes ||= self.class.special_classes
+      @handler = nil
+    end
+
+    #########################################################################
+    # Public API
+    #########################################################################
+
+    # Returns whether the database is opened.
+    def opened?
+      not @handler.nil?
+    end
+
+    # The DB open mode.
+    def readonly_data?
+      @readonly
+    end
+
+    # Creates the database at specified +path+, which allows
+    # for Rod::Model#store calls to be performed.
+    #
+    # The database is created for all classes, which have this
+    # database configured via Rod::Model#database_class call
+    # (this configuration is by default inherited in subclasses,
+    # so it have to be called only in the root class of given model).
+    #
+    # WARNING: all files in the DB directory are removed during DB creation!
+    def create_database(path)
+      raise DatabaseError.new("Database already opened.") unless @handler.nil?
+      @readonly = false
+      self.classes.each{|s| s.send(:build_structure)}
+      @path = canonicalize_path(path)
+      # XXX maybe should be more careful?
+      if File.exist?(@path)
+        Dir.glob("#{@path}**/*").each do |file_name|
+          File.delete(file_name) unless File.directory?(file_name)
+        end
+      else
+        Dir.mkdir(@path)
+      end
+      generate_c_code(@path, classes)
+      @handler = _init_handler(@path)
+      _create(@handler)
+    end
+
+    # Opens the database at +path+ for readonly mode. This allows
+    # for Rod::Model.count, Rod::Model.each, and similar calls.
+    #
+    # By default the database is opened in +readonly+ mode. You
+    # can change it by passing +false+ as the second argument.
+    def open_database(path,readonly=true)
+      raise DatabaseError.new("Database already opened.") unless @handler.nil?
+      @readonly = readonly
+      self.classes.each{|s| s.send(:build_structure)}
+      @path = canonicalize_path(path)
+      generate_c_code(@path, classes)
+      metadata = {}
+      File.open(@path + DATABASE_FILE) do |input|
+        metadata = YAML::load(input)
+      end
+      unless valid_version?(metadata["Rod"][:version])
+        raise RodException.new("Incompatible versions - library #{VERSION} vs. file #{metatdata["Rod"][:version]}")
+      end
+      @handler = _init_handler(@path)
+      self.classes.each do |klass|
+        meta = metadata[klass.name]
+        if meta.nil?
+          # new class
+          next
+        end
+        set_count(klass,meta[:count])
+        file_size = File.new(klass.path_for_data(@path)).size
+        unless file_size % _page_size == 0
+          raise DatabaseError.new("Size of data file of #{klass} is invalid: #{file_size}")
+        end
+        set_page_count(klass,file_size / _page_size)
+      end
+      _open(@handler)
+    end
+
+    # Closes the database.
+    #
+    # If the +purge_classes+ flag is set to true, the information about the classes
+    # linked with this database is removed. This is important for testing, when
+    # classes with same names have different definitions.
+    def close_database(purge_classes=false)
+      raise DatabaseError.new("Database not opened.") if @handler.nil?
+
+      unless readonly_data?
+        unless referenced_objects.select{|k, v| not v.empty?}.size == 0
+          raise DatabaseError.new("Not all associations have been stored: #{referenced_objects.size} objects")
+        end
+        metadata = {}
+        rod_data = metadata["Rod"] = {}
+        rod_data[:version] = VERSION
+        self.classes.each do |klass|
+          meta = metadata[klass.name] = {}
+          meta[:count] = count(klass)
+          next if special_class?(klass)
+          # fields
+          fields = meta[:fields] = {} unless klass.fields.empty?
+          klass.fields.each do |field,options|
+            fields[field] = {}
+            fields[field][:options] = options
+            write_index(klass,field,options) if options[:index]
+          end
+          # singular_associations
+          has_one = meta[:has_one] = {} unless klass.singular_associations.empty?
+          klass.singular_associations.each do |name,options|
+            has_one[name] = {}
+            has_one[name][:options] = options
+            write_index(klass,name,options) if options[:index]
+          end
+          # plural_associations
+          has_many = meta[:has_many] = {} unless klass.plural_associations.empty?
+          klass.plural_associations.each do |name,options|
+            has_many[name] = {}
+            has_many[name][:options] = options
+            write_index(klass,name,options) if options[:index]
+          end
+        end
+        File.open(@path + DATABASE_FILE,"w") do |out|
+          out.puts(YAML::dump(metadata))
+        end
+      end
+      _close(@handler)
+      @handler = nil
+      # clear cached data
+      self.clear_cache
+      if purge_classes
+        @classes = self.class.special_classes
+      end
+    end
+
+    # Clears the cache of the database.
+    def clear_cache
+      classes.each{|c| c.cache.send(:__get_hash__).clear}
+    end
+
+    #########################################################################
+    # 'Private' API
+    #########################################################################
+
+    # "Stack" of objects which are referenced by other objects during store,
+    # but are not yet stored.
+    def referenced_objects
+      @referenced_objects ||= {}
+    end
+
+
+    # Adds the +klass+ to the set of classes linked with this database.
+    def add_class(klass)
+      @classes << klass unless @classes.include?(klass)
+    end
+
+    # Remove the +klass+ from the set of classes linked with this database.
+    def remove_class(klass)
+      unless @classes.include?(klass)
+        raise DatabaseError.new("Class #{klass} is not linked with #{self}!")
+      end
+      @classes.delete(klass)
+    end
+
+    # Returns join index with +index+ and +offset+.
+    def join_index(offset, index)
+      _join_element_index(offset, index, @handler)
+    end
+
+    # Returns polymorphic join index with +index+ and +offset+.
+    # This is the rod_id of the object referenced via
+    # a polymorphic has many association for one instance.
+    def polymorphic_join_index(offset, index)
+      _polymorphic_join_element_index(offset, index, @handler)
+    end
+
+    # Returns polymorphic join class id with +index+ and +offset+.
+    # This is the class_id (name_hash) of the object referenced via
+    # a polymorphic has many association for one instance.
+    def polymorphic_join_class(offset, index)
+      _polymorphic_join_element_class(offset, index, @handler)
+    end
+
+    # Sets the +object_id+ of the join element with +offset+ and +index+.
+    def set_join_element_id(offset,index,object_id)
+      raise DatabaseError.new("Readonly database.") if readonly_data?
+      _set_join_element_offset(offset, index, object_id, @handler)
+    end
+
+    # Sets the +object_id+ and +class_id+ of the
+    # polymorphic join element with +offset+ and +index+.
+    def set_polymorphic_join_element_id(offset,index,object_id,class_id)
+      raise DatabaseError.new("Readonly database.") if readonly_data?
+      _set_polymorphic_join_element_offset(offset, index, object_id,
+                                           class_id, @handler)
+    end
+
+    # Returns the string of given +length+ starting at given +offset+.
+    def read_string(length, offset)
+      # TODO the encoding should be stored in the DB
+      # or configured globally
+      _read_string(length, offset, @handler).force_encoding("utf-8")
+    end
+
+    # Stores the string in the DB encoding it to utf-8.
+    def set_string(value)
+      raise DatabaseError.new("Readonly database.") if readonly_data?
+      _set_string(value.encode("utf-8"),@handler)
+    end
+
+    # Returns the number of objects for given +klass+.
+    def count(klass)
+      send("_#{klass.struct_name}_count",@handler)
+    end
+
+    # Sets the number of objects for given +klass+.
+    def set_count(klass,value)
+      send("_#{klass.struct_name}_count=",@handler,value)
+    end
+
+    # Sets the number of pages allocated for given +klass+.
+    def set_page_count(klass,value)
+      send("_#{klass.struct_name}_page_count=",@handler,value)
+    end
+
+    # Reads index of +field+ (with +options+) for +klass+.
+    def read_index(klass,field,options)
+      case options[:index]
+      when :flat,true
+        begin
+          File.open(klass.path_for_index(@path,field,options)) do |input|
+            return {} if input.size == 0
+            return Marshal.load(input)
+          end
+        rescue Errno::ENOENT
+          return {}
+        end
+      when :segmented
+        return SegmentedIndex.new(klass.path_for_index(@path,field,options))
+      else
+        raise RodException.new("Invalid index type '#{options[:index]}'.")
+      end
+    end
+
+    # Store index of +field+ (with +options+) of +klass+ in the database.
+    # There are two types of indices:
+    # * +:flat+ - marshalled index is stored in one file
+    # * +:segmented+ - marshalled index is stored in many files
+    def write_index(klass,property,options)
+      raise DatabaseError.new("Readonly database.") if readonly_data?
+      class_index = klass.index_for(property,options)
+      class_index.each do |key,ids|
+        unless ids.is_a?(CollectionProxy)
+          proxy = CollectionProxy.new(ids[1]) do |index|
+            [join_index(ids[0],index), klass]
+          end
+        else
+          proxy = ids
+        end
+        offset = _allocate_join_elements(proxy.size,@handler)
+        proxy.each_id.with_index do |rod_id,index|
+          set_join_element_id(offset, index, rod_id)
+        end
+        class_index[key] = [offset,proxy.size]
+      end
+      case options[:index]
+      when :flat,true
+        File.open(klass.path_for_index(@path,property,options),"w") do |out|
+          out.puts(Marshal.dump(class_index))
+        end
+      when :segmented
+        path = klass.path_for_index(@path,property,options)
+        if class_index.is_a?(Hash)
+          index = SegmentedIndex.new(path)
+          class_index.each{|k,v| index[k] = v}
+        else
+          index = class_index
+        end
+        index.save
+        index = nil
+      else
+        raise RodException.new("Invalid index type '#{options[:index]}'.")
+      end
+    end
+
+    # Store the object in the database.
+    def store(klass,object)
+      raise DatabaseError.new("Readonly database.") if readonly_data?
+      send("_store_" + klass.struct_name,object,@handler)
+      # set fields' values
+      object.class.fields.each do |name,options|
+        # rod_id is set during _store
+        object.update_field(name) unless name == "rod_id"
+      end
+      # set ids of objects referenced via singular associations
+      object.class.singular_associations.each do |name,options|
+        object.update_singular_association(name,object.send(name))
+      end
+      # set ids of objects referenced via plural associations
+      object.class.plural_associations.each do |name,options|
+        elements = object.send(name) || []
+        if options[:polymorphic]
+          offset = _allocate_polymorphic_join_elements(elements.size,@handler)
+        else
+          offset = _allocate_join_elements(elements.size,@handler)
+        end
+        object.update_count_and_offset(name,elements.size,offset)
+        object.update_plural_association(name,elements)
+      end
+    end
+
+    # Prints the layout of the pages in memory and other
+    # internal data of the model.
+    def print_layout
+      raise DatabaseError.new("Database not opened.") if @handler.nil?
+      _print_layout(@handler)
+    end
+
+    # Prints the last error of system call.
+    def print_system_error
+      _print_system_error
+    end
+
+    protected
+
+    # Checks if the version of the library is valid.
+    # Consult https://github.com/apohllo/rod/wiki for versioning scheme.
+    def valid_version?(version)
+      file = version.split(".")
+      library = VERSION.split(".")
+      return false if file[0] != library[0] || file[1] != library[1]
+      if library[1].to_i.even?
+        return true
+      else
+        return file[2] == library[2]
+      end
+    end
+
+    # Returns collected subclasses.
+    def classes
+      @classes.sort{|c1,c2| c1.to_s <=> c2.to_s}
+    end
+
+    # Retruns the path to the DB as a name of a directory.
+    def canonicalize_path(path)
+      path + "/" unless path[-1] == "/"
+    end
+
+    # Special classes used by the database.
+    def self.special_classes
+      [JoinElement, PolymorphicJoinElement, StringElement]
+    end
+  end
+end