makoto-dm-tokyo-cabinet-adapter 0.0.2

data/README

@@ -0,0 +1,48 @@
+An experimentation to add ORM on top of Tokyo Cabinet 
+
+What Is Tokyo Cabinet?
+- Modern implementation of DBM(key/value hash style(HDB), but supports fixed length hash(FDB), and b tree(BDB))
+- High concurrency/ high scalability (developed by a developer at Mixi, Japanese equivalent of Facebook)
+- More detail at http://tokyocabinet.sourceforge.net/index.html
+
+How dm-tokyo-cabinet-adapter stores data into Tokyo Cabinet.
+
+- Each object is stored as "ObjName.bdb" file: object id as key and entire data marshalled as value
+- Each attribute is stored as "ObjNameAttribute.bdb" file: attribute name as key and reference object id as value
+- The above architecture can also be considerd "ObjeName.bdb" as table and "ObjNameAttribute.bdb" as indexes for each attributes.
+- Currently implements basic CRUD, association, and eql finder.
+
+Motivation behind the development.
+
+- To experiment what you can do with basic hash based database. 
+  Interesting post at http://groups.google.com/group/merb/browse_thread/thread/a8c6b154576c6270
+- To learn internal of DataMapper and how to implement ORM/adapter 
+
+How to install
+
+1. Install Tokyo Cabinet http://tokyocabinet.sourceforge.net
+2. Install Ruby Binding  http://tokyocabinet.sourceforge.net/rubydoc/
+3. Install dm-tokyo-cabinet-adapter
+3.1 download dm-tokyo-cabinet-adapter
+3.2 cd to the dir
+3.3 gem build dm-tokyo-cabinet-adapter.gemspec 
+3.4 gem install dm-tokyo-cabinet-adapter-0.0.2.gem
+4. Create data dir
+5. Setup database.yml like below
+
+:development: 
+  :adapter: tokyo_cabinet
+  :data_path: <%= Pathname(__FILE__).dirname.expand_path + 'data' %>
+
+6. The rest is usual way to setup datamapper on Merb.
+
+Benchmarking results
+http://gist.github.com/25946
+
+My current implementation is a lot slower than MySQL and sqlite, but changing some data storage strategies speed up the performance dramatically, so there are lots of potential for optimization. 
+
+Further research topics/TODO
+- Implement outstanding tasks such as multi conditions, other finder conditions (<, <=, like), Data Types
+- At this moment, it's only uses B tree, as it covers the wide range of functionality (range search, duplicate values, transaction support, and so on). Consider replacing part to HDB or FDB for compact storage and speed.
+- Are there any ways to retrieve first/last key on FDB/HDB?
+- Can TC support "not equal" operation?

data/dm-tokyo-cabinet-adapter.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.name = %q{dm-tokyo-cabinet-adapter}
+  s.version = "0.0.2"
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Makoto Inoue"]
+  s.date = %q{2008-11-15}
+  s.description = %q{A DataMapper adapter for Tokyo Cabinet}
+  s.email = %q{inouemak@googlemail.com}
+  s.files = ["lib/tokyo_cabinet_adapter.rb", "README", "dm-tokyo-cabinet-adapter.gemspec", "spec/tokyo_cabinet_adapter_spec.rb", "spec/spec_helper.rb"]
+  s.homepage = %q{http://github.com/makoto/dm-tokyo-cabinet-adapter}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{A DatMapper adapter for Tokyo Cabinet}
+  s.test_files = ["spec/tokyo_cabinet_adapter_spec.rb", "spec/spec_helper.rb"]
+end
+
+

data/lib/tokyo_cabinet_adapter.rb

@@ -0,0 +1,225 @@
+require 'rubygems'
+require 'dm-core'
+require 'tokyocabinet'
+include TokyoCabinet
+
+module DataMapper
+  module Adapters
+    class TokyoCabinetAdapter < AbstractAdapter
+      
+      def create(resources)
+        resource = resources[0]
+        attributes = resource.attributes
+        
+        item_id = access_data(resource.model) do |item|
+          #Getting the latest id
+          #TODO:Find out how to get last id using FDB, rather than BDB
+          cur = BDBCUR::new(item)
+          cur.last
+          attributes[:id] = cur.key.to_i + 1
+          
+          item.put(attributes[:id], Marshal.dump(attributes))
+          attributes[:id]
+        end
+
+        resource.instance_variable_set(:@id, item_id)
+        
+        add_index(attributes, resource, item_id)
+        
+        # Seems required to return 1 to update @new_record instance variable at DataMapper::Resource.
+        # Not quite sure how it works.
+        1
+      end
+
+      def read_many(query)
+        results = parse_query(query)
+
+        if results
+          results = results.sort_by do |result|
+            result[query.order.first.property.name] if result
+          end
+          results = results.reverse if query.order.first.direction == :desc
+        end
+
+        if results # to handle results == nil
+          Collection.new(query) do |collection|
+            results.each do |result|
+              data = map_into_query_field(query, result)
+              if data # to handle results == [nil]
+                collection.load(data)
+              end
+            end
+          end
+        else
+          []
+        end
+      end
+
+      def read_one(query)
+        results = parse_query(query)
+        data = (results.class == Array ? results.first : results)
+
+        if data
+          data = map_into_query_field(query, data)
+          query.model.load(data, query)
+        end
+      end
+
+      def update(attributes, query)
+        item_id = get_id(query)
+        
+        old_attributes = get_items_from_id(query, item_id)
+        delete_index(old_attributes, query, item_id)
+        
+        # Converting {#<Property:User:name>=>"peter", #<Property:User:age>=>22} to {:age=>22, :name=>"peter"}
+        new_attributes = attributes.inject({}){|total,current|  total[current[0].name] = current[1]; total}
+        add_index(new_attributes, query, item_id)
+        
+        access_data(query.model) do |item|
+          raw_data = item.get(item_id)
+          if raw_data
+            record = Marshal.load(raw_data)
+
+            attributes.each do |key, value|
+              record[key.name.to_sym] = value
+            end
+
+            item.put(item_id, Marshal.dump(record))              
+          end
+        end
+        # Seems required to return 1 to update @new_record instance variable at DataMapper::Resource.
+        # Not quite sure how it works.
+        1
+      end
+      
+      def delete(query)
+        item_id = get_id(query)        
+        attributes = get_items_from_id(query, item_id)
+        
+        delete_index(attributes, query, item_id)
+        
+        access_data(query.model) do |item|
+          item.out(item_id)
+        end
+
+        # Seems required to return 1 to update @new_record instance variable at DataMapper::Resource.
+        # Not quite sure how it works.
+        1
+      end
+      
+    private
+      # Access Index file if property is given. If not, access data file
+      def access_data(model, property = nil, &block)
+        item = BDB::new
+        attribute = property.to_s.capitalize if property
+        item.open(data_path + "#{model}#{attribute}.bdb", BDB::OWRITER | BDB::OCREAT)
+        
+        result = yield(item)
+        
+        item.close        
+
+        result
+      end
+      
+      def data_path
+        data_path = DataMapper.repository.adapter.uri[:data_path].to_s + "/"
+      end
+            
+      def get_id(query)
+        unless query.conditions.empty?
+          query.conditions.first.last
+        end
+      end
+      
+      def get_items_from_id(query, values)
+        result = []
+        values_in_array = (values.class == Array ? values : [values])
+        access_data(query.model) do |item|
+          result = values_in_array.map do |value|
+              raw_data = item.get(value)
+              if raw_data
+                Marshal.load(raw_data)
+              end
+          end
+        end
+        
+        values.class == Array ? result : result.first
+      end
+      
+      def parse_query(query)
+        results = []
+        
+        unless query.conditions.empty?
+          operator, property, value = query.conditions.first
+          
+          if property.name == :id # Model.get
+            results = get_items_from_id(query, value)
+          else # Model.first w argument
+            case operator
+            when :eql
+            then
+              item_ids = access_data(query.model, property.name) do |item|
+                value = value.first if value.class == Array                
+                item.getlist(value)
+              end
+            when :not # TODO: Think about better way to extract, as this is going through data one by one
+            then  NotImplementedError{"The below code is not working as order is not always correct"}
+            else
+              raise NotImplementedError("#{operator} is not implmented yet")
+            end
+            results = get_items_from_id(query, item_ids)
+          end
+        else # Model.all w/o argument
+          access_data(query.model) do |item|
+            #Getting the first id
+            #TODO:Find out how to get first id using FDB, rather than BDB
+            raw_data = BDBCUR::new(item)
+            if raw_data.first              
+              while key = raw_data.key
+                results << Marshal.load(raw_data.val)
+                raw_data.next
+              end
+            end
+          end
+        end
+        results
+      end
+      
+      def map_into_query_field(query, data)
+        if data
+          query.fields.map do |property|
+            data[property.field.to_sym]
+          end
+        end
+      end
+
+      def delete_index(attributes, query, item_id)
+        # Don't need id attribute and attribut with no data.
+        attributes.reject{|k,v| k == :id || v == nil}.each do | k, v|
+          access_data(query.model, k) do |item|
+            items = item.getlist(v)
+            items = items - [item_id]
+            item.out(v)
+            if items.size > 0
+              item.putlist(v, items)
+            end
+          end
+        end
+      end
+      
+      def add_index(attributes, query, item_id)
+        attributes.each do |key, value|
+          # Creating index for each attributes except id
+          unless key == :id
+            access_data(query.model, key) do |item|
+              item.putlist(value, [item_id])
+            end
+          end
+        end
+      end
+      
+    end # class AbstractAdapter
+  end # module Adapters
+end # module DataMapper
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'dm-core'
+
+require 'pathname'
+require Pathname(__FILE__).dirname.parent.expand_path + 'lib/tokyo_cabinet_adapter'
+
+DataMapper.setup(:default, {
+  :adapter => 'tokyo_cabinet',
+  :data_path => Pathname(__FILE__).dirname.parent.expand_path + 'data'
+})
+
+class Post
+  include DataMapper::Resource
+
+  property :id,         Serial
+  property :title,      String
+  
+  belongs_to :user
+end
+
+class User
+  include DataMapper::Resource
+  
+  property :id,   Serial
+  property :name, String
+  property :age,  Integer
+  
+  has n, :posts
+end

data/spec/tokyo_cabinet_adapter_spec.rb

@@ -0,0 +1,231 @@
+require 'pathname'
+require Pathname(__FILE__).dirname.expand_path + 'spec_helper'
+
+describe DataMapper::Adapters::TokyoCabinetAdapter do
+  before(:each) do
+    db_files = Dir.glob(DataMapper.repository.adapter.uri[:data_path].to_s + "/*.*db")
+    FileUtils.rm(db_files)
+  end
+  
+  describe "Repository" do
+    it "should return adapter name" do
+      DataMapper.repository.adapter.uri[:adapter].should == 'tokyo_cabinet'
+    end
+    
+    it "should return data path" do
+      DataMapper.repository.adapter.uri[:data_path].should == Pathname(__FILE__).dirname.parent.expand_path + 'data'
+    end
+  end
+    
+  describe "CRUD" do
+    before(:each) do
+      @user = User.create(:name => 'tom')
+    end
+    describe "create" do
+      it "should assign id and attributes" do
+        user = User.create
+        user.should be_an_instance_of(User)
+        user.id.should_not == nil
+      end
+      
+      it "should increment id" do
+        first_user = User.create
+        second_user = User.create
+        first_user.id.should == second_user.id - 1
+      end
+    end
+
+    describe "get" do
+      it "should get an item" do
+        User.get(@user.id).should == @user
+      end
+
+      it "should raise error if item does not exist" do
+        non_existance_number = 100
+        lambda{User.get!(non_existance_number)}.should raise_error(DataMapper::ObjectNotFoundError)
+      end
+    end
+
+    describe "update" do
+      before(:each) do
+        @user.name = 'peter'
+        @user.age = 22
+
+        @user.save
+      end
+      it "should update an item" do
+        User.get(@user.id) == @user
+      end
+
+      it "should reflect index" do
+        User.first(:name => @user.name).should == @user
+      end
+    end
+
+    describe "destroy" do
+      before(:each) do
+        @user.destroy
+      end
+      it "should destroy an item" do
+        lambda{User.get!(@user.id)}.should raise_error(DataMapper::ObjectNotFoundError)
+      end
+      it "should reflect index" do
+        User.first(:name => @user.name).should == nil
+        User.all(:name => @user.name).should == []
+      end
+    end
+  end
+
+  describe 'Finder' do
+    describe "when no data" do
+      it "first should return nil" do
+        User.first(:name => 'someone').should == nil
+      end
+      
+      it "all should return []" do
+        User.all(:name => 'someone').should == []
+      end
+    end
+    
+    describe "when data" do
+      before(:each) do
+        @tom = User.create(:name => 'tom')
+        @peter = User.create(:name => 'peter')
+        @post = Post.create
+      end
+
+      it 'should get one record per model' do
+        User.first.should == @tom
+        Post.first.should == @post
+      end
+
+      it 'should return collection of all records per model' do
+        Post.all.should have(1).post
+        User.all.should have(2).users
+      end
+    end
+    
+  end
+
+  describe "Matcher" do
+    describe "first" do
+      describe "eql" do
+        before(:each) do
+          @tom = User.create(:name => 'tom', :age => 32)
+          @peter = User.create(:name => 'peter', :age => 32)
+        end
+        
+        it "should return a record " do
+          User.first(:name => 'tom').should == @tom
+        end
+        it "should return first record when searched by an attribute which allows duplicate entry" do
+          User.first(:age => 32).should == @tom        
+        end
+      end
+
+      describe "not" do
+        before(:each) do
+          @tom = User.create(:name => 'tom', :age => 2)
+          @peter = User.create(:name => 'peter', :age => 3)
+          @mark = User.create(:name => 'mark', :age => 5)
+        end
+        it "should return a record for string when non matching value comes first" do
+          pending()
+          User.first(:name.not => 'tom').should == @peter
+        end
+        
+        it "should return a record for string when matching value comes first" do
+          pending()
+          User.first(:name.not => 'peter').should == @tom
+        end
+        it "should return a record for numeric" do
+          pending
+          User.first(:age.not => 2).should == @peter
+        end
+        
+      end
+      it 'should get a record by not matcher'
+      it 'should get a record by gt matcher'
+      it 'should get a record by gte matcher'
+      it 'should get a record by lt matcher'
+      it 'should get a record by lte matcher'
+      it 'should get a record with multiple matchers'
+    end
+    describe "all" do
+      before(:each) do
+        @tom = User.create(:name => 'tom', :age => 2)
+        @peter = User.create(:name => 'peter', :age => 3)
+        @mark = User.create(:name => 'mark', :age => 5)
+        @andy = User.create(:name => 'andy', :age => 5)
+      end
+      it 'should get records by eql matcher' do
+        User.all(:age => 5).should == [@mark, @andy]
+      end
+      it 'should get records by not matcher'
+      it 'should get records by gt matcher'
+      it 'should get records by gte matcher'
+      it 'should get records by lt matcher'
+      it 'should get records by lte matcher'
+      it 'should get records with multiple matchers'
+    end
+  end
+
+  describe "DataType" do
+    before(:each) do
+      @dave = User.create(:name => 'dave', :age => 5)
+      @charles = User.create(:name => 'charles', :age => 15)
+      @bob = User.create(:name => 'bob', :age => 3)
+      @andy = User.create(:name => 'andy', :age => 4)
+    end
+
+    describe "sorting" do
+      # Tokyo Cabinet itself does not provide sorting, so done at ruby level.
+      it "should order by alphabet asc" do
+        User.all(:order => [:name]).should == [@andy, @bob, @charles, @dave]
+      end
+
+      it "should order by alphabet desc" do
+        User.all(:order => [:name.desc]).should == [@andy, @bob, @charles, @dave].reverse
+      end
+
+      it "should order by numeric asc" do
+        User.all(:order => [:age]).should == [@bob, @andy, @dave, @charles]
+      end
+
+      it "should order by numeric desc" do
+        User.all(:order => [:age.desc]).should == [@bob, @andy, @dave, @charles].reverse
+      end
+    end
+  end
+  
+  describe 'associations' do
+    before(:each) do
+      @user = User.create(:name => 'tom')
+      @post = Post.create(:title => 'Good morning', :user => @user)
+    end
+  
+    describe "Adding association" do
+     it 'should work with belongs_to associations' do 
+       User.get(@user.id).posts.should include(@post)
+     end
+
+     it 'should work with has n associations' do
+       Post.get(@post.id).user.should == @user
+     end
+    end
+    describe "Appending association" do
+      before(:each) do
+        @post2 = Post.create(:title => 'Good morning', :user => @user)
+        @user.posts << @post2
+      end
+      it 'should work with belongs_to associations' do 
+        User.get(@user.id).posts.should == [@post, @post2]
+      end
+
+      it 'should work with has n associations' do
+        Post.get(@post.id).user.should == @user
+        Post.get(@post2.id).user.should == @user
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification 
+name: makoto-dm-tokyo-cabinet-adapter
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Makoto Inoue
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-15 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: A DataMapper adapter for Tokyo Cabinet
+email: inouemak@googlemail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/tokyo_cabinet_adapter.rb
+- README
+- dm-tokyo-cabinet-adapter.gemspec
+- spec/tokyo_cabinet_adapter_spec.rb
+- spec/spec_helper.rb
+has_rdoc: false
+homepage: http://github.com/makoto/dm-tokyo-cabinet-adapter
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A DatMapper adapter for Tokyo Cabinet
+test_files: 
+- spec/tokyo_cabinet_adapter_spec.rb
+- spec/spec_helper.rb