rkv 0.0.1 → 0.2.0

data/.gitignore

@@ -19,3 +19,6 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+
+#Tokyocabinet test DB
+test.tcb

data/README.rdoc

@@ -1,4 +1,12 @@
-rkv - Ruby Key Value - an adapter on top of various key-value stores
+rkv - Ruby Key Value - an adapter on top of various key-value and column stores
+
+== Philosophy
+
+The API is a nested (multidimensional) Hash.  This allows effective mapping to both column stores and flat KV stores.
+
+For example, if the Cassandra adapter is used, the first dimension is the column family, the second is the row within the CF and the third is the column within the row.
+
+For flat KV stores, a key hierarchy is used for efficient mapping.
 
 == Install
 
@@ -6,25 +14,30 @@ Installing the gem:
 
   gem install rkv
 
-
 == Usage
 
   require 'rubygems'
   require 'rkv'
   
-  client = Rkv::Store.new(:cassandra, { :servers =>  ['127.0.0.1:9160'], :keyspace => "Blog" })
-  store = client[:Users]
+  store = Rkv.open(:cassandra, { :servers => ['127.0.0.1:9160'], :keyspace => "Blog" })
+  # store = Rkv.open(:memory, :recurse => {"Users" => "*"})
+  # store = Rkv.open(:tokyocabinet, :file => "test.tcb", :recurse => {"Users" => "*"})
+
+  users = store["Users"]
   
-  store["5"] = { :name => "Dev Random", :role => "admin" }
-  puts store["3"].inspect
+  user = users["5"]
+  user["name"] = "Dev Random"
+  user["role"] = "admin"
+
+  puts users["3"].inspect
   
-  store.batch do
-    store[id1] = { :name => "John Doe" }
-    store[id2] = { :name => "Jane Doe" }
+  users.batch do
+    users[id1]["name"] = "John Doe"
+    users[id2]["name"] = "Jane Doe"
   end
 
-  store = client[:UserRelationships]
-  store[id1, "posts"] = post_id1
+  user_rels = client[:UserRelationships]
+  user_rels[id1]["posts"][Cassandra::UUID.new] = post_id1
 
 == Notes
 
@@ -34,17 +47,29 @@ Keystores have different features.  Operations are mapped to the best of the key
 
 RDoc: http://rdoc.info/projects/devrandom/rkv
 
+See spec/integration/stores_spec.rb for an example program that runs identically on all stores.
+
+Git:
+* http://gitorious.org/cluster-storage/rkv
+* http://github.com/devrandom/rkv
+
 == Supported KV stores
 
 * Cassandra
+* Memory
+* TokyoCabinet
 
 == Planned KV stores
 
-* Keyspace
 * Voldemort
 * Redis
+* Keyspace
+* SQL
 
 == Roadmap
 
-* Create a keystore feature support matrix
+* Store support
+* Performance testing
+* Keystore feature support matrix
+* Object mapper?
 

data/Rakefile

@@ -25,6 +25,7 @@ end
 Spec::Rake::SpecTask.new(:rcov) do |spec|
   spec.libs << 'lib' << 'spec'
   spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov_opts = ['-x.gem/gems,spec']
   spec.rcov = true
 end
 

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.2.0

data/lib/rkv.rb

@@ -0,0 +1 @@
+require 'rkv/rkv'

data/lib/rkv/rkv.rb

@@ -0,0 +1,52 @@
+module Rkv
+  module Store
+    class BaseAdapter
+      def batch
+        # default batch just passes control to block
+        yield
+      end
+
+      def close
+      end
+
+      def ==(other)
+        my_keys = keys.sort
+        o_keys = keys.sort
+        return false unless my_keys == o_keys
+        my_keys.each do |key|
+          puts "#{key}" unless self[key] == other[key]
+          return false unless self[key] == other[key]
+        end
+        return true
+      end
+    end
+  end
+
+  @@backends = {}
+  def self.open(backend_name, opts)
+    get_backend(backend_name).open(opts)
+  end
+
+  private
+
+  def self.get_backend(backend_name)
+    backend = @@backends[backend_name]
+    return backend if backend
+    return load_backend(backend_name)
+  end
+
+  def self.load_backend(backend_name)
+    my_require "rkv/store/#{backend_name}_adapter"
+    backend = Rkv::Store.const_get(camelize(backend_name) + "Adapter")
+    @@backends[backend_name] = backend
+    return backend
+  end
+
+  def self.my_require(path)
+    require path
+  end
+
+  def self.camelize(str)
+    str.to_s.split(/[^a-z0-9]/i).map{|word| word.capitalize}.join
+  end
+end

data/lib/rkv/store/cassandra_adapter.rb

@@ -0,0 +1,191 @@
+require 'rubygems'
+require 'cassandra'
+
+module Rkv
+  module Store
+    class CassandraAdapter < BaseAdapter
+
+      attr_accessor :consistency
+
+      attr_accessor :opts
+
+      attr_accessor :recurse
+
+      def self.open(opts)
+        self.new(opts)
+      end
+
+      def initialize(opts)
+        @format = opts[:format] || :pass
+        @consistency = opts[:consistency] || :safe
+        @backend = Cassandra.new(opts[:keyspace], opts[:servers])
+        @columns = {}
+        @recurse = opts[:recurse] || :default
+        @opts = {
+          :consistency => (consistency == :safe) ? Cassandra::Consistency::QUORUM : Cassandra::Consistency::ONE
+        }
+      end
+
+      def [](key)
+        _must_recurse(key)
+        return @columns[key] if @columns[key]
+        @columns[key] = ColumnAdapter.new(self, key)
+        return @columns[key]
+      end
+
+      def []=(key, value)
+        _must_recurse(key)
+        raise ArgumentError.new("this store does not allow storing at top level - subscript me")
+      end
+
+      private
+
+      def _must_recurse(key)
+        raise ArgumentError.new("at this depth - key must end with slash, or :recurse option must be specified") unless _recurse?(key)
+      end
+
+      def _recurse?(key)
+        return false if @recurse.nil?
+        return (@recurse == :default || @recurse == "*" || @recurse[key] || key.end_with?("/"))
+      end
+
+      public
+
+      def backend
+        @backend
+      end
+
+      class ColumnAdapter < BaseAdapter
+        attr_reader :store
+
+        attr_reader :id
+
+        attr_reader :recurse
+
+        def initialize(store, column_key)
+          @store = store
+          @id = column_key.to_sym
+          @recurse = @store.recurse
+          if Hash === @recurse
+            @recurse = @recurse[column_key]
+          elsif "*" == @recurse
+            @recurse = {}
+          end
+        end
+
+        def [](key)
+          _must_recurse(key)
+          RowAdapter.new(self, key, nil)
+        end
+
+        def delete(key)
+          if _recurse?(key)
+            @store.backend.remove(@id, key, @store.opts)
+          end
+        end
+
+        def []=(key, value)
+          _must_recurse(key)
+          # TODO allow storing of hashes here
+        end
+
+        private
+
+        def _must_recurse(key)
+          raise ArgumentError.new("at this depth - key must end with slash, or :recurse option must be specified") unless _recurse?(key)
+        end
+
+        def _recurse?(key)
+          return false if @recurse.nil?
+          return (@recurse == :default || @recurse == "*" || @recurse[key] || key.end_with?("/"))
+        end
+      end
+
+
+      class RowAdapter < BaseAdapter
+        def initialize(column, id, prefix)
+          @column = column
+          @store = column.store
+          @prefix = prefix
+          @id = id
+
+          @recurse = @column.recurse
+          if Hash === @recurse
+            @recurse = @recurse[id]
+          elsif "*" == @recurse
+            @recurse = {}
+          end
+        end
+
+        def [](key)
+          if _recurse?(key)
+            RowAdapter.new(@column, @id, (@prefix || "") + key)
+          else
+            _get[key]
+          end
+        end
+
+        def []=(key, val)
+          key = @prefix + key if @prefix
+          # TODO recurse
+          @store.backend.insert(@column.id, @id, { key => val } , @store.opts)
+        end
+
+        def delete(key)
+          if _recurse?(key)
+            key = @prefix + key if @prefix
+            key = key + "/" unless key.end_with?("/")
+            ohash = @store.backend.get(@column.id, @id, @store.opts)
+            ohash.each_key do |okey|
+              next unless okey == key || okey.start_with?(key)
+              @store.backend.remove(@column.id, @id, okey, @store.opts)
+            end
+          else
+            key = @prefix + key if @prefix
+            @store.backend.remove(@column.id, @id, key, @store.opts)
+          end
+        end
+
+        def method_missing(meth, *args, &block)
+          _get.send(meth, *args, &block)
+        end
+
+        def inspect
+          _get.inspect
+        end
+
+        def to_s
+          _get.to_s
+        end
+
+        private
+        def _recurse?(key)
+          return false if @recurse.nil?
+          return (@recurse == :default || @recurse == "*" || @recurse[key] || key.end_with?("/"))
+        end
+
+        def _get
+          # TODO cache
+          ohash = @store.backend.get(@column.id, @id, @store.opts)
+          res = {}
+          ohash.each_pair do |okey, value|
+            if @prefix
+              next unless okey.start_with?(@prefix)
+              okey[@prefix] = ""
+            end
+            keys = okey.split("/")
+            last_key = keys.pop
+            ptr = res
+            keys.each do |key|
+              ptr[key] ||= {}
+              ptr = ptr[key]
+            end
+            ptr[last_key] = value
+          end
+          res
+        end
+      end
+    end
+  end
+end
+

data/lib/rkv/store/memory_adapter.rb

@@ -0,0 +1,63 @@
+require 'rubygems'
+
+module Rkv
+  module Store
+    class MemoryAdapter < BaseAdapter
+      attr_accessor :recurse
+
+      def self.open(opts)
+        self.new(opts)
+      end
+
+      def initialize(opts)
+        @recurse = opts[:recurse] || :default
+        @content = {}
+      end
+
+      def [](key)
+        if _recurse?(key)
+          key = key[0..-2] if key.end_with?("/")
+          res = @content[key]
+          return res if res
+          recurse = @recurse
+          if Hash === @recurse
+            recurse = recurse[key]
+          elsif "*" == @recurse
+            recurse = nil
+          end
+          res = MemoryAdapter.new(:recurse => recurse)
+          @content[key] = res
+          return res
+        else
+          return @content[key]
+        end
+      end
+
+      def delete(key)
+        key = key[0..-2] if key.end_with?("/")
+        @content.delete(key)
+      end
+
+      def []=(key, value)
+        # TODO hash value
+        @content[key] = value
+      end
+
+      def inspect
+        @content.inspect
+      end
+
+      def method_missing(meth, *args, &block)
+        @content.send(meth, *args, &block)
+      end
+
+      private
+
+      def _recurse?(key)
+        return false if @recurse.nil?
+        return (@recurse == :default || @recurse == "*" || @recurse[key] || key.end_with?("/"))
+      end
+    end
+  end
+end
+

data/lib/rkv/store/tokyocabinet_adapter.rb

@@ -0,0 +1,124 @@
+require 'rubygems'
+require 'tokyocabinet'
+
+module Rkv
+  module Store
+    class TokyocabinetAdapter < BaseAdapter
+      attr_accessor :recurse
+
+      def self.open(opts)
+        self.new(opts)
+      end
+
+      def tc_raise
+          ecode = @bdb.ecode
+          raise "TokyoCabinet - #{@bdb.errmsg(ecode)}"
+      end
+
+      def self.new(opts)
+        @recurse = opts[:recurse] || :default
+        @bdb = TokyoCabinet::BDB::new
+
+        # open the database
+        unless @bdb.open(opts[:file], TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT)
+          tc_raise
+        end
+
+        TCHash.new(@bdb, @recurse, "")
+      end
+
+      class TCHash < BaseAdapter
+        def initialize(bdb, recurse, prefix)
+          @bdb = bdb
+          @recurse = recurse
+          @prefix = prefix
+        end
+
+        def close
+          bdb.close
+        end
+
+        def [](key)
+          if _recurse?(key)
+            key = key[0..-2] if key.end_with?("/")
+            new_prefix = "#{@prefix}#{key}/"
+            recurse = @recurse
+            if Hash === @recurse
+              recurse = recurse[key]
+            elsif "*" == @recurse
+              recurse = nil
+            end
+            return TCHash.new(@bdb, recurse, new_prefix)
+          else
+            return _get[key]
+          end
+        end
+
+        def delete(key)
+          full_key = "#{@prefix}#{key}"
+          if _recurse?(key)
+            full_key = full_key[0..-2] if full_key.end_with?("/")
+            cur = TokyoCabinet::BDBCUR.new(@bdb)
+            return unless cur.jump(full_key)
+            while true
+              ckey = cur.key
+              return unless ckey.start_with?(full_key)
+              cur.out
+              return unless cur.next
+            end
+          else
+            @bdb.delete(full_key)
+          end
+        end
+
+        def []=(key, value)
+          # TODO hash value
+          full_key = "#{@prefix}#{key}"
+          @bdb[full_key] = value
+        end
+
+        def inspect
+          _get().inspect
+        end
+
+        def method_missing(meth, *args, &block)
+          _get().send(meth, *args, &block)
+        end
+
+        private
+
+        def _recurse?(key)
+          return true if key.end_with?("/")
+          return false if @recurse.nil?
+          return (@recurse == :default || @recurse == "*" || @recurse[key])
+        end
+
+        def _get
+          cur = TokyoCabinet::BDBCUR.new(@bdb)
+
+          res = {}
+
+          return res unless cur.jump(@prefix)
+
+          while true
+            ckey = cur.key
+            break unless ckey.start_with?(@prefix)
+            value = cur.val
+            ckey[@prefix] = ""
+            keys = ckey.split("/")
+            last_key = keys.pop
+            ptr = res
+            keys.each do |key|
+              ptr[key] ||= {}
+              ptr = ptr[key]
+            end
+            ptr[last_key] = value
+            break unless cur.next
+          end
+          res
+        end
+      end
+    end
+  end
+end
+

data/rkv.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{rkv}
-  s.version = "0.0.1"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Miron Cuperman"]
-  s.date = %q{2010-01-09}
+  s.date = %q{2010-01-11}
   s.description = %q{Ruby Key Value - an adapter on top of various key-value stores, supporting Cassandra and others}
   s.email = %q{c1.github@niftybox.net}
   s.extra_rdoc_files = [
@@ -20,7 +20,14 @@ Gem::Specification.new do |s|
      "README.rdoc",
      "Rakefile",
      "VERSION",
+     "lib/rkv.rb",
+     "lib/rkv/rkv.rb",
+     "lib/rkv/store/cassandra_adapter.rb",
+     "lib/rkv/store/memory_adapter.rb",
+     "lib/rkv/store/tokyocabinet_adapter.rb",
      "rkv.gemspec",
+     "spec/integration/stores_spec.rb",
+     "spec/lib/rkv_store_spec.rb",
      "spec/spec.opts",
      "spec/spec_helper.rb"
   ]
@@ -30,7 +37,9 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Ruby Key Value - an adapter on top of various key-value stores}
   s.test_files = [
-    "spec/spec_helper.rb"
+    "spec/lib/rkv_store_spec.rb",
+     "spec/spec_helper.rb",
+     "spec/integration/stores_spec.rb"
   ]
 
   if s.respond_to? :specification_version then

data/spec/integration/stores_spec.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+def test_store(store)
+  userid = "__test1"
+  users = store["Users"]
+  users.delete(userid)
+  user = users[userid]
+  user.batch do
+    user["x1"] = "3"
+    user["x2"] = "4"
+  end
+  users[userid].should == {"x1" => "3", "x2" => "4"}
+  test = []
+  user.each_pair { |k,v| test << [k,v] }
+  test.should == [["x1", "3"], ["x2", "4"]]
+  user.delete("x1")
+  user["abc/"]["def"] = "xyz"
+  user["abc/"]["lmn"] = "xyz"
+
+  users[userid].should == {"abc"=>{"lmn"=>"xyz", "def"=>"xyz"}, "x2"=>"4"}
+  users[userid]["abc/"].should == {"lmn"=>"xyz", "def"=>"xyz"}
+  user["abc/"].delete("def")
+  users[userid].should == {"abc"=>{"lmn"=>"xyz"}, "x2"=>"4"}
+  user.delete("abc/")
+  users[userid].should == {"x2"=>"4"}
+  users.delete(userid)
+  users[userid].should == {}
+end
+
+def open_store(store_name)
+  store = nil
+  case store_name
+  when :memory then
+    store = Rkv.open(:memory, :recurse => {"Users" => "*"})
+  when :cassandra then
+    store = Rkv.open(:cassandra, :servers => "127.0.0.1:9160", :keyspace => 'Twitter', :recurse => {"Users" => "*"})
+  when :tokyocabinet then
+    File.unlink "test.tcb"
+    store = Rkv.open(:tokyocabinet, :file => "test.tcb", :recurse => {"Users" => "*"})
+  else
+    raise "Unknown store #{store_name}"
+  end
+  store
+end
+
+describe Rkv do
+  [:memory, :cassandra, :tokyocabinet].each do |store_name|
+    it "should work with #{store_name}" do
+      store = nil
+      begin
+        store = open_store(store_name)
+      rescue Exception => e
+        puts "#{e.message}\n#{e.backtrace}"
+      end
+      if store
+        test_store(store)
+      else
+        fail "failed to load store - is it installed?"
+      end
+    end
+  end
+end

data/spec/lib/rkv_store_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+module Rkv
+  module Store
+    class TestAdapter
+      def self.open(opts)
+        return self.new
+      end
+    end
+  end
+end
+
+describe Rkv::Store do
+  before :each do
+    @opts = {:servers => ['dummy1:8888']}
+    @store = Rkv::Store::TestAdapter.new
+  end
+
+  it "should load backend" do
+    Rkv.should_receive(:my_require).with("rkv/store/test_adapter").and_return(true)
+    Rkv::Store::TestAdapter.should_receive(:open).with(@opts).and_return(@store)
+    store = Rkv.open(:test, @opts)
+    store.should == @store
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rkv
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.2.0
 platform: ruby
 authors: 
 - Miron Cuperman
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-09 00:00:00 -08:00
+date: 2010-01-11 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -26,7 +26,14 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- lib/rkv.rb
+- lib/rkv/rkv.rb
+- lib/rkv/store/cassandra_adapter.rb
+- lib/rkv/store/memory_adapter.rb
+- lib/rkv/store/tokyocabinet_adapter.rb
 - rkv.gemspec
+- spec/integration/stores_spec.rb
+- spec/lib/rkv_store_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 has_rdoc: true
@@ -58,4 +65,6 @@ signing_key:
 specification_version: 3
 summary: Ruby Key Value - an adapter on top of various key-value stores
 test_files: 
+- spec/lib/rkv_store_spec.rb
 - spec/spec_helper.rb
+- spec/integration/stores_spec.rb