ruby-tokyotyrant 0.3.1 → 0.4

data/README.rdoc

@@ -4,10 +4,11 @@ This is a c extension for Ruby to access TokyoTyrant databases.  It currently su
 
 == Install
 
-  # install tokyocabinet (1.4.30) and tokyotyrant (requires 1.1.33)
+  # install tokyocabinet (1.4.34) and tokyotyrant (requires 1.1.35)
   # after installing tc and tt on linux I had to /sbin/ldconfig (as root)
-  gem sources -a http://gems.github.com
-  sudo gem install actsasflinn-ruby-tokyotyrant
+  sudo gem install gemcutter
+  sudo gem tumble
+  sudo gem install ruby-tokyotyrant
 
 == Performance
 
@@ -196,6 +197,32 @@ This is not in production but the initial benchmarks are very interesting. Resul
   t.run(:echo, 'hello', 'world')
   # => "hello\tworld"
 
+=== Balanced Nodes with Consistent Hashing
+
+  # usage is similar to single node
+  require 'tokyo_tyrant/balancer'
+
+  servers = ['127.0.0.1:1978',
+             '127.0.0.1:1979',
+             '127.0.0.1:1980',
+             '127.0.0.1:1981']
+
+  tb = TokyoTyrant::Balancer::Table.new(servers)
+
+  # store server is determined by key which is consistent
+  tb[:foo] = { 'foo' => 'bar' }
+  tb[:bar] = { 'bar' => 'baz' }
+
+  # retrieval server is determined by key which is consistent
+  tb[:foo]
+  # => { 'foo' => 'bar' }
+
+  # aggregate from all nodes
+  tb.size
+
+  # parallel_search based querying across all nodes
+  tb.find{ |q| q.condition(:foo, :streq, 'bar') }
+
 == Contributors
 
 * Flinn Mueller (actsasflinn) author/maintainer

data/Rakefile

@@ -20,7 +20,7 @@ CLEAN.include('pkg', 'tmp')
 
 gemspec = Gem::Specification.new do |s|
   s.name = 'ruby-tokyotyrant'
-  s.version = '0.3.1'
+  s.version = '0.4'
   s.authors = [ 'Flinn' ]
   s.email = 'flinn@actsasflinn.com'
   s.homepage = 'http://github.com/actsasflinn/ruby-tokyotyrant/'
@@ -35,9 +35,12 @@ gemspec = Gem::Specification.new do |s|
              'Rakefile',
              'README.rdoc'] +
              Dir['ext/**/*.[rb|c|h]'] +
+             Dir['lib/**/*.rb'] +
              Dir['spec/**/*'] +
              Dir['benchmarks/**/*']
   s.extensions << "ext/extconf.rb"
+
+  s.add_runtime_dependency('fast_hash_ring', '>= 0.1.1')
 end
 
 task :gemspec do
@@ -50,13 +53,14 @@ Rake::GemPackageTask.new(gemspec) do |pkg|
   pkg.need_tar = true
 end
 
-Rake::PackageTask.new('ruby-tokyotyrant', '0.1') do |pkg|
+Rake::PackageTask.new('ruby-tokyotyrant', '0.4') do |pkg|
   pkg.need_zip = true
   pkg.package_files = FileList[
     'COPYING',
     'Rakefile',
     'README.rdoc',
     'ext/**/*',
+    'lib/**/*.[rb]',
     'spec/**/*',
     'benchmarks/**/*'
   ].to_a

data/benchmarks/balancer.rb

@@ -0,0 +1,101 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+require 'date'
+
+#
+# the data
+#
+ 
+colnames = %w{ name sex birthday divisions }
+ 
+$year = (1909 .. 2009).to_a
+$month = (1..12).to_a
+$day = (1..28).to_a # not bothering with month diffs
+ 
+def rbdate
+  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+end
+ 
+def rdiv
+  case rand(3)
+  when 0
+    'dev'
+  when 1
+    'brd'
+  else
+    'brd,dev'
+  end
+end
+ 
+def rgen
+  (rand(2) == 1 ? 'male' : 'female')
+end
+ 
+data = [
+  [ 'Alphonse Armalite', 'male', DateTime.new(1972, 10, 14), 'brd,dev' ],
+  [ 'Brutus Beromunster', 'male', DateTime.new(1964, 07, 14), 'dev' ],
+  [ 'Crystel Chucknorris', 'female', DateTime.new(1980, 07, 12), 'brd' ],
+  [ 'Desree Dylan', 'female', DateTime.new(1954, 07, 13), 'brd,dev' ]
+]
+ 
+10_000.times do |i|
+  data << [ Faker::Name.name, rgen, rbdate, rdiv]  
+end
+ 
+$find_name_list = []
+100.times { $find_name_list << data[rand(data.size)][0] }
+ 
+data.collect! { |e|
+  (0..colnames.length - 1).inject({}) { |h, i| h[colnames[i]] = e[i]; h }
+}
+
+data_h = {}
+i = 0
+data1 = data.collect { |e|
+  i = i + 1
+  h = e.dup
+  h['birthday'] = h['birthday'].to_s
+  data_h[i.to_s] = h
+  h
+}
+
+require 'tokyo_tyrant'
+
+t = TokyoTyrant::Table.new('127.0.0.1', 45001)
+t.clear
+
+2.times { puts }
+puts 'TokyoTyrant Single'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| t[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = t[i.to_s] }
+  end
+end
+
+require 'lib/tokyo_tyrant/balancer'
+
+servers = ['127.0.0.1:45001',
+           '127.0.0.1:45002',
+           '127.0.0.1:45003']
+
+tb = TokyoTyrant::Balancer::Table.new(servers)
+tb.clear
+
+2.times { puts }
+puts 'TokyoTyrant Balancer'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| tb[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = tb[i.to_s] }
+  end
+end

data/benchmarks/db.rb

@@ -4,10 +4,11 @@ require 'faker'
 
 data = []
 
-10_000.times do |i|
+100_000.times do |i|
   data << Faker::Name.name
 end
 
+=begin
 require 'rufus/tokyo/tyrant'
 
 r = Rufus::Tokyo::Tyrant.new('127.0.0.1', 45000)
@@ -44,9 +45,10 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
     data.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
   end
 end
+=end
 
 require 'tokyo_tyrant'
-t = TokyoTyrant::DB.new('127.0.0.1', 45000)
+t = TokyoTyrant::DB.new('127.0.0.1', 1978)
 t.clear
 
 2.times { puts }
@@ -62,6 +64,23 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   end
 end
 
+m = TokyoTyrant::MDB.new
+m.clear
+
+2.times { puts }
+puts 'MDB'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| m[i] = e }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = m[i] }
+  end
+end
+
+=begin
 require 'memcached'
 m = Memcached.new('127.0.0.1:45000')
 m.flush
@@ -112,3 +131,4 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
     data.each_with_index { |e, i| nothing = mc.get(i.to_s) }
   end
 end
+=end

data/ext/tokyo_tyrant.c

@@ -128,6 +128,7 @@ VALUE eTokyoTyrantError;
 VALUE cDB;
 VALUE cTable;
 VALUE cQuery;
+VALUE cMDB;
 
 void Init_tokyo_tyrant(){
   mTokyoTyrant = rb_define_module("TokyoTyrant");
@@ -144,4 +145,7 @@ void Init_tokyo_tyrant(){
 
   cQuery = rb_define_class_under(mTokyoTyrant, "Query", rb_cObject);
   init_query();
+
+  cMDB = rb_define_class_under(mTokyoTyrant, "MDB", rb_cObject);
+  init_utils();
 }

data/ext/tokyo_tyrant.h

@@ -11,6 +11,7 @@
 #include <tokyo_tyrant_db.h>
 #include <tokyo_tyrant_table.h>
 #include <tokyo_tyrant_query.h>
+#include <tokyo_utils.h>
 
 #define RDBVNDATA "@rdb"
 #define RDBQRYVNDATA "@rdbquery"
@@ -19,6 +20,7 @@
 #define TTPUTKEEP  1
 #define TTPUTCAT   2
 #define TTPUTNR    3
+#define MDBVNDATA "@mdb"
 
 #if !defined(RSTRING_PTR)
 #define RSTRING_PTR(TC_s) (RSTRING(TC_s)->ptr)
@@ -35,6 +37,7 @@ extern VALUE eTokyoTyrantError;
 extern VALUE cDB;
 extern VALUE cTable;
 extern VALUE cQuery;
+extern VALUE cMDB;
 
 extern VALUE StringRaw(const char *buf, int bsiz);
 extern VALUE StringValueEx(VALUE vobj);

data/ext/tokyo_tyrant_module.c

@@ -1,4 +1,4 @@
-#include <tokyo_tyrant_db.h>
+#include <tokyo_tyrant_module.h>
 
 extern TCRDB *mTokyoTyrant_getdb(VALUE vself){
   TCRDB *db;
@@ -438,7 +438,7 @@ void init_mod(){
   rb_define_alias(mTokyoTyrant, "clear", "vanish");
   rb_define_method(mTokyoTyrant, "copy", mTokyoTyrant_copy, 1);
   rb_define_method(mTokyoTyrant, "restore", mTokyoTyrant_restore, 2);
-  rb_define_method(mTokyoTyrant, "setmst", mTokyoTyrant_setmst, 2);
+  rb_define_method(mTokyoTyrant, "setmst", mTokyoTyrant_setmst, 4);
   rb_define_method(mTokyoTyrant, "rnum", mTokyoTyrant_rnum, 0);
   rb_define_alias(mTokyoTyrant, "count", "rnum");
   rb_define_method(mTokyoTyrant, "empty?", mTokyoTyrant_empty, 0);

data/ext/tokyo_utils.c

@@ -0,0 +1,362 @@
+#include <tokyo_utils.h>
+
+extern TCMDB *cMDB_getdb(VALUE vself){
+  TCMDB *db;
+  Data_Get_Struct(rb_iv_get(vself, MDBVNDATA), TCMDB, db);
+  return db;
+}
+
+static void cMDB_free(TCMDB *db){
+  tcmdbdel(db);
+}
+
+static VALUE cMDB_bnum(VALUE vself){
+  return rb_iv_get(vself, "@bnum");
+}
+
+static VALUE cMDB_initialize(int argc, VALUE *argv, VALUE vself){
+  VALUE bnum;
+  TCMDB *db;
+
+  rb_scan_args(argc, argv, "0", &bnum);
+  if(NIL_P(bnum)){
+    db = tcmdbnew();
+  } else {
+    db = tcmdbnew2(bnum);
+  }
+
+  rb_iv_set(vself, "@bnum", bnum);
+
+  rb_iv_set(vself, MDBVNDATA, Data_Wrap_Struct(rb_cObject, 0, cMDB_free, db));
+
+  return Qtrue;
+}
+
+static VALUE cMDB_put_method(VALUE vself, VALUE vkey, VALUE vstr, int method){
+  bool res;
+  TCMDB *db = cMDB_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  vstr = StringValueEx(vstr);
+
+  res = true;
+  switch(method){
+    case TTPUT:
+      tcmdbput(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vstr), RSTRING_LEN(vstr));
+      break;
+    case TTPUTKEEP:
+      res = tcmdbputkeep(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vstr), RSTRING_LEN(vstr));
+      break;
+    case TTPUTCAT:
+      tcmdbputcat(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vstr), RSTRING_LEN(vstr));
+      break;
+    default:
+      res = false;
+      break;
+  }
+
+  return res ? Qtrue : Qfalse;
+}
+
+static VALUE cMDB_put(VALUE vself, VALUE vkey, VALUE vstr){
+  return cMDB_put_method(vself, vkey, vstr, TTPUT);
+}
+
+static VALUE cMDB_putkeep(VALUE vself, VALUE vkey, VALUE vstr){
+  return cMDB_put_method(vself, vkey, vstr, TTPUTKEEP);  
+}
+
+static VALUE cMDB_putcat(VALUE vself, VALUE vkey, VALUE vstr){
+  return cMDB_put_method(vself, vkey, vstr, TTPUTCAT);
+}
+
+static VALUE cMDB_get(VALUE vself, VALUE vkey){
+  VALUE vval;
+  char *buf;
+  int bsiz;
+  TCMDB *db = cMDB_getdb(vself);
+
+  // this is ugly
+  vkey = StringValueEx(vkey);
+  if(!(buf = tcmdbget(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), &bsiz))){
+    return Qnil;
+  } else {
+    vval = StringRaw(buf, bsiz);
+  }
+
+  tcfree(buf);
+  return vval;
+}
+
+static VALUE cMDB_vsiz(VALUE vself, VALUE vkey){
+  TCMDB *db = cMDB_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  return INT2NUM(tcmdbvsiz(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey)));
+}
+
+static VALUE cMDB_fetch(int argc, VALUE *argv, VALUE vself){
+  VALUE vkey, vrv, vforce;
+  rb_scan_args(argc, argv, "11", &vkey, &vforce);
+  if(rb_block_given_p() != Qtrue) rb_raise(rb_eArgError, "no block given");
+  if(vforce == Qnil) vforce = Qfalse;
+
+  if(vforce != Qfalse || (vrv = cMDB_get(vself, vkey)) == Qnil){
+    vrv = rb_yield(vkey);
+    cMDB_put(vself, vkey, vrv);
+  }
+  return vrv;
+}
+
+static VALUE cMDB_each(VALUE vself){
+  VALUE vrv;
+  if(rb_block_given_p() != Qtrue) rb_raise(rb_eArgError, "no block given");
+  TCMDB *db = cMDB_getdb(vself);
+  vrv = Qnil;
+  tcmdbiterinit(db);
+  int ksiz;
+  char *kbuf;
+  while((kbuf = tcmdbiternext(db, &ksiz)) != NULL){
+    int vsiz;
+    char *vbuf = tcmdbget(db, kbuf, ksiz, &vsiz);
+    vrv = rb_yield_values(2, rb_str_new2(kbuf), StringRaw(vbuf, vsiz));
+    tcfree(vbuf);
+    tcfree(kbuf);
+  }
+  return vrv;
+}
+
+static VALUE cMDB_each_value(VALUE vself){
+  VALUE vrv;
+  if(rb_block_given_p() != Qtrue) rb_raise(rb_eArgError, "no block given");
+  TCMDB *db = cMDB_getdb(vself);
+  vrv = Qnil;
+  tcmdbiterinit(db);
+  int ksiz;
+  char *kbuf;
+  while((kbuf = tcmdbiternext(db, &ksiz)) != NULL){
+    int vsiz;
+    char *vbuf = tcmdbget(db, kbuf, ksiz, &vsiz);
+    vrv = rb_yield_values(1, StringRaw(vbuf, vsiz));
+    tcfree(vbuf);
+    tcfree(kbuf);
+  }
+  return vrv;
+}
+
+static VALUE cMDB_values(VALUE vself){
+  VALUE vary;
+  TCMDB *db = cMDB_getdb(vself);
+  vary = rb_ary_new2(tcmdbrnum(db));
+  tcmdbiterinit(db);
+  int ksiz;
+  char *kbuf;
+  while((kbuf = tcmdbiternext(db, &ksiz)) != NULL){
+    int vsiz;
+    char *vbuf = tcmdbget(db, kbuf, ksiz, &vsiz);
+    rb_ary_push(vary, StringRaw(vbuf, vsiz));
+    tcfree(vbuf);
+    tcfree(kbuf);
+  }
+  return vary;
+}
+
+static VALUE cMDB_out(VALUE vself, VALUE vkey){
+  TCMDB *db = cMDB_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  return tcmdbout(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey)) ? Qtrue : Qfalse;
+}
+
+static VALUE cMDB_check(VALUE vself, VALUE vkey){
+  TCMDB *db = cMDB_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  return tcmdbvsiz(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey)) >= 0 ? Qtrue : Qfalse;
+}
+
+static VALUE cMDB_iterinit(VALUE vself){
+  TCMDB *db = cMDB_getdb(vself);
+
+  tcmdbiterinit(db);
+  return Qtrue;
+}
+
+static VALUE cMDB_iternext(VALUE vself){
+  VALUE vval;
+  char *vbuf;
+  TCMDB *db = cMDB_getdb(vself);
+
+  if(!(vbuf = tcmdbiternext2(db))) return Qnil;
+  vval = rb_str_new2(vbuf);
+  tcfree(vbuf);
+
+  return vval;
+}
+
+static VALUE cMDB_fwmkeys(int argc, VALUE *argv, VALUE vself){
+  VALUE vprefix, vmax, vary;
+  TCLIST *keys;
+  int max;
+  TCMDB *db = cMDB_getdb(vself);
+  rb_scan_args(argc, argv, "11", &vprefix, &vmax);
+
+  vprefix = StringValueEx(vprefix);
+  max = (vmax == Qnil) ? -1 : NUM2INT(vmax);
+  keys = tcmdbfwmkeys(db, RSTRING_PTR(vprefix), RSTRING_LEN(vprefix), max);
+  vary = listtovary(keys);
+  tclistdel(keys);
+  return vary;
+}
+
+static VALUE cMDB_keys(VALUE vself){
+  /*
+  VALUE vary;
+  char *kxstr;
+  TCMDB *db = cMDB_getdb(vself);
+  vary = rb_ary_new2(tcmdbrnum(db));
+  tcmdbiterinit(db);
+  while((kxstr = tcmdbiternext2(db)) != NULL){
+    rb_ary_push(vary, rb_str_new2(kxstr));
+  }
+  return vary;
+  */
+
+  // Using forward matching keys with an empty string is 100x faster than iternext+get
+  VALUE vary;
+  TCLIST *keys;
+  char *prefix;
+  TCMDB *db = cMDB_getdb(vself);
+  prefix = "";
+  keys = tcmdbfwmkeys2(db, prefix, -1);
+  vary = listtovary(keys);
+  tclistdel(keys);
+  return vary;
+}
+
+static VALUE cMDB_addint(VALUE vself, VALUE vkey, int inum){
+  TCMDB *db = cMDB_getdb(vself);
+  vkey = StringValueEx(vkey);
+
+  inum = tcmdbaddint(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), inum);
+  return inum == INT_MIN ? Qnil : INT2NUM(inum);
+}
+
+static VALUE cMDB_add_int(int argc, VALUE *argv, VALUE vself){
+  VALUE vkey, vnum;
+  int inum = 1;
+
+  rb_scan_args(argc, argv, "11", &vkey, &vnum);
+  vkey = StringValueEx(vkey);
+  if(NIL_P(vnum)) vnum = INT2NUM(inum);
+
+  return cMDB_addint(vself, vkey, NUM2INT(vnum));
+}
+
+static VALUE cMDB_get_int(VALUE vself, VALUE vkey){
+  return cMDB_addint(vself, vkey, 0);
+}
+
+static VALUE cMDB_adddouble(VALUE vself, VALUE vkey, double dnum){
+  TCMDB *db = cMDB_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  dnum = tcmdbadddouble(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), dnum);
+  return isnan(dnum) ? Qnil : rb_float_new(dnum);
+}
+
+static VALUE cMDB_add_double(int argc, VALUE *argv, VALUE vself){
+  VALUE vkey, vnum;
+  double dnum = 1.0;
+
+  rb_scan_args(argc, argv, "11", &vkey, &vnum);
+  vkey = StringValueEx(vkey);
+  if(NIL_P(vnum)) vnum = rb_float_new(dnum);
+
+  return cMDB_adddouble(vself, vkey, NUM2DBL(vnum));
+}
+
+static VALUE cMDB_get_double(VALUE vself, VALUE vkey){
+  return cMDB_adddouble(vself, vkey, 0.0);
+}
+
+static VALUE cMDB_vanish(VALUE vself){
+  TCMDB *db = cMDB_getdb(vself);
+
+  tcmdbvanish(db);
+  return Qtrue;
+}
+
+static VALUE cMDB_rnum(VALUE vself){
+  TCMDB *db = cMDB_getdb(vself);
+
+  return LL2NUM(tcmdbrnum(db));
+}
+
+static VALUE cMDB_empty(VALUE vself){
+  TCMDB *db = cMDB_getdb(vself);
+
+  return tcmdbrnum(db) < 1 ? Qtrue : Qfalse;
+}
+
+static VALUE cMDB_size(VALUE vself){
+  TCMDB *db = cMDB_getdb(vself);
+
+  return LL2NUM(tcmdbmsiz(db));
+}
+
+static VALUE cMDB_each_key(VALUE vself){
+  VALUE vrv;
+  char *kxstr;
+  if(rb_block_given_p() != Qtrue) rb_raise(rb_eArgError, "no block given");
+  TCMDB *db = cMDB_getdb(vself);
+  vrv = Qnil;
+  tcmdbiterinit(db);
+  while((kxstr = tcmdbiternext2(db)) != NULL){
+    vrv = rb_yield_values(1, rb_str_new2(kxstr));
+  }
+  return vrv;
+}
+
+void init_utils(){
+  rb_define_private_method(cMDB, "initialize", cMDB_initialize, -1);
+  rb_define_method(cMDB, "bnum", cMDB_bnum, 0);
+  rb_define_method(cMDB, "put", cMDB_put, 2);
+  rb_define_alias(cMDB, "[]=", "put");
+  rb_define_method(cMDB, "putkeep", cMDB_putkeep, 2);
+  rb_define_method(cMDB, "putcat", cMDB_putcat, 2);
+  rb_define_method(cMDB, "get", cMDB_get, 1);
+  rb_define_alias(cMDB, "[]", "get");
+  rb_define_method(cMDB, "vsiz", cMDB_vsiz, 1);
+  rb_define_method(cMDB, "out", cMDB_out, 1);
+  rb_define_method(cMDB, "check", cMDB_check, 1);
+  rb_define_alias(cMDB, "has_key?", "check");
+  rb_define_alias(cMDB, "key?", "check");
+  rb_define_alias(cMDB, "include?", "check");
+  rb_define_alias(cMDB, "member?", "check");
+  rb_define_method(cMDB, "iterinit", cMDB_iterinit, 0);
+  rb_define_method(cMDB, "iternext", cMDB_iternext, 0);
+  rb_define_method(cMDB, "fwmkeys", cMDB_fwmkeys, -1);
+  rb_define_method(cMDB, "keys", cMDB_keys, 0);
+  rb_define_method(cMDB, "add_int", cMDB_add_int, -1);
+  rb_define_alias(cMDB, "addint", "add_int");
+  rb_define_alias(cMDB, "increment", "add_int");
+  rb_define_method(cMDB, "get_int", cMDB_get_int, 1);
+  rb_define_method(cMDB, "add_double", cMDB_add_double, -1);
+  rb_define_alias(cMDB, "adddouble", "add_double");
+  rb_define_method(cMDB, "get_double", cMDB_get_double, 1);
+  rb_define_method(cMDB, "vanish", cMDB_vanish, 0);
+  rb_define_alias(cMDB, "clear", "vanish");
+  rb_define_method(cMDB, "rnum", cMDB_rnum, 0);
+  rb_define_alias(cMDB, "count", "rnum");
+  rb_define_alias(cMDB, "length", "rnum");
+  rb_define_method(cMDB, "size", cMDB_size, 0);
+  rb_define_method(cMDB, "empty?", cMDB_empty, 0);
+  rb_define_method(cMDB, "each_key", cMDB_each_key, 0);
+
+  rb_define_method(cMDB, "fetch", cMDB_fetch, -1);
+  rb_define_method(cMDB, "each", cMDB_each, 0);
+  rb_define_alias(cMDB, "each_pair", "each");
+  rb_define_method(cMDB, "each_value", cMDB_each_value, 0);
+  rb_define_method(cMDB, "values", cMDB_values, 0);
+}

data/ext/tokyo_utils.h

@@ -0,0 +1,8 @@
+#ifndef RUBY_TOKYO_UTILS
+#define RUBY_TOKYO_UTILS
+
+#include <tokyo_tyrant.h>
+
+void init_utils();
+
+#endif

data/lib/tokyo_tyrant/balancer.rb

@@ -0,0 +1,189 @@
+require 'tokyo_tyrant'
+# require 'hash_ring'
+require 'fast_hash_ring'
+
+module TokyoTyrant
+  module Balancer
+    class Base
+      def initialize(servers = [], weights = {})
+        servers.collect! do |server|
+          host, port = server.split(':')
+          klass.new(host, port.to_i)
+        end
+        @servers = servers
+        # @ring = HashRing.new(servers, weights)
+        @ring = FastHashRing.new(servers, weights)
+      end
+
+      def ring
+        @ring
+      end
+
+      def servers
+        @servers
+      end
+
+      def db_for_key(key)
+        ring.get_node(key)
+      end
+
+      # Delegate Methods
+      def get(key)
+        db = db_for_key(key)
+        db.get(key)
+      end
+      alias :[] :get
+
+      def add_int(key, i = 1)
+        db = db_for_key(key)
+        db.add_int(key, i)
+      end
+      alias :addint    :add_int
+      alias :increment :add_int
+
+      def get_int(key)
+        db = db_for_key(key)
+        db.get_int(key)
+      end
+
+      def add_double(key, i = 1.0)
+        db = db_for_key(key)
+        db.add_double(key, i)
+      end
+      alias :adddouble :add_double
+
+      def get_double(key)
+        db = db_for_key(key)
+        db.get_double(key)
+      end
+
+      def put(key, value)
+        db = db_for_key(key)
+        db.put(key, value)
+      end
+      alias :[]= :put
+
+      def putkeep(key, value)
+        db = db_for_key(key)
+        db.putkeep(key, value)
+      end
+
+      def putcat(key, value)
+        db = db_for_key(key)
+        db.putcat(key, value)
+      end
+
+      def putshl(key, value, width)
+        db = db_for_key(key)
+        db.putshl(key, value, width)
+      end
+
+      def putnr(key, value)
+        db = db_for_key(key)
+        db.putnr(key, value)
+      end
+
+      def vsiz(key)
+        db = db_for_key(key)
+        db.vsiz(key)
+      end
+
+      def fetch(key, &block)
+        db = db_for_key(key)
+        db.fetch(key, &block)
+      end
+
+      def out(key)
+        db = db_for_key(key)
+        db.out(key)
+      end
+      alias :delete :out
+
+      # Aggregate Methods
+      def close
+        @servers.all?{ |server| server.close }
+      end
+
+      def rnum
+        @servers.collect{ |server| server.rnum }.inject(nil){ |sum,x| sum ? sum+x : x }
+      end
+      alias :count  :rnum
+      alias :size   :rnum
+      alias :length :rnum
+
+      def empty?
+        @servers.all?{ |server| server.empty? }
+      end
+
+      def vanish
+        @servers.all?{ |server| server.vanish }
+      end
+      alias :clear :vanish
+
+      def sync
+        @servers.each{ |server| server.sync }
+      end
+
+      def optimize(*args)
+        @servers.all?{ |server| server.optimize(*args) }
+      end
+
+      def check(key)
+        @servers.any?{ |server| server.check(key) }
+      end
+      alias :has_key? :check
+      alias :key?     :check
+      alias :include? :check
+      alias :member?  :check
+
+      def set_index(name, type)
+        @servers.all?{ |server| server.set_index(name, type) }
+      end
+
+      def fwmkeys(prefix, max = -1)
+        @servers.collect{ |server| server.fwmkeys(prefix, max) }.flatten
+      end
+
+      def delete_keys_with_prefix(prefix, max = -1)
+        @servers.each{ |server| server.delete_keys_with_prefix(prefix, max) }
+        nil
+      end
+      alias :dfwmkeys :delete_keys_with_prefix
+
+      def keys
+        @servers.collect{ |server| server.keys }.flatten.uniq
+      end
+
+      def values
+        @servers.collect{ |server| server.values }.flatten.uniq
+      end
+    end
+  end
+end
+
+module TokyoTyrant
+  module Balancer
+    class DB < Base
+      def klass
+        TokyoTyrant::DB
+      end
+    end
+  end
+end
+
+module TokyoTyrant
+  module Balancer
+    class Table < Base
+      def klass
+        TokyoTyrant::Table
+      end
+
+      def find(&block)
+        queries = @servers.collect{ |server|
+          server.prepare_query(&block)
+        }
+        TokyoTyrant::Query.parallel_search(*queries)
+      end
+    end
+  end
+end

data/spec/spec_base.rb

@@ -9,6 +9,7 @@ require 'rubygems'
 require 'fileutils'
 require 'bacon'
 require 'tokyo_tyrant'
+require 'lib/tokyo_tyrant/balancer'
 
 $root.class.glob('spec/*_spec.rb').each {|l| load l}
 

data/spec/tokyo_tyrant_balancer_db_spec.rb

@@ -0,0 +1,160 @@
+require 'pathname'
+require Pathname(__FILE__).dirname.join('spec_base') unless $root
+
+describe TokyoTyrant::Balancer::DB, "with an open database" do
+
+  before do
+    @db = TokyoTyrant::Balancer::DB.new(['127.0.0.1:45000'])
+    @db.clear
+  end
+
+  it "should not be nil" do
+    @db.should.not.be.nil
+  end
+
+  it "should close" do
+    @db.close.should.be.true
+    begin
+      @db.close
+    rescue => e
+      e.message.should == 'close error: invalid operation'
+    end
+  end
+
+  it "should optimize" do
+    @db.optimize.should.be.true
+  end
+
+  it "should save a value" do
+    @db['salad'] = 'bacon bits'
+    @db['salad'].should == 'bacon bits'
+  end
+  
+  it "should return a value" do
+    @db['salad'] = 'bacon bits'
+    @db['salad'].should == 'bacon bits'
+  end
+
+  it "should accept binary data" do
+    s = "mango#{0.chr}salsa"
+    @db.put(s, s).should.be.true
+    @db[s].should.equal(s)
+  end
+
+  it "should out a value" do
+    k = 'tomato'
+    @db[k] = 'green'
+    @db.out(k).should.be.true
+    @db[k].should.be.nil
+    @db.out(k).should.be.false
+  end
+
+  it "should get a value size" do
+    k = 'cereal'
+    v = 'granola'
+    @db[k] = v
+    @db.vsiz(k).should == v.size
+  end
+
+  it "should check a key" do
+    k = 'fruit'
+    @db[k] = 'banana'
+    @db.check(k).should.be.true
+    @db.out(k)
+    @db.check(k).should.be.false
+  end
+
+  it "should get forward matching keys" do
+    @db['apples/royalgala'] = '4173'
+    @db['apples/grannysmith'] = '4139'
+    @db['bananas/yellow'] = '4011'
+    @db['oranges/shamouti'] = '3027'
+    @db['grapefruit/deepred'] = '4288'
+    @db.fwmkeys('apples').sort.should == ["apples/grannysmith", "apples/royalgala"]
+  end
+
+  it "should delete forward matching keys" do
+    @db['apples/royalgala'] = '4173'
+    @db['apples/grannysmith'] = '4139'
+    @db['bananas/yellow'] = '4011'
+    @db['oranges/shamouti'] = '3027'
+    @db['grapefruit/deepred'] = '4288'
+    @db.delete_keys_with_prefix('apples').should == nil
+    @db.fwmkeys('apples').should.be.empty
+    @db.keys.sort.should == ['bananas/yellow', 'grapefruit/deepred', 'oranges/shamouti']
+  end
+
+  it "should get all keys" do
+    keys = %w[appetizers entree dessert]
+    values = %w[chips chicken\ caeser\ salad ice\ cream]
+    keys.each_with_index do |k,i|
+      @db[k] = values[i]
+    end
+    @db.keys.should == keys
+  end
+
+  it "should get all values" do
+    keys = %w[appetizers entree dessert]
+    values = %w[chips chicken\ caeser\ salad ice\ cream]
+    keys.each_with_index do |k,i|
+      @db[k] = values[i]
+    end
+    @db.values.should == values
+  end
+
+  it "should vanish all records" do
+    @db['chocolate'] = 'dark'
+    @db['tea'] = 'earl grey'
+    @db.empty?.should.be.false
+    @db.vanish.should.be.true
+    @db.empty?.should.be.true
+  end
+
+  it "should count records" do
+    @db['hummus'] = 'chickpeas'
+    @db['falafel'] = 'chickpeas'
+    @db.rnum.should == 2
+  end
+
+  it "should fetch a record" do
+    @db.out('beer')
+    @db.fetch('beer'){ 'heineken' }.should == 'heineken'
+    @db['beer'].should == 'heineken'
+    @db.fetch('beer'){ 'becks' }.should == 'heineken'
+  end
+
+  it "should add serialized integer values" do
+    key = 'counter'
+    @db.out(key)
+    @db.add_int(key, 1).should == 1
+    @db.add_int(key, 1).should == 2
+    @db.get_int(key).should == 2
+  end
+
+  it "should increment integer values" do
+    key = 'counter'
+    @db.out(key)
+    @db.increment(key).should == 1
+    @db.increment(key, 2).should == 3
+    @db.get_int(key).should == 3
+  end
+
+  it "should add serialized double values" do
+    key = 'counter'
+    @db.out(key)
+    @db.add_double(key, 1.0).should.be.close?(1.0, 0.005)
+    @db.add_double(key, 1.0).should.be.close?(2.0, 0.005)
+    @db.get_double(key).should.be.close?(2.0, 0.005)
+  end
+
+  it "should serialize objects that respond to to_tokyo_tyrant" do
+    class Thing
+      def to_tokyo_tyrant
+        "success"
+      end
+    end
+
+    @db["to_tokyo_tyrant"] = Thing.new
+    @db["to_tokyo_tyrant"].should == "success"
+  end
+end

data/spec/tokyo_tyrant_balancer_table_spec.rb

@@ -0,0 +1,177 @@
+require 'pathname'
+require Pathname(__FILE__).dirname.join('spec_base') unless $root
+
+describe TokyoTyrant::Balancer::Table, "with an open database" do
+
+  before do
+    @db = TokyoTyrant::Balancer::Table.new(['127.0.0.1:45001'])
+    @db.clear
+  end
+
+  it "should not be nil" do
+    @db.should.not.be.nil
+  end
+
+  it "should close" do
+    @db.close.should.be.true
+    begin
+      @db.close
+    rescue => e
+      e.message.should == 'close error: invalid operation'
+    end
+  end
+
+  it "should optimize" do
+    @db.optimize.should.be.true
+  end
+
+  it "should save a value" do
+    value = { 'lettuce' => 'Red Leaf', 'dressing' => 'ranch', 'extra' => 'bacon bits' }
+    @db['salad'] = value
+    @db['salad'].should == value
+  end
+  
+  it "should return a value" do
+    value = { 'lettuce' => 'Red Leaf', 'dressing' => 'ranch', 'extra' => 'bacon bits' }
+    @db['salad'] = value
+    @db['salad'].should == value
+  end
+
+  it "should accept binary data" do
+    s = "mango#{0.chr}salsa"
+    h = { s => s }
+    @db.put(s, h).should.be.true
+    @db[s].should.equal(h)
+  end
+
+  it "should out a value" do
+    k = 'tomato'
+    @db[k] = { 'color' => 'red', 'variety' => 'beefy' }
+    @db.out(k).should.be.true
+    @db[k].should.be.nil
+    @db.out(k).should.be.false
+  end
+
+  it "should check a key" do
+    k = 'fruit'
+    @db[k] = { 'name' => 'banana', 'code' => '4011' }
+    @db.check(k).should.be.true
+    @db.out(k)
+    @db.check(k).should.be.false
+  end
+
+  it "should get forward matching keys" do
+    @db['apples/royalgala'] = { 'code' => '4173', 'color' => 'red-yellow' }
+    @db['apples/grannysmith'] = { 'code' => '4139', 'color' => 'green' }
+    @db['bananas/yellow'] = { 'code' => '4011', 'color' => 'yellow' }
+    @db['oranges/shamouti'] = { 'code' => '3027', 'color' => 'orange' }
+    @db['grapefruit/deepred'] = { 'code' => '4288', 'color' => 'yellow/pink' }
+    @db.fwmkeys('apples').sort.should == ["apples/grannysmith", "apples/royalgala"]
+  end
+
+  it "should delete forward matching keys" do
+    @db['apples/royalgala'] = { 'code' => '4173', 'color' => 'red-yellow' }
+    @db['apples/grannysmith'] = { 'code' => '4139', 'color' => 'green' }
+    @db['bananas/yellow'] = { 'code' => '4011', 'color' => 'yellow' }
+    @db['oranges/shamouti'] = { 'code' => '3027', 'color' => 'orange' }
+    @db['grapefruit/deepred'] = { 'code' => '4288', 'color' => 'yellow/pink' }
+    @db.delete_keys_with_prefix('apples').should == nil
+    @db.fwmkeys('apples').should.be.empty
+    @db.keys.sort.should == ['bananas/yellow', 'grapefruit/deepred', 'oranges/shamouti']
+  end
+
+  it "should get all keys" do
+    keys = %w[appetizers entree dessert]
+    values = [{ 'cheap' => 'chips', 'expensive' => 'sample everything platter' },
+              { 'cheap' => 'hoagie', 'expensive' => 'steak' },
+              { 'cheap' => 'water ice', 'expensive' => 'cheesecake' }]
+
+    keys.each_with_index do |k,i|
+      @db[k] = values[i]
+    end
+    @db.keys.should == keys
+  end
+
+  it "should get all values" do
+    keys = %w[appetizers entree dessert]
+    values = [{ 'cheap' => 'chips', 'expensive' => 'sample everything platter' },
+              { 'cheap' => 'hoagie', 'expensive' => 'steak' },
+              { 'cheap' => 'water ice', 'expensive' => 'cheesecake' }]
+
+    keys.each_with_index do |k,i|
+      @db[k] = values[i]
+    end
+    @db.values.should == values
+  end
+
+  it "should vanish all records" do
+    @db['chocolate'] = { 'type' => 'dark' }
+    @db['tea'] = { 'type' => 'earl grey' }
+    @db.empty?.should.be.false
+    @db.vanish.should.be.true
+    @db.empty?.should.be.true
+  end
+
+  it "should count records" do
+    @db['hummus'] = { 'ingredients' => 'chickpeas,garlic' }
+    @db['falafel'] = { 'ingredients' => 'chickpeas,herbs' }
+    @db.rnum.should == 2
+  end
+
+  it "should fetch a record" do
+    @db.out('beer')
+    @db.fetch('beer'){{ 'import' => 'heineken' }}.should == { 'import' => 'heineken' }
+    @db['beer'].should == { 'import' => 'heineken' }
+    @db.fetch('beer'){{ 'import' => 'becks' }}.should == { 'import' => 'heineken' }
+  end
+
+  it "should add serialized integer values" do
+    key = 'counter'
+    @db.out(key)
+    @db[key] = { 'title' => 'Bean Counter' }
+    @db.add_int(key, 1).should == 1
+    @db.add_int(key, 1).should == 2
+    @db.get_int(key).should == 2
+    @db[key].should == { 'title' => 'Bean Counter', '_num' => "2" }
+  end
+
+  it "should increment integer values" do
+    key = 'counter'
+    @db.out(key)
+    @db[key] = { 'title' => 'Bean Counter' }
+    @db.increment(key).should == 1
+    @db.increment(key, 2).should == 3
+    @db.get_int(key).should == 3
+    @db[key].should == { 'title' => 'Bean Counter', '_num' => "3" }
+  end
+
+  it "should add serialized double values" do
+    key = 'counter'
+    @db.out(key)
+    @db[key] = { 'title' => 'Bean Counter' }
+    @db.add_double(key, 1.0).should.be.close?(1.0, 0.005)
+    @db.add_double(key, 1.0).should.be.close?(2.0, 0.005)
+    @db.get_double(key).should.be.close?(2.0, 0.005)
+    @db[key].should == { 'title' => 'Bean Counter', '_num' => "2" }
+  end
+
+  it "should set an index" do
+    key = 'counter'
+    50.times do |i|
+      @db["key#{i}"] = { 'title' => %w{rice beans corn}.sort_by{rand}.first,
+                         'description' => 'a whole protein' }
+    end
+    @db.set_index('title', :lexical).should.be.true
+  end
+
+  it "should serialize objects that respond to to_tokyo_tyrant" do
+    class Thing
+      def to_tokyo_tyrant
+        "success"
+      end
+    end
+
+    @db["to_tokyo_tyrant"] = { "thing" => Thing.new }
+    @db["to_tokyo_tyrant"].should == { "thing" => "success" }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ruby-tokyotyrant
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
+  version: "0.4"
 platform: ruby
 authors: 
 - Flinn
@@ -9,10 +9,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-17 00:00:00 -04:00
+date: 2009-11-29 00:00:00 -05:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fast_hash_ring
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.1
+    version: 
 description: 
 email: flinn@actsasflinn.com
 executables: []
@@ -35,15 +44,21 @@ files:
 - ext/tokyo_tyrant_query.h
 - ext/tokyo_tyrant_table.c
 - ext/tokyo_tyrant_table.h
+- ext/tokyo_utils.c
+- ext/tokyo_utils.h
+- lib/tokyo_tyrant/balancer.rb
 - spec/ext.lua
 - spec/plu_db.rb
 - spec/spec.rb
 - spec/spec_base.rb
 - spec/start_tyrants.sh
 - spec/stop_tyrants.sh
+- spec/tokyo_tyrant_balancer_db_spec.rb
+- spec/tokyo_tyrant_balancer_table_spec.rb
 - spec/tokyo_tyrant_query_spec.rb
 - spec/tokyo_tyrant_spec.rb
 - spec/tokyo_tyrant_table_spec.rb
+- benchmarks/balancer.rb
 - benchmarks/bulk_db.rb
 - benchmarks/bulk_table.rb
 - benchmarks/db.rb