ruby-tokyotyrant 0.4 → 0.5

data/README.rdoc

@@ -6,9 +6,7 @@ This is a c extension for Ruby to access TokyoTyrant databases.  It currently su
 
   # install tokyocabinet (1.4.34) and tokyotyrant (requires 1.1.35)
   # after installing tc and tt on linux I had to /sbin/ldconfig (as root)
-  sudo gem install gemcutter
-  sudo gem tumble
-  sudo gem install ruby-tokyotyrant
+  sudo gem install ruby-tokyotyrant --source=http://gemcutter.org
 
 == Performance
 
@@ -39,6 +37,28 @@ This is not in production but the initial benchmarks are very interesting. Resul
   db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
   db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}
 
+=== B+ Tree DB
+
+  # start tyrant like so:
+  # ttserver example.tcb
+
+  require 'tokyo_tyrant'
+  bdb = TokyoTyrant::BDB.new('127.0.0.1', 1978)
+
+  bdb.putdup('foo', 'bar')
+  # => true
+
+  bdb.putlist({ 'foo' => ['baz', 'bat']})
+  # => []
+
+  bdb.getlist('foo')
+  # => {"foo"=>["bar", "baz", "bat"]}
+
+  bdb.each{ |k,v| puts [k, v].inspect }
+  # ["foo", "bar"]
+  # ["foo", "baz"]
+  # ["foo", "bat"]
+
 === Table DB
 
   # start tyrant like so:

data/Rakefile

@@ -20,7 +20,7 @@ CLEAN.include('pkg', 'tmp')
 
 gemspec = Gem::Specification.new do |s|
   s.name = 'ruby-tokyotyrant'
-  s.version = '0.4'
+  s.version = '0.5'
   s.authors = [ 'Flinn' ]
   s.email = 'flinn@actsasflinn.com'
   s.homepage = 'http://github.com/actsasflinn/ruby-tokyotyrant/'
@@ -35,12 +35,10 @@ gemspec = Gem::Specification.new do |s|
              'Rakefile',
              'README.rdoc'] +
              Dir['ext/**/*.[rb|c|h]'] +
-             Dir['lib/**/*.rb'] +
+             Dir['ext/tokyo_tyrant/**/*.rb'] +
              Dir['spec/**/*'] +
              Dir['benchmarks/**/*']
   s.extensions << "ext/extconf.rb"
-
-  s.add_runtime_dependency('fast_hash_ring', '>= 0.1.1')
 end
 
 task :gemspec do
@@ -53,7 +51,7 @@ Rake::GemPackageTask.new(gemspec) do |pkg|
   pkg.need_tar = true
 end
 
-Rake::PackageTask.new('ruby-tokyotyrant', '0.4') do |pkg|
+Rake::PackageTask.new('ruby-tokyotyrant', '0.5') do |pkg|
   pkg.need_zip = true
   pkg.package_files = FileList[
     'COPYING',

data/benchmarks/bulk_db.rb

@@ -1,3 +1,26 @@
+=begin
+Tokyo Tyrant Bulk Operations Benchmark
+
+
+TokyoTyrant::RDB (Ruby) mget
+                          user     system      total        real
+inserting data        7.770000   1.870000   9.640000 ( 12.598373)
+reading data          1.540000   0.250000   1.790000 (  2.290628)
+
+
+TokyoTyrant (c) mput/mget
+                          user     system      total        real
+inserting data        0.020000   0.000000   0.020000 (  0.037996)
+reading data          0.060000   0.010000   0.070000 (  0.120651)
+
+
+Memcached (C) set/get_multi
+                          user     system      total        real
+inserting data*       0.160000   0.140000   0.300000 (  0.884097)
+reading data          0.090000   0.000000   0.090000 (  0.140217)
+* bulk operation not supported
+=end
+
 require 'benchmark'
 require 'rubygems'
 require 'faker'

data/benchmarks/bulk_table.rb

@@ -1,3 +1,31 @@
+=begin
+Tokyo Tyrant Bulk Table Operations Benchmark
+
+
+TokyoTyrant (C)
+                          user     system      total        real
+bulk writing data     0.180000   0.000000   0.180000 (  0.292081)
+bulk reading data     0.120000   0.020000   0.140000 (  0.216074)
+
+
+TokyoTyrant::RDB (Ruby)
+                          user     system      total        real
+bulk inserting data* 42.410000   4.130000  46.540000 ( 63.657929)
+bulk reading data     2.360000   0.280000   2.640000 (  3.709915)
+
+
+Mongo
+                          user     system      total        real
+bulk writing data    27.860000   0.470000  28.330000 ( 37.619586)
+bulk reading data     0.650000   0.100000   0.750000 (  1.371567)
+
+Notes:
+* To supply a hash for bulk operations TokyoTyrant::RDB creates an array and join hash columns.
+* This operation is included in the benchmark because the same code exists within the c-ext.
+Rufus::Tokyo::TyrantTable does not support the misc method, therefor cannot handle a putlist/getlist
+Memcached does not support bulk writing
+=end
+
 require 'benchmark'
 require 'rubygems'
 require 'faker'
@@ -10,7 +38,7 @@ $month = (1..12).to_a
 $day = (1..28).to_a # not bothering with month diffs
  
 def rbdate
-  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+  Date.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
 end
  
 def rdiv
@@ -30,9 +58,12 @@ end
 
 data = {}
 10_000.times do |i|
-  data[i] = { :name => Faker::Name.name, :sex => rgen, :birthday => rbdate.to_s, :divisions => rdiv }
+  data[i.to_s] = { :_id => i.to_s, :name => Faker::Name.name, :sex => rgen, :birthday => rbdate.to_s, :divisions => rdiv }
 end
 
+data_keys = data.keys
+data_values = data.values
+
 require 'tokyo_tyrant'
 t = TokyoTyrant::Table.new('127.0.0.1', 45001)
 t.clear
@@ -46,7 +77,7 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   end
 
   b.report('bulk reading data') do
-    nothing = t.mget(0..9999)
+    nothing = t.mget(data_keys)
   end
 end
 
@@ -80,6 +111,23 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   end
 end
 
+require 'mongo'
+db = Mongo::Connection.new.db("benchmark")
+2.times { puts }
+puts 'Mongo'
+mdb = db["data"]
+mdb.remove
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('bulk writing data') do
+    nothing = mdb.insert(data_values)
+  end
+
+  b.report('bulk reading data') do
+    nothing = mdb.find(:_id => {'$in' => data_keys}).to_a
+  end
+end
+
 puts "\nNotes:"
 puts "* To supply a hash for bulk operations TokyoTyrant::RDB creates an array and join hash columns."
 puts "* This operation is included in the benchmark because the same code exists within the c-ext."

data/benchmarks/db.rb

@@ -1,14 +1,56 @@
+=begin
+Tokyo::Tyrant (Ruby FFI)
+                          user     system      total        real
+inserting data        0.140000   0.180000   0.320000 (  1.208150)
+reading data          0.240000   0.170000   0.410000 (  1.467640)
+
+
+TokyoTyrant::RDB (Ruby)
+                          user     system      total        real
+inserting data        0.300000   0.170000   0.470000 (  1.481900)
+reading data          0.410000   0.210000   0.620000 (  1.705723)
+
+
+TokyoTyrant (C)
+                          user     system      total        real
+inserting data        0.130000   0.170000   0.300000 (  1.251842)
+reading data          0.120000   0.160000   0.280000 (  1.539256)
+
+
+Memcached (C) to Tyrant
+                          user     system      total        real
+inserting data        0.200000   0.150000   0.350000 (  1.814691)
+reading data          0.180000   0.160000   0.340000 (  1.358712)
+
+
+Memcached (C) to Memcached
+                          user     system      total        real
+inserting data        0.190000   0.140000   0.330000 (  1.022183)
+reading data          0.160000   0.140000   0.300000 (  0.908084)
+
+
+MemCache (Ruby)
+                          user     system      total        real
+inserting data        1.200000   0.210000   1.410000 (  2.329328)
+reading data          1.510000   0.220000   1.730000 (  2.721509)
+
+
+Voldemort (Ruby)
+                          user     system      total        real
+inserting data       18.290000   1.050000  19.340000 ( 37.562370)
+reading data          9.040000   0.530000   9.570000 ( 15.145843)
+=end
+
 require 'benchmark'
 require 'rubygems'
 require 'faker'
 
 data = []
 
-100_000.times do |i|
+10_000.times do |i|
   data << Faker::Name.name
 end
 
-=begin
 require 'rufus/tokyo/tyrant'
 
 r = Rufus::Tokyo::Tyrant.new('127.0.0.1', 45000)
@@ -45,10 +87,9 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
     data.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
   end
 end
-=end
 
 require 'tokyo_tyrant'
-t = TokyoTyrant::DB.new('127.0.0.1', 1978)
+t = TokyoTyrant::DB.new('127.0.0.1', 45000)
 t.clear
 
 2.times { puts }
@@ -64,23 +105,6 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   end
 end
 
-m = TokyoTyrant::MDB.new
-m.clear
-
-2.times { puts }
-puts 'MDB'
-
-Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
-  b.report('inserting data') do
-    data.each_with_index { |e, i| m[i] = e }
-  end
-
-  b.report('reading data') do
-    data.each_with_index { |e, i| nothing = m[i] }
-  end
-end
-
-=begin
 require 'memcached'
 m = Memcached.new('127.0.0.1:45000')
 m.flush
@@ -131,4 +155,19 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
     data.each_with_index { |e, i| nothing = mc.get(i.to_s) }
   end
 end
-=end
+
+require 'voldemort-rb'
+vdb = VoldemortClient.new("test", "localhost:6666")
+2.times { puts }
+puts 'Voldemort (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| vdb.put(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = vdb.get(i.to_s) }
+  end
+end
+

data/benchmarks/table.rb

@@ -1,3 +1,40 @@
+=begin
+Tokyo::TyrantTable (Ruby FFI)
+                          user     system      total        real
+inserting data        0.840000   0.190000   1.030000 (  3.214565)
+reading data          1.310000   0.210000   1.520000 (  3.024142)
+
+
+TokyoTyrant::RDB (Ruby)
+                          user     system      total        real
+inserting data        1.170000   0.240000   1.410000 (  2.859321)
+reading data          2.250000   0.660000   2.910000 (  4.752291)
+
+
+TokyoTyrant (C)
+                          user     system      total        real
+inserting data        0.260000   0.180000   0.440000 (  1.637843)
+reading data          0.300000   0.170000   0.470000 (  1.631104)
+
+
+Memcached (C)
+                          user     system      total        real
+inserting data        0.300000   0.160000   0.460000 (  1.230830)
+reading data          0.310000   0.140000   0.450000 (  1.171531)
+
+
+MemCache (Ruby)
+                          user     system      total        real
+inserting data        2.080000   0.220000   2.300000 (  3.379047)
+reading data          2.220000   0.200000   2.420000 (  4.018160)
+
+
+Mongo
+                          user     system      total        real
+inserting data        1.590000   0.130000   1.720000 (  2.310027)
+reading data          4.220000   0.380000   4.600000 (  7.178335)
+=end
+
 require 'benchmark'
 require 'rubygems'
 require 'faker'
@@ -7,14 +44,14 @@ require 'date'
 # the data
 #
  
-colnames = %w{ name sex birthday divisions }
+colnames = %w{ _id name sex birthday divisions }
  
 $year = (1909 .. 2009).to_a
 $month = (1..12).to_a
 $day = (1..28).to_a # not bothering with month diffs
  
 def rbdate
-  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+  Date.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
 end
  
 def rdiv
@@ -31,16 +68,16 @@ end
 def rgen
   (rand(2) == 1 ? 'male' : 'female')
 end
- 
+
 data = [
-  [ 'Alphonse Armalite', 'male', DateTime.new(1972, 10, 14), 'brd,dev' ],
-  [ 'Brutus Beromunster', 'male', DateTime.new(1964, 07, 14), 'dev' ],
-  [ 'Crystel Chucknorris', 'female', DateTime.new(1980, 07, 12), 'brd' ],
-  [ 'Desree Dylan', 'female', DateTime.new(1954, 07, 13), 'brd,dev' ]
+  ['alphonse-armalite', 'Alphonse Armalite', 'male', Date.new(1972, 10, 14).to_s, 'brd,dev'],
+  ['brutus-beromunster', 'Brutus Beromunster', 'male', Date.new(1964, 07, 14).to_s, 'dev'],
+  ['crystel-chucknorris', 'Crystel Chucknorris', 'female', Date.new(1980, 07, 12).to_s, 'brd'],
+  ['desree-dylan', 'Desree Dylan', 'female', Date.new(1954, 07, 13).to_s, 'brd,dev']
 ]
  
 10_000.times do |i|
-  data << [ Faker::Name.name, rgen, rbdate, rdiv]  
+  data << [i.to_s, Faker::Name.name, rgen, rbdate.to_s, rdiv]
 end
  
 $find_name_list = []
@@ -112,14 +149,6 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   b.report('reading data') do
     data1.each_with_index { |e, i| nothing = t[i] }
   end
-
-  b.report('bulk writing data') do
-    nothing = t.mput(data_h)
-  end
-
-  b.report('bulk reading data') do
-    nothing = t.mget(0..9999)
-  end
 end
 
 require 'memcached'
@@ -137,10 +166,6 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
   b.report('reading data') do
     data1.each_with_index { |e, i| nothing = m.get(i.to_s) }
   end
-
-  b.report('bulk reading data') do
-    nothing = m.get((0..9999).to_a.collect{|i| i.to_s})
-  end
 end
 
 require 'memcache'
@@ -159,3 +184,20 @@ Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
     data1.each_with_index { |e, i| nothing = mc.get(i.to_s) }
   end
 end
+
+require 'mongo'
+db = Mongo::Connection.new.db("benchmark")
+2.times { puts }
+puts 'Mongo'
+mdb = db["data"]
+mdb.remove
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| mdb.insert(e) }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = mdb.find_one({ "_id" => i.to_s }) }
+  end
+end

data/ext/tokyo_tyrant.c

@@ -123,22 +123,94 @@ extern TCLIST *vhashtolist(VALUE vhash){
   return list;
 }
 
+extern TCLIST *vhashtoputlist(VALUE vhash){
+  VALUE vkey, vval, vkeys, vvals;
+  TCLIST *list;
+  int i, j;
+  vkeys = rb_funcall(vhash, rb_intern("keys"), 0);
+  list = tclistnew();
+  for(i = 0; i < RARRAY_LEN(vkeys); i++){
+    vkey = rb_ary_entry(vkeys, i);
+    vkey = StringValueEx(vkey);
+
+    vvals = rb_hash_aref(vhash, vkey);
+    if (TYPE(vvals) == T_ARRAY){
+      for(j = 0; j < RARRAY_LEN(vvals); j++){
+        vval = rb_ary_entry(vvals, j);
+        vval = StringValueEx(vval);
+        tclistpush(list, RSTRING_PTR(vkey), RSTRING_LEN(vkey));
+        tclistpush(list, RSTRING_PTR(vval), RSTRING_LEN(vval));
+      }
+    } else {
+      vval = StringValueEx(vvals);
+      tclistpush(list, RSTRING_PTR(vkey), RSTRING_LEN(vkey));
+      tclistpush(list, RSTRING_PTR(vval), RSTRING_LEN(vval));
+    }
+  }
+  return list;
+}
+
+extern VALUE listtovhash(TCLIST *list){
+  VALUE vhash, vkey, vval, vvals;
+  const char *key, *val;
+  int i, num, ksiz, vsiz;
+  vhash = rb_hash_new();
+
+  num = tclistnum(list);
+  for(i = 0; i < num; i += 2){
+    key = tclistval(list, i, &ksiz);
+    val = tclistval(list, i + 1, &vsiz);
+    vkey = rb_str_new(key, ksiz);
+    vval = rb_str_new(val, vsiz);
+
+    vvals = rb_hash_aref(vhash, vkey);
+    if (TYPE(vvals) != T_ARRAY) vvals = rb_ary_new();
+    vvals = rb_ary_push(vvals, vval);
+    rb_hash_aset(vhash, vkey, vvals);
+  }
+  return vhash;
+}
+
 VALUE mTokyoTyrant;
+
 VALUE eTokyoTyrantError;
+VALUE eTokyoTyrantErrorInvalid;
+VALUE eTokyoTyrantErrorNoHost;
+VALUE eTokyoTyrantErrorRefused;
+VALUE eTokyoTyrantErrorSend;
+VALUE eTokyoTyrantErrorReceive;
+VALUE eTokyoTyrantErrorKeep;
+VALUE eTokyoTyrantErrorNoRecord;
+VALUE eTokyoTyrantErrorMisc;
+
 VALUE cDB;
+VALUE cBDB;
 VALUE cTable;
 VALUE cQuery;
-VALUE cMDB;
+VALUE cConstistentHash;
 
 void Init_tokyo_tyrant(){
   mTokyoTyrant = rb_define_module("TokyoTyrant");
+
   eTokyoTyrantError = rb_define_class("TokyoTyrantError", rb_eStandardError);
+  eTokyoTyrantErrorInvalid = rb_define_class("TokyoTyrantErrorInvalid", eTokyoTyrantError);
+  eTokyoTyrantErrorNoHost = rb_define_class("TokyoTyrantErrorNoHost", eTokyoTyrantError);
+  eTokyoTyrantErrorRefused = rb_define_class("TokyoTyrantErrorRefused", eTokyoTyrantError);
+  eTokyoTyrantErrorSend = rb_define_class("TokyoTyrantErrorSend", eTokyoTyrantError);
+  eTokyoTyrantErrorReceive = rb_define_class("TokyoTyrantErrorReceive", eTokyoTyrantError);
+  eTokyoTyrantErrorKeep = rb_define_class("TokyoTyrantErrorKeep", eTokyoTyrantError);
+  eTokyoTyrantErrorNoRecord = rb_define_class("TokyoTyrantErrorNoRecord", eTokyoTyrantError);
+  eTokyoTyrantErrorMisc = rb_define_class("TokyoTyrantErrorMisc", eTokyoTyrantError);
+
   init_mod();
 
   cDB = rb_define_class_under(mTokyoTyrant, "DB", rb_cObject);
   rb_include_module(cDB, mTokyoTyrant);
   init_db();
 
+  cBDB = rb_define_class_under(mTokyoTyrant, "BDB", cDB);
+  init_bdb();
+
   cTable = rb_define_class_under(mTokyoTyrant, "Table", rb_cObject);
   rb_include_module(cTable, mTokyoTyrant);
   init_table();
@@ -146,6 +218,6 @@ void Init_tokyo_tyrant(){
   cQuery = rb_define_class_under(mTokyoTyrant, "Query", rb_cObject);
   init_query();
 
-  cMDB = rb_define_class_under(mTokyoTyrant, "MDB", rb_cObject);
-  init_utils();
+  cConstistentHash = rb_define_class_under(mTokyoTyrant, "ConstistentHash", rb_cObject);
+  init_consistent_hash();
 }