actsasflinn-ruby-tokyotyrant 0.1.2 → 0.1.3

data/benchmarks/bulk_db.rb

@@ -0,0 +1,74 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+
+puts "Tokyo Tyrant Bulk Operations Benchmark"
+
+data   = {}
+data_e = {}
+data_a = []
+
+10_000.times do |i|
+  data[i.to_s]   = Faker::Name.name
+  data_e[i.to_s] = nil
+  data_a << i.to_s
+  data_a << data[i.to_s]
+end
+
+require 'tokyotyrant'
+
+rdb = TokyoTyrant::RDB::new
+rdb.open("127.0.0.1", 1978)
+rdb.clear
+nothing = nil
+
+2.times { puts }
+puts 'TokyoTyrant::RDB (Ruby) mget'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    rdb.misc('putlist', data_a)
+  end
+
+  b.report('reading data') do
+    rdb.mget(data_e)
+  end
+end
+
+require 'tokyo_tyrant'
+t = TokyoTyrant::DB.new('127.0.0.1', 1978)
+t.clear
+nothing = nil
+
+2.times { puts }
+puts 'TokyoTyrant (c) mput/mget'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data ') do
+    t.mput(data)
+  end
+
+  b.report('reading data') do
+    nothing = t.mget(0..9999)
+  end
+end
+
+require 'memcached'
+m = Memcached.new('127.0.0.1:11211')
+m.flush
+nothing = nil
+
+2.times { puts }
+puts 'Memcached (C) set/get_multi'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data*') do
+    data.each_pair { |k, v| m.set(k, v) }
+  end
+
+  b.report('reading data') do
+    nothing = m.get(data.keys)
+  end
+end
+
+puts "* bulk operation not supported"

data/benchmarks/bulk_table.rb

@@ -0,0 +1,85 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+require 'date'
+
+$year = (1909 .. 2009).to_a
+$month = (1..12).to_a
+$day = (1..28).to_a # not bothering with month diffs
+ 
+def rbdate
+  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+end
+ 
+def rdiv
+  case rand(3)
+  when 0
+    'dev'
+  when 1
+    'brd'
+  else
+    'brd,dev'
+  end
+end
+
+def rgen
+  (rand(2) == 1 ? 'male' : 'female')
+end
+
+data = {}
+10_000.times do |i|
+  data[i] = { :name => Faker::Name.name, :sex => rgen, :birthday => rbdate.to_s, :divisions => rdiv }
+end
+
+require 'tokyo_tyrant'
+t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+t.clear
+
+2.times { puts }
+puts 'TokyoTyrant (C)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('bulk writing data') do
+    nothing = t.mput(data)
+  end
+
+  b.report('bulk reading data') do
+    nothing = t.mget(0..9999)
+  end
+end
+
+require 'tokyotyrant'
+
+rdb = TokyoTyrant::RDB::new
+rdb.open("127.0.0.1", 1978)
+rdb.clear
+
+2.times { puts }
+puts 'TokyoTyrant::RDB (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('bulk inserting data*') do
+    # is this fair to put in the benchmark? yes because it happens within the c-ext
+    data_a = []
+    data.each_pair{ |i,v|
+      data_a << i.to_s
+      data_a << v.collect{ |k,v| [k.to_s,v] }.join("\0")
+    }
+    rdb.misc('putlist', data_a)
+  end
+
+  b.report('bulk reading data ') do
+    l = rdb.misc('getlist', data.keys)
+    h = Hash[*l]
+    h.each_pair do |k,v|
+      a = v.split("\0")
+      h[k] = Hash[*a]
+    end
+  end
+end
+
+puts "\nNotes:"
+puts "* To supply a hash for bulk operations TokyoTyrant::RDB creates an array and join hash columns."
+puts "* This operation is included in the benchmark because the same code exists within the c-ext."
+puts "Rufus::Tokyo::TyrantTable does not support the misc method, therefor cannot handle a putlist/getlist"
+puts "Memcached does not support bulk writing"

data/benchmarks/db.rb

@@ -0,0 +1,114 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+
+data = []
+
+10_000.times do |i|
+  data << Faker::Name.name
+end
+
+require 'rufus/tokyo/tyrant'
+
+r = Rufus::Tokyo::Tyrant.new('127.0.0.1', 1978)
+r.clear
+
+2.times { puts }
+puts 'Tokyo::Tyrant (Ruby FFI)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| r[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = r[i.to_s] }
+  end
+end
+
+require 'tokyotyrant'
+
+rdb = TokyoTyrant::RDB::new
+rdb.open("127.0.0.1", 1978)
+rdb.clear
+
+2.times { puts }
+puts 'TokyoTyrant::RDB (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| rdb.put(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
+  end
+end
+
+require 'tokyo_tyrant'
+t = TokyoTyrant::DB.new('127.0.0.1', 1978)
+t.clear
+
+2.times { puts }
+puts 'TokyoTyrant (C)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| t[i] = e }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = t[i] }
+  end
+end
+
+require 'memcached'
+m = Memcached.new('127.0.0.1:1978')
+m.flush
+
+2.times { puts }
+puts 'Memcached (C) to Tyrant'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| m.set(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = m.get(i.to_s) }
+  end
+end
+
+require 'memcached'
+m = Memcached.new('127.0.0.1:11211')
+m.flush
+
+2.times { puts }
+puts 'Memcached (C) to Memcached'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| m.set(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = m.get(i.to_s) }
+  end
+end
+
+require 'memcache'
+mc = MemCache.new('127.0.0.1:11211')
+mc.flush_all
+
+2.times { puts }
+puts 'MemCache (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data.each_with_index { |e, i| mc.set(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data.each_with_index { |e, i| nothing = mc.get(i.to_s) }
+  end
+end

data/benchmarks/table.rb

@@ -0,0 +1,161 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+require 'date'
+
+#
+# the data
+#
+ 
+colnames = %w{ name sex birthday divisions }
+ 
+$year = (1909 .. 2009).to_a
+$month = (1..12).to_a
+$day = (1..28).to_a # not bothering with month diffs
+ 
+def rbdate
+  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+end
+ 
+def rdiv
+  case rand(3)
+  when 0
+    'dev'
+  when 1
+    'brd'
+  else
+    'brd,dev'
+  end
+end
+ 
+def rgen
+  (rand(2) == 1 ? 'male' : 'female')
+end
+ 
+data = [
+  [ 'Alphonse Armalite', 'male', DateTime.new(1972, 10, 14), 'brd,dev' ],
+  [ 'Brutus Beromunster', 'male', DateTime.new(1964, 07, 14), 'dev' ],
+  [ 'Crystel Chucknorris', 'female', DateTime.new(1980, 07, 12), 'brd' ],
+  [ 'Desree Dylan', 'female', DateTime.new(1954, 07, 13), 'brd,dev' ]
+]
+ 
+10_000.times do |i|
+  data << [ Faker::Name.name, rgen, rbdate, rdiv]  
+end
+ 
+$find_name_list = []
+100.times { $find_name_list << data[rand(data.size)][0] }
+ 
+data.collect! { |e|
+  (0..colnames.length - 1).inject({}) { |h, i| h[colnames[i]] = e[i]; h }
+}
+
+data_h = {}
+i = 0
+data1 = data.collect { |e|
+  i = i + 1
+  h = e.dup
+  h['birthday'] = h['birthday'].to_s
+  data_h[i.to_s] = h
+  h
+}
+
+require 'rufus/tokyo/tyrant'
+
+r = Rufus::Tokyo::TyrantTable.new('127.0.0.1', 1978)
+r.clear
+
+2.times { puts }
+puts 'Tokyo::TyrantTable (Ruby FFI)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| r[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = r[i.to_s] }
+  end
+end
+
+require 'tokyotyrant'
+
+rdb = TokyoTyrant::RDBTBL::new
+rdb.open("127.0.0.1", 1978)
+rdb.clear
+
+2.times { puts }
+puts 'TokyoTyrant::RDB (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| rdb.put(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
+  end
+end
+
+require 'tokyo_tyrant'
+t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+t.clear
+
+2.times { puts }
+puts 'TokyoTyrant (C)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| t[i] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = t[i] }
+  end
+
+  b.report('bulk writing data') do
+    nothing = t.mput(data_h)
+  end
+
+  b.report('bulk reading data') do
+    nothing = t.mget(0..9999)
+  end
+end
+
+require 'memcached'
+m = Memcached.new('127.0.0.1:11211')
+m.flush
+
+2.times { puts }
+puts 'Memcached (C)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| m.set(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = m.get(i.to_s) }
+  end
+
+  b.report('bulk reading data') do
+    nothing = m.get((0..9999).to_a.collect{|i| i.to_s})
+  end
+end
+
+require 'memcache'
+mc = MemCache.new('127.0.0.1:11211')
+mc.flush_all
+
+2.times { puts }
+puts 'MemCache (Ruby)'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| mc.set(i.to_s, e) }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = mc.get(i.to_s) }
+  end
+end

data/ext/tokyo_tyrant.c

@@ -0,0 +1,162 @@
+#include <tokyo_tyrant.h>
+
+extern VALUE StringValueEx(VALUE vobj){
+  char kbuf[NUMBUFSIZ];
+  int ksiz;
+  switch(TYPE(vobj)){
+  case T_FIXNUM:
+    ksiz = sprintf(kbuf, "%d", (int)FIX2INT(vobj));
+    return rb_str_new(kbuf, ksiz);
+  case T_BIGNUM:
+    ksiz = sprintf(kbuf, "%lld", (long long)NUM2LL(vobj));
+    return rb_str_new(kbuf, ksiz);
+  case T_SYMBOL:
+    return rb_str_new2(rb_id2name(SYM2ID(vobj)));
+  case T_TRUE:
+    ksiz = sprintf(kbuf, "true");
+    return rb_str_new(kbuf, ksiz);
+  case T_FALSE:
+    ksiz = sprintf(kbuf, "false");
+    return rb_str_new(kbuf, ksiz);
+// I don't like this, I'd rather an empty string
+//  case T_NIL:
+//    ksiz = sprintf(kbuf, "nil");
+//    return rb_str_new(kbuf, ksiz);
+  default:
+    if (rb_respond_to(vobj, rb_intern("to_s"))) {
+      return rb_convert_type(vobj, T_STRING, "String", "to_s");
+    }
+  }
+  return StringValue(vobj);
+}
+
+extern TCLIST *varytolist(VALUE vary){
+  VALUE vval;
+  TCLIST *list;
+  int i, num;
+  num = RARRAY_LEN(vary);
+  list = tclistnew2(num);
+  for(i = 0; i < num; i++){
+    vval = rb_ary_entry(vary, i);
+    vval = StringValueEx(vval);
+    tclistpush(list, RSTRING_PTR(vval), RSTRING_LEN(vval));
+  }
+  return list;
+}
+
+extern VALUE listtovary(TCLIST *list){
+  VALUE vary;
+  const char *vbuf;
+  int i, num, vsiz;
+  num = tclistnum(list);
+  vary = rb_ary_new2(num);
+  for(i = 0; i < num; i++){
+    vbuf = tclistval(list, i, &vsiz);
+    rb_ary_push(vary, rb_str_new(vbuf, vsiz));
+  }
+  return vary;
+}
+
+extern TCMAP *vhashtomap(VALUE vhash){
+  VALUE vkeys, vkey, vval;
+  TCMAP *map;
+  int i, num;
+  map = tcmapnew2(31);
+  vkeys = rb_funcall(vhash, rb_intern("keys"), 0);
+  num = RARRAY_LEN(vkeys);
+  for(i = 0; i < num; i++){
+    vkey = rb_ary_entry(vkeys, i);
+    vval = rb_hash_aref(vhash, vkey);
+    vkey = StringValueEx(vkey);
+    vval = StringValueEx(vval);
+    tcmapput(map, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vval), RSTRING_LEN(vval));
+  }
+  return map;
+}
+
+extern VALUE maptovhash(TCMAP *map){
+  const char *kbuf, *vbuf;
+  int ksiz, vsiz;
+  VALUE vhash;
+  vhash = rb_hash_new();
+  tcmapiterinit(map);
+  while((kbuf = tcmapiternext(map, &ksiz)) != NULL){
+    vbuf = tcmapiterval(kbuf, &vsiz);
+    rb_hash_aset(vhash, rb_str_new(kbuf, ksiz), rb_str_new(vbuf, vsiz));
+  }
+  return vhash;
+}
+
+extern VALUE maptovhashsym(TCMAP *map){
+  const char *kbuf, *vbuf;
+  int ksiz, vsiz;
+  VALUE vhash;
+  vhash = rb_hash_new();
+  tcmapiterinit(map);
+  while((kbuf = tcmapiternext(map, &ksiz)) != NULL){
+    vbuf = tcmapiterval(kbuf, &vsiz);
+    rb_hash_aset(vhash, ID2SYM(rb_intern(kbuf)), rb_str_new(vbuf, vsiz));
+  }
+  return vhash;
+}
+
+extern TCMAP *varytomap(VALUE vary){
+  int i;
+  TCLIST *keys;
+  TCMAP *recs = tcmapnew();
+  keys = varytolist(vary);
+  for(i = 0; i < tclistnum(keys); i++){
+    int ksiz;
+    const char *kbuf = tclistval(keys, i, &ksiz);
+    tcmapput(recs, kbuf, ksiz, "", 0);
+  }
+  return recs;
+}
+
+extern TCLIST *vhashtolist(VALUE vhash){
+  /*
+  Seems like something like this might work just as well
+  vary = rb_hash_to_a(vhash);
+  vary = rb_ary_flatten(vary);
+  args = varytolist(vary);
+  */
+
+  VALUE vkeys, vkey, vval;
+  TCLIST *list;
+  int i, num;
+  vkeys = rb_funcall(vhash, rb_intern("keys"), 0);
+  num = RARRAY_LEN(vkeys);
+  list = tclistnew2(num);
+  for(i = 0; i < num; i++){
+    vkey = rb_ary_entry(vkeys, i);
+    vval = rb_hash_aref(vhash, vkey);
+    vkey = StringValueEx(vkey);
+    vval = StringValueEx(vval);
+    tclistpush(list, RSTRING_PTR(vkey), RSTRING_LEN(vkey));
+    tclistpush(list, RSTRING_PTR(vval), RSTRING_LEN(vval));
+  }
+  return list;
+}
+
+VALUE mTokyoTyrant;
+VALUE eTokyoTyrantError;
+VALUE cDB;
+VALUE cTable;
+VALUE cQuery;
+
+void Init_tokyo_tyrant(){
+  mTokyoTyrant = rb_define_module("TokyoTyrant");
+  eTokyoTyrantError = rb_define_class("TokyoTyrantError", rb_eStandardError);
+  init_mod();
+
+  cDB = rb_define_class_under(mTokyoTyrant, "DB", rb_cObject);
+  rb_include_module(cDB, mTokyoTyrant);
+  init_db();
+
+  cTable = rb_define_class_under(mTokyoTyrant, "Table", rb_cObject);
+  rb_include_module(cTable, mTokyoTyrant);
+  init_table();
+
+  cQuery = rb_define_class_under(mTokyoTyrant, "Query", rb_cObject);
+  init_query();
+}