ruby-tokyotyrant 0.4 → 0.5

data/ext/tokyo_tyrant.h

@@ -6,12 +6,15 @@
 #include <math.h>
 #include <time.h>
 #include <locale.h>
+#include <stdio.h>
+#include <unistd.h>
 #include <tcrdb.h>
 #include <tokyo_tyrant_module.h>
 #include <tokyo_tyrant_db.h>
+#include <tokyo_tyrant_bdb.h>
 #include <tokyo_tyrant_table.h>
 #include <tokyo_tyrant_query.h>
-#include <tokyo_utils.h>
+#include <tokyo_tyrant_consistent_hash.h>
 
 #define RDBVNDATA "@rdb"
 #define RDBQRYVNDATA "@rdbquery"
@@ -20,7 +23,6 @@
 #define TTPUTKEEP  1
 #define TTPUTCAT   2
 #define TTPUTNR    3
-#define MDBVNDATA "@mdb"
 
 #if !defined(RSTRING_PTR)
 #define RSTRING_PTR(TC_s) (RSTRING(TC_s)->ptr)
@@ -33,11 +35,22 @@
 #endif
 
 extern VALUE mTokyoTyrant;
+
 extern VALUE eTokyoTyrantError;
+extern VALUE eTokyoTyrantErrorInvalid;
+extern VALUE eTokyoTyrantErrorNoHost;
+extern VALUE eTokyoTyrantErrorRefused;
+extern VALUE eTokyoTyrantErrorSend;
+extern VALUE eTokyoTyrantErrorReceive;
+extern VALUE eTokyoTyrantErrorKeep;
+extern VALUE eTokyoTyrantErrorNoRecord;
+extern VALUE eTokyoTyrantErrorMisc;
+
 extern VALUE cDB;
+extern VALUE cBDB;
 extern VALUE cTable;
 extern VALUE cQuery;
-extern VALUE cMDB;
+extern VALUE cConstistentHash;
 
 extern VALUE StringRaw(const char *buf, int bsiz);
 extern VALUE StringValueEx(VALUE vobj);
@@ -47,5 +60,7 @@ extern TCMAP *vhashtomap(VALUE vhash);
 extern VALUE maptovhash(TCMAP *map);
 extern TCMAP *varytomap(VALUE vhash);
 extern TCLIST *vhashtolist(VALUE vhash);
+extern TCLIST *vhashtoputlist(VALUE vhash);
+extern VALUE listtovhash(TCLIST *list);
 
 #endif

data/{lib → ext}/tokyo_tyrant/balancer.rb

@@ -1,18 +1,15 @@
 require 'tokyo_tyrant'
-# require 'hash_ring'
-require 'fast_hash_ring'
 
 module TokyoTyrant
   module Balancer
     class Base
-      def initialize(servers = [], weights = {})
+      def initialize(servers = [])
         servers.collect! do |server|
           host, port = server.split(':')
           klass.new(host, port.to_i)
         end
         @servers = servers
-        # @ring = HashRing.new(servers, weights)
-        @ring = FastHashRing.new(servers, weights)
+        @ring = TokyoTyrant::ConstistentHash.new(servers)
       end
 
       def ring
@@ -24,7 +21,7 @@ module TokyoTyrant
       end
 
       def db_for_key(key)
-        ring.get_node(key)
+        ring.db_for_key(key)
       end
 
       # Delegate Methods

data/ext/tokyo_tyrant/rails/tokyo_tyrant_store.rb

@@ -0,0 +1,106 @@
+require 'tokyo_tyrant/balancer'
+
+module ActiveSupport
+  module Cache
+    # A cache store implementation which stores data in Tokyo Tyrant:
+    # http://1978th.net/tokyotyrant/
+    #
+    # Special features:
+    # - Clustering and load balancing. One can specify multiple servers,
+    #   and TokyoTyrantStore will load balance between all available servers.
+    # - Per-request in memory cache for all communication with the Tokyo Tyrant server(s).
+    class TokyoTyrantStore < Store
+      def self.build_servers(*addresses)
+        addresses = addresses.flatten
+        options = addresses.extract_options!
+        addresses = ["127.0.0.1:1978"] if addresses.empty?
+        TokyoTyrant::Balancer::DB.new(addresses, options)
+      end
+
+      # Creates a new TokyoTyrantStore object, with the given server
+      # addresses. Each address is either a host name, or a host-with-port string
+      # in the form of "host_name:port". For example:
+      #
+      #   ActiveSupport::Cache::TokyoTyrantStore.new("localhost", "server-downstairs.localnetwork:8229")
+      #
+      # If no addresses are specified, then TokyoTyrantStore will connect to
+      # localhost port 1978 (the default tokyo tyrant port).
+      def initialize(*addresses)
+        if addresses.first.respond_to?(:get)
+          @data = addresses.first
+        else
+          @data = self.class.build_servers(*addresses)
+        end
+
+        extend Strategy::LocalCache
+      end
+
+      def read(key, options = nil) # :nodoc:
+        super
+        @data.get(key)
+      rescue TokyoTyrantError => e
+        logger.error("TokyoTyrantError (#{e}): #{e.message}")
+        nil
+      end
+
+      # Writes a value to the cache.
+      #
+      # Possible options:
+      # - +:unless_exist+ - set to true if you don't want to update the cache
+      #   if the key is already set.
+      #   the cache. See ActiveSupport::Cache::Store#write for an example.
+      def write(key, value, options = nil)
+        super
+        method = options && options[:unless_exist] ? :putkeep : :put
+        @data.send(method, key, value)
+      rescue TokyoTyrantError => e
+        logger.error("TokyoTyrantError (#{e}): #{e.message}")
+        false
+      end
+
+      def delete(key, options = nil) # :nodoc:
+        super
+        @data.delete(key)
+      rescue TokyoTyrantError => e
+        logger.error("TokyoTyrantError (#{e}): #{e.message}")
+        false
+      end
+
+      def exist?(key, options = nil) # :nodoc:
+        @data.has_key?(key)
+      end
+
+      def increment(key, amount = 1) # :nodoc:
+        log("incrementing", key, amount)
+        @data.add_int(key, amount)
+      rescue TokyoTyrantError
+        nil
+      end
+
+      def decrement(key, amount = 1) # :nodoc:
+        log("decrement", key, amount)
+        @data.add_int(key, amount * -1)
+      rescue TokyoTyrantError
+        nil
+      end
+
+      def integer(key) # :nodoc:
+        log("integer", key)
+        @data.get_int(integer)
+      rescue TokyoTyrantError
+        nil
+      end
+
+      def delete_matched(matcher, options = nil) # :nodoc:
+        super
+        @data.delete_keys_with_prefix(matcher)
+      rescue TokyoTyrantError
+        nil
+      end
+
+      def clear
+        @data.clear
+      end
+    end
+  end
+end

data/ext/tokyo_tyrant_bdb.c

@@ -0,0 +1,138 @@
+#include <tokyo_tyrant_bdb.h>
+
+static VALUE cBDB_put_method(VALUE vself, VALUE vkey, VALUE vval, char *command, bool bang){
+  VALUE vres;
+  TCLIST *list, *result;
+  TCRDB *db = mTokyoTyrant_getdb(vself);
+
+  vkey = StringValueEx(vkey);
+  vval = StringValueEx(vval);
+
+  list = tclistnew2(2);
+  tclistpush(list, RSTRING_PTR(vkey), RSTRING_LEN(vkey));
+  tclistpush(list, RSTRING_PTR(vval), RSTRING_LEN(vval));
+  if ((result = tcrdbmisc(db, command, 0, list)) != NULL){
+    if (tclistnum(result) == 0){
+      vres = Qtrue;
+    } else {
+      vres = listtovary(result);
+    }
+  } else {
+    if (bang) mTokyoTyrant_exception(vself, NULL);
+    vres = Qfalse;
+  }
+  tclistdel(list);
+  return vres;
+}
+
+static VALUE cBDB_putlist(VALUE vself, VALUE vhash){
+  VALUE vary;
+  TCLIST *list, *result;
+  TCRDB *db = mTokyoTyrant_getdb(vself);
+  Check_Type(vhash, T_HASH);
+
+  list = vhashtoputlist(vhash);
+  if ((result = tcrdbmisc(db, "putlist", 0, list)) != NULL){
+    vary = listtovary(result);
+    tclistdel(result);
+  } else {
+    vary = rb_ary_new();
+  }
+  tclistdel(list);
+
+  return vary;
+}
+
+static VALUE cBDB_getlist(int argc, VALUE *argv, VALUE vself){
+  VALUE vkeys, vvalue, vhash;
+  TCLIST *list, *result;
+  TCRDB *db = mTokyoTyrant_getdb(vself);
+  rb_scan_args(argc, argv, "*", &vkeys);
+
+  // I really hope there is a better way to do this
+  if (RARRAY_LEN(vkeys) == 1) {
+    vvalue = rb_ary_entry(vkeys, 0);
+    switch (TYPE(vvalue)){
+      case T_STRING:
+      case T_FIXNUM:
+        break;
+      case T_ARRAY:
+        vkeys = vvalue;
+        break;
+      case T_OBJECT:
+        vkeys = rb_convert_type(vvalue, T_ARRAY, "Array", "to_a");
+        break;
+    }
+  }
+  Check_Type(vkeys, T_ARRAY);
+
+  list = varytolist(vkeys);
+  result = tcrdbmisc(db, "getlist", RDBMONOULOG, list);
+  tclistdel(list);
+  vhash = listtovhash(result);
+  tclistdel(result);
+  return vhash;
+}
+
+static VALUE cBDB_each(VALUE vself){
+  VALUE vrv;
+  const char *kbuf, *vbuf;
+  int ksiz, vsiz;
+  TCLIST *result;
+  TCRDB *db = mTokyoTyrant_getdb(vself);
+  vrv = Qnil;
+
+  if(rb_block_given_p() != Qtrue) rb_raise(rb_eArgError, "no block given");
+
+  tcrdbiterinit(db);
+  tcrdbmisc(db, "iterinit", RDBMONOULOG, tclistnew());
+  while((result = tcrdbmisc(db, "iternext", RDBMONOULOG, tclistnew())) != NULL){
+    if (tclistnum(result) == 2) {
+      kbuf = tclistval(result, 0, &ksiz);
+      vbuf = tclistval(result, 1, &vsiz);
+      vrv = rb_yield_values(2, rb_str_new(kbuf, ksiz), rb_str_new(vbuf, vsiz));
+    }
+    tclistdel(result);
+  }
+  return vrv;
+}
+
+static VALUE cBDB_values(VALUE vself){
+  VALUE vary;
+  const char *vbuf;
+  int vsiz;
+  TCLIST *result;
+  TCRDB *db = mTokyoTyrant_getdb(vself);
+
+  vary = rb_ary_new();
+  tcrdbiterinit(db);
+  tcrdbmisc(db, "iterinit", RDBMONOULOG, tclistnew());
+  while((result = tcrdbmisc(db, "iternext", RDBMONOULOG, tclistnew())) != NULL){
+    if (tclistnum(result) == 2){
+      vbuf = tclistval(result, 1, &vsiz);
+      vary = rb_ary_push(vary, rb_str_new(vbuf, vsiz));
+    }
+    tclistdel(result);
+  }
+
+  return vary;
+}
+
+static VALUE cBDB_putdup(VALUE vself, VALUE vkey, VALUE vval){
+  return cBDB_put_method(vself, vkey, vval, "putdup", false);
+}
+
+static VALUE cBDB_putdup_bang(VALUE vself, VALUE vkey, VALUE vval){
+  return cBDB_put_method(vself, vkey, vval, "putdup", true);
+}
+
+void init_bdb(){
+  rb_define_method(cBDB, "putlist", cBDB_putlist, 1);
+  rb_define_alias(cBDB, "mput", "putlist");
+  rb_define_method(cBDB, "getlist", cBDB_getlist, -1);
+  rb_define_alias(cBDB, "mget", "putlist");
+  rb_define_method(cBDB, "each", cBDB_each, 0);
+  rb_define_method(cBDB, "values", cBDB_values, 0);
+  rb_define_method(cBDB, "putdup", cBDB_putdup, 2);
+  rb_define_method(cBDB, "putdup!", cBDB_putdup_bang, 2);
+}

data/ext/tokyo_tyrant_bdb.h

@@ -0,0 +1,8 @@
+#ifndef RUBY_TOKYOTYRANT_BDB
+#define RUBY_TOKYOTYRANT_BDB
+
+#include <tokyo_tyrant.h>
+
+void init_bdb();
+
+#endif

data/ext/tokyo_tyrant_consistent_hash.c

@@ -0,0 +1,34 @@
+#include <tokyo_tyrant_consistent_hash.h>
+
+static void cConstistentHash_free(TCCHIDX *idx){
+  tcchidxdel(idx);
+}
+
+static VALUE cConstistentHash_initialize(VALUE self, VALUE dbs){
+  TCCHIDX *idx;
+
+  Check_Type(dbs, T_ARRAY);
+  idx = tcchidxnew(RARRAY_LEN(dbs));
+
+  rb_iv_set(self, "@dbs", dbs);
+  rb_iv_set(self, "@idx", Data_Wrap_Struct(rb_cObject, 0, cConstistentHash_free, idx));
+
+  return Qtrue;
+}
+
+static VALUE cConstistentHash_db_for_key(VALUE self, VALUE key){
+  VALUE dbs;
+  TCCHIDX *idx;
+  int hash;
+
+  Data_Get_Struct(rb_iv_get(self, "@idx"), TCCHIDX, idx);
+  hash = tcchidxhash(idx, RSTRING_PTR(key), RSTRING_LEN(key));
+  dbs = rb_iv_get(self, "@dbs");
+
+  return rb_ary_entry(dbs, hash);
+}
+
+void init_consistent_hash(){
+  rb_define_private_method(cConstistentHash, "initialize", cConstistentHash_initialize, 1);
+  rb_define_method(cConstistentHash, "db_for_key", cConstistentHash_db_for_key, 1);
+}

data/ext/tokyo_tyrant_consistent_hash.h

@@ -0,0 +1,8 @@
+#ifndef RUBY_TOKYOTYRANT_CONSISTENT_HASH
+#define RUBY_TOKYOTYRANT_CONSISTENT_HASH
+
+#include <tokyo_tyrant.h>
+
+void init_consistent_hash();
+
+#endif

data/ext/tokyo_tyrant_db.c

@@ -25,7 +25,7 @@ static VALUE cDB_put_method(VALUE vself, VALUE vkey, VALUE vstr, int method){
       break;
   }
 
-  if(!res) mTokyoTyrant_exception(vself);
+  if(!res) mTokyoTyrant_exception(vself, NULL);
 
   return Qtrue;
 }
@@ -68,7 +68,7 @@ static VALUE cDB_putshl(VALUE vself, VALUE vkey, VALUE vstr, VALUE vwidth){
 
   res = tcrdbputshl(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), RSTRING_PTR(vstr), RSTRING_LEN(vstr), FIXNUM_P(vwidth));
 
-  if(!res) mTokyoTyrant_exception(vself);
+  if(!res) mTokyoTyrant_exception(vself, NULL);
 
   return Qtrue;
 }
@@ -83,7 +83,7 @@ static VALUE cDB_get(VALUE vself, VALUE vkey){
   vkey = StringValueEx(vkey);
   if(!(buf = tcrdbget(db, RSTRING_PTR(vkey), RSTRING_LEN(vkey), &bsiz))){
     if ((ecode = tcrdbecode(db))) {
-      if (ecode != TTENOREC) mTokyoTyrant_exception(vself);
+      if (ecode != TTENOREC) mTokyoTyrant_exception(vself, NULL);
     }
     return Qnil;
   } else {

data/ext/tokyo_tyrant_module.c

@@ -1,4 +1,4 @@
-#include <tokyo_tyrant_module.h>
+#include <tokyo_tyrant_db.h>
 
 extern TCRDB *mTokyoTyrant_getdb(VALUE vself){
   TCRDB *db;
@@ -6,12 +6,44 @@ extern TCRDB *mTokyoTyrant_getdb(VALUE vself){
   return db;
 }
 
-extern void mTokyoTyrant_exception(VALUE vself){
+extern void mTokyoTyrant_exception(VALUE vself, void *message){
+  VALUE exception;
   int ecode;
   TCRDB *db = mTokyoTyrant_getdb(vself);
+  if (message == NULL) message = "%s";
 
   ecode = tcrdbecode(db);
-  rb_raise(eTokyoTyrantError, tcrdberrmsg(ecode));
+  // Is there a better way to do this?
+  switch(ecode){
+    case TTEINVALID:
+      exception = eTokyoTyrantErrorInvalid;
+      break;
+    case TTENOHOST:
+      exception = eTokyoTyrantErrorNoHost;
+      break;
+    case TTEREFUSED:
+      exception = eTokyoTyrantErrorRefused;
+      break;
+    case TTESEND:
+      exception = eTokyoTyrantErrorSend;
+      break;
+    case TTERECV:
+      exception = eTokyoTyrantErrorReceive;
+      break;
+    case TTEKEEP:
+      exception = eTokyoTyrantErrorKeep;
+      break;
+    case TTENOREC:
+      exception = eTokyoTyrantErrorNoRecord;
+      break;
+    case TTEMISC:
+      exception = eTokyoTyrantErrorMisc;
+      break;
+    default:
+      exception = eTokyoTyrantError;
+      break;
+  }
+  rb_raise(exception, message, tcrdberrmsg(ecode));
 }
 
 static void mTokyoTyrant_free(TCRDB *db){
@@ -23,34 +55,36 @@ static VALUE mTokyoTyrant_server(VALUE vself){
 }
 
 static VALUE mTokyoTyrant_close(VALUE vself){
-  int ecode;
   TCRDB *db = mTokyoTyrant_getdb(vself);
 
-  if(!tcrdbclose(db)){
-    ecode = tcrdbecode(db);
-    rb_raise(eTokyoTyrantError, "close error: %s", tcrdberrmsg(ecode));
-  }
+  if(!tcrdbclose(db)) mTokyoTyrant_exception(vself, "close error: %s");
   return Qtrue;
 }
 
 static VALUE mTokyoTyrant_connect(VALUE vself){
-  VALUE host, port, timeout, retry, server;
-  int ecode;
+  VALUE vhost, vport, vtimeout, vretry, vserver;
+  int port;
+  char *host;
   TCRDB *db = mTokyoTyrant_getdb(vself);
 
-  host = rb_iv_get(vself, "@host");
-  port = rb_iv_get(vself, "@port");
-  timeout = rb_iv_get(vself, "@timeout");
-  retry = rb_iv_get(vself, "@retry");
+  vhost = rb_iv_get(vself, "@host");
+  vport = rb_iv_get(vself, "@port");
+  vtimeout = rb_iv_get(vself, "@timeout");
+  vretry = rb_iv_get(vself, "@retry");
+  host = RSTRING_PTR(vhost);
+  port = FIX2INT(vport);
+
+  if(port == 0 && access(host, R_OK) < 0 && access(host, W_OK) < 0){
+    rb_raise(rb_eArgError, "Can't open unix socket: %s", host);
+  }
 
-  if((!tcrdbtune(db, NUM2DBL(timeout), retry == Qtrue ? RDBTRECON : 0)) ||
-     (!tcrdbopen(db, RSTRING_PTR(host), FIX2INT(port)))){
-    ecode = tcrdbecode(db);
-    rb_raise(eTokyoTyrantError, "open error: %s", tcrdberrmsg(ecode));
+  if((!tcrdbtune(db, NUM2DBL(vtimeout), vretry == Qtrue ? RDBTRECON : 0)) ||
+     (!tcrdbopen(db, host, port))){
+    mTokyoTyrant_exception(vself, "open error: %s");
   }
 
-  server = rb_str_new2(tcrdbexpr(db));
-  rb_iv_set(vself, "@server", server);
+  vserver = rb_str_new2(tcrdbexpr(db));
+  rb_iv_set(vself, "@server", vserver);
 
   return Qtrue;
 }
@@ -338,9 +372,18 @@ static VALUE mTokyoTyrant_size(VALUE vself){
 }
 
 static VALUE mTokyoTyrant_stat(VALUE vself){
+  VALUE vhash;
+  char *stats;
+  TCMAP *map;
   TCRDB *db = mTokyoTyrant_getdb(vself);
+  vhash = rb_hash_new();
 
-  return rb_str_new2(tcrdbstat(db));
+  if ((stats = tcrdbstat(db)) != NULL){
+    map = tcstrsplit3(stats, "\t\n");
+    vhash = maptovhash(map);
+    tcmapdel(map);
+  }
+  return vhash;
 }
 
 static VALUE mTokyoTyrant_misc(int argc, VALUE *argv, VALUE vself){
@@ -348,13 +391,21 @@ static VALUE mTokyoTyrant_misc(int argc, VALUE *argv, VALUE vself){
   TCLIST *list, *args;
   TCRDB *db = mTokyoTyrant_getdb(vself);
   rb_scan_args(argc, argv, "13", &vname, &vopts, &vargs);
+  if (vopts == Qnil) vopts = INT2NUM(0);
+  if (vargs == Qnil) vargs = rb_ary_new();
 
+  Check_Type(vargs, T_ARRAY);
   args = varytolist(vargs);
   vname = StringValueEx(vname);
 
-  list = tcrdbmisc(db, RSTRING_PTR(vname), NUM2INT(vopts), args);
-  vary = listtovary(list);
-  tclistdel(list);
+  if ((list = tcrdbmisc(db, RSTRING_PTR(vname), NUM2INT(vopts), args)) != NULL){
+    vary = listtovary(list);
+    tclistdel(list);
+  } else {
+    vary = rb_ary_new();
+  }
+  tclistdel(args);
+
   return vary;
 }