zookeeper 1.0.6-java → 1.1.0-java

data/ext/{zookeeper_c.c → zkrb.c}

@@ -8,11 +8,64 @@
  * approach to isolate the ZK state machine from the ruby interpreter via an
  * event queue.  It's similar to the ZookeeperFFI version except that it
  * actually works on MRI 1.8.
+ *
+ *----------------
+ * (slyphon)
+ *
+ * Wickman's implementation was linked against the 'mt' version of the zookeeper
+ * library, which is multithreaded at the zookeeper level and is subsequently
+ * much more difficult to get to behave properly with the ruby runtime (which
+ * he did, and I could never have written).
+ *
+ * The current implementation has been converted to use the 'st' version of the
+ * zookeeper library, which is single threaded and requires a ruby-side event
+ * loop. This is essentially a ruby port of the code running in the 'mt'
+ * library, with one important difference: It's running in ruby-land. The
+ * reason this change is so important is that it's virtually impossible to
+ * provide a fork-safe library when you have native threads you don't own
+ * running around. If you fork when a thread holds a mutex, and that thread
+ * is not the fork-caller, that mutex can never be unlocked, and is therefore
+ * a ticking time-bomb in the child. The only way to guarantee safety is to 
+ * either replace all of your mutexes and conditions and such after a fork
+ * (which is what we do on the ruby side), or avoid the problem altogether
+ * and not use a multithreaded library on the backend. Since we can't replace
+ * mutexes in the zookeeper code, we opt for the latter solution.
+ *
+ * The ruby code runs the event loop in a thread that will never cause a fork()
+ * to occur. This way, when fork() is called, the event thread will be dead
+ * in the child, guaranteeing that the child can safely be cleaned up.
+ *
+ * In that cleanup, there is a nasty (and brutishly effective) hack that makes
+ * the fork case work. We keep track of the pid that allocated the
+ * zkrb_instance_data_t, and if at destruction time we see that a fork has
+ * happened, we reach inside the zookeeper handle (zk->zh), and close the
+ * open socket it's got before calling zookeeper_close. This prevents
+ * corruption of the client/server state. Without this code, zookeeper_close
+ * in the child would actually send an "Ok, we're closing" message with the
+ * parent's session id, causing the parent to hit an assert() case in 
+ * zookeeper_process, and cause a SIGABRT. With this code in place, we get back
+ * a ZCONNECTIONLOSS from zookeeper_close in the child (which we ignore), and
+ * the parent continues on.
+ * 
+ * You will notice below we undef 'THREADED', which would be set if we were
+ * using the 'mt' library. We also conditionally include additional cases
+ * ('SYNC', 'SYNC_WATCH') inside of some of the methods defined here. These
+ * would be valid when running the 'mt' library, but since we have a ruby layer
+ * to provide a sync front-end to an async backend, these cases should never be
+ * hit, and instead will raise exceptions.
+ *
+ * NOTE: This file depends on exception classes defined in lib/zookeeper/exceptions.rb
+ *
  */
 
-#define THREADED
-
 #include "ruby.h"
+
+#ifdef ZKRB_RUBY_187
+#include "rubyio.h"
+#else
+#include "ruby/io.h"
+#endif
+
 #include "c-client-src/zookeeper.h"
 #include <errno.h>
 #include <stdio.h>
@@ -21,15 +74,18 @@
 #include <sys/fcntl.h>
 #include <pthread.h>
 #include <inttypes.h>
+#include <time.h>
 
-#include "zookeeper_lib.h"
+#include "common.h"
+#include "event_lib.h"
 #include "zkrb_wrapper.h"
 #include "dbg.h"
 
-
 static VALUE mZookeeper = Qnil;         // the Zookeeper module
 static VALUE CZookeeper = Qnil;         // the Zookeeper::CZookeeper class
 static VALUE ZookeeperClientId = Qnil;
+static VALUE mZookeeperExceptions = Qnil;    // the Zookeeper::Exceptions module
+static VALUE eHandleClosedException = Qnil;  // raised when we don't have a valid handle in FETCH_DATA_PTR
 
 // slyphon: possibly add a lock to this for synchronizing during get_next_event
 
@@ -41,6 +97,8 @@ struct zkrb_instance_data {
   pid_t             orig_pid;
 };
 
+typedef struct zkrb_instance_data zkrb_instance_data_t;
+
 typedef enum {
   SYNC  = 0,
   ASYNC = 1,
@@ -48,41 +106,109 @@ typedef enum {
   ASYNC_WATCH = 3
 } zkrb_call_type;
 
+inline static const char* call_type_to_str(zkrb_call_type ct) {
+  switch (ct) {
+    case SYNC:
+      return "SYNC";
+    case ASYNC:
+      return "ASYNC";
+    case SYNC_WATCH:
+      return "SYNC_WATCH";
+    case ASYNC_WATCH:
+      return "ASYNC_WATCH";
+  }
+}
+
+inline static void assert_valid_params(VALUE reqid, VALUE path) {
+  switch (TYPE(reqid)) {
+    case T_FIXNUM:
+    case T_BIGNUM:
+      break;
+    default:
+      rb_raise(rb_eTypeError, "reqid must be Fixnum/Bignum");  
+  }
+
+  Check_Type(path, T_STRING);
+}
+
+inline static zkrb_call_type get_call_type(VALUE async, VALUE watch) {
+  if RTEST(async) {
+    return RTEST(watch) ? ASYNC_WATCH : ASYNC;
+  } else {
+    return RTEST(watch) ? SYNC_WATCH : SYNC;
+  }
+}
+
+inline static void raise_invalid_call_type_err(zkrb_call_type call_type) {
+  rb_raise(rb_eRuntimeError, "hit the default case, call_type: %s", call_type_to_str(call_type));
+}
+
 #define IS_SYNC(zkrbcall) ((zkrbcall)==SYNC || (zkrbcall)==SYNC_WATCH)
 #define IS_ASYNC(zkrbcall) ((zkrbcall)==ASYNC || (zkrbcall)==ASYNC_WATCH)
 
+#define FETCH_DATA_PTR(SELF, ZK)                                        \
+  zkrb_instance_data_t * ZK;                                            \
+  Data_Get_Struct(rb_iv_get(SELF, "@_data"), zkrb_instance_data_t, ZK); \
+  if ((ZK)->zh == NULL)                                                 \
+    rb_raise(eHandleClosedException, "zookeeper handle is closed")
+
+#define STANDARD_PREAMBLE(SELF, ZK, REQID, PATH, ASYNC, WATCH, CALL_TYPE) \
+  assert_valid_params(REQID, PATH); \
+  FETCH_DATA_PTR(SELF, ZK); \
+  zkrb_call_type CALL_TYPE = get_call_type(ASYNC, WATCH); \
 
-static int destroy_zkrb_instance(struct zkrb_instance_data* ptr) {
+#define CTX_ALLOC(ZK,REQID) zkrb_calling_context_alloc(NUM2LL(REQID), ZK->queue)
+
+static void hexbufify(char *dest, const char *src, int len) {
+  int i=0;
+
+  for (i=0; i < len; i++) {
+    sprintf(&dest[i*2], "%x", src[i]);
+  }
+}
+
+static int destroy_zkrb_instance(zkrb_instance_data_t* zk) {
   int rv = ZOK;
 
-  zkrb_debug("destroy_zkrb_instance, zk_local_ctx: %p, zh: %p, queue: %p", ptr, ptr->zh, ptr->queue);
+  zkrb_debug("destroy_zkrb_instance, zk_local_ctx: %p, zh: %p, queue: %p", zk, zk->zh, zk->queue);
 
-  if (ptr->zh) {
-    const void *ctx = zoo_get_context(ptr->zh);
+  if (zk->zh) {
+    const void *ctx = zoo_get_context(zk->zh);
     /* Note that after zookeeper_close() returns, ZK handle is invalid */
-    zkrb_debug("obj_id: %lx, calling zookeeper_close", ptr->object_id);
-    rv = zookeeper_close(ptr->zh);
-    zkrb_debug("obj_id: %lx, zookeeper_close returned %d", ptr->object_id, rv); 
-    free((void *) ctx);
+    zkrb_debug("obj_id: %lx, calling zookeeper_close", zk->object_id);
+
+    if (zk->orig_pid != getpid()) {
+      zkrb_debug("FORK DETECTED! orig_pid: %d, current pid: %d, "
+          "using socket-closing hack before zookeeper_close", zk->orig_pid, getpid());
+
+      int fd = ((int *)zk->zh)[0];  // nasty, brutish, and wonderfully effective hack (see above)
+      close(fd);
+    }
+
+    rv = zookeeper_close(zk->zh);
+
+    zkrb_debug("obj_id: %lx, zookeeper_close returned %d", zk->object_id, rv); 
+    zkrb_calling_context_free((zkrb_calling_context *) ctx);
   }
 
-#warning [wickman] TODO: fire off warning if queue is not empty
-  if (ptr->queue) {
-    zkrb_debug("obj_id: %lx, freeing queue pointer: %p", ptr->object_id, ptr->queue);
-    zkrb_queue_free(ptr->queue);
+  zk->zh = NULL;
+
+// [wickman] TODO: fire off warning if queue is not empty
+  if (zk->queue) {
+    zkrb_debug("obj_id: %lx, freeing queue pointer: %p", zk->object_id, zk->queue);
+    zkrb_queue_free(zk->queue);
   }
 
-  ptr->zh = NULL;
-  ptr->queue = NULL;
+  zk->queue = NULL;
 
   return rv;
 }
 
-static void free_zkrb_instance_data(struct zkrb_instance_data* ptr) {
+static void free_zkrb_instance_data(zkrb_instance_data_t* ptr) {
   destroy_zkrb_instance(ptr);
 }
 
-static void print_zkrb_instance_data(struct zkrb_instance_data* ptr) {
+static void print_zkrb_instance_data(zkrb_instance_data_t* ptr) {
   fprintf(stderr, "zkrb_instance_data (%p) {\n", ptr);
   fprintf(stderr, "      zh = %p\n",           ptr->zh);
   fprintf(stderr, "        { state = %d }\n",  zoo_state(ptr->zh));
@@ -92,52 +218,16 @@ static void print_zkrb_instance_data(struct zkrb_instance_data* ptr) {
   fprintf(stderr, "}\n");
 }
 
-// cargo culted from io.c
-static VALUE zkrb_new_instance _((VALUE));
-
-static VALUE zkrb_new_instance(VALUE args) {
-    return rb_class_new_instance(2, (VALUE*)args+1, *(VALUE*)args);
-}
-
-static int zkrb_dup(int orig) {
-    int fd;
-
-    fd = dup(orig);
-    if (fd < 0) {
-        if (errno == EMFILE || errno == ENFILE || errno == ENOMEM) {
-            rb_gc();
-            fd = dup(orig);
-        }
-        if (fd < 0) {
-            rb_sys_fail(0);
-        }
-    }
-    return fd;
-}
-
-#if 0
-static VALUE create_selectable_io(zkrb_queue_t *q) {
-  // rb_cIO is the ruby IO class?
-
-  int pipe, state, read_fd;
-  VALUE args[3], reader;
-
-  read_fd = zkrb_dup(q->pipe_read);
-
-  args[0] = rb_cIO;
-  args[1] = INT2NUM(read_fd);
-  args[2] = INT2FIX(O_RDONLY);
-  reader = rb_protect(zkrb_new_instance, (VALUE)args, &state);
+#define session_timeout_msec(self) rb_iv_get(self, "@_session_timeout_msec")
 
-  if (state) {
-    rb_jump_tag(state);
-  }
+inline static void zkrb_debug_clientid_t(const clientid_t *cid) {
+  int pass_len = sizeof(cid->passwd);
+  int hex_len = 2 * pass_len;
+  char buf[hex_len];
+  hexbufify(buf, cid->passwd, pass_len);
 
-  return reader;
+  zkrb_debug("myid, client_id: %"PRId64", passwd: %*s", cid->client_id, hex_len, buf);
 }
-#endif
-
-#define session_timeout_msec(self) rb_iv_get(self, "@_session_timeout_msec")
 
 static VALUE method_zkrb_init(int argc, VALUE* argv, VALUE self) {
   VALUE hostPort;
@@ -163,8 +253,8 @@ static VALUE method_zkrb_init(int argc, VALUE* argv, VALUE self) {
   }
 
   VALUE data;
-  struct zkrb_instance_data *zk_local_ctx;
-  data = Data_Make_Struct(CZookeeper, struct zkrb_instance_data, 0, free_zkrb_instance_data, zk_local_ctx);
+  zkrb_instance_data_t *zk_local_ctx;
+  data = Data_Make_Struct(CZookeeper, zkrb_instance_data_t, 0, free_zkrb_instance_data, zk_local_ctx);
 
   zk_local_ctx->queue = zkrb_queue_alloc();
 
@@ -190,7 +280,6 @@ static VALUE method_zkrb_init(int argc, VALUE* argv, VALUE self) {
   zkrb_debug("method_zkrb_init, zk_local_ctx: %p, zh: %p, queue: %p, calling_ctx: %p",
       zk_local_ctx, zk_local_ctx->zh, zk_local_ctx->queue, ctx);
 
-#warning [wickman] TODO handle this properly on the Ruby side rather than C side
   if (!zk_local_ctx->zh) {
     rb_raise(rb_eRuntimeError, "error connecting to zookeeper: %d", errno);
   }
@@ -200,66 +289,47 @@ static VALUE method_zkrb_init(int argc, VALUE* argv, VALUE self) {
   rb_iv_set(self, "@_data", data);
   rb_funcall(self, rb_intern("zkc_set_running_and_notify!"), 0);
 
-error:
   return Qnil;
 }
 
-#define FETCH_DATA_PTR(X, Y)                                             \
-  struct zkrb_instance_data * Y;                                         \
-  Data_Get_Struct(rb_iv_get(X, "@_data"), struct zkrb_instance_data, Y); \
-  if ((Y)->zh == NULL)                                                   \
-    rb_raise(rb_eRuntimeError, "zookeeper handle is closed")
-
-#define STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, cb_ctx, w_ctx, call_type) \
-  if (TYPE(reqid) != T_FIXNUM && TYPE(reqid) != T_BIGNUM) {                   \
-    rb_raise(rb_eTypeError, "reqid must be Fixnum/Bignum");                   \
-    return Qnil;                                                              \
-  }                                                                           \
-  Check_Type(path, T_STRING);                                                 \
-  struct zkrb_instance_data * zk;                                             \
-  Data_Get_Struct(rb_iv_get(self, "@_data"), struct zkrb_instance_data, zk);  \
-  if (!zk->zh)                                                                \
-    rb_raise(rb_eRuntimeError, "zookeeper handle is closed");                 \
-  zkrb_calling_context* cb_ctx =                                              \
-    (async != Qfalse && async != Qnil) ?                                      \
-       zkrb_calling_context_alloc(NUM2LL(reqid), zk->queue) :                 \
-       NULL;                                                                  \
-  zkrb_calling_context* w_ctx =                                               \
-    (watch != Qfalse && watch != Qnil) ?                                      \
-       zkrb_calling_context_alloc(NUM2LL(reqid), zk->queue) :                 \
-       NULL;                                                                  \
-  int a_  = (async != Qfalse && async != Qnil);                               \
-  int w_  = (watch != Qfalse && watch != Qnil);                               \
-  zkrb_call_type call_type;                                                   \
-  if (a_) { if (w_) { call_type = ASYNC_WATCH; } else { call_type = ASYNC; } } \
-     else { if (w_) { call_type =  SYNC_WATCH; } else { call_type = SYNC; } }
-
 static VALUE method_get_children(VALUE self, VALUE reqid, VALUE path, VALUE async, VALUE watch) {
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, call_type);
 
+  VALUE output = Qnil;
   struct String_vector strings;
   struct Stat stat;
 
   int rc;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
-      rc = zkrb_call_zoo_get_children2(zk->zh, RSTRING_PTR(path), 0, &strings, &stat);
+      rc = zkrb_call_zoo_get_children2(
+              zk->zh, RSTRING_PTR(path), 0, &strings, &stat);
       break;
 
     case SYNC_WATCH:
-      rc = zkrb_call_zoo_wget_children2(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, &strings, &stat);
+      rc = zkrb_call_zoo_wget_children2(
+              zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), &strings, &stat);
       break;
+#endif
 
     case ASYNC:
-      rc = zkrb_call_zoo_aget_children2(zk->zh, RSTRING_PTR(path), 0, zkrb_strings_stat_callback, data_ctx);
+      rc = zkrb_call_zoo_aget_children2(
+              zk->zh, RSTRING_PTR(path), 0, zkrb_strings_stat_callback, CTX_ALLOC(zk, reqid));
       break;
 
     case ASYNC_WATCH:
-      rc = zkrb_call_zoo_awget_children2(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, zkrb_strings_stat_callback, data_ctx);
+      rc = zkrb_call_zoo_awget_children2(
+              zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), zkrb_strings_stat_callback, CTX_ALLOC(zk, reqid));
+      break;
+
+    default:
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     rb_ary_push(output, zkrb_string_vector_to_ruby(&strings));
@@ -269,30 +339,38 @@ static VALUE method_get_children(VALUE self, VALUE reqid, VALUE path, VALUE asyn
 }
 
 static VALUE method_exists(VALUE self, VALUE reqid, VALUE path, VALUE async, VALUE watch) {
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, call_type);
 
+  VALUE output = Qnil;
   struct Stat stat;
 
   int rc;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_exists(zk->zh, RSTRING_PTR(path), 0, &stat);
       break;
 
     case SYNC_WATCH:
-      rc = zkrb_call_zoo_wexists(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, &stat);
+      rc = zkrb_call_zoo_wexists(zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), &stat);
       break;
+#endif
 
     case ASYNC:
-      rc = zkrb_call_zoo_aexists(zk->zh, RSTRING_PTR(path), 0, zkrb_stat_callback, data_ctx);
+      rc = zkrb_call_zoo_aexists(zk->zh, RSTRING_PTR(path), 0, zkrb_stat_callback, CTX_ALLOC(zk, reqid));
       break;
 
     case ASYNC_WATCH:
-      rc = zkrb_call_zoo_awexists(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, zkrb_stat_callback, data_ctx);
+      rc = zkrb_call_zoo_awexists(zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), zkrb_stat_callback, CTX_ALLOC(zk, reqid));
+      break;
+
+    default:
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     rb_ary_push(output, zkrb_stat_to_rarray(&stat));
@@ -302,20 +380,21 @@ static VALUE method_exists(VALUE self, VALUE reqid, VALUE path, VALUE async, VAL
 
 // this method is *only* called asynchronously
 static VALUE method_sync(VALUE self, VALUE reqid, VALUE path) {
-  VALUE async = Qtrue;
-  VALUE watch = Qfalse;
-  int rc;
+  int rc = ZOK;
 
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  // don't use STANDARD_PREAMBLE here b/c we don't need to determine call_type
+  assert_valid_params(reqid, path);
+  FETCH_DATA_PTR(self, zk);
 
-  rc = zkrb_call_zoo_async(zk->zh, RSTRING_PTR(path), zkrb_string_callback, data_ctx);
+  rc = zkrb_call_zoo_async(zk->zh, RSTRING_PTR(path), zkrb_string_callback, CTX_ALLOC(zk, reqid));
 
   return INT2FIX(rc);
 }
 
+
 static VALUE method_create(VALUE self, VALUE reqid, VALUE path, VALUE data, VALUE async, VALUE acls, VALUE flags) {
-  VALUE watch = Qfalse;
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, Qfalse, call_type);
+  VALUE output = Qnil;
 
   if (data != Qnil) Check_Type(data, T_STRING);
   Check_Type(flags, T_FIXNUM);
@@ -328,28 +407,29 @@ static VALUE method_create(VALUE self, VALUE reqid, VALUE path, VALUE data, VALU
 
   int rc;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       // casting data_len to int is OK as you can only store 1MB in zookeeper
       rc = zkrb_call_zoo_create(zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, aclptr, FIX2INT(flags), realpath, sizeof(realpath));
-
       break;
-    case ASYNC:
-      rc = zkrb_call_zoo_acreate(zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, aclptr, FIX2INT(flags), zkrb_string_callback, data_ctx);
+#endif
 
+    case ASYNC:
+      rc = zkrb_call_zoo_acreate(zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, aclptr, FIX2INT(flags), zkrb_string_callback, CTX_ALLOC(zk, reqid));
       break;
+
     default:
-      /* TODO(wickman) raise proper argument error */
-      return Qnil;
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-
   if (aclptr) {
     deallocate_ACL_vector(aclptr);
     free(aclptr);
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     return rb_ary_push(output, rb_str_new2(realpath));
@@ -358,21 +438,24 @@ static VALUE method_create(VALUE self, VALUE reqid, VALUE path, VALUE data, VALU
 }
 
 static VALUE method_delete(VALUE self, VALUE reqid, VALUE path, VALUE version, VALUE async) {
-  VALUE watch = Qfalse;
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, Qfalse, call_type);
   Check_Type(version, T_FIXNUM);
 
   int rc = 0;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_delete(zk->zh, RSTRING_PTR(path), FIX2INT(version));
       break;
+#endif
+
     case ASYNC:
-      rc = zkrb_call_zoo_adelete(zk->zh, RSTRING_PTR(path), FIX2INT(version), zkrb_void_callback, data_ctx);
+      rc = zkrb_call_zoo_adelete(zk->zh, RSTRING_PTR(path), FIX2INT(version), zkrb_void_callback, CTX_ALLOC(zk, reqid));
       break;
+
     default:
-      /* TODO(wickman) raise proper argument error */
-      return Qnil;
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
@@ -382,34 +465,46 @@ static VALUE method_delete(VALUE self, VALUE reqid, VALUE path, VALUE version, V
 #define MAX_ZNODE_SIZE 1048576
 
 static VALUE method_get(VALUE self, VALUE reqid, VALUE path, VALUE async, VALUE watch) {
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, call_type);
+
+  VALUE output = Qnil;
 
-  /* ugh */
-  char * data = malloc(MAX_ZNODE_SIZE);
   int data_len = MAX_ZNODE_SIZE;
   struct Stat stat;
 
+  char * data = xmalloc(MAX_ZNODE_SIZE); /* ugh */
+
   int rc;
 
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_get(zk->zh, RSTRING_PTR(path), 0, data, &data_len, &stat);
       break;
 
     case SYNC_WATCH:
-      rc = zkrb_call_zoo_wget(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, data, &data_len, &stat);
+      rc = zkrb_call_zoo_wget(
+              zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), data, &data_len, &stat);
       break;
+#endif
 
     case ASYNC:
-      rc = zkrb_call_zoo_aget(zk->zh, RSTRING_PTR(path), 0, zkrb_data_callback, data_ctx);
+      rc = zkrb_call_zoo_aget(zk->zh, RSTRING_PTR(path), 0, zkrb_data_callback, CTX_ALLOC(zk, reqid));
       break;
 
     case ASYNC_WATCH:
-      rc = zkrb_call_zoo_awget(zk->zh, RSTRING_PTR(path), zkrb_state_callback, watch_ctx, zkrb_data_callback, data_ctx);
+      // first ctx is a watch, second is the async callback
+      rc = zkrb_call_zoo_awget(
+            zk->zh, RSTRING_PTR(path), zkrb_state_callback, CTX_ALLOC(zk, reqid), zkrb_data_callback, CTX_ALLOC(zk, reqid));
+      break;
+
+    default:
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     if (data_len == -1)
@@ -418,36 +513,42 @@ static VALUE method_get(VALUE self, VALUE reqid, VALUE path, VALUE async, VALUE
       rb_ary_push(output, rb_str_new(data, data_len));
     rb_ary_push(output, zkrb_stat_to_rarray(&stat));
   }
-  free(data);
 
+  xfree(data);
   return output;
 }
 
 static VALUE method_set(VALUE self, VALUE reqid, VALUE path, VALUE data, VALUE async, VALUE version) {
-  VALUE watch = Qfalse;
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, Qfalse, call_type);
 
+  VALUE output = Qnil;
   struct Stat stat;
+
   if (data != Qnil) Check_Type(data, T_STRING);
+
   const char *data_ptr = (data == Qnil) ? NULL : RSTRING_PTR(data);
   size_t      data_len = (data == Qnil) ? -1   : RSTRING_LEN(data);
 
   int rc;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_set2(zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, FIX2INT(version), &stat);
       break;
+#endif
+
     case ASYNC:
-      rc = zkrb_call_zoo_aset(zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, FIX2INT(version),
-                            zkrb_stat_callback, data_ctx);
+      rc = zkrb_call_zoo_aset(
+            zk->zh, RSTRING_PTR(path), data_ptr, (int)data_len, FIX2INT(version), zkrb_stat_callback, CTX_ALLOC(zk, reqid));
       break;
+
     default:
-      /* TODO(wickman) raise proper argument error */
-      return Qnil;
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     rb_ary_push(output, zkrb_stat_to_rarray(&stat));
@@ -456,21 +557,25 @@ static VALUE method_set(VALUE self, VALUE reqid, VALUE path, VALUE data, VALUE a
 }
 
 static VALUE method_set_acl(VALUE self, VALUE reqid, VALUE path, VALUE acls, VALUE async, VALUE version) {
-  VALUE watch = Qfalse;
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, Qfalse, call_type);
+
   struct ACL_vector * aclptr = zkrb_ruby_to_aclvector(acls);
 
   int rc;
   switch (call_type) {
+    
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_set_acl(zk->zh, RSTRING_PTR(path), FIX2INT(version), aclptr);
       break;
+#endif
+
     case ASYNC:
-      rc = zkrb_call_zoo_aset_acl(zk->zh, RSTRING_PTR(path), FIX2INT(version), aclptr, zkrb_void_callback, data_ctx);
+      rc = zkrb_call_zoo_aset_acl(zk->zh, RSTRING_PTR(path), FIX2INT(version), aclptr, zkrb_void_callback, CTX_ALLOC(zk, reqid));
       break;
+
     default:
-      /* TODO(wickman) raise proper argument error */
-      return Qnil;
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
@@ -481,27 +586,31 @@ static VALUE method_set_acl(VALUE self, VALUE reqid, VALUE path, VALUE acls, VAL
 }
 
 static VALUE method_get_acl(VALUE self, VALUE reqid, VALUE path, VALUE async) {
-  VALUE watch = Qfalse;
-  STANDARD_PREAMBLE(self, zk, reqid, path, async, watch, data_ctx, watch_ctx, call_type);
+  STANDARD_PREAMBLE(self, zk, reqid, path, async, Qfalse, call_type);
 
+  VALUE output = Qnil;
   struct ACL_vector acls;
   struct Stat stat;
 
   int rc;
   switch (call_type) {
+
+#ifdef THREADED
     case SYNC:
       rc = zkrb_call_zoo_get_acl(zk->zh, RSTRING_PTR(path), &acls, &stat);
       break;
+#endif
+
     case ASYNC:
-      rc = zkrb_call_zoo_aget_acl(zk->zh, RSTRING_PTR(path), zkrb_acl_callback, data_ctx);
+      rc = zkrb_call_zoo_aget_acl(zk->zh, RSTRING_PTR(path), zkrb_acl_callback, CTX_ALLOC(zk, reqid));
       break;
+
     default:
-      /* TODO(wickman) raise proper argument error */
-      return Qnil;
+      raise_invalid_call_type_err(call_type);
       break;
   }
 
-  VALUE output = rb_ary_new();
+  output = rb_ary_new();
   rb_ary_push(output, INT2FIX(rc));
   if (IS_SYNC(call_type) && rc == ZOK) {
     rb_ary_push(output, zkrb_acl_vector_to_ruby(&acls));
@@ -515,7 +624,7 @@ static VALUE method_get_acl(VALUE self, VALUE reqid, VALUE path, VALUE async) {
 #define is_closed(self) RTEST(rb_iv_get(self, "@_closed"))
 #define is_shutting_down(self) RTEST(rb_iv_get(self, "@_shutting_down"))
 
-static VALUE method_get_next_event(VALUE self, VALUE blocking) {
+static VALUE method_zkrb_get_next_event(VALUE self, VALUE blocking) {
   // dbg.h 
   check_debug(!is_closed(self), "we are closed, not trying to get event");
 
@@ -523,7 +632,7 @@ static VALUE method_get_next_event(VALUE self, VALUE blocking) {
   FETCH_DATA_PTR(self, zk);
 
   for (;;) {
-    check_debug(!is_closed(self), "we're closed in the middle of method_get_next_event, bailing");
+    check_debug(!is_closed(self), "we're closed in the middle of method_zkrb_get_next_event, bailing");
 
     zkrb_event_t *event = zkrb_dequeue(zk->queue, 1);
 
@@ -534,7 +643,7 @@ static VALUE method_get_next_event(VALUE self, VALUE blocking) {
       } 
       else {
         // if we're shutting down, don't enter this section, we don't want to block
-        check_debug(!is_shutting_down(self), "method_get_next_event, we're shutting down, don't enter blocking section");
+        check_debug(!is_shutting_down(self), "method_zkrb_get_next_event, we're shutting down, don't enter blocking section");
 
         int fd = zk->queue->pipe_read;
         ssize_t bytes_read = 0;
@@ -565,6 +674,117 @@ static VALUE method_get_next_event(VALUE self, VALUE blocking) {
     return Qnil;
 }
 
+// the single threaded version of this call. will go away when we do direct
+// event delivery (soon)
+static VALUE method_zkrb_get_next_event_st(VALUE self) {
+  VALUE rval = Qnil;
+
+  if (is_closed(self)) {
+    zkrb_debug("we are closed, not gonna try to get an event");
+    return Qnil;
+  }
+
+  FETCH_DATA_PTR(self, zk);
+
+  zkrb_event_t *event = zkrb_dequeue(zk->queue, 0);
+
+  if (event != NULL) {
+    rval = zkrb_event_to_ruby(event);
+    zkrb_event_free(event);
+
+    int fd = zk->queue->pipe_read;
+
+    // we don't care in this case. this is just until i can remove the self
+    // pipe from the queue
+    char b[128];
+    while(read(fd, b, sizeof(b)) == sizeof(b)){}
+  }
+
+  return rval;
+}
+
+inline static int get_self_pipe_read_fd(VALUE self) {
+  rb_io_t *fptr;
+  VALUE pipe_read = rb_iv_get(self, "@pipe_read");
+
+  if NIL_P(pipe_read)
+      rb_raise(rb_eRuntimeError, "@pipe_read was nil!");
+
+  GetOpenFile(pipe_read, fptr);
+
+  rb_io_check_readable(fptr);
+
+#ifdef ZKRB_RUBY_187
+  return fileno(fptr->f);
+#else
+  return fptr->fd;
+#endif
+}
+
+
+static VALUE method_zkrb_iterate_event_loop(VALUE self) {
+  FETCH_DATA_PTR(self, zk);
+
+  fd_set rfds, wfds, efds;
+  FD_ZERO(&rfds); FD_ZERO(&wfds); FD_ZERO(&efds); 
+
+  int fd=0, interest=0, events=0, rc=0, maxfd=0;
+  struct timeval tv;
+  
+  zookeeper_interest(zk->zh, &fd, &interest, &tv);
+
+  if (fd != -1) {
+    if (interest & ZOOKEEPER_READ) {
+      FD_SET(fd, &rfds);
+    } else {
+      FD_CLR(fd, &rfds);
+    }
+    if (interest & ZOOKEEPER_WRITE) {
+      FD_SET(fd, &wfds);
+    } else {
+      FD_CLR(fd, &wfds);
+    }
+  } else {
+    fd = 0;
+  }
+
+  // add our self-pipe to the read set, allow us to wake up in case our attention is needed
+  int pipe_r_fd = get_self_pipe_read_fd(self);
+
+  FD_SET(pipe_r_fd, &rfds);
+
+  maxfd = (pipe_r_fd > fd) ? pipe_r_fd : fd;
+
+  rc = rb_thread_select(maxfd+1, &rfds, &wfds, &efds, &tv);
+
+  if (rc > 0) {
+    if (FD_ISSET(fd, &rfds)) {
+      events |= ZOOKEEPER_READ;
+    } 
+    if (FD_ISSET(fd, &wfds)) {
+      events |= ZOOKEEPER_WRITE;
+    }
+
+    // we got woken up by the self-pipe
+    if (FD_ISSET(pipe_r_fd, &rfds)) {
+      // flush the pipe (from mt_adaptor.c)
+      char b[1];
+      read(pipe_r_fd, b, 1); // one event has awoken us, so we clear one event from the pipe
+    }
+
+    rc = zookeeper_process(zk->zh, events);
+  }
+  else if (rc == 0) {
+    zkrb_debug("timed out waiting for descriptor to be ready");
+  }
+  else {
+    log_err("select returned: %d", rc);
+  }
+
+  return INT2FIX(rc);
+}
+
+
 static VALUE method_has_events(VALUE self) {
   VALUE rb_event;
   FETCH_DATA_PTR(self, zk);
@@ -575,6 +795,7 @@ static VALUE method_has_events(VALUE self) {
 
 
 // wake up the event loop, used when shutting down
+#if 0
 static VALUE method_wake_event_loop_bang(VALUE self) {
   FETCH_DATA_PTR(self, zk); 
 
@@ -583,12 +804,12 @@ static VALUE method_wake_event_loop_bang(VALUE self) {
 
   return Qnil;
 };
+#endif
 
 static VALUE method_close_handle(VALUE self) {
   FETCH_DATA_PTR(self, zk);
 
   if (ZKRBDebugging) {
-/*    fprintf(stderr, "CLOSING ZK INSTANCE: obj_id %lx", FIX2LONG(rb_obj_id(self)));*/
     zkrb_debug_inst(self, "CLOSING_ZK_INSTANCE");
     print_zkrb_instance_data(zk);
   }
@@ -597,15 +818,12 @@ static VALUE method_close_handle(VALUE self) {
   // has been called
   rb_iv_set(self, "@_closed", Qtrue);
 
-  zkrb_debug_inst(self, "calling destroy_zkrb_instance");
-
   /* Note that after zookeeper_close() returns, ZK handle is invalid */
   int rc = destroy_zkrb_instance(zk);
 
   zkrb_debug("destroy_zkrb_instance returned: %d", rc);
 
-  return Qnil;
-/*  return INT2FIX(rc);*/
+  return INT2FIX(rc);
 }
 
 static VALUE method_deterministic_conn_order(VALUE self, VALUE yn) {
@@ -634,12 +852,19 @@ static VALUE method_recv_timeout(VALUE self) {
 /*  return UINT2NUM(id->client_id);*/
 /*}*/
 
-
 // returns a CZookeeper::ClientId object with the values set for session_id and passwd
 static VALUE method_client_id(VALUE self) {
   FETCH_DATA_PTR(self, zk);
+  char buf[32];
   const clientid_t *cid = zoo_client_id(zk->zh);
 
+  if (strlen(cid->passwd) != 16) { 
+    zkrb_debug("passwd is not null-termniated");
+  } else {
+    hexbufify(buf, cid->passwd, 16);
+    zkrb_debug("password in hex is: %s", buf);
+  }
+
   VALUE session_id = LL2NUM(cid->client_id);
   VALUE passwd = rb_str_new2(cid->passwd);
 
@@ -663,21 +888,28 @@ static VALUE method_zerror(VALUE self, VALUE errc) {
 }
 
 static void zkrb_define_methods(void) {
-#define DEFINE_METHOD(method, args) { \
-    rb_define_method(CZookeeper, #method, method_ ## method, args); }
-#define DEFINE_CLASS_METHOD(method, args) { \
-    rb_define_singleton_method(CZookeeper, #method, method_ ## method, args); }
+#define DEFINE_METHOD(M, ARGS) { \
+    rb_define_method(CZookeeper, #M, method_ ## M, ARGS); }
+
+#define DEFINE_CLASS_METHOD(M, ARGS) { \
+    rb_define_singleton_method(CZookeeper, #M, method_ ## M, ARGS); }
+
+// defines a method with a zkrb_ prefix, the actual C method does not have this prefix
+#define DEFINE_ZKRB_METHOD(M, ARGS) { \
+    rb_define_method(CZookeeper, zkrb_ ## M, method_ ## M, ARGS); }
 
   // the number after the method name should be actual arity of C function - 1
   DEFINE_METHOD(zkrb_init, -1);
-  DEFINE_METHOD(get_children, 4);
-  DEFINE_METHOD(exists, 4);
-  DEFINE_METHOD(create, 6);
-  DEFINE_METHOD(delete, 4);
-  DEFINE_METHOD(get, 4);
-  DEFINE_METHOD(set, 5);
-  DEFINE_METHOD(set_acl, 5);
-  DEFINE_METHOD(get_acl, 3);
+
+  rb_define_method(CZookeeper, "zkrb_get_children", method_get_children,  4);
+  rb_define_method(CZookeeper, "zkrb_exists",       method_exists,        4);
+  rb_define_method(CZookeeper, "zkrb_create",       method_create,        6);
+  rb_define_method(CZookeeper, "zkrb_delete",       method_delete,        4);
+  rb_define_method(CZookeeper, "zkrb_get",          method_get,           4);
+  rb_define_method(CZookeeper, "zkrb_set",          method_set,           5);
+  rb_define_method(CZookeeper, "zkrb_set_acl",      method_set_acl,       5);
+  rb_define_method(CZookeeper, "zkrb_get_acl",      method_get_acl,       3);
+
   DEFINE_METHOD(client_id, 0);
   DEFINE_METHOD(close_handle, 0);
   DEFINE_METHOD(deterministic_conn_order, 1);
@@ -685,12 +917,15 @@ static void zkrb_define_methods(void) {
   DEFINE_METHOD(recv_timeout, 1);
   DEFINE_METHOD(zkrb_state, 0);
   DEFINE_METHOD(sync, 2);
+  DEFINE_METHOD(zkrb_iterate_event_loop, 0);
+  DEFINE_METHOD(zkrb_get_next_event_st, 0);
 
   // TODO
   // DEFINE_METHOD(add_auth, 3);
 
   // methods for the ruby-side event manager
-  DEFINE_METHOD(get_next_event, 1);
+  DEFINE_METHOD(zkrb_get_next_event, 1);
+  DEFINE_METHOD(zkrb_get_next_event_st, 0);
   DEFINE_METHOD(has_events, 0);
 
   // Make these class methods?
@@ -699,7 +934,6 @@ static void zkrb_define_methods(void) {
   rb_define_singleton_method(CZookeeper, "set_zkrb_debug_level", klass_method_zkrb_set_debug_level, 1);
 
   rb_attr(CZookeeper, rb_intern("selectable_io"), 1, 0, Qtrue);
-  rb_define_method(CZookeeper, "wake_event_loop!", method_wake_event_loop_bang, 0);
 
 }
 
@@ -723,6 +957,10 @@ void Init_zookeeper_c() {
   ZKRBDebugging = 0;
 
   mZookeeper = rb_define_module("Zookeeper");
+  mZookeeperExceptions = rb_define_module_under(mZookeeper, "Exceptions");
+
+  // this will likely fail if the load order is screwed up
+  eHandleClosedException = rb_const_get(mZookeeperExceptions, rb_intern("HandleClosedException"));
 
   /* initialize CZookeeper class */
   CZookeeper = rb_define_class_under(mZookeeper, "CZookeeper", rb_cObject);
@@ -732,6 +970,7 @@ void Init_zookeeper_c() {
   rb_define_method(ZookeeperClientId, "initialize", zkrb_client_id_method_initialize, 0);
   rb_define_attr(ZookeeperClientId, "session_id", 1, 1);
   rb_define_attr(ZookeeperClientId, "passwd", 1, 1);
+
 }
 
 // vim:ts=2:sw=2:sts=2:et