slyphon-zookeeper 0.1.7 → 0.2.0

data/.gitignore

@@ -6,3 +6,4 @@
 Makefile
 pkg
 zookeeper.gemspec
+Gemfile.lock

data/ext/zookeeper_base.rb

@@ -16,7 +16,8 @@ class ZookeeperBase < CZookeeper
   ZOO_LOG_LEVEL_WARN   = 2
   ZOO_LOG_LEVEL_INFO   = 3
   ZOO_LOG_LEVEL_DEBUG  = 4
-  
+
+ 
   def reopen(timeout = 10, watcher=nil)
     watcher ||= @default_watcher
 
@@ -26,13 +27,15 @@ class ZookeeperBase < CZookeeper
       set_default_global_watcher(&watcher)
     end
 
-    init(@host)
+    @start_stop_mutex.synchronize do
+      init(@host)
 
-    if timeout > 0
-      time_to_stop = Time.now + timeout
-      until state == Zookeeper::ZOO_CONNECTED_STATE
-        break if Time.now > time_to_stop
-        sleep 0.1
+      if timeout > 0
+        time_to_stop = Time.now + timeout
+        until state == Zookeeper::ZOO_CONNECTED_STATE
+          break if Time.now > time_to_stop
+          sleep 0.1
+        end
       end
     end
 
@@ -46,9 +49,12 @@ class ZookeeperBase < CZookeeper
     @current_req_id = 1
     @host = host
 
+    @start_stop_mutex = Mutex.new
+
     watcher ||= get_default_global_watcher
 
-    @_running = nil # used by the C layer
+    @_running = nil   # used by the C layer
+    @_closed = false  # also used by the C layer
 
     yield self if block_given?
 
@@ -76,12 +82,32 @@ class ZookeeperBase < CZookeeper
   end
 
   def close
-    @_running = false
-    wake_event_loop!
-    
-    @dispatcher.join
+    @start_stop_mutex.synchronize do
+      @_running = false if @_running
+    end
+
+    if @dispatcher
+      wake_event_loop! unless @_closed
+      @dispatcher.join 
+    end
+
+    @start_stop_mutex.synchronize do
+      unless @_closed
+        close_handle
+        
+        # this is set up in the C init method, but it's easier to 
+        # do the teardown here
+        begin
+          @selectable_io.close if @selectable_io
+        rescue IOError
+        end
+      end
+    end
+  end
 
-    super
+  def set_debug_level(int)
+    warn "DEPRECATION WARNING: #{self.class.name}#set_debug_level, it has moved to the class level and will be removed in a future release"
+    self.class.set_debug_level(int)
   end
 
   # set the watcher object/proc that will receive all global events (such as session/state events)
@@ -92,16 +118,20 @@ class ZookeeperBase < CZookeeper
     end
   end
 
-protected
+  def closed?
+    @start_stop_mutex.synchronize { false|@_closed }
+  end
+
   def running?
-    false|@_running
+    @start_stop_mutex.synchronize { false|@_running }
   end
 
+protected
   def setup_dispatch_thread!
     @dispatcher = Thread.new do
       while running?
         begin                     # calling user code, so protect ourselves
-          dispatch_next_callback 
+          dispatch_next_callback
         rescue Exception => e
           $stderr.puts "Error in dispatch thread, #{e.class}: #{e.message}\n" << e.backtrace.map{|n| "\t#{n}"}.join("\n")
         end

data/ext/zookeeper_c.c

@@ -17,11 +17,16 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <sys/fcntl.h>
+#include <pthread.h>
 
 #include "zookeeper_lib.h"
 
+
 static VALUE Zookeeper = Qnil;
 
+// slyphon: possibly add a lock to this for synchronizing during get_next_event
+
 struct zkrb_instance_data {
   zhandle_t            *zh;
   clientid_t          myid;
@@ -69,9 +74,53 @@ static void print_zkrb_instance_data(struct zkrb_instance_data* ptr) {
   fprintf(stderr, "}\n");
 }
 
+// cargo culted from io.c
+static VALUE zkrb_new_instance _((VALUE));
+
+static VALUE zkrb_new_instance(VALUE args) {
+    return rb_class_new_instance(2, (VALUE*)args+1, *(VALUE*)args);
+}
+
+static int zkrb_dup(int orig) {
+    int fd;
+
+    fd = dup(orig);
+    if (fd < 0) {
+        if (errno == EMFILE || errno == ENFILE || errno == ENOMEM) {
+            rb_gc();
+            fd = dup(orig);
+        }
+        if (fd < 0) {
+            rb_sys_fail(0);
+        }
+    }
+    return fd;
+}
+
+static VALUE create_selectable_io(zkrb_queue_t *q) {
+  // rb_cIO is the ruby IO class?
+
+  int pipe, state, read_fd;
+  VALUE args[3], reader;
+
+  read_fd = zkrb_dup(q->pipe_read);
+
+  args[0] = rb_cIO;
+  args[1] = INT2NUM(read_fd);
+  args[2] = INT2FIX(O_RDONLY);
+  reader = rb_protect(zkrb_new_instance, (VALUE)args, &state);
+
+  if (state) {
+    rb_jump_tag(state);
+  }
+
+  return reader;
+}
+
 static VALUE method_init(int argc, VALUE* argv, VALUE self) {
   VALUE hostPort;
   VALUE options;
+
   rb_scan_args(argc, argv, "11", &hostPort, &options);
 
   if (NIL_P(options)) {
@@ -121,8 +170,10 @@ static VALUE method_init(int argc, VALUE* argv, VALUE self) {
   }
 
   rb_iv_set(self, "@data", data);
+  rb_iv_set(self, "@selectable_io", create_selectable_io(zk_local_ctx->queue));
   rb_iv_set(self, "@_running", Qtrue);
 
+
   return Qnil;
 }
 
@@ -421,7 +472,10 @@ static int is_running(VALUE self) {
   return RTEST(rval);
 }
 
-static VALUE method_get_next_event(VALUE self) {
+
+/* slyphon: NEED TO PROTECT THIS AGAINST SHUTDOWN */
+
+static VALUE method_get_next_event(VALUE self, VALUE blocking) {
   char buf[64];
   FETCH_DATA_PTR(self, zk);
 
@@ -439,19 +493,35 @@ static VALUE method_get_next_event(VALUE self) {
 
     /* Wait for an event using rb_thread_select() on the queue's pipe */
     if (event == NULL) {
-      int fd = zk->queue->pipe_read;
-      fd_set rset;
+      if (NIL_P(blocking) || (blocking == Qfalse)) { 
+        return Qnil; // no event for us
+      } 
+      else {
+        int fd = zk->queue->pipe_read;
+        ssize_t bytes_read = 0;
+
+        fd_set rset;        // a file descriptor set for use w/ select()
+
+        FD_ZERO(&rset);     // FD_ZERO clears the set
+        FD_SET(fd, &rset);  // FD_SET adds fd to the rset
 
-      FD_ZERO(&rset);
-      FD_SET(fd, &rset);
+        // first arg is nfds: "the highest-numbered file descriptor in any of the three sets, plus 1"
+        // why? F*** you, that's why!
 
-      if (rb_thread_select(fd + 1, &rset, NULL, NULL, NULL) == -1)
-        rb_raise(rb_eRuntimeError, "select failed: %d", errno);
+        if (rb_thread_select(fd + 1, &rset, NULL, NULL, NULL) == -1)
+          rb_raise(rb_eRuntimeError, "select failed: %d", errno);
 
-      if (read(fd, buf, sizeof(buf)) == -1)
-        rb_raise(rb_eRuntimeError, "read failed: %d", errno);
+        bytes_read = read(fd, buf, sizeof(buf));
 
-      continue;
+        if (bytes_read == -1) {
+          rb_raise(rb_eRuntimeError, "read failed: %d", errno);
+        }
+        else if (ZKRBDebugging) {
+          fprintf(stderr, "read %d bytes from the queue's pipe\n", bytes_read);
+        }
+
+        continue;
+      }
     }
 
     VALUE hash = zkrb_event_to_ruby(event);
@@ -493,11 +563,22 @@ static VALUE method_wake_event_loop_bang(VALUE self) {
 //   return Qnil;
 // }
 
-static VALUE method_close(VALUE self) {
+static VALUE method_close_handle(VALUE self) {
   FETCH_DATA_PTR(self, zk);
 
+  if (ZKRBDebugging) {
+    fprintf(stderr, "CLOSING ZK INSTANCE:");
+    print_zkrb_instance_data(zk);
+  }
+  
+  // this is a value on the ruby side we can check to see if destroy_zkrb_instance
+  // has been called
+  rb_iv_set(self, "@_closed", Qtrue);
+
+
   /* Note that after zookeeper_close() returns, ZK handle is invalid */
   int rc = destroy_zkrb_instance(zk);
+
   return INT2FIX(rc);
 }
 
@@ -521,8 +602,7 @@ static VALUE method_recv_timeout(VALUE self) {
   return INT2NUM(zoo_recv_timeout(zk->zh));
 }
 
-// how do you make a class method??
-static VALUE method_set_debug_level(VALUE self, VALUE level) {
+static VALUE klass_method_set_debug_level(VALUE klass, VALUE level) {
   Check_Type(level, T_FIXNUM);
   ZKRBDebugging = (FIX2INT(level) == ZOO_LOG_LEVEL_DEBUG);
   zoo_set_debug_level(FIX2INT(level));
@@ -549,7 +629,7 @@ static void zkrb_define_methods(void) {
   DEFINE_METHOD(set_acl, 5);
   DEFINE_METHOD(get_acl, 3);
   DEFINE_METHOD(client_id, 0);
-  DEFINE_METHOD(close, 0);
+  DEFINE_METHOD(close_handle, 0);
   DEFINE_METHOD(deterministic_conn_order, 1);
   DEFINE_METHOD(is_unrecoverable, 0);
   DEFINE_METHOD(recv_timeout, 1);
@@ -559,13 +639,16 @@ static void zkrb_define_methods(void) {
   // DEFINE_METHOD(async, 1);
 
   // methods for the ruby-side event manager
-  DEFINE_METHOD(get_next_event, 0);
+  DEFINE_METHOD(get_next_event, 1);
   DEFINE_METHOD(has_events, 0);
 
   // Make these class methods?
-  DEFINE_METHOD(set_debug_level, 1);
   DEFINE_METHOD(zerror, 1);
 
+  rb_define_singleton_method(Zookeeper, "set_debug_level", klass_method_set_debug_level, 1);
+
+  rb_attr(Zookeeper, rb_intern("selectable_io"), 1, 0, Qtrue);
+
   rb_define_method(Zookeeper, "wake_event_loop!", method_wake_event_loop_bang, 0);
 }
 

data/ext/zookeeper_lib.c

@@ -140,18 +140,26 @@ void zkrb_event_free(zkrb_event_t *event) {
     }
     case ZKRB_STRINGS: {
       struct zkrb_strings_completion *strings_ctx = event->completion.strings_completion;
-      int k;
-      for (k = 0; k < strings_ctx->values->count; ++k) free(strings_ctx->values->data[k]);
-      free(strings_ctx->values);
+      if (strings_ctx->values != NULL) {
+        int k;
+        for (k = 0; k < strings_ctx->values->count; ++k) free(strings_ctx->values->data[k]);
+        free(strings_ctx->values);
+      }
       free(strings_ctx);
       break;
     }
     case ZKRB_STRINGS_STAT: {
       struct zkrb_strings_stat_completion *strings_stat_ctx = event->completion.strings_stat_completion;
-      int k;
-      for (k = 0; k < strings_stat_ctx->values->count; ++k) free(strings_stat_ctx->values->data[k]);
-      free(strings_stat_ctx->values);
-      free(strings_stat_ctx->stat);
+      if (strings_stat_ctx->values != NULL) {
+        int k;
+        for (k = 0; k < strings_stat_ctx->values->count; ++k) free(strings_stat_ctx->values->data[k]);
+        free(strings_stat_ctx->values);
+      }
+
+      if (strings_stat_ctx->stat != NULL) {
+        free(strings_stat_ctx->stat);
+      }
+
       free(strings_stat_ctx);
       break;
     }
@@ -200,33 +208,40 @@ VALUE zkrb_event_to_ruby(zkrb_event_t *event) {
       break;
     }
     case ZKRB_STAT: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_STAT\n");
       struct zkrb_stat_completion *stat_ctx = event->completion.stat_completion;
       rb_hash_aset(hash, GET_SYM("stat"), stat_ctx->stat ? zkrb_stat_to_rarray(stat_ctx->stat) : Qnil);
       break;
     }
     case ZKRB_STRING: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_STRING\n");
       struct zkrb_string_completion *string_ctx = event->completion.string_completion;
       rb_hash_aset(hash, GET_SYM("string"), string_ctx->value ? rb_str_new2(string_ctx->value) : Qnil);
       break;
     }
     case ZKRB_STRINGS: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_STRINGS\n");
       struct zkrb_strings_completion *strings_ctx = event->completion.strings_completion;
       rb_hash_aset(hash, GET_SYM("strings"), strings_ctx->values ? zkrb_string_vector_to_ruby(strings_ctx->values) : Qnil);
       break;
     }
     case ZKRB_STRINGS_STAT: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_STRINGS_STAT\n");
       struct zkrb_strings_stat_completion *strings_stat_ctx = event->completion.strings_stat_completion;
       rb_hash_aset(hash, GET_SYM("strings"), strings_stat_ctx->values ? zkrb_string_vector_to_ruby(strings_stat_ctx->values) : Qnil);
       rb_hash_aset(hash, GET_SYM("stat"), strings_stat_ctx->stat ? zkrb_stat_to_rarray(strings_stat_ctx->stat) : Qnil);
       break;
     }
     case ZKRB_ACL: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_ACL\n");
       struct zkrb_acl_completion *acl_ctx = event->completion.acl_completion;
       rb_hash_aset(hash, GET_SYM("acl"), acl_ctx->acl ? zkrb_acl_vector_to_ruby(acl_ctx->acl) : Qnil);
       rb_hash_aset(hash, GET_SYM("stat"), acl_ctx->stat ? zkrb_stat_to_rarray(acl_ctx->stat) : Qnil);
       break;
     }
     case ZKRB_WATCHER: {
+      if (ZKRBDebugging) fprintf(stderr, "zkrb_event_to_ruby ZKRB_WATCHER\n");
+      struct zkrb_acl_completion *acl_ctx = event->completion.acl_completion;
       struct zkrb_watcher_completion *watcher_ctx = event->completion.watcher_completion;
       rb_hash_aset(hash, GET_SYM("type"), INT2FIX(watcher_ctx->type));
       rb_hash_aset(hash, GET_SYM("state"), INT2FIX(watcher_ctx->state));
@@ -422,6 +437,7 @@ void zkrb_strings_stat_callback(
   struct zkrb_strings_stat_completion *sc = malloc(sizeof(struct zkrb_strings_stat_completion));
   sc->stat = NULL;
   if (stat != NULL) { sc->stat = malloc(sizeof(struct Stat)); memcpy(sc->stat, stat, sizeof(struct Stat)); }
+
   sc->values = (strings != NULL) ? zkrb_clone_string_vector(strings) : NULL;
 
   ZKH_SETUP_EVENT(queue, event);

data/java/zookeeper_base.rb

@@ -59,6 +59,53 @@ class ZookeeperBase
     end
   end
 
+  class QueueWithPipe
+    attr_writer :clear_reads_on_pop
+
+    def initialize
+      r, w = IO.pipe
+      @pipe = { :read => r, :write => w }
+      @queue = Queue.new
+
+      # with the EventMachine client, we want to let EM handle clearing the
+      # event pipe, so we set this to false
+      @clear_reads_on_pop = true
+    end
+
+    def push(obj)
+      rv = @queue.push(obj)
+      @pipe[:write].write('0')
+      logger.debug { "pushed #{obj.inspect} onto queue and wrote to pipe" }
+      rv
+    end
+
+    def pop(non_blocking=false)
+      rv = @queue.pop(non_blocking)
+
+      # if non_blocking is true and an exception is raised, this won't get called
+      @pipe[:read].read(1) if clear_reads_on_pop?
+
+      rv
+    end
+
+    def close
+      @pipe.values.each { |io| io.close unless io.closed? }
+    end
+
+    def selectable_io
+      @pipe[:read]
+    end
+
+    private
+      def clear_reads_on_pop?
+        @clear_reads_on_pop
+      end
+
+      def logger
+        Zookeeper.logger
+      end
+  end
+
   # used for internal dispatching
   module JavaCB #:nodoc:
     class Callback
@@ -78,13 +125,24 @@ class ZookeeperBase
       include JZK::AsyncCallback::DataCallback
 
       def processResult(rc, path, queue, data, stat)
-        queue.push({
+        logger.debug { "#{self.class.name}#processResult rc: #{rc}, req_id: #{req_id}, path: #{path}, queue: #{queue.inspect}, data: #{data.inspect}, stat: #{stat.inspect}" }
+
+        hash = {
           :rc     => rc,
           :req_id => req_id,
           :path   => path,
-          :data   => String.from_java_bytes(data),
-          :stat   => stat.to_hash,
-        })
+          :data   => (data && String.from_java_bytes(data)),
+          :stat   => (stat && stat.to_hash),
+        }
+
+#         if rc == Zookeeper::ZOK
+#           hash.merge!({
+#             :data   => String.from_java_bytes(data),
+#             :stat   => stat.to_hash,
+#           })
+#         end
+
+        queue.push(hash)
       end
     end
 
@@ -92,6 +150,7 @@ class ZookeeperBase
       include JZK::AsyncCallback::StringCallback
 
       def processResult(rc, path, queue, str)
+        logger.debug { "#{self.class.name}#processResult rc: #{rc}, req_id: #{req_id}, path: #{path}, queue: #{queue.inspect}, str: #{str.inspect}" }
         queue.push(:rc => rc, :req_id => req_id, :path => path, :string => str)
       end
     end
@@ -100,7 +159,7 @@ class ZookeeperBase
       include JZK::AsyncCallback::StatCallback
 
       def processResult(rc, path, queue, stat)
-        logger.debug { "StatCallback#processResult rc: #{rc.inspect}, path: #{path.inspect}, queue: #{queue.inspect}, stat: #{stat.inspect}" }
+        logger.debug { "#{self.class.name}#processResult rc: #{rc.inspect}, req_id: #{req_id}, path: #{path.inspect}, queue: #{queue.inspect}, stat: #{stat.inspect}" }
         queue.push(:rc => rc, :req_id => req_id, :stat => (stat and stat.to_hash), :path => path)
       end
     end
@@ -109,7 +168,16 @@ class ZookeeperBase
       include JZK::AsyncCallback::Children2Callback
 
       def processResult(rc, path, queue, children, stat)
-        queue.push(:rc => rc, :req_id => req_id, :path => path, :strings => children.to_a, :stat => stat.to_hash)
+        logger.debug { "#{self.class.name}#processResult rc: #{rc}, req_id: #{req_id}, path: #{path}, queue: #{queue.inspect}, children: #{children.inspect}, stat: #{stat.inspect}" }
+        hash = {
+          :rc       => rc, 
+          :req_id   => req_id, 
+          :path     => path, 
+          :strings  => (children && children.to_a), 
+          :stat     => (stat and stat.to_hash),
+        }
+
+        queue.push(hash)
       end
     end
 
@@ -117,9 +185,9 @@ class ZookeeperBase
       include JZK::AsyncCallback::ACLCallback
       
       def processResult(rc, path, queue, acl, stat)
-        logger.debug { "ACLCallback#processResult #{rc.inspect} #{path.inspect} #{queue.inspect} #{acl.inspect} #{stat.inspect}" }
+        logger.debug { "ACLCallback#processResult rc: #{rc.inspect}, req_id: #{req_id}, path: #{path.inspect}, queue: #{queue.inspect}, acl: #{acl.inspect}, stat: #{stat.inspect}" }
         a = Array(acl).map { |a| a.to_hash }
-        queue.push(:rc => rc, :req_id => req_id, :path => path, :acl => a, :stat => stat.to_hash)
+        queue.push(:rc => rc, :req_id => req_id, :path => path, :acl => a, :stat => (stat && stat.to_hash))
       end
     end
 
@@ -127,6 +195,7 @@ class ZookeeperBase
       include JZK::AsyncCallback::VoidCallback
 
       def processResult(rc, path, queue)
+        logger.debug { "#{self.class.name}#processResult rc: #{rc}, req_id: #{req_id}, queue: #{queue.inspect}" }
         queue.push(:rc => rc, :req_id => req_id, :path => path)
       end
     end
@@ -140,7 +209,7 @@ class ZookeeperBase
       end
 
       def process(event)
-        logger.debug { "WatcherCallback got event: #{event.to_hash}" }
+        logger.debug { "WatcherCallback got event: #{event.to_hash.inspect}" }
         hash = event.to_hash.merge(:req_id => req_id)
         @event_queue.push(hash)
       end
@@ -156,26 +225,32 @@ class ZookeeperBase
       set_default_global_watcher(&watcher)
     end
 
-    @jzk = JZK::ZooKeeper.new(@host, DEFAULT_SESSION_TIMEOUT, JavaCB::WatcherCallback.new(@event_queue))
+    @start_stop_mutex.synchronize do
+      @jzk = JZK::ZooKeeper.new(@host, DEFAULT_SESSION_TIMEOUT, JavaCB::WatcherCallback.new(@event_queue))
 
-    if timeout > 0
-      time_to_stop = Time.now + timeout
-      until connected?
-        break if Time.now > time_to_stop
-        sleep 0.1
+      if timeout > 0
+        time_to_stop = Time.now + timeout
+        until connected?
+          break if Time.now > time_to_stop
+          sleep 0.1
+        end
       end
     end
 
     state
   end
 
-  def initialize(host, timeout=10, watcher=nil)
+  def initialize(host, timeout=10, watcher=nil, options={})
     @host = host
-    @event_queue = Queue.new
+    @event_queue = QueueWithPipe.new
     @current_req_id = 0
     @req_mutex = Monitor.new
     @watcher_reqs = {}
     @completion_reqs = {}
+    @_running = nil
+    @_closed  = false
+    @options = {}
+    @start_stop_mutex = Mutex.new
 
     watcher ||= get_default_global_watcher
 
@@ -184,6 +259,7 @@ class ZookeeperBase
 
     reopen(timeout, watcher)
     return nil unless connected?
+    @_running = true
     setup_dispatch_thread!
   end
 
@@ -203,6 +279,22 @@ class ZookeeperBase
     state == JZK::ZooKeeper::States::ASSOCIATING
   end
 
+  def running?
+    @_running
+  end
+
+  def closed?
+    @_closed
+  end
+
+  def self.set_debug_level(*a)
+    # IGNORED IN JRUBY
+  end
+
+  def set_debug_level(*a)
+    # IGNORED IN JRUBY
+  end
+
   def get(req_id, path, callback, watcher)
     handle_keeper_exception do
       watch_cb = watcher ? create_watcher(req_id, path) : false
@@ -332,20 +424,36 @@ class ZookeeperBase
 
   class DispatchShutdownException < StandardError; end
 
+  def wake_event_loop!
+    @event_queue.push(KILL_TOKEN)    # ignored by dispatch_next_callback
+  end
+
   def close
     @req_mutex.synchronize do
-      if @jzk
-        @jzk.close
-        wait_until { !connected? }
-      end
+      @_running = false if @_running
+    end
+        
+    # XXX: why is wake_event_loop! here?
+    if @dispatcher 
+      wake_event_loop!
+      @dispatcher.join 
+    end
 
-      if @dispatcher 
-        @dispatcher[:running] = false
-        @event_queue.push(KILL_TOKEN)    # ignored by dispatch_next_callback
+    unless @_closed
+      @start_stop_mutex.synchronize do
+        @_closed = true
+        close_handle
       end
+
+      @event_queue.close
     end
+  end
 
-    @dispatcher.join
+  def close_handle
+    if @jzk
+      @jzk.close
+      wait_until { !connected? }
+    end
   end
 
   # set the watcher object/proc that will receive all global events (such as session/state events)
@@ -360,6 +468,23 @@ class ZookeeperBase
     end
   end
 
+  # by accessing this selectable_io you indicate that you intend to clear it
+  # when you have delivered an event by reading one byte per event.
+  #
+  def selectable_io
+    @event_queue.clear_reads_on_pop = false
+    @event_queue.selectable_io
+  end
+
+  def get_next_event(blocking=true)
+    @event_queue.pop(!blocking).tap do |event|
+      logger.debug { "get_next_event delivering event: #{event.inspect}" }
+      raise DispatchShutdownException if event == KILL_TOKEN
+    end
+  rescue ThreadError
+    nil
+  end
+ 
   protected
     def handle_keeper_exception
       yield
@@ -376,28 +501,23 @@ class ZookeeperBase
     end
    
     def create_watcher(req_id, path)
+      logger.debug { "creating watcher for req_id: #{req_id} path: #{path}" }
       lambda do |event|
+        logger.debug { "watcher for req_id #{req_id}, path: #{path} called back" }
         h = { :req_id => req_id, :type => event.type.int_value, :state => event.state.int_value, :path => path }
         @event_queue.push(h)
       end
     end
 
-    def get_next_event
-      @event_queue.pop.tap do |event|
-        raise DispatchShutdownException if event == KILL_TOKEN
-      end
-    end
-    
     # method to wait until block passed returns true or timeout (default is 10 seconds) is reached 
     def wait_until(timeout=10, &block)
       time_to_stop = Time.now + timeout
       until yield do 
         break if Time.now > time_to_stop
-        sleep 0.3
+        sleep 0.1
       end
     end
 
-  protected
     # TODO: Make all global puts configurable
     def get_default_global_watcher
       Proc.new { |args|
@@ -406,13 +526,10 @@ class ZookeeperBase
       }
     end
 
-  private
     def setup_dispatch_thread!
       logger.debug {  "starting dispatch thread" }
       @dispatcher = Thread.new do
-        Thread.current[:running] = true
-
-        while Thread.current[:running]
+        while running?
           begin
             dispatch_next_callback 
           rescue DispatchShutdownException