zk 0.9.0 → 0.9.1

data/lib/z_k.rb

@@ -27,10 +27,19 @@ module ZK
 
   KILL_TOKEN = Object.new unless defined?(KILL_TOKEN) 
 
-  # The logger used by the ZK library. uses a Logger to +/dev/null+ by default
+  unless @logger
+    @logger = Logger.new($stderr).tap { |n| n.level = Logger::ERROR }
+  end
+
+  # The logger used by the ZK library. uses a Logger stderr with Logger::ERROR
+  # level. The only thing that should ever be logged are exceptions that are
+  # swallowed by background threads.
+  #
+  # You can change this logger by setting ZK#logger= to an object that
+  # implements the stdllb Logger API.
   #
   def self.logger
-    @logger ||= Logger.new('/dev/null')
+    @logger
   end
 
   # Assign the Logger instance to be used by ZK

data/lib/z_k/client/base.rb

@@ -97,6 +97,15 @@ module ZK
       end
 
       # close the underlying connection and clear all pending events.
+      #
+      # @note it is VERY IMPORTANT that when using the threaded client that you
+      #   __NOT CALL THIS ON THE EVENT DISPATCH THREAD__. If you do call it when
+      #   `event_dispatch_thread?` is true, you will get a big fat Kernel.warn
+      #   and nothing will happen. There is currently no safe way to have the event
+      #   thread cause a shutdown in the main thread (with good reason). There is also
+      #   no way to get an exception from the event thread (they're currently
+      #   just logged and swallowed), so this is the least horrible remedy available.
+      #   
       def close!
         event_handler.clear!
         wrap_state_closed_error { cnx.close unless cnx.closed? }
@@ -317,9 +326,9 @@ module ZK
       def get(path, opts={})
         h = { :path => path }.merge(opts)
 
-        setup_watcher!(:data, h)
-
-        rv = check_rc(cnx.get(h), h)
+        rv = setup_watcher!(:data, h) do
+          check_rc(cnx.get(h), h)
+        end
 
         opts[:callback] ? rv : rv.values_at(:data, :stat)
       end
@@ -429,33 +438,19 @@ module ZK
       #
       #
       def stat(path, opts={})
-        # ===== exist node asynchronously
-        #
-        #   class StatCallback
-        #     def process_result(return_code, path, context, stat)
-        #       # do processing here
-        #     end
-        #   end
-        #  
-        #   callback = StatCallback.new
-        #   context = Object.new
-        #
-        #   zk.exists?("/path", :callback => callback, :context => context)
-
-
         h = { :path => path }.merge(opts)
 
-        setup_watcher!(:data, h)
+        setup_watcher!(:data, h) do
+          rv = cnx.stat(h)
 
-        rv = cnx.stat(h)
+          return rv if opts[:callback] 
 
-        return rv if opts[:callback] 
-
-        case rv[:rc] 
-        when Zookeeper::ZOK, Zookeeper::ZNONODE
-          rv[:stat]
-        else
-          check_rc(rv, h) # throws the appropriate error
+          case rv[:rc] 
+          when Zookeeper::ZOK, Zookeeper::ZNONODE
+            rv[:stat]
+          else
+            check_rc(rv, h) # throws the appropriate error
+          end
         end
       end
 
@@ -539,9 +534,10 @@ module ZK
 
         h = { :path => path }.merge(opts)
 
-        setup_watcher!(:child, h)
+        rv = setup_watcher!(:child, h) do
+          check_rc(cnx.get_children(h), h)
+        end
 
-        rv = check_rc(cnx.get_children(h), h)
         opts[:callback] ? rv : rv[:children]
       end
 
@@ -771,8 +767,9 @@ module ZK
       end
 
       # @private
-      def assert_we_are_not_on_the_event_dispatch_thread!
-        raise Exceptions::EventDispatchThreadException, "blocking method called on dispatch thread" if event_dispatch_thread?
+      def assert_we_are_not_on_the_event_dispatch_thread!(msg=nil)
+        msg ||= "blocking method called on dispatch thread"
+        raise Exceptions::EventDispatchThreadException, msg if event_dispatch_thread?
       end
 
       protected
@@ -788,8 +785,8 @@ module ZK
         end
 
         # @private
-        def setup_watcher!(watch_type, opts)
-          event_handler.setup_watcher!(watch_type, opts)
+        def setup_watcher!(watch_type, opts, &b)
+          event_handler.setup_watcher!(watch_type, opts, &b)
         end
 
         # used in #inspect, doesn't raise an error if we're not connected

data/lib/z_k/client/continuation_proxy.rb

@@ -21,7 +21,7 @@ module ZK
         end
       end
 
-      call_with_continuation :create, :get, :set, :stat, :children, :delete, :get_acl, :set_acl
+      call_with_continuation :get, :set, :stat, :children, :delete, :get_acl, :set_acl
 
       def initialize(zookeeper_cnx=nil)
         @zookeeper_cnx = zookeeper_cnx
@@ -29,6 +29,14 @@ module ZK
         @dropboxen = []
       end
 
+      # create is "special" because the return hash doesn't have a :path, it has a :string
+      def create(opts)
+        logger.debug { "_call_continue(create, #{opts.inspect})" }
+        _call_continue(:create, opts).tap do |rval|
+          rval[:path] = rval.delete(:string)
+        end
+      end
+
       # called by the multiplxed client to wake up threads that are waiting for
       # results (with an exception)
       # @private
@@ -66,7 +74,7 @@ module ZK
 
           _with_drop_box do |db|
             cb = lambda do |hash|
-              logger.debug { "#{self.class}##{__method__} block pushing: #{hash.inspect}" } 
+#               logger.debug { "#{self.class}##{__method__} block pushing: #{hash.inspect}" } 
               db.push(hash)
             end
 
@@ -74,9 +82,11 @@ module ZK
 
             @zookeeper_cnx.__send__(meth, opts)
 
-            db.pop.tap do |obj|
-              logger.debug { "#{self.class}##{__method__} popped and returning: #{obj.inspect}" } 
-            end
+#             db.pop.tap do |obj|
+#               logger.debug { "#{self.class}##{__method__} popped and returning: #{obj.inspect}" } 
+#             end
+            
+            db.pop
           end
         end
 

data/lib/z_k/client/threaded.rb

@@ -50,6 +50,16 @@ module ZK
 
       # @see ZK::Client::Base#close!
       def close!
+        if event_dispatch_thread?
+          msg = ["ZK ERROR: You called #{self.class}#close! on the event dispatch thread!!",
+                 "This will cause the client to deadlock and possibly your main thread as well!"]
+
+          warn_msg = [nil, msg, nil, "See ZK error log output (stderr by default) for a backtrace", nil].join("\n")
+
+          Kernel.warn(warn_msg)
+          assert_we_are_not_on_the_event_dispatch_thread!(msg.join(' '))
+        end
+
         @threadpool.shutdown
         super
         nil

data/lib/z_k/event_handler.rb

@@ -153,26 +153,52 @@ module ZK
       end
     end
 
+    # returns true if there's a pending watch of type for path
+    # @private
+    def restricting_new_watches_for?(watch_type, path)
+      synchronize do
+        if set = @outstanding_watches[watch_type]
+          return set.include?(path)
+        end
+      end
+
+      false
+    end
+
     # implements not only setting up the watcher callback, but deduplicating 
     # event delivery. Keeps track of in-flight watcher-type+path requests and
     # doesn't re-register the watcher with the server until a response has been
     # fired. This prevents one event delivery to *every* callback per :watch => true
     # argument.
     #
+    # due to somewhat poor design, we destructively modify opts before we yield
+    # and the client implictly knows this
+    #
     # @private
     def setup_watcher!(watch_type, opts)
-      return unless opts.delete(:watch)
+      return yield unless opts.delete(:watch)
 
       synchronize do
         set = @outstanding_watches.fetch(watch_type)
         path = opts[:path]
 
         if set.add?(path)
-          # this path has no outstanding watchers, let it do its thing
-          opts[:watcher] = watcher_callback 
+          # if we added the path to the set, blocking further registration of
+          # watches and an exception is raised then we rollback
+          begin
+            # this path has no outstanding watchers, let it do its thing
+            opts[:watcher] = watcher_callback 
+
+            yield opts
+          rescue Exception
+            set.delete(path)
+            raise
+          end
         else
-          # outstanding watch for path and data pair already exists, so ignore
-#           logger.debug { "outstanding watch request for path #{path.inspect} and watcher type #{watch_type.inspect}, not re-registering" }
+          # we did not add the path to the set, which means we are not
+          # responsible for removing a block on further adds if the operation
+          # fails, therefore, we just yield
+          yield opts
         end
       end
     end

data/lib/z_k/version.rb

@@ -1,3 +1,3 @@
 module ZK
-  VERSION = "0.9.0"
+  VERSION = "0.9.1"
 end

data/spec/informal/close-in-event-thread.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'zk'
+
+LOG = Logger.new($stderr).tap { |n| n.level = Logger::DEBUG }
+
+ZK.logger = LOG
+Zookeeper.logger = LOG
+
+class CloseInEventThread
+  include ZookeeperConstants
+
+  def initialize
+    @zk = ZK.new
+    @q = Queue.new
+  end
+
+  def run
+    @zk.on_connecting do |event|
+      if @ok_do_it
+        logger.debug { "ok, calling close, in event thread? #{@zk.event_dispatch_thread?}" }
+        @zk.close! 
+        logger.debug { "close! returned, continuing" }
+        @q.push(:OK)
+      else
+        logger.debug { "on_connecting, got event #{event}" }
+      end
+    end
+
+    @ok_do_it = true
+    logger.debug { "push bogus ZOO_CONNECTING_STATE event into queue" }
+    @zk.__send__(:cnx).event_queue.push(:req_id => -1, :type => -1, :state => ZOO_CONNECTING_STATE, :path => '')
+
+    rval = @q.pop
+
+    logger.debug { "got #{rval.inspect}" }
+
+    @zk.close!
+  end
+
+  def logger
+    LOG
+  end
+end
+
+CloseInEventThread.new.run
+

data/spec/informal/what-the-fork.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'zk'
+
+def new_stderr_logger
+  Logger.new($stderr).tap { |l| l.level = Logger::DEBUG }
+end
+
+ZK.logger = new_stderr_logger
+
+class WhatTheFork
+  attr_reader :logger
+
+  def initialize
+    @zk = ZK.new
+    @base_path = '/what-the-fork'
+    @path_to_delete = "#{@base_path}/delete_me"
+  end
+
+  def setup_logs!
+    Zookeeper.logger = ZK.logger = @logger = new_stderr_logger
+  end
+
+  def run
+    setup_logs!
+
+    @zk.mkdir_p(@path_to_delete)
+
+    @zk.on_connected do |event|
+      _debug  "on_connected: #{event.inspect}"
+    end
+
+    @zk.on_connecting do |event|
+      _debug "on_connecting: #{event.inspect}"
+    end
+
+    @zk.on_expired_session do |event|
+      _debug "on_expired_session: #{event.inspect}"
+    end
+
+    fork_it!
+
+    @zk.block_until_node_deleted(@path_to_delete)
+    _debug "exiting main process!"
+  end
+
+  def fork_it!
+    pid = fork do
+      setup_logs!
+
+      _debug "closing zk"
+      @zk.close!
+      _debug "closed zk"
+      @zk = ZK.new
+      _debug "created new zk"
+
+      @zk.delete(@path_to_delete)
+
+      _debug "deleted path #{@path_to_delete}, closing new zk instance"
+
+      @zk.close!
+
+      _debug  "EXITING!!"
+      exit 0
+    end
+
+    _, stat = Process.waitpid2(pid)
+    _debug "child exited, stat: #{stat.inspect}"
+  ensure
+    if pid
+      _debug "ensuring #{pid} is really dead"
+      Process.kill(9, pid) rescue Errno::ESRCH
+    end
+  end
+
+  def _debug(str)
+    logger.debug { str }
+  end
+end
+
+WhatTheFork.new.run if __FILE__ == $0
+

data/spec/shared/client_contexts.rb

@@ -1,12 +1,29 @@
 shared_context 'threaded client connection' do
   before do
     @connection_string = "localhost:#{ZK_TEST_PORT}"
+    @base_path = '/zktests'
     @zk = ZK::Client::Threaded.new(@connection_string).tap { |z| wait_until { z.connected? } }
-    @zk.rm_rf('/test')
+    @zk.rm_rf(@base_path)
   end
 
   after do
-    @zk.rm_rf('/test')
+    @zk.rm_rf(@base_path)
+    @zk.close!
+
+    wait_until(2) { @zk.closed? }
+  end
+end
+
+shared_context 'multiplexed client connection' do
+  before do
+    @connection_string = "localhost:#{ZK_TEST_PORT}"
+    @base_path = '/zktests'
+    @zk = ZK::Client::Multiplexed.new(@connection_string).tap { |z| wait_until { z.connected? } }
+    @zk.rm_rf(@base_path)
+  end
+
+  after do
+    @zk.rm_rf(@base_path)
     @zk.close!
 
     wait_until(2) { @zk.closed? }

data/spec/shared/client_examples.rb

@@ -6,6 +6,7 @@ shared_examples_for 'client' do
     end
     
     it %[should create all intermediate paths for the path givem] do
+      @zk.rm_rf('/test')
       @zk.should_not be_exists(@bogus_path)
       @zk.should_not be_exists(File.dirname(@bogus_path))
       @zk.mkdir_p(@bogus_path)

data/spec/watch_spec.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), %w[spec_helper])
+require 'spec_helper'
 
 describe ZK do
   describe do
@@ -8,15 +8,18 @@ describe ZK do
 
       @path = "/_testWatch"
       wait_until { @zk.connected? }
+
+      # make sure we start w/ clean state
+      @zk.rm_rf(@path)
     end
 
     after do
-      if @zk.connected?
-        @zk.close! 
-        wait_until { !@zk.connected? }
-      end
-
       mute_logger do
+        if @zk.connected?
+          @zk.close! 
+          wait_until { !@zk.connected? }
+        end
+
         ZK.open(@cnx_str) { |zk| zk.rm_rf(@path) }
       end
     end
@@ -25,7 +28,7 @@ describe ZK do
       locker = Mutex.new
       callback_called = false
 
-      @zk.watcher.register(@path) do |event|
+      @zk.register(@path) do |event|
         locker.synchronize do
           callback_called = true
         end
@@ -52,7 +55,7 @@ describe ZK do
     it %[should only deliver an event once to each watcher registered for exists?] do
       events = []
 
-      sub = @zk.watcher.register(@path) do |ev|
+      sub = @zk.register(@path) do |ev|
         logger.debug "got event #{ev}"
         events << ev
       end
@@ -73,7 +76,7 @@ describe ZK do
 
       @zk.create(@path, 'one', :mode => :ephemeral)
 
-      sub = @zk.watcher.register(@path) do |ev|
+      sub = @zk.register(@path) do |ev|
         logger.debug "got event #{ev}"
         events << ev
       end
@@ -96,7 +99,7 @@ describe ZK do
 
       @zk.create(@path, '')
 
-      sub = @zk.watcher.register(@path) do |ev|
+      sub = @zk.register(@path) do |ev|
         logger.debug "got event #{ev}"
         events << ev
       end
@@ -112,6 +115,40 @@ describe ZK do
 
       events.length.should == 1
     end
+
+    it %[should restrict_new_watches_for? if a successul watch has been set] do
+      @zk.stat(@path, :watch => true)
+      @zk.event_handler.should be_restricting_new_watches_for(:data, @path)
+    end
+
+    it %[should not a block on new watches after an operation fails] do
+      # this is a situation where we did get('/blah', :watch => true) but
+      # got an exception, the next watch set should work
+
+      events = []
+
+      sub = @zk.register(@path) do |ev|
+        logger.debug { "got event #{ev}" }
+        events << ev
+      end
+
+      # get a path that doesn't exist with a watch
+
+      lambda { @zk.get(@path, :watch => true) }.should raise_error(ZK::Exceptions::NoNode)
+
+      @zk.event_handler.should_not be_restricting_new_watches_for(:data, @path)
+
+      @zk.stat(@path, :watch => true)
+
+      @zk.event_handler.should be_restricting_new_watches_for(:data, @path)
+
+      @zk.create(@path, '')
+
+      wait_while { events.empty? }
+
+      events.should_not be_empty
+
+    end
   end
 
   describe 'state watcher' do
@@ -141,10 +178,6 @@ describe ZK do
           m.should_receive(:state).and_return(ZookeeperConstants::ZOO_CONNECTED_STATE)
         end
       end
-
-      it %[should only fire the callback once] do
-        pending "not sure if this is the behavior we want"
-      end
     end
   end
 end

data/spec/z_k/election_spec.rb

@@ -295,6 +295,7 @@ describe ZK::Election do
         end
 
         it %[should have seen both the death and life events] do
+          pending 'this test is flapping'
           @got_life_event.should be_true
           @got_death_event.should be_true
         end

data/spec/z_k/locker_spec.rb

@@ -494,7 +494,7 @@ describe 'ZK::Locker Multiplexed client', :client => :multiplexed do
 
   before do
     wait_until{ connections.all?(&:connected?) }
-    pending "Mutliplexed client locking is broken"
+#     pending "Mutliplexed client locking is broken"
   end
 
   after do
@@ -507,5 +507,3 @@ describe 'ZK::Locker Multiplexed client', :client => :multiplexed do
   it_should_behave_like 'shared-exclusive interaction'
 end # ZK::Locker
 
-
-

data/spec/zookeeper_spec.rb

@@ -1,17 +1,8 @@
-require File.join(File.dirname(__FILE__), %w[spec_helper])
+require 'spec_helper'
 
-describe ZK do
+shared_examples_for 'ZK basic' do
   before do
-    @zk = ZK.new("localhost:#{ZK_TEST_PORT}")
-
-    @base_path = "/zktests"
-    @zk.rm_rf(@base_path)
-    @zk.mkdir_p(@base_path)
-  end
-
-  after do
-    @zk.rm_rf(@base_path)
-    @zk.close!
+    @zk.create(@base_path) rescue ZK::Exceptions::NodeExists
   end
 
   describe ZK, "with no paths" do
@@ -125,3 +116,15 @@ describe ZK do
     end
   end
 end
+
+describe 'basic multiplexed', :client => :multiplexed do
+  include_context 'multiplexed client connection'
+  it_should_behave_like 'ZK basic'
+end
+
+describe 'basic threaded', :client => :threaded do
+  include_context 'threaded client connection'
+  it_should_behave_like 'ZK basic'
+end
+
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: zk
 version: !ruby/object:Gem::Version 
-  hash: 59
+  hash: 57
   prerelease: 
   segments: 
   - 0
   - 9
-  - 0
-  version: 0.9.0
+  - 1
+  version: 0.9.1
 platform: ruby
 authors: 
 - Jonathan D. Simms
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-23 00:00:00 Z
+date: 2012-04-26 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: slyphon-zookeeper
@@ -81,7 +81,9 @@ files:
 - lib/z_k/threadpool.rb
 - lib/z_k/version.rb
 - lib/zk.rb
+- spec/informal/close-in-event-thread.rb
 - spec/informal/lock_with_dead_session.rb
+- spec/informal/what-the-fork.rb
 - spec/log4j.properties
 - spec/message_queue_spec.rb
 - spec/shared/client_contexts.rb
@@ -141,7 +143,9 @@ signing_key:
 specification_version: 3
 summary: A high-level wrapper around the zookeeper driver
 test_files: 
+- spec/informal/close-in-event-thread.rb
 - spec/informal/lock_with_dead_session.rb
+- spec/informal/what-the-fork.rb
 - spec/log4j.properties
 - spec/message_queue_spec.rb
 - spec/shared/client_contexts.rb