zk 0.8.9 → 0.9.0

data/lib/z_k/client/conveniences.rb

@@ -1,5 +1,8 @@
 module ZK
   module Client
+    # EXTENSIONS
+    #
+    # convenience methods for dealing with zookeeper (rm -rf, mkdir -p, etc)
     module Conveniences
       # Queue an operation to be run on an internal threadpool. You may either
       # provide an object that responds_to?(:call) or pass a block. There is no
@@ -9,10 +12,13 @@ module ZK
       #
       # An ArgumentError will be raised if +callable+ does not <tt>respond_to?(:call)</tt>
       #
-      # ==== Arguments
-      # * <tt>callable</tt>: an object that <tt>respond_to?(:call)</tt>, takes precedence
-      #   over a given block
+      # @param [#call] callable an object that `respond_to?(:call)`, takes
+      #   precedence over a given block
+      #
+      # @yield [] the block that should be run in the threadpool, if `callable`
+      #   isn't given
       #
+      # @private
       def defer(callable=nil, &block)
         @threadpool.defer(callable, &block)
       end
@@ -28,18 +34,9 @@ module ZK
         false
       end
 
-
-      #--
-      #
-      # EXTENSIONS
-      #
-      # convenience methods for dealing with zookeeper (rm -rf, mkdir -p, etc)
-      #
-      #++
-      
       # creates a new locker based on the name you send in
       #
-      # see ZK::Locker::ExclusiveLocker
+      # @see ZK::Locker::ExclusiveLocker
       #
       # returns a ZK::Locker::ExclusiveLocker instance using this Client and provided
       # lock name

data/lib/z_k/client/drop_box.rb

@@ -0,0 +1,98 @@
+module ZK
+  module Client
+    # A simple threadsafe way of having a thread deliver a single value
+    # to another thread. 
+    #
+    # Each thread making requests will have a thread-local continuation
+    # that can be accessed via DropBox.current and one can use
+    # DropBox.with_current that will clear the result once the given block
+    # exits (allowing for reuse)
+    #
+    # (this class is in no way related to dropbox.com or Dropbox Inc.)
+    # @private
+    class DropBox
+      UNDEFINED = Object.new              unless defined?(UNDEFINED)
+      KILL_TOKEN = Object.new             unless defined?(KILL_TOKEN)
+      IMPOSSIBLE_TO_CONTINUE = Object.new unless defined?(IMPOSSIBLE_TO_CONTINUE)
+
+      # represents an exception to raise. if the pop method sees the value as an instance
+      # of this class, it will call the raise! method
+      class ExceptionValue
+        def initialize(exception_class, message)
+          @exception_class, @message = exception_class, message
+        end
+
+        def raise!
+          raise @exception_class, @message
+        end
+      end
+
+      THREAD_LOCAL_KEY = :__zk_client_continuation_current__ unless defined?(THREAD_LOCAL_KEY)
+
+      # @private
+      attr_reader :value
+
+      # sets the thread-local instance to nil, used by tests
+      # @private
+      def self.remove_current
+        Thread.current[THREAD_LOCAL_KEY] = nil
+      end
+
+      # access the thread-local DropBox instance for the current thread
+      def self.current
+        Thread.current[THREAD_LOCAL_KEY] ||= self.new()
+      end
+
+      # yields the current thread's DropBox instance and clears its value
+      # after the block returns
+      def self.with_current
+        yield current
+      ensure
+        current.clear
+      end
+
+      def initialize
+        @mutex = Mutex.new
+        @cond = ConditionVariable.new
+        @value = UNDEFINED  # allows us to return nil
+      end
+
+      def push(obj)
+        @mutex.synchronize do
+          @value = obj
+          @cond.signal
+        end
+      end
+
+      def pop
+        @mutex.synchronize do
+          @cond.wait(@mutex)
+          @value.kind_of?(ExceptionValue) ? @value.raise! : @value
+        end
+      end
+
+      def clear
+        @mutex.synchronize do
+          @value = UNDEFINED
+        end
+      end
+
+      # we are done if value is defined, use clear to reset
+      def done?
+        @value != UNDEFINED
+      end
+
+      # called when you need the waiting thread to receive an exception
+      # returns nil if a value has already been set
+      def oh_noes!(exception_class, message)
+        @mutex.synchronize do
+          return if done?
+          @value = ExceptionValue.new(exception_class, message)
+          @cond.signal
+        end
+        true
+      end
+    end
+  end
+end
+

data/lib/z_k/client/multiplexed.rb

@@ -0,0 +1,28 @@
+module ZK
+  module Client
+    # This client is an experimental implementation of a threaded and
+    # multiplexed client. The idea is that each synchronous request represents 
+    # a continuation. This way, you can have multiple requests pending with the
+    # server simultaneously, and the responses will be delivered on the event
+    # thread (but run in the calling thread). This allows for higher throughput
+    # for multi-threaded applications.
+    #
+    # Asynchronous requests are not supported through this client.
+    #
+    class Multiplexed < Threaded
+      def close!
+        @cnx.connection_closed!
+        super
+      end
+
+      protected
+        def create_connection(*args)
+          ContinuationProxy.new.tap do |cp|
+            on_expired_session { cp.expired_session! } # hook up client's session expired event listener
+            cp.zookeeper_cnx = super(*args)
+          end
+        end
+    end
+  end
+end
+

data/lib/z_k/client/threaded.rb

@@ -2,35 +2,65 @@ module ZK
   module Client
     # This is the default client that ZK will use. In the zk-eventmachine gem,
     # there is an Evented client.
+    #
     class Threaded < Base
       include StateMixin
       include Unixisms
       include Conveniences
 
-      # Create a new client and connect to the zookeeper server. 
+      DEFAULT_THREADPOOL_SIZE = 1
+
+      # @note The `:timeout` argument here is *not* the session_timeout for the
+      #   connection. rather it is the amount of time we wait for the connection
+      #   to be established. The session timeout exchanged with the server is 
+      #   set to 10s by default in the C implemenation, and as of version 0.8.0 
+      #   of slyphon-zookeeper has yet to be exposed as an option. That feature
+      #   is planned. 
+      #
+      # @param [String] host (see ZK::Client::Base#initialize)
+      #
+      # @option opts [Fixnum] :threadpool_size the size of the threadpool that
+      #   should be used to deliver events. In ZK 0.8.x this was set to 5, which
+      #   means that events could be delivered concurrently. As of 0.9, this will
+      #   be set to 1, so it's very important to _not block the event thread_.
       #
-      # +host+ should be a string of comma-separated host:port pairs. You can
-      # also supply an optional "chroot" suffix that will act as an implicit 
-      # prefix to all paths supplied.
+      # @option opts [Fixnum] :timeout how long we will wait for the connection
+      #   to be established. 
       #
-      # example:
-      #    
-      #   ZK::Client.new("zk01:2181,zk02:2181/chroot/path")
+      # @yield [self] calls the block with the new instance after the event
+      #   handler has been set up, but before any connections have been made.
+      #   This allows the client to register watchers for session events like
+      #   `connected`. You *cannot* perform any other operations with the client 
+      #   as you will get a NoMethodError (the underlying connection is nil).
       #
-      def initialize(host, opts={})
-        @event_handler = EventHandler.new(self)
+      def initialize(host, opts={}, &b)
+        super(host, opts)
+
+        @session_timeout = opts.fetch(:timeout, DEFAULT_TIMEOUT) # maybe move this into superclass?
+        @event_handler   = EventHandler.new(self)
+
         yield self if block_given?
-        @cnx = ::Zookeeper.new(host, DEFAULT_TIMEOUT, @event_handler.get_default_watcher_block)
-        @threadpool = Threadpool.new
+
+        @cnx = create_connection(host, @session_timeout, @event_handler.get_default_watcher_block)
+
+        tp_size = opts.fetch(:threadpool_size, DEFAULT_THREADPOOL_SIZE)
+
+        @threadpool = Threadpool.new(tp_size)
       end
 
-      # closes the underlying connection and deregisters all callbacks
+      # @see ZK::Client::Base#close!
       def close!
         @threadpool.shutdown
         super
         nil
       end
+
+      protected
+        # allows for the Mutliplexed client to wrap the connection in its ContinuationProxy
+        # @private
+        def create_connection(*args)
+          ::Zookeeper.new(*args)
+        end
     end
   end
 end
-

data/lib/z_k/client/unixisms.rb

@@ -5,11 +5,10 @@ module ZK
 
       # Creates all parent paths and 'path' in zookeeper as persistent nodes with
       # zero data.
-      #
-      # ==== Arguments
-      # * <tt>path</tt>: An absolute znode path to create
-      #
-      # ==== Examples
+      # 
+      # @param [String] path An absolute znode path to create
+      # 
+      # @example
       #
       #   zk.exists?('/path')
       #   # => false
@@ -17,10 +16,10 @@ module ZK
       #   zk.mkdir_p('/path/to/blah')
       #   # => "/path/to/blah"  
       #
-      #--
-      # TODO: write a non-recursive version of this. ruby doesn't have TCO, so
-      # this could get expensive w/ psychotically long paths
       def mkdir_p(path)
+        # TODO: write a non-recursive version of this. ruby doesn't have TCO, so
+        # this could get expensive w/ psychotically long paths
+
         create(path, '', :mode => :persistent)
       rescue Exceptions::NodeExists
         return
@@ -49,23 +48,84 @@ module ZK
         end
       end
 
-      # see ZK::Find for explanation
+      # Acts in a similar way to ruby's Find class. Performs a depth-first
+      # traversal of every node under the given paths, and calls the given
+      # block with each path found. Like the ruby Find class, you can call
+      # {ZK::Find.prune} to avoid descending further into a given sub-tree
+      #
+      # @example list the paths under a given node
+      #
+      #   zk = ZK.new
+      #   
+      #   paths = %w[
+      #     /root
+      #     /root/alpha
+      #     /root/bravo
+      #     /root/charlie
+      #     /root/charlie/rose
+      #     /root/charlie/manson
+      #     /root/charlie/manson/family
+      #     /root/charlie/manson/murders
+      #     /root/charlie/brown
+      #     /root/delta
+      #     /root/delta/blues
+      #     /root/delta/force
+      #     /root/delta/burke
+      #   ]
+      #   
+      #   paths.each { |p| zk.create(p) }
+      #   
+      #   zk.find('/root') do |path|
+      #     puts path
+      #   
+      #     ZK::Find.prune if path == '/root/charlie/manson'
+      #   end
+      #
+      #   # this produces the output:
+      #
+      #   # /root
+      #   # /root/alpha
+      #   # /root/bravo
+      #   # /root/charlie
+      #   # /root/charlie/brown
+      #   # /root/charlie/manson
+      #   # /root/charlie/rose
+      #   # /root/delta
+      #   # /root/delta/blues
+      #   # /root/delta/burke
+      #   # /root/delta/force
+      #
+      # @param [Array[String]] paths a list of paths to recursively 
+      #   yield the sub-paths of
+      #
+      # @see ZK::Find#find 
       def find(*paths, &block)
         ZK::Find.find(self, *paths, &block)
       end
 
-      # will block the caller until +abs_node_path+ has been removed
-      #
-      # @private this method is of dubious value and may be removed in a later
-      #   version
+      # Will _safely_ block the caller until `abs_node_path` has been removed.
+      # This is trickier than it first appears. This method will wake the caller
+      # if a session event occurs that would ensure the event would never be
+      # delivered, and also checks to make sure that the caller is not calling
+      # from the event distribution thread (which would cause a deadlock).
       #
       # @note this is dangerous to use in callbacks! there is only one
       #   event-delivery thread, so if you use this method in a callback or
       #   watcher, you *will* deadlock!
+      #
+      # @raise [Exceptions::InterruptedSession] If a session event occurs while we're
+      #   blocked waiting for the node to be deleted, an exception that
+      #   mixes in the InterruptedSession module will be raised. 
+      #
       def block_until_node_deleted(abs_node_path)
-        queue = Queue.new
         subs = []
 
+        assert_we_are_not_on_the_event_dispatch_thread!
+
+        raise ArgumentError, "argument must be String-ish, not: #{abs_node_path.inspect}" unless abs_node_path
+
+        queue = Queue.new
+
         node_deletion_cb = lambda do |event|
           if event.node_deleted?
             queue.enq(:deleted) 

data/lib/z_k/client.rb

@@ -24,9 +24,12 @@ module ZK
   end
 end
 
+require 'z_k/client/drop_box'
 require 'z_k/client/state_mixin'
 require 'z_k/client/unixisms'
 require 'z_k/client/conveniences'
 require 'z_k/client/base'
 require 'z_k/client/threaded'
+require 'z_k/client/continuation_proxy'
+require 'z_k/client/multiplexed'
 

data/lib/z_k/event_handler.rb

@@ -32,51 +32,7 @@ module ZK
       end
     end
 
-    # register a path with the handler
-    #
-    # your block will be called with all events on that path.
-    #
-    # @note All watchers are one-shot handlers. After an event is delivered to
-    #   your handler, you *must* re-watch the node to receive more events. This
-    #   leads to a pattern you will find throughout ZK code that avoids races,
-    #   see the example below "avoiding a race"
-    #
-    # @example avoiding a race waiting for a node to be deleted
-    #
-    #   # we expect that '/path/to/node' exists currently and want to be notified
-    #   # when it's deleted
-    #
-    #   # register a handler that will be called back when an event occurs on
-    #   # node
-    #   # 
-    #   node_subscription = zk.event_handler.register('/path/to/node') do |event|
-    #     if event.node_deleted?
-    #       do_something_when_node_deleted
-    #     end
-    #   end
-    #
-    #   # check to see if our condition is true *while* setting a watch on the node
-    #   # if our condition happens to be true while setting the watch
-    #   #
-    #   unless exists?('/path/to/node', :watch => true)
-    #     node_subscription.unsubscribe   # cancel the watch
-    #     do_something_when_node_deleted  # call the callback
-    #   end
-    #
-    #
-    # @param [String] path the path you want to listen to
-    #
-    # @param [Block] block the block to execute when a watch event happpens
-    #
-    # @yield [event] We will call your block with the watch event object (which
-    #   has the connection the event occurred on as its #zk attribute)
-    #
-    # @return [ZooKeeper::EventHandlerSubscription] the subscription object
-    #   you can use to to unsubscribe from an event
-    #
-    # @see ZooKeeper::WatcherEvent
-    # @see ZK::EventHandlerSubscription
-    #
+    # @see ZK::Client::Base#register
     def register(path, &block)
 #       logger.debug { "EventHandler#register path=#{path.inspect}" }
       EventHandlerSubscription.new(self, path, block).tap do |subscription|
@@ -136,6 +92,10 @@ module ZK
     alias :unsubscribe :unregister
 
     # called from the client-registered callback when an event fires
+    #
+    # @note this is *ONLY* dealing with asynchronous callbacks! watchers
+    #   and session events go through here, NOT anything else!!
+    #
     # @private
     def process(event)
 #       logger.debug { "EventHandler#process dispatching event: #{event.inspect}" }# unless event.type == -1

data/lib/z_k/event_handler_subscription.rb

@@ -1,29 +1,36 @@
 module ZK
   # the subscription object that is passed back from subscribing
   # to events.
-  # @see ZooKeeperEventHandler#subscribe
+  # @see ZK::Client::Base#register
   class EventHandlerSubscription
-    attr_accessor :event_handler, :path, :callback
+    # the event handler associated with this subscription
+    # @return [EventHandler]
+    attr_accessor :event_handler
+
+    # the path this subscription is for
+    # @return [String]
+    attr_accessor :path
+    
+    # the block associated with the path
+    # @return [Proc]
+    attr_accessor :callback
 
     # @private
-    # :nodoc:
     def initialize(event_handler, path, callback)
       @event_handler, @path, @callback = event_handler, path, callback
     end
 
     # unsubscribe from the path or state you were watching
-    # @see ZooKeeperEventHandler#subscribe
+    # @see ZK::Client::Base#register
     def unsubscribe
       @event_handler.unregister(self)
     end
     alias :unregister :unsubscribe
 
     # @private
-    # :nodoc:
     def call(event)
       callback.call(event)
     end
-
   end
 end
 

data/lib/z_k/exceptions.rb

@@ -35,10 +35,15 @@ module ZK
       end
     end
 
+    # This module is mixed into the session-related exceptions to allow
+    # one to rescue that group of exceptions. It is also mixed into the related
+    # ZookeeperException objects
+    module InterruptedSession
+    end
+
     class SystemError             < KeeperException; end
     class RunTimeInconsistency    < KeeperException; end
     class DataInconsistency       < KeeperException; end
-    class ConnectionLoss          < KeeperException; end
     class MarshallingError        < KeeperException; end
     class Unimplemented           < KeeperException; end
     class OperationTimeOut        < KeeperException; end
@@ -50,11 +55,23 @@ module ZK
     class NoChildrenForEphemerals < KeeperException; end
     class NodeExists              < KeeperException; end
     class NotEmpty                < KeeperException; end
-    class SessionExpired          < KeeperException; end
     class InvalidCallback         < KeeperException; end
     class InvalidACL              < KeeperException; end
     class AuthFailed              < KeeperException; end
 
+    class ConnectionLoss < KeeperException
+      include InterruptedSession
+    end
+
+    class SessionExpired < KeeperException
+      include InterruptedSession
+    end
+
+    # mixes in InterruptedSession, and can be raised on its own
+    class InterruptedSessionException < KeeperException
+      include InterruptedSession
+    end
+
     ERROR_MAP = {
       SYSTEMERROR             => SystemError,
       RUNTIMEINCONSISTENCY    => RunTimeInconsistency,
@@ -100,6 +117,9 @@ module ZK
     # raised for certain operations when using a chrooted connection, but the
     # root doesn't exist.
     class NonExistentRootError < ZKError; end
+
+    # raised when someone performs a blocking ZK operation on the event dispatch thread. 
+    class EventDispatchThreadException < ZKError; end
   end
 end
 

data/lib/z_k/extensions.rb

@@ -17,7 +17,7 @@ module ZK
           #
           # *args, if given, will be passed on *after* the callback
           #
-          # example:
+          # @example
           #   
           #   WatcherCallback.create do |cb|
           #     puts "watcher callback called with argument: #{cb.inspect}"
@@ -116,7 +116,6 @@ module ZK
         MEMBERS.all? { |m| self.__send__(m) == other.__send__(m) }
       end
     end
-
   end     # Extensions
 end       # ZK
 
@@ -125,6 +124,15 @@ ZookeeperCallbacks::Callback.send(:include, ZK::Extensions::Callbacks::Callback)
 ZookeeperCallbacks::WatcherCallback.send(:include, ZK::Extensions::Callbacks::WatcherCallbackExt)
 ZookeeperStat::Stat.send(:include, ZK::Extensions::Stat)
 
+# Include the InterruptedSession module in key ZookeeperExceptions to allow
+# clients to catch a single error type when waiting on a node (for example)
+
+[:ConnectionClosed, :NotConnected, :SessionExpired, :SessionMoved, :ConnectionLoss].each do |class_name|
+  ZookeeperExceptions::ZookeeperException.const_get(class_name).tap do |klass|
+    klass.__send__(:include, ZK::Exceptions::InterruptedSession)
+  end
+end
+
 class ::Exception
   unless method_defined?(:to_std_format)
     def to_std_format

data/lib/z_k/find.rb

@@ -3,7 +3,10 @@ module ZK
     # like ruby's Find module, will call the given block with each _absolute_ znode path 
     # under +paths+. you can call ZK::Find.prune if you want to not recurse
     # deeper under the current directory path.
-    def find(zk, *paths) #:yield: znode_path
+    #
+    # @yield [String] each znode path under the list of paths given.
+    #
+    def find(zk, *paths) 
       paths.collect!{|d| d.dup}
 
       while p = paths.shift
@@ -11,7 +14,7 @@ module ZK
           yield p.dup.taint
           next unless zk.exists?(p)
 
-          zk.children(p).each do |ch| 
+          zk.children(p).sort.reverse.each do |ch| 
             paths.unshift ZK.join(p, ch).untaint
           end
         end

data/lib/z_k/locker.rb

@@ -28,16 +28,21 @@ module ZK
     class LockerBase
       include ZK::Logging
 
-      attr_accessor :zk #:nodoc:
+      # @private
+      attr_accessor :zk
 
       # our absolute lock node path
       #
       # ex. '/_zklocking/foobar/__blah/lock000000007'
-      attr_reader :lock_path #;nodoc:
+      #
+      # @private
+      attr_reader :lock_path
 
-      attr_reader :root_lock_path #:nodoc:
+      # @private
+      attr_reader :root_lock_path
 
-      def self.digit_from_lock_path(path) #:nodoc:
+      # @private
+      def self.digit_from_lock_path(path)
         path[/0*(\d+)$/, 1].to_i
       end