zk 1.5.1 → 1.5.2

data/Gemfile

@@ -33,6 +33,10 @@ group :development do
   gem 'guard-shell',    :require => false
   gem 'guard-bundler',  :require => false
   gem 'growl',          :require => false
+
+  if RUBY_PLATFORM =~ /darwin/i
+    gem 'rb-readline', :platform => :ruby
+  end
 end
 
 group :test do

data/Guardfile

@@ -17,14 +17,18 @@ guard 'rspec', :version => 2 do
 
   watch(%r{^lib/(.+)\.rb$}) do |m| 
     case m[1]
-    when %r{^zk/event_handler$}
+    when 'zk/event_handler'
       "spec/zk/watch_spec.rb"
-    when %r{^zk/client/threaded.rb$}
+
+    when 'zk/client/threaded'
       ["spec/zk/client_spec.rb", "spec/zk/zookeeper_spec.rb"]
-    when %r{^zk/locker/}
-      "spec/zk/locker_spec.rb"
-    when %r{^zk\.rb$}
+
+    when %r{^(?:zk/locker/locker_base|spec/shared/locker)}
+      Dir["spec/zk/locker/*_spec.rb"]
+
+    when 'zk' # .rb
       'spec'  # run all tests
+
     else
       "spec/#{m[1]}_spec.rb"
     end

data/README.markdown

@@ -67,7 +67,7 @@ In addition to all of that, I would like to think that the public API the ZK::Cl
 ## NEWS ##
 ### v1.5.1 ###
 
-* Added a `:retry_duration` option to client constructor which will allows the user to specify for how long in the case of a connection loss, should an operation wait for the connection to be re-established before retrying the operation. This can be set at a global level and overridden on a per-call basis. The default is to not retry (which may change at a later date). Generally speaking, a timeout of > 30s is probably excessive, and care should be taken because during a connection loss, the server-side state may change without you being aware of it (i.e. events will not be delivered). 
+* Added a `:retry_duration` option to the Threaded client constructor which will allows the user to specify for how long in the case of a connection loss, should an operation wait for the connection to be re-established before retrying the operation. This can be set at a global level and overridden on a per-call basis. The default is to not retry (which may change at a later date). Generally speaking, a timeout of > 30s is probably excessive, and care should be taken because during a connection loss, the server-side state may change without you being aware of it (i.e. events will not be delivered). 
 
 * Small fork-hook implementation fix. Previously we were using WeakRefs so that hooks would not prevent an object from being garbage collected. This has been replaced with a finalizer which is more deterministic.
 

data/RELEASES.markdown

@@ -1,5 +1,13 @@
 This file notes feature differences and bugfixes contained between releases. 
 
+### v1.5.2 ###
+
+* Fix locker cleanup code to avoid a nasty race when a session is lost, see [issue #34](https://github.com/slyphon/zk/issues/34)
+
+* Fix potential deadlock in ForkHook code so the mutex is unlocked in the case of an exception
+
+* Do not hang forever when shutting down and the shutdown thread does not exit (wait 30 seconds).
+
 ### v1.5.1 ###
 
 * Added a `:retry_duration` option to client constructor which will allows the user to specify for how long in the case of a connection loss, should an operation wait for the connection to be re-established before retrying the operation. This can be set at a global level and overridden on a per-call basis. The default is to not retry (which may change at a later date). Generally speaking, a timeout of > 30s is probably excessive, and care should be taken because during a connection loss, the server-side state may change without you being aware of it (i.e. events will not be delivered). 

data/lib/zk/client/threaded.rb

@@ -171,6 +171,8 @@ module ZK
 
         @retry_duration = opts.fetch(:retry_duration, nil).to_i
 
+        yield self if block_given?
+
         @fork_subs = [
           ForkHook.prepare_for_fork(method(:pause_before_fork_in_parent)),
           ForkHook.after_fork_in_parent(method(:resume_after_fork_in_parent)),
@@ -179,11 +181,10 @@ module ZK
 
         ObjectSpace.define_finalizer(self, self.class.finalizer(@fork_subs))
 
-        yield self if block_given?
-
         connect if opts.fetch(:connect, true)
       end
 
+      # @private
       def self.finalizer(hooks)
         proc { hooks.each(&:unregister) }
       end
@@ -259,7 +260,11 @@ module ZK
           @cond.broadcast
         end
 
-        [@event_handler, @threadpool, @cnx].each(&:pause_before_fork_in_parent)
+        # the compact is here because the @cnx *may* be nil when this callback is fired by the
+        # ForkHook (in the case of ZK.open). The race is between the GC calling the finalizer
+        [@event_handler, @threadpool, @cnx].compact.each(&:pause_before_fork_in_parent)
+      ensure
+        logger.debug { "#{self.class}##{__method__} returning" }
       end
 
       # @private
@@ -270,7 +275,7 @@ module ZK
 
           logger.debug { "#{self.class}##{__method__}" }
 
-          [@cnx, @event_handler, @threadpool].each(&:resume_after_fork_in_parent)
+          [@cnx, @event_handler, @threadpool].compact.each(&:resume_after_fork_in_parent)
 
           @cond.broadcast
         end
@@ -304,6 +309,8 @@ module ZK
         #
         shutdown_thread = Thread.new do
           @threadpool.shutdown(10)
+
+          # this will call #close
           super
 
           @mutex.synchronize do
@@ -313,7 +320,7 @@ module ZK
           end
         end
 
-        on_tpool ? shutdown_thread : shutdown_thread.join 
+        on_tpool ? shutdown_thread : shutdown_thread.join(30)
       end
 
       # {see Base#close}

data/lib/zk/fork_hook.rb

@@ -18,6 +18,9 @@ module ZK
       @mutex.lock
       logger.debug { "#{__method__}" }      
       safe_call(@hooks[:prepare])
+    rescue Exception => e
+      @mutex.unlock rescue nil    # if something goes wrong in a hook, then release the lock
+      raise e
     end
 
     # @private

data/lib/zk/locker/locker_base.rb

@@ -48,11 +48,14 @@ module ZK
       def initialize(client, name, root_lock_node=nil) 
         @zk = client
         @root_lock_node = root_lock_node || Locker.default_root_lock_node
-        @path = name
-        @locked = false
-        @waiting = false
-        @lock_path = nil
+
+        @path           = name
+        @locked         = false
+        @waiting        = false
+        @lock_path      = nil
+        @parent_stat    = nil
         @root_lock_path = "#{@root_lock_node}/#{@path.gsub("/", "__")}"
+
         @mutex  = Monitor.new
         @cond   = @mutex.new_cond
         @node_deletion_watcher = nil
@@ -119,19 +122,21 @@ module ZK
       # @return [true] if we held the lock and this method has
       #   unlocked it successfully
       #
-      # @return [false] we did not own the lock
+      # @return [false] if we did not own the lock.
+      #
+      # @note There is more than one way you might not "own the lock" 
+      #   see [issue #34](https://github.com/slyphon/zk/issues/34)
       #
       def unlock
+        rval = false
         synchronize do
           if @locked
-            cleanup_lock_path!
+            rval = cleanup_lock_path!
             @locked = false
             @node_deletion_watcher = nil
-            true
-          else
-            false # i know, i know, but be explicit
           end
         end
+        rval
       end
 
       # (see #unlock)
@@ -220,6 +225,7 @@ module ZK
           raise LockAssertionFailedError, "not connected"                             unless zk.connected?
           raise LockAssertionFailedError, "lock_path was #{lock_path.inspect}"        unless lock_path
           raise LockAssertionFailedError, "the lock path #{lock_path} did not exist!" unless zk.exists?(lock_path)
+          raise LockAssertionFailedError, "the parent node was replaced!"             unless root_lock_path_same?
           raise LockAssertionFailedError, "we do not actually hold the lock"          unless got_lock?
         end
       end
@@ -248,6 +254,8 @@ module ZK
           end
         end
 
+        # root_lock_path is /_zklocking/foobar
+        #
         def create_root_path!
           zk.mkdir_p(@root_lock_path)
         end
@@ -262,9 +270,14 @@ module ZK
         # prefix is the string that will appear in front of the sequence num,
         # defaults to 'lock'
         #
+        # this method also saves the stat of root_lock_path at the time of creation
+        # to ensure we don't accidentally remove a lock we don't own. see 
+        # [rule #34](https://github.com/slyphon/zk/issues/34)...er, *issue* #34.
+        #
         def create_lock_path!(prefix='lock')
           synchronize do
-            @lock_path = @zk.create("#{root_lock_path}/#{prefix}", "", :mode => :ephemeral_sequential)
+            @lock_path = @zk.create("#{root_lock_path}/#{prefix}", :mode => :ephemeral_sequential)
+            @parent_stat = @zk.stat(root_lock_path)
           end
 
           logger.debug { "got lock path #{@lock_path}" }
@@ -274,12 +287,43 @@ module ZK
           retry
         end
 
+        # if the root_lock_path has the same stat .ctime as the one
+        # we cached when we created our lock path, then we can be sure
+        # that we actually own the lock_path 
+        #
+        # see [issue #34](https://github.com/slyphon/zk/issues/34)
+        #
+        def root_lock_path_same?
+          synchronize do
+            return false unless @parent_stat
+
+            cur_stat = zk.stat(root_lock_path)  
+            cur_stat.exists? and (cur_stat.ctime == @parent_stat.ctime)
+          end
+        end
+
+        # we make a best-effort to clean up, this case is rife with race
+        # conditions if there is a lot of contention for the locks, so if we
+        # can't remove a path or if that path happens to not be empty we figure
+        # either we got pwned or that someone else will run this same method
+        # later and get to it
+        #
         def cleanup_lock_path!
-          logger.debug { "removing lock path #{@lock_path}" }
-          zk.delete(@lock_path)
+          rval = false
+
+          synchronize do
+            if root_lock_path_same?
+              logger.debug { "removing lock path #{@lock_path}" }
+
+              zk.delete(@lock_path, :ignore => :no_node)
+              zk.delete(root_lock_path, :ignore => [:not_empty, :no_node])
+              rval = true
+            end
+
+            @lock_path = @parent_stat = nil
+          end
 
-          zk.delete(root_lock_path, :ignore => :not_empty) 
-          @lock_path = nil
+          rval
         end
     end # LockerBase
   end # Locker

data/lib/zk/version.rb

@@ -1,3 +1,3 @@
 module ZK
-  VERSION = "1.5.1"
+  VERSION = "1.5.2"
 end

data/spec/message_queue_spec.rb

@@ -1,10 +1,11 @@
 require File.join(File.dirname(__FILE__), %w[spec_helper])
 
 describe ZK::MessageQueue do
+  include_context 'connection opts'
 
   before(:each) do
-    @zk = ZK.new("localhost:#{ZK.test_port}")
-    @zk2 = ZK.new("localhost:#{ZK.test_port}")
+    @zk = ZK.new(connection_host)
+    @zk2 = ZK.new(connection_host)
     wait_until{ @zk.connected? && @zk2.connected? }
     @queue_name = "_specQueue"
     @consume_queue = @zk.queue(@queue_name)

data/spec/shared/client_contexts.rb

@@ -10,7 +10,7 @@ shared_context 'threaded client connection' do
   before do
 #     logger.debug { "threaded client connection - begin before hook" }
 
-    @connection_string = "localhost:#{ZK.test_port}"
+    @connection_string = connection_host
     @base_path = '/zktests'
     @zk = ZK::Client::Threaded.new(*connection_args).tap { |z| wait_until { z.connected? } }
     @threadpool_exception = nil

data/spec/shared/locker_contexts.rb

@@ -0,0 +1,53 @@
+shared_context 'locker non-chrooted' do
+  include_context 'connection opts'
+
+  let(:zk)  { ZK.new(*connection_args) }
+  let(:zk2) { ZK.new(*connection_args) }
+  let(:zk3) { ZK.new(*connection_args) }
+
+  let(:connections) { [zk, zk2, zk3] }
+
+  let(:path) { "lock_path" }
+  let(:root_lock_path) { "#{ZK::Locker.default_root_lock_node}/#{path}" }
+
+  before do
+    wait_until{ connections.all?(&:connected?) }
+    zk.rm_rf(ZK::Locker.default_root_lock_node)
+  end
+
+  after do
+    connections.each { |c| c.close! }
+    wait_until { !connections.any?(&:connected?) }
+    ZK.open(*connection_args) { |z| z.rm_rf(ZK::Locker.default_root_lock_node) }
+  end
+end
+
+shared_context 'locker chrooted' do
+  include_context 'connection opts'
+
+  let(:chroot_path) { '/_zk_chroot_' }
+  let(:path) { "lock_path" }
+
+  let(:zk)  { ZK.new("#{connection_host}#{chroot_path}", connection_opts) }
+  let(:zk2) { ZK.new("#{connection_host}#{chroot_path}", connection_opts) }
+  let(:zk3) { ZK.new("#{connection_host}#{chroot_path}", connection_opts) }
+  let(:connections) { [zk, zk2, zk3] }
+  let(:root_lock_path) { "#{ZK::Locker.default_root_lock_node}/#{path}" }
+
+  before do
+    ZK.open(*connection_args) do |zk|
+      zk.mkdir_p(chroot_path)
+    end
+
+    wait_until{ connections.all?(&:connected?) }
+  end
+
+  after do
+    connections.each { |c| c.close! }
+    wait_until { !connections.any?(&:connected?) }
+
+    ZK.open(*connection_args) do |zk|
+      zk.rm_rf(chroot_path)
+    end
+  end
+end

data/spec/shared/locker_examples.rb

@@ -0,0 +1,55 @@
+# basic shared exmples for locker specs (both exclusive and shared)
+
+# these assume they're being executed in the 'locker chrooted' or 'locker
+# non-chrooted' contexts
+#
+shared_examples_for 'LockerBase#assert!' do
+  it %[should raise LockAssertionFailedError if its connection is no longer connected?] do
+    zk.close!
+    lambda { locker.assert! }.should raise_error(ZK::Exceptions::LockAssertionFailedError)
+  end
+
+  it %[should raise LockAssertionFailedError if locked? is false] do
+    locker.should_not be_locked
+    lambda { locker.assert! }.should raise_error(ZK::Exceptions::LockAssertionFailedError)
+  end
+
+  it %[should raise LockAssertionFailedError lock_path does not exist] do
+    locker.lock
+    lambda { locker.assert! }.should_not raise_error
+
+    zk.delete(locker.lock_path)
+    lambda { locker.assert! }.should raise_error(ZK::Exceptions::LockAssertionFailedError)
+  end
+
+  it %[should raise LockAssertionFailedError if our parent node's ctime is different than what we think it should be] do
+    locker.lock.should be_true
+
+    zk.rm_rf(File.dirname(locker.lock_path)) # remove the parent node
+    zk.mkdir_p(locker.lock_path)
+
+    lambda { locker.assert! }.should raise_error(ZK::Exceptions::LockAssertionFailedError)
+  end
+end
+
+shared_examples_for 'LockerBase#unlock' do
+  it %[should not delete a lock path it does not own] do
+    locker.lock.should be_true
+
+    zk.rm_rf(File.dirname(locker.lock_path)) # remove the parent node
+    zk.mkdir_p(File.dirname(locker.lock_path))
+
+    locker2.lock.should be_true
+
+    locker2.lock_path.should == locker.lock_path
+
+    lambda { locker2.assert! }.should_not raise_error
+
+    lock_path = locker.lock_path
+
+    locker.unlock.should be_false
+
+    zk.stat(lock_path).should exist
+  end
+end
+

data/spec/support/logging.rb

@@ -1,35 +1,49 @@
 module ZK
   TEST_LOG_PATH = File.join(ZK::ZK_ROOT, 'test.log')
-end
 
-layout = Logging.layouts.pattern(
-  :pattern => '%.1l, [%d #%p] %30.30c{2}:  %m\n',
-  :date_pattern => '%Y-%m-%d %H:%M:%S.%6N' 
-)
-
-appender = ENV['ZK_DEBUG'] ? Logging.appenders.stderr : Logging.appenders.file(ZK::TEST_LOG_PATH)
-appender.layout = layout
-#appender.immediate_at = "debug,info,warn,error,fatal"
-appender.auto_flushing = 25
-appender.flush_period = 5
-
-%w[ZK ClientForker spec Zookeeper].each do |name|
-  ::Logging.logger[name].tap do |log|
-    log.appenders = [appender]
-    log.level = :debug
+  def self.logging_gem_setup
+    layout = ::Logging.layouts.pattern(
+      :pattern => '%.1l, [%d #%p] %30.30c{2}:  %m\n',
+      :date_pattern => '%Y-%m-%d %H:%M:%S.%6N'
+    )
+
+
+    appender = ENV['ZK_DEBUG'] ? ::Logging.appenders.stderr : ::Logging.appenders.file(ZK::TEST_LOG_PATH)
+    appender.layout = layout
+    appender.immediate_at = "debug,info,warn,error,fatal"
+#     appender.auto_flushing = true
+    appender.auto_flushing = 25
+    appender.flush_period = 5
+
+    %w[ZK ClientForker spec Zookeeper].each do |name|
+      ::Logging.logger[name].tap do |log|
+        log.appenders = [appender]
+        log.level = :debug
+      end
+    end
+
+    # this logger is kinda noisy
+    ::Logging.logger['ZK::EventHandler'].level = :info
+
+    Zookeeper.logger = ::Logging.logger['Zookeeper']
+    Zookeeper.logger.level = ENV['ZOOKEEPER_DEBUG'] ? :debug : :warn
+
+    ZK::ForkHook.after_fork_in_child { ::Logging.reopen }
   end
-end
 
-# this logger is kinda noisy
-Logging.logger['ZK::EventHandler'].level = :info
 
-Zookeeper.logger = Logging.logger['Zookeeper']
-Zookeeper.logger.level = ENV['ZOOKEEPER_DEBUG'] ? :debug : :warn
+  def self.stdlib_logger_setup
+    require 'logger'
+    log = ::Logger.new($stderr).tap {|l| l.level = ::Logger::DEBUG }
+    ZK.logger = log
+    Zookeeper.logger = log
+  end
+end
 
-ZK::ForkHook.after_fork_in_child { ::Logging.reopen }
+ZK.logging_gem_setup
+# ZK.stdlib_logger_setup
 
 # Zookeeper.logger = ZK.logger.clone_new_log(:progname => 'zoo')
-
 # Zookeeper.logger = ZK.logger
 # Zookeeper.set_debug_level(4)