zk 1.2.0 → 1.3.0

data/Gemfile

@@ -7,9 +7,9 @@ source :rubygems
 # this is the last known commit that we tested against and is passing.
 # keep closer track of this stuff to make bisecting easier and travis more
 # accurate
-#
-# git 'git://github.com/slyphon/zookeeper.git', :ref => 'e8b181a37ee' do
-#   gem 'zookeeper', '>= 1.0.0.beta.1'
+
+# git 'git://github.com/slyphon/zookeeper.git', :tag => 'dev/zk/00000' do
+#   gem 'zookeeper', '~> 1.0.0'
 # end
 
 

data/README.markdown

@@ -63,6 +63,14 @@ In addition to all of that, I would like to think that the public API the ZK::Cl
 
 ## NEWS ##
 
+### v1.3.0 ###
+
+Phusion Passenger and Unicorn users are encouraged to upgrade!
+
+* __fork()__: ZK should now work reliably after a fork() if you call `reopen()` ASAP in the child process (before continuing any ZK work). Additionally, your event-handler (blocks set up with `zk.register`) will still work in the child. You will have to make calls like `zk.stat(path, :watch => true)` to tell ZooKeeper to notify you of events (as the child will have a new session), but everything should work.
+
+* See the fork-handling documentation [on the wiki](http://github.com/slyphon/zk/wiki/Forking).
+
 ### v1.2.0 ###
 
 You are __STRONGLY ENCOURAGED__ to go and look at the [CHANGELOG](http://git.io/tPbNBw) from the zookeeper 1.0.0 release

data/docs/examples/events_01.rb

@@ -13,7 +13,10 @@ class Events
   end
 
   def run
-    @zk.delete(@path) rescue ZK::Exceptions::NoNode
+    begin
+      @zk.delete(@path) 
+    rescue ZK::Exceptions::NoNode
+    end
 
     @zk.register(@path) do |event|
       if event.node_changed? or event.node_created?

data/docs/examples/events_02.rb

@@ -23,7 +23,11 @@ class Events
       end
     end
 
-    @zk.delete(@path) rescue ZK::Exceptions::NoNode
+    begin
+      @zk.delete(@path) 
+    rescue ZK::Exceptions::NoNode
+    end
+
     @zk.stat(@path, watch: true)
     @zk.create(@path, 'Hello, events!')
 

data/lib/zk/client/base.rb

@@ -85,7 +85,9 @@ module ZK
       #
       # @abstract Overridden in subclasses
       def initialize(host, opts={})
-        # no-op
+        # keep track of the process we were in when we started
+        @host = host
+        @pid  = Process.pid
       end
 
       private
@@ -113,17 +115,40 @@ module ZK
       #
       # @return [Symbol] state of connection after operation
       def reopen(timeout=nil)
-        timeout ||= @session_timeout # XXX: @session_timeout ?
-        cnx.reopen(timeout)
-        @threadpool.start!  # restart the threadpool if previously stopped by close!
-        state
+#         timeout ||= @session_timeout # XXX: @session_timeout ?
+#         cnx.reopen(timeout)
+#         @threadpool.start!    
+#         state
       end
 
       # close the underlying connection and clear all pending events.
       #
       def close!
         event_handler.clear!
-        wrap_state_closed_error { cnx.close unless cnx.closed? }
+        wrap_state_closed_error { cnx.close if cnx && !cnx.closed? }
+      end
+
+      # Connect to the server/cluster. This is called automatically by the
+      # constructor by default. 
+      def connect(opts={})
+      end
+       
+      # this method will wait until the underlying connection is connected.
+      # please note that when a connection is established, the underlying 
+      # zookeeper gem performs this operation. you should only use this
+      # method if you have received a connecting event and want to wait
+      # until the connection has been re-established. 
+      #
+      # this method will block until you reach the connected? state or timeout
+      # seconds have passed. if we enter another state, you will not be
+      # awakened in the current implementation, so this method is somewhat
+      # unsafe.
+      #
+      # use this with caution
+      #
+      # @private
+      def wait_until_connected(timeout=10)
+        cnx.wait_until_connected(timeout)
       end
 
       # Create a node with the given path. The node data will be the given data.
@@ -718,7 +743,11 @@ module ZK
           num =
             case level
             when String, Symbol
-              ZookeeperBase.const_get(:"ZOO_LOG_LEVEL_#{level.to_s.upcase}") rescue NameError
+              begin
+                ZookeeperBase.const_get(:"ZOO_LOG_LEVEL_#{level.to_s.upcase}")
+              rescue NameError
+                nil
+              end
             when Integer
               level
             end
@@ -865,6 +894,11 @@ module ZK
       end
 
       protected
+        # does the current pid match the one that created us?
+        def forked?
+          Process.pid != @pid
+        end
+
         # @private
         def check_rc(hash, inputs=nil)
           code = hash[:rc]

data/lib/zk/client/threaded.rb

@@ -110,9 +110,13 @@ module ZK
       #   * see {https://github.com/slyphon/zk/wiki/EventDeliveryModel the wiki} for a
       #     discussion and demonstration of the effect of this setting.
       #
-      # @option opts [Fixnum] :timeout how long we will wait for the connection
-      #   to be established. If timeout is nil, we will wait forever: *use
-      #   carefully*.
+      # @option opts [Fixnum] :timeout used as a default for calls to {#reopen}
+      #   and {#connect} (including the initial default immediate connection)
+      #
+      # @option opts [true,false] :connect (true) Immediately connect to the
+      #   server. It may be useful to pass false if you wish to do callback
+      #   setup without passing a block. You must then call {#connect} 
+      #   explicitly.
       #
       # @yield [self] calls the block with the new instance after the event
       #   handler and threadpool have been set up, but before any connections
@@ -130,24 +134,57 @@ module ZK
         tp_size = opts.fetch(:threadpool_size, DEFAULT_THREADPOOL_SIZE)
         @threadpool = Threadpool.new(tp_size)
 
-        @session_timeout = opts.fetch(:timeout, DEFAULT_TIMEOUT) # maybe move this into superclass?
+        @connection_timeout = opts.fetch(:timeout, DEFAULT_TIMEOUT) # maybe move this into superclass?
         @event_handler   = EventHandler.new(self, opts)
 
         @reconnect = opts.fetch(:reconnect, true)
 
-        @mutex = Mutex.new
+        @mutex = Monitor.new
 
         @close_requested = false
 
         yield self if block_given?
 
-        @cnx = create_connection(host, @session_timeout, @event_handler.get_default_watcher_block)
+        @mutex.synchronize do
+          connect if opts.fetch(:connect, true)
+        end
+      end
+
+      # @option opts [Fixnum] :timeout how long we will wait for the connection
+      #   to be established. If timeout is nil, we will wait forever: *use
+      #   carefully*.
+       def connect(opts={})
+        @mutex.synchronize do
+          return if @cnx
+          timeout = opts.fetch(:timeout, @connection_timeout)
+          @cnx = create_connection(@host, timeout, @event_handler.get_default_watcher_block)
+        end
       end
 
       # (see Base#reopen)
       def reopen(timeout=nil)
+        # If we've forked, then we can call all sorts of normally dangerous 
+        # stuff because we're the only thread. 
+        if forked?
+          # ok, just to sanity check here
+          raise "[BUG] we hit the fork-reopening code in JRuby!!" if defined?(::JRUBY_VERSION)
+
+          logger.debug { "#{self.class}##{__method__} reopening everything, fork detected!" }
+
+          @mutex = Monitor.new
+          @threadpool.reopen_after_fork!      # prune dead threadpool threads after a fork()
+          @event_handler.reopen_after_fork!
+          @pid = Process.pid
+
+          old_cnx, @cnx = @cnx, nil
+          old_cnx.close! if old_cnx # && !old_cnx.closed?
+        else
+          logger.debug { "#{self.class}##{__method__} not reopening, no fork detected" }
+        end
+
         @mutex.synchronize { @close_requested = false }
-        super
+        connect
+        state
       end
 
       # (see Base#close!)

data/lib/zk/core_ext.rb

@@ -99,3 +99,4 @@ class ::Module
   end
 end
 
+

data/lib/zk/election.rb

@@ -239,7 +239,8 @@ module ZK
       protected
         # the inauguration, as it were
         def acknowledge_win!
-          @zk.create(leader_ack_path, @data, :ephemeral => true) rescue Exceptions::NodeExists
+          @zk.create(leader_ack_path, @data, :ephemeral => true) 
+        rescue Exceptions::NodeExists
         end
 
         # return the list of ephemeral vote nodes

data/lib/zk/event_handler.rb

@@ -32,19 +32,30 @@ module ZK
     # :nodoc:
     def initialize(zookeeper_client, opts={})
       @zk = zookeeper_client
-      @callbacks = Hash.new { |h,k| h[k] = [] }
 
       @thread_opt = opts.fetch(:thread, :single)
+      EventHandlerSubscription.class_for_thread_option(@thread_opt) # this is side-effecty, will raise an ArgumentError if given a bad value. 
 
-      # this is side-effecty, will raise an ArgumentError if given a bad
-      # value. 
-      EventHandlerSubscription.class_for_thread_option(@thread_opt)
+      @mutex = nil
 
-      @mutex = Monitor.new
+      @callbacks = Hash.new { |h,k| h[k] = [] }
 
       @outstanding_watches = VALID_WATCH_TYPES.inject({}) do |h,k|
         h.tap { |x| x[k] = Set.new }
       end
+
+      reopen_after_fork!
+    end
+    
+    # do not call this method. it is inteded for use only when we've forked and 
+    # all other threads are dead.
+    #
+    # @private
+    def reopen_after_fork!
+      logger.debug { "#{self.class}##{__method__} reopening callbacks" }
+      @mutex = Monitor.new
+      @callbacks.values.flatten.each { |cb| cb.reopen_after_fork! if cb.respond_to?(:reopen_after_fork!) }
+      nil
     end
 
     # @see ZK::Client::Base#register
@@ -292,7 +303,6 @@ module ZK
           end
         end
       end
-
   end # EventHandler
 end # ZK 
 

data/lib/zk/event_handler_subscription/base.rb

@@ -33,18 +33,6 @@ module ZK
         @interests = prep_interests(opts[:only])
       end
 
-      # unsubscribe from the path or state you were watching
-      # @see ZK::Client::Base#register
-#       def unsubscribe
-#         @event_handler.unregister(self)
-#       end
-#       alias :unregister :unsubscribe
-
-#       # @private
-#       def call(event)
-#         callback.call(event)
-#       end
-
       # the Actor returns true for this
       # @private
       def async?

data/lib/zk/locker/locker_base.rb

@@ -256,7 +256,12 @@ module ZK
         def cleanup_lock_path!
           logger.debug { "removing lock path #{@lock_path}" }
           zk.delete(@lock_path)
-          zk.delete(root_lock_path) rescue NotEmpty
+
+          begin
+            zk.delete(root_lock_path) 
+          rescue NotEmpty
+          end
+
           @lock_path = nil
         end
     end # LockerBase

data/lib/zk/message_queue.rb

@@ -120,9 +120,17 @@ module ZK
         locks << lock
       end
       children.each do |path|
-        @zk.delete("#{full_queue_path}/#{path}") rescue ZK::Exceptions::NoNode
+        begin
+          @zk.delete("#{full_queue_path}/#{path}") 
+        rescue ZK::Exceptions::NoNode
+        end
       end
-      @zk.delete(full_queue_path) rescue ZK::Exceptions::NoNode
+
+      begin
+        @zk.delete(full_queue_path) 
+      rescue ZK::Exceptions::NoNode
+      end
+
       locks.each do |lock|
         lock.unlock!
       end

data/lib/zk/subscription.rb

@@ -20,7 +20,7 @@ module ZK
         raise ArgumentError, "block must repsond_to?(:call)" unless block.respond_to?(:call)
         @parent = parent
         @callable = block
-        @mutex = Monitor.new
+        reopen_after_fork!
       end
 
       def unregister
@@ -33,6 +33,11 @@ module ZK
         callable.call(*args)
       end
 
+      # @private
+      def reopen_after_fork!
+        @mutex = Monitor.new
+      end
+
       protected
         def synchronize
           @mutex.synchronize { yield }
@@ -45,6 +50,7 @@ module ZK
       included do
         alias_method_chain :unsubscribe, :threaded_callback
         alias_method_chain :callable, :threaded_callback_wrapper
+        alias_method_chain :reopen_after_fork!, :threaded_refresh
       end
 
       def unsubscribe_with_threaded_callback
@@ -53,6 +59,11 @@ module ZK
           unsubscribe_without_threaded_callback
         end
       end
+
+      def reopen_after_fork_with_threaded_refresh!
+        reopen_after_fork_without_threaded_refresh!
+        @threaded_callback = ThreadedCallback.new(@callable)
+      end
     
       def callable_with_threaded_callback_wrapper(*args)
         synchronize do 

data/lib/zk/threaded_callback.rb

@@ -11,9 +11,11 @@ module ZK
     def initialize(callback=nil, &blk)
       @callback = callback || blk
       @mutex = Monitor.new
-      @queue = Queue.new
-      @running = true
-      setup_dispatch_thread
+
+      @mutex.synchronize do
+        @running = true
+        reopen_after_fork!
+      end
     end
 
     def running?
@@ -36,9 +38,22 @@ module ZK
       @queue.push(args)
     end
 
+    # called after a fork to replace a dead delivery thread
+    # special case, there should be ONLY ONE THREAD RUNNING, 
+    # (the one that survived the fork)
+    #
+    # @private
+    def reopen_after_fork!
+      return unless @running
+      return if @thread and @thread.alive?
+      @mutex = Monitor.new
+      @queue = Queue.new
+      @thread = spawn_dispatch_thread()
+    end
+
     protected
-      def setup_dispatch_thread
-        @thread ||= Thread.new do
+      def spawn_dispatch_thread
+        Thread.new do
           while running?
             args = @queue.pop
             break if args == KILL_TOKEN

data/lib/zk/threadpool.rb

@@ -64,6 +64,17 @@ module ZK
       true
     end
 
+    # like the start! method, but checks for dead threads in the threadpool
+    # (which will happen after a fork())
+    # @private
+    def reopen_after_fork!
+      return false unless @running
+      @mutex = Monitor.new
+      @threadqueue = Queue.new
+      prune_dead_threads
+      spawn_threadpool
+    end
+
     # register a block to be called back with unhandled exceptions that occur
     # in the threadpool. 
     # 
@@ -138,21 +149,41 @@ module ZK
         logger.error { "#{msg}: #{e.to_std_format}" }
       end
 
-      def spawn_threadpool #:nodoc:
-        until @threadpool.size >= @size.to_i
-          thread = Thread.new do
-            while @running
-              begin
-                op = @threadqueue.pop
-                break if op == KILL_TOKEN
-                op.call
-              rescue Exception => e
-                dispatch_to_error_handler(e)
+      def prune_dead_threads
+        @mutex.synchronize do
+          threads, @threadpool = @threadpool, []
+          return if threads.empty?
+
+          while th = threads.shift
+            begin
+              if th.join(0).nil?
+                @threadpool << th
               end
+            rescue Exception => e
+              logger.error { "Caught exception pruning threads in the threadpool" }
+              logger.error { e.to_std_format }
             end
           end
+        end
+      end
 
-          @threadpool << thread
+      def spawn_threadpool #:nodoc:
+        @mutex.synchronize do
+          until @threadpool.size >= @size.to_i
+            thread = Thread.new do
+              while @running
+                begin
+                  op = @threadqueue.pop
+                  break if op == KILL_TOKEN
+                  op.call
+                rescue Exception => e
+                  dispatch_to_error_handler(e)
+                end
+              end
+            end
+
+            @threadpool << thread
+          end
         end
       end
   end

data/lib/zk/version.rb

@@ -1,3 +1,3 @@
 module ZK
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/zk.rb

@@ -13,6 +13,9 @@ require 'set'
 require 'time'
 require 'date'
 
+module ZK
+end
+
 require 'zk/core_ext'
 require 'zk/logging'
 require 'zk/exceptions'
@@ -44,7 +47,7 @@ module ZK
   end
 
   unless @logger
-    @logger = Logger.new($stderr).tap { |n| n.level = Logger::ERROR }
+    @logger = Logger.new($stderr).tap { |n| n.level = ENV['ZK_DEBUG'] ? Logger::DEBUG : Logger::ERROR }
   end
  
   @default_host   = 'localhost' unless @default_host

data/spec/informal/what-the-fork.rb

@@ -69,7 +69,10 @@ class WhatTheFork
   ensure
     if pid
       _debug "ensuring #{pid} is really dead"
-      Process.kill(9, pid) rescue Errno::ESRCH
+      begin
+        Process.kill(9, pid) 
+      rescue Errno::ESRCH
+      end
     end
   end
 

data/spec/shared/client_examples.rb

@@ -239,6 +239,8 @@ shared_examples_for 'client' do
 
         logger.debug  { "taunt them a second time" }
         ensure_event_delivery!
+
+        logger.debug { "events: #{@events.inspect}" }
       end
     end
 

data/spec/support/event_catcher.rb

@@ -29,8 +29,18 @@ class EventCatcher
     end
   end
 
-  def add(sym,obj)
+  def add(*args)
     synchronize do
+      case args.length
+      when 2
+        sym, obj = args
+      when 1
+        obj = args.first
+        sym = obj.interest_key
+      else
+        raise ArgumentError, "Dunno how to handle args: #{args.inspect}" 
+      end
+
       logger.debug { "adding #{sym.inspect} #{obj.inspect}" }
       events[sym] << obj
       cond(sym).broadcast
@@ -39,6 +49,7 @@ class EventCatcher
       cond(:all).broadcast
     end
   end
+  alias << add
 
   def wait_for(ev_name, timeout=5)
     cond(ev_name).wait(timeout)

data/spec/support/pendings.rb

@@ -35,6 +35,18 @@ module Pendings
     end
   end
 
+  def pending_rbx(msg)
+    if ZK.rubinius?
+      if block_given?
+        pending(msg) { yield }
+      else
+        pending(msg)
+      end
+    else
+      yield if block_given?
+    end
+  end
+  alias pending_rubinius pending_rbx
 
   def pending_in_travis(msg)
     # defined in the somewhat ill-named 00_test_port_attr.rb

data/spec/zk/client_spec.rb

@@ -31,5 +31,90 @@ describe ZK::Client::Threaded do
       end
     end
   end
-end
+
+  unless defined?(::JRUBY_VERSION)
+    describe :forked do
+      include_context 'connection opts'
+
+      before do
+        @base_path = '/zktests'
+        @pids_root = "#{@base_path}/pid"
+
+        ZK.open(*connection_args) do |z| 
+          z.rm_rf(@base_path)
+          z.mkdir_p(@pids_root)
+        end
+      end
+
+      after do
+        if @pid
+          begin
+            Process.kill('TERM', @pid)
+            pid, st = Process.wait2(@pid)
+            logger.debug { "child #{@pid} exited with status: #{st.inspect}" }
+          rescue Errno::ESRCH
+          end
+        end
+
+        @zk.close! if @zk
+        ZK.open(*connection_args) { |z| z.rm_rf(@base_path) }
+      end
+
+      it %[should deliver callbacks in the child] do
+        pending_in_travis "skip this test, flaky in travis"
+        pending_rbx('fails in rubinius') do
+        
+          logger.debug { "Process.pid of parent: #{Process.pid}" }
+
+          @zk = ZK.new do |z|
+            z.on_connected do
+              logger.debug { "on_connected fired, writing pid to path #{@pids_root}/#{$$}" }
+              begin
+                z.create("#{@pids_root}/#{Process.pid}", Process.pid.to_s)
+              rescue ZK::Exceptions::NodeExists
+              end
+            end
+          end
+
+          @parent_pid = $$
+
+          @zk.create("#{@pids_root}/#{$$}", $$.to_s)
+
+          event_catcher = EventCatcher.new
+
+          @zk.register(@pids_root, :only => :child) do |event|
+            event_catcher << event
+          end
+
+          @pid = fork do
+            @zk.reopen
+            @zk.wait_until_connected
+
+            wait_until(3) { @zk.exists?("#{@pids_root}/#{$$}") }
+
+            logger.debug { "in child: child pid path exists?: %p" % [@zk.exists?("#{@pids_root}/#{$$}")] }
+
+            exit! 0
+          end
+
+          _, stat = Process.wait2(@pid)
+
+          stat.should_not be_signaled
+          stat.should be_exited
+          stat.should be_success
+
+          event_catcher.synchronize do
+            unless event_catcher.child.empty?
+              event_catcher.wait_for_child
+              event_catcher.child.should_not be_empty
+            end
+          end
+
+          @zk.should be_exists("#{@pids_root}/#{@pid}")
+
+        end
+      end # should deliver callbacks in the child
+    end # forked
+  end # # jruby guard
+end # ZK::Client::Threaded
 

data/spec/zk/pool_spec.rb

@@ -28,7 +28,10 @@ describe ZK::Pool do
 
       report_realtime("closing") do
         ZK.open("localhost:#{ZK.test_port}") do |zk|
-          zk.delete('/test_pool') rescue ZK::Exceptions::NoNode
+          begin
+            zk.delete('/test_pool')
+          rescue ZK::Exceptions::NoNode
+          end
         end
       end
     end
@@ -119,7 +122,10 @@ describe ZK::Pool do
       @path = '/_testWatch'
 
       @connection_pool.with_connection do |zk|
-        zk.delete(@path) rescue ZK::Exceptions::NoNode
+        begin
+          zk.delete(@path) 
+        rescue ZK::Exceptions::NoNode
+        end
       end
 
       @connection_pool.with_connection do |zk|

data/spec/zookeeper_spec.rb

@@ -2,7 +2,10 @@ require 'spec_helper'
 
 shared_examples_for 'ZK basic' do
   before do
-    @zk.create(@base_path) rescue ZK::Exceptions::NodeExists
+    begin
+      @zk.create(@base_path)
+    rescue ZK::Exceptions::NodeExists
+    end
   end
 
   describe ZK, "with no paths" do

data/zk.gemspec

@@ -12,8 +12,7 @@ Gem::Specification.new do |s|
   s.summary     = %q{A high-level wrapper around the zookeeper driver}
   s.description = s.summary + "\n"
 
-#   s.add_runtime_dependency 'slyphon-zookeeper', '~> 1.0.0'
-  s.add_runtime_dependency 'zookeeper', '~> 1.0.0.beta.1'
+  s.add_runtime_dependency 'zookeeper', '~> 1.0.2'
   s.add_runtime_dependency 'backports', '~> 2.5.1'
 
   s.files         = `git ls-files`.split("\n")

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: zk
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 27
   prerelease: 
   segments: 
   - 1
-  - 2
+  - 3
   - 0
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Jonathan D. Simms
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-07 00:00:00 Z
+date: 2012-05-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: zookeeper
@@ -26,14 +26,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 1711417137
+        hash: 19
         segments: 
         - 1
         - 0
-        - 0
-        - beta
-        - 1
-        version: 1.0.0.beta.1
+        - 2
+        version: 1.0.2
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency