zookeeper 1.0.6 → 1.1.0

data/.dotfiles/rvmrc

@@ -1 +1,2 @@
 rvm ruby-1.9.3@zookeeper --create
+export ZK_DEV=1

data/.travis.yml

@@ -20,3 +20,8 @@ rvm:
 
 bundler_args: --without development docs coverage
 
+# blacklist
+branches:
+  except:
+#     - 'dev/zookeeper-st'
+

data/CHANGELOG

@@ -1,3 +1,16 @@
+v1.1.0 Rewrite C backend to use zookeeper_st, the async library
+
+  * In order to ensure fork safety, a rewrite of the backend was necessary.
+    It was impossible to guarantee with the mt lib that a lock would not
+    be held by a thread when fork() was called, which opened up the possibility
+    for corruption and other badness.
+
+    This version contains a Continuation class, which allows us to present a 
+    synchronous front-end to the asynchronous backend. All features are still
+    supported, no special action is necessary to prepare for a fork, and the
+    post-fork procedure is the same as before: call reopen() in the child,
+    continue on in the parent like nothing happened.
+
 v1.0.6 Only include backports if RUBY_VERSION is 1.8.x
 
   * 'backports' pollutes too much, use sparingly

data/Gemfile

@@ -30,6 +30,9 @@ end
 
 group :development do
   gem 'pry'
+  gem 'guard',        :require => false
+  gem 'guard-rspec',  :require => false
+  gem 'guard-shell',  :require => false
 end
 
 # vim:ft=ruby

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec' do
+  watch(%r{^spec/.+_spec.rb$})
+  watch(%r{^lib/(.+)\.rb$}) { |m| %w[spec/zookeeper_spec.rb spec/chrooted_connection_spec.rb] }
+  watch(%r{^ext/zookeeper_c.bundle}) { %w[spec/c_zookeeper_spec.rb] }
+end
+

data/README.markdown

@@ -6,6 +6,10 @@ An interface to the Zookeeper cluster coordination server.
 
 For a higher-level interface with a more convenient API and features such as locks, have a look at [ZK](https://github.com/slyphon/zk) (also available is [ZK-EventMachine](https://github.com/slyphon/zk-eventmachine) for those who prefer async).
 
+## Fork Safety! ##
+
+As of 1.1.0, this library is fork-safe (which was not easy to accomplish). This means you can use it without worry in unicorn, resque, and whatever other fork-philic frameworks you sick little monkeys are using this week. The only rule is that after a fork(), you need to call `#reopen` on the client ASAP, because if you try to peform any other action, an exception will be raised. Other than that, there is no special action that is needed in the parent.
+
 ## Big Plans for 1.0 ##
 
 The 1.0 release will feature a reorganization of the heirarchy. There will be a single top-level `Zookeeper` namespace (as opposed to the current layout, with 5-6 different top-level constants), and for the next several releases, there will be a backwards compatible require for users that still need to use the old names.
@@ -36,18 +40,18 @@ Connect to a server:
 ## Idioms
 
 The following methods are initially supported:
-* get
-* set
-* get\_children
-* stat
-* create
-* delete
-* get\_acl
-* set\_acl
-
-All support async callbacks.  get, get\_children and stat support both watchers and callbacks.
-
-Calls take a dictionary of parameters.  With the exception of set\_acl, the only required parameter is :path.  Each call returns a dictionary with at minimum two keys :req\_id and :rc.
+* `get`
+* `set`
+* `get_children`
+* `stat`
+* `create`
+* `delete`
+* `get_acl`
+* `set_acl`
+
+All support async callbacks. `get`, `get_children` and `stat` support both watchers and callbacks.
+
+Calls take a dictionary of parameters. With the exception of set\_acl, the only required parameter is `:path`. Each call returns a dictionary with at minimum two keys :req\_id and :rc.
 
 ### A Bit about this repository ###
 

data/Rakefile

@@ -100,6 +100,11 @@ end
 
 task 'spec:run' => 'build:clean' unless defined?(::JRUBY_VERSION)
 
+task 'ctags' do
+  sh 'bundle-ctags'
+end
+
 # because i'm a creature of habit
 task 'mb:test_all' => 'zk:test_all'
 
+

data/cause-abort.rb

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+require 'zookeeper'
+require File.expand_path('../spec/support/zookeeper_spec_helpers', __FILE__)
+
+class CauseAbort
+  include Zookeeper::Logger
+  include Zookeeper::SpecHelpers
+
+  attr_reader :path, :pids_root, :data
+
+  def initialize
+    @path = "/_zktest_"
+    @pids_root = "#{@path}/pids"
+    @data = 'underpants'
+  end
+
+  def before
+    @zk = Zookeeper.new('localhost:2181')
+    rm_rf(@zk, path)
+    logger.debug { "----------------< BEFORE: END >-------------------" }
+  end
+
+  def process_alive?(pid)
+    Process.kill(0, @pid)
+    true
+  rescue Errno::ESRCH
+    false
+  end
+
+  def wait_for_child_safely(pid, timeout=5)
+    time_to_stop = Time.now + timeout
+
+    until Time.now > time_to_stop
+      if a = Process.wait2(@pid, Process::WNOHANG)
+        return a.last
+      else
+        sleep(0.01)
+      end
+    end
+
+    nil
+  end
+
+  def try_pause_and_resume
+    @zk.pause
+    logger.debug { "paused" }
+    @zk.resume
+    logger.debug { "resumed" }
+    @zk.close
+    logger.debug { "closed" }
+  end
+
+  def run_test
+    logger.debug { "----------------< TEST: BEGIN >-------------------" }
+    @zk.wait_until_connected
+
+    mkdir_p(@zk, pids_root)
+
+    # the parent's pid path
+    @zk.create(:path => "#{pids_root}/#{$$}", :data => $$.to_s)
+
+    @latch = Zookeeper::Latch.new
+    @event = nil
+
+    cb = proc do |h|
+      logger.debug { "watcher called back: #{h.inspect}" }
+      @event = h
+      @latch.release
+    end
+
+    @zk.stat(:path => "#{pids_root}/child", :watcher => cb)
+
+    logger.debug { "------------------->   FORK   <---------------------------" }
+
+    @pid = fork do
+      rand_sleep = rand()
+
+      $stderr.puts "sleeping for rand_sleep: #{rand_sleep}"
+      sleep(rand_sleep)
+
+      logger.debug { "reopening connection in child: #{$$}" }
+      @zk.reopen
+      logger.debug { "creating path" }
+      rv = @zk.create(:path => "#{pids_root}/child", :data => $$.to_s)
+      logger.debug { "created path #{rv[:path]}" }
+      @zk.close
+
+      logger.debug { "close finished" }
+      exit!(0)
+    end
+
+    event_waiter_th = Thread.new do
+      @latch.await(5) unless @event 
+      @event
+    end
+
+    logger.debug { "waiting on child #{@pid}" }
+
+    status = wait_for_child_safely(@pid)
+    raise "Child process did not exit, likely hung"  unless status
+
+    if event_waiter_th.join(5) == event_waiter_th
+      logger.warn { "event waiter has not received events" }
+    end
+
+    exit(@event.nil? ? 1 : 0)
+  end
+
+  def run
+    before
+    run_test
+#     try_pause_and_resume
+  end
+end
+
+CauseAbort.new.run

data/ext/Rakefile

@@ -1,20 +1,41 @@
+require 'rbconfig'
 
-task :clean do
-  if File.exists?('Makefile')
-    sh 'make clean'
-  else
-    $stderr.puts "nothing to clean, no Makefile"
+LIB_ZK_SO = 'lib/libzookeeper_mt_gem.la'
+
+TARBALL = FileList['zkc-*.tar.gz'].first
+
+raise "Where is the zkc tarball!?" unless TARBALL
+
+namespace :zkrb do
+  task :clean do
+    if File.exists?('Makefile')
+      sh 'make clean'
+      rm 'Makefile' # yep, regenerate this
+    else
+      $stderr.puts "nothing to clean, no Makefile"
+    end
+  end
+
+  task :clobber => :clean do
+    rm_rf %w[Makefile c lib bin include]
   end
 end
 
+task :clean   => 'zkrb:clean'
+task :clobber => 'zkrb:clobber'
+
 GENERATE_GVL_CODE_RB = 'generate_gvl_code.rb'
 
-file 'c' do
-  if tarball = Dir['zkc-*.tar.gz'].first
-    sh "tar -zxf #{tarball}"
-  else
-    raise "couldn't find the tarball! wtf?!"
-  end
+# file 'c' do
+#   if tarball = Dir['zkc-*.tar.gz'].first
+#     sh "tar -zxf #{tarball}"
+#   else
+#     raise "couldn't find the tarball! wtf?!"
+#   end
+# end
+
+file 'c' => TARBALL do
+  sh "tar -zxf #{TARBALL}"
 end
 
 file GENERATE_GVL_CODE_RB => 'c'
@@ -29,20 +50,12 @@ end
 
 ZKRB_WRAPPER = %w[zkrb_wrapper.c zkrb_wrapper.h]
 
-
 task :wrappers => ZKRB_WRAPPER
 
-
-task :clobber => :clean do
-  rm_rf %w[Makefile c lib bin include]
+file 'Makefile' do
+  sh "ruby extconf.rb"
 end
 
-task :build_zkc do
-  sh 'ruby extconf.rb'
-end
-
-file 'Makefile' => :build_zkc
-
 task :build => [ZKRB_WRAPPER, 'Makefile'].flatten do
   sh 'make'
 end

data/ext/c_zookeeper.rb

@@ -1,13 +1,13 @@
 require_relative '../lib/zookeeper/logger'
 require_relative '../lib/zookeeper/common'
 require_relative '../lib/zookeeper/constants'
+require_relative '../lib/zookeeper/exceptions' # zookeeper_c depends on exceptions defined in here
 require_relative 'zookeeper_c'
 
 # require File.expand_path('../zookeeper_c', __FILE__)
 
-# TODO: see if we can get the destructor to handle thread/event queue teardown
-#       when we're garbage collected
 module Zookeeper
+# NOTE: this class extending (opening) the class defined in zkrb.c
 class CZookeeper
   include Forked
   include Constants
@@ -30,6 +30,15 @@ class CZookeeper
     set_zkrb_debug_level(value)
   end
 
+  # wrap these calls in our sync->async special sauce
+  %w[get set exists create delete get_acl set_acl get_children].each do |sym|
+    class_eval(<<-EOS, __FILE__, __LINE__+1)
+      def #{sym}(*args)
+        submit_and_block(:#{sym}, *args)
+      end
+    EOS
+  end
+
   def initialize(host, event_queue, opts={})
     @host = host
     @event_queue = event_queue
@@ -40,7 +49,7 @@ class CZookeeper
     # used by the C layer. CZookeeper sets this to true when the init method
     # has completed. once this is set to true, it stays true.
     #
-    # you should grab the @start_stop_mutex before messing with this flag
+    # you should grab the @mutex before messing with this flag
     @_running = nil
 
     # This is set to true after destroy_zkrb_instance has been called and all
@@ -55,40 +64,40 @@ class CZookeeper
 
     @_session_timeout_msec = DEFAULT_SESSION_TIMEOUT_MSEC
 
-    if cid = opts[:client_id]
-      raise ArgumentError, "opts[:client_id] must be a CZookeeper::ClientId" unless cid.kind_of?(ClientId)
-      raise ArgumentError, "session_id must not be nil" if cid.session_id.nil?
-    end
-
-    @_client_id = opts[:client_id]
-
-    @start_stop_mutex = Monitor.new
+    @mutex = Monitor.new
     
     # used to signal that we're running
-    @running_cond = @start_stop_mutex.new_cond
+    @running_cond = @mutex.new_cond
 
     # used to signal we've received the connected event
-    @connected_cond = @start_stop_mutex.new_cond
+    @connected_cond = @mutex.new_cond
+
+    @pipe_read, @pipe_write = IO.pipe
     
     @event_thread = nil
 
-    setup_event_thread!
+    # hash of in-flight Continuation instances
+    @reg = Continuation::Registry.new
+
+    log_level = ENV['ZKC_DEBUG'] ? ZOO_LOG_LEVEL_DEBUG : ZOO_LOG_LEVEL_ERROR
+
+    zkrb_init(@host)#, :zkc_log_level => log_level)
 
-    zkrb_init(@host)
+    start_event_thread
 
     logger.debug { "init returned!" }
   end
 
   def closed?
-    @start_stop_mutex.synchronize { !!@_closed }
+    @mutex.synchronize { !!@_closed }
   end
 
   def running?
-    @start_stop_mutex.synchronize { !!@_running }
+    @mutex.synchronize { !!@_running }
   end
 
   def shutting_down?
-    @start_stop_mutex.synchronize { !!@_shutting_down }
+    @mutex.synchronize { !!@_shutting_down }
   end
 
   def connected?
@@ -116,13 +125,34 @@ class CZookeeper
     if forked?
       fn_close.call
     else
-      shut_down!
-      stop_event_thread!
-      @start_stop_mutex.synchronize(&fn_close)
+      stop_event_thread
+      @mutex.synchronize(&fn_close)
     end
 
+    [@pipe_read, @pipe_write].each { |io| io.close unless io.closed? }
+
     nil
   end
+  
+  # call this to stop the event loop, you can resume with the
+  # resume method
+  #
+  # requests may still be added during this time, but they will not be
+  # processed until you call resume
+  def pause
+    logger.debug { "#{self.class}##{__method__}" }
+    @mutex.synchronize { stop_event_thread }
+  end
+
+  # call this if 'pause' was previously called to start the event loop again
+  def resume
+    logger.debug { "#{self.class}##{__method__}" }
+
+    @mutex.synchronize do 
+      @_shutting_down = nil
+      start_event_thread
+    end
+  end
 
   def state
     return ZOO_CLOSED_STATE if closed?
@@ -137,107 +167,188 @@ class CZookeeper
   # if timeout is nil, we never time out, and wait forever for CONNECTED state
   #
   def wait_until_connected(timeout=10)
-    @start_stop_mutex.synchronize do
-      wait_until_running(timeout)
-      @connected_cond.wait(timeout) unless connected?
-    end
+    # this begin/ensure/end style is recommended by tarceri
+    # no need to create a context for every mutex grab
 
+    wait_until_running(timeout)
+
+    Thread.pass until connected? or is_unrecoverable
     connected?
   end
 
+
   private
+    # submits a job for processing 
+    # blocks the caller until result has returned
+    def submit_and_block(meth, *args)
+      cnt = Continuation.new(meth, *args)
+      @reg.synchronized { |r| r.pending << cnt }
+      wake_event_loop!
+      cnt.value
+    end
+    
+    # this method is part of the reopen/close code, and is responsible for
+    # shutting down the dispatch thread. 
+    #
+    # @event_thread will be nil when this method exits
+    #
+    def stop_event_thread
+      if @event_thread
+        logger.debug { "#{self.class}##{__method__}" }
+        shut_down!
+        wake_event_loop!
+        @event_thread.join 
+        @event_thread = nil
+      end
+    end
+
+    # starts the event thread running if not already started
+    # returns false if already running
+    def start_event_thread
+      return false if @event_thread
+      @event_thread = Thread.new(&method(:event_thread_body))
+    end
+
     # will wait until the client has entered the running? state
     # or until timeout seconds have passed.
     #
     # returns true if we're running, false if we timed out
-    def wait_until_running(timeout=5)
-      @start_stop_mutex.synchronize do
+    def wait_until_running(timeout=5) 
+      @mutex.lock
+      begin
         return true if @_running
         @running_cond.wait(timeout)
         !!@_running
+      ensure
+        @mutex.unlock
       end
     end
 
-    def setup_event_thread!
-      @event_thread ||= Thread.new(&method(:_event_thread_body))
-    end
+    def event_thread_body
+      Thread.current.abort_on_exception = true
+      logger.debug { "#{self.class}##{__method__} starting event thread" }
 
-    def _event_thread_body
-      logger.debug { "event_thread waiting until running: #{@_running}" }
+      event_thread_await_running
+
+      # this is the main loop
+      until (@_shutting_down or @_closed or is_unrecoverable)
+        submit_pending_calls    if @reg.pending?
+#         log_realtime("zkrb_iterate_event_loop") do
+          zkrb_iterate_event_loop # XXX: check rc here
+#         end
+        iterate_event_delivery
+      end
 
-      @start_stop_mutex.synchronize do
-        @running_cond.wait_until { @_running }
+      # ok, if we're exiting the event loop, and we still have a valid connection
+      # and there's still completions we're waiting to hear about, then we
+      # should pump the handle before leaving this loop
+      if @_shutting_down and not (@_closed or is_unrecoverable or @reg.in_flight.empty?)
+        logger.debug { "we're in shutting down state, ensuring we have no in-flight completions" }
 
-        if @_shutting_down
-          logger.error { "event thread saw @_shutting_down, bailing without entering loop" }
-          return
+        until @reg.in_flight.empty?
+          zkrb_iterate_event_loop
+          iterate_event_delivery
         end
+
+        logger.debug { "finished completions" }
       end
 
-      logger.debug { "event_thread running: #{@_running}" }
+    rescue ShuttingDownException
+      logger.error { "event thread saw @_shutting_down, bailing without entering loop" }
+    ensure
+      logger.debug { "#{self.class}##{__method__} exiting" }
+    end
 
-      until @_shutting_down
-        begin
-          _iterate_event_delivery
-        rescue GotNilEventException
-          logger.debug { "#{self.class}##{__method__}: event delivery thread is exiting" }
-          break
-        end
+    def submit_pending_calls
+      # this is ok, because the calling thread only ever *adds* to this hash,
+      # and the keys are always unique
+
+      pending = nil
+
+      @reg.lock
+      begin
+        pending, @reg.pending = @reg.pending, []
+      ensure
+        @reg.unlock
       end
+
+      return if pending.empty?
+
+      logger.debug { "#{self.class}##{__method__} " }
+
+      while cntn = pending.shift
+        cntn.submit(self)
+        @reg.in_flight[cntn.req_id] = cntn   # in_flight is only ever touched by us
+      end
+    end
+
+    def wake_event_loop!
+      logger.debug { "#{self.class}##{__method__}" }
+      @pipe_write && @pipe_write.write("\001")
     end
 
-    def _iterate_event_delivery
-      get_next_event(true).tap do |hash|
-        raise GotNilEventException if hash.nil?
+    def iterate_event_delivery
+      while hash = zkrb_get_next_event_st()
+        logger.debug { "#{self.class}##{__method__} got #{hash.inspect} " }
 
-        # TODO: should push notify_connected! down so that it's common to both java and C impl.
-        if hash.values_at(:req_id, :type, :state) == CONNECTED_EVENT_VALUES
+        # notify when we get this event so we know we're connected
+        if hash.values_at(:req_id, :type, :state) == CONNECTED_EVENT_VALUES[1..2]
           notify_connected!
         end
 
+        cntn = @reg.in_flight.delete(hash[:req_id])
+
+        if cntn and not cntn.user_callback?     # this is one of "our" continuations 
+          cntn.call(hash)                       # so we handle delivering it
+          next                                  # and skip handing it to the dispatcher
+        end
+
+        # otherwise, the event was a session event (ZKRB_GLOBAL_CB_REQ)
+        # or a user-provided callback
         @event_queue.push(hash)
       end
     end
 
-    # use this method to set the @_shutting_down flag to true
-    def shut_down!
-      logger.debug { "#{self.class}##{__method__}" }
+    def event_thread_await_running
+      logger.debug { "event_thread waiting until running: #{@_running}" }
 
-      @start_stop_mutex.synchronize do
-        @_shutting_down = true
+      @mutex.lock
+      begin
+        @running_cond.wait_until { @_running or @_shutting_down }
+        logger.debug { "event_thread running: #{@_running}" }
+
+        raise ShuttingDownException if @_shutting_down
+      ensure
+        @mutex.unlock
       end
     end
 
-    # this method is part of the reopen/close code, and is responsible for
-    # shutting down the dispatch thread. 
-    #
-    # @dispatch will be nil when this method exits
-    #
-    def stop_event_thread!
+    # use this method to set the @_shutting_down flag to true
+    def shut_down!
       logger.debug { "#{self.class}##{__method__}" }
 
-      if @event_thread
-        unless @_closed
-          wake_event_loop! # this is a C method
-        end
-        @event_thread.join 
-        @event_thread = nil
-      end
+      @mutex.synchronize { @_shutting_down = true }
     end
 
     # called by underlying C code to signal we're running
     def zkc_set_running_and_notify!
       logger.debug { "#{self.class}##{__method__}" }
 
-      @start_stop_mutex.synchronize do
+      @mutex.lock
+      begin
         @_running = true
         @running_cond.broadcast
+      ensure
+        @mutex.unlock
       end
     end
 
     def notify_connected!
-      @start_stop_mutex.synchronize do
+      @mutex.lock
+      begin
         @connected_cond.broadcast
+      ensure
+        @mutex.unlock
       end
     end
 end