zk 1.0.0 → 1.1.0

data/lib/zk/threaded_callback.rb

@@ -0,0 +1,56 @@
+module ZK
+  # A class that encapsulates the queue + thread that calls a callback.
+  # Repsonds to `call` but places call on a queue to be delivered by a thread.
+  # You will not have a useful return value from `call` so this is only useful
+  # for background processing.
+  class ThreadedCallback
+    include ZK::Logging
+
+    attr_reader :callback
+
+    def initialize(callback=nil, &blk)
+      @callback = callback || blk
+      @mutex = Monitor.new
+      @queue = Queue.new
+      @running = true
+      setup_dispatch_thread
+    end
+
+    def running?
+      @mutex.synchronize { @running }
+    end
+
+    # how long to wait on thread shutdown before we return
+    def shutdown(timeout=2)
+      @mutex.synchronize do
+        @running = false
+        @queue.push(KILL_TOKEN)
+        return unless @thread 
+        unless @thread.join(2)
+          logger.error { "#{self.class} timed out waiting for dispatch thread, callback: #{callback.inspect}" }
+        end
+      end
+    end
+
+    def call(*args)
+      @queue.push(args)
+    end
+
+    protected
+      def setup_dispatch_thread
+        @thread ||= Thread.new do
+          while running?
+            args = @queue.pop
+            break if args == KILL_TOKEN
+            begin
+              callback.call(*args)
+            rescue Exception => e
+              logger.error { "error caught in handler for path: #{path.inspect}, interests: #{interests.inspect}" }
+              logger.error { e.to_std_format }
+            end
+          end
+        end
+      end
+  end
+end
+

data/lib/zk/threadpool.rb

@@ -20,7 +20,7 @@ module ZK
       @threadpool = []
       @threadqueue = ::Queue.new
 
-      @mutex = Mutex.new
+      @mutex = Monitor.new
 
       @error_callbacks = []
 
@@ -86,9 +86,7 @@ module ZK
     def shutdown(timeout=2)
       @mutex.synchronize do
         return unless @running
-
         @running = false
-
         @threadqueue.clear
         @size.times { @threadqueue << KILL_TOKEN }
 

data/lib/zk/version.rb

@@ -1,3 +1,3 @@
 module ZK
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/spec/event_catcher_spec.rb

@@ -0,0 +1,29 @@
+# quick tests for one of the support classes
+require 'spec_helper'
+
+describe EventCatcher do
+  subject { EventCatcher.new }
+
+  describe :wait_for do
+    it %[should wake when an event is delivered] do
+      th = Thread.new do
+        subject.synchronize do
+          logger.debug { "about to wait for created" }
+          subject.wait_for_created
+          logger.debug { "woke up, created must have been delivered" }
+        end
+        true
+      end
+
+      th.run
+      Thread.pass until th.status == 'sleep'
+
+      logger.debug { "th.status: #{th.status}" }
+
+      subject.add(:created, 'blah')
+
+      th.join(2).value.should be_true
+    end
+  end
+end
+

data/spec/message_queue_spec.rb

@@ -3,8 +3,8 @@ require File.join(File.dirname(__FILE__), %w[spec_helper])
 describe ZK::MessageQueue do
 
   before(:each) do
-    @zk = ZK.new("localhost:#{ZK_TEST_PORT}")
-    @zk2 = ZK.new("localhost:#{ZK_TEST_PORT}")
+    @zk = ZK.new("localhost:#{ZK.test_port}")
+    @zk2 = ZK.new("localhost:#{ZK.test_port}")
     wait_until{ @zk.connected? && @zk2.connected? }
     @queue_name = "_specQueue"
     @consume_queue = @zk.queue(@queue_name)

data/spec/shared/client_contexts.rb

@@ -1,20 +1,30 @@
+shared_context 'connection opts' do
+  let(:connection_opts) { { :thread => :per_callback, :timeout => 5 } }
+  let(:connection_host) { "localhost:#{ZK.test_port}" }
+  let(:connection_args) { [connection_host, connection_opts] }
+end
+
 shared_context 'threaded client connection' do
+  include_context 'connection opts'
+
   before do
-    @connection_string = "localhost:#{ZK_TEST_PORT}"
+    @connection_string = "localhost:#{ZK.test_port}"
     @base_path = '/zktests'
-    @zk = ZK::Client::Threaded.new(@connection_string).tap { |z| wait_until { z.connected? } }
-    @zk.on_exception { |e| raise e }
+    @zk = ZK::Client::Threaded.new(*connection_args).tap { |z| wait_until { z.connected? } }
+    @threadpool_exception = nil
+    @zk.on_exception { |e| @threadpool_exception = e }
     @zk.rm_rf(@base_path)
   end
 
   after do
-    if @zk.closed?
-      ZK.open(@connection_string) { |z| z.rm_rf(@base_path) }
-    else
-      @zk.rm_rf(@base_path)
-      @zk.close!
-      wait_until(2) { @zk.closed? }
-    end
+#     raise @threadpool_exception if @threadpool_exception
+    @zk.reopen if @zk.closed?
+    wait_until(5) { @zk.connected? }
+    
+    @zk.rm_rf(@base_path)
+    @zk.close!
+    wait_until(5) { @zk.closed? }
   end
 end
 
+

data/spec/shared/client_examples.rb

@@ -175,7 +175,7 @@ shared_examples_for 'client' do
 
   describe 'session_id and session_passwd' do
     it %[should expose the underlying session_id] do
-      @zk.session_id.should be_kind_of(Fixnum)
+      @zk.session_id.should be_kind_of(Integer)
     end
 
     it %[should expose the underlying session_passwd] do

data/spec/spec_helper.rb

@@ -8,11 +8,9 @@ Bundler.require(:development, :test)
 require 'zk'
 require 'benchmark'
 
-ZK_TEST_PORT = 2181 unless defined?(ZK_TEST_PORT)
-
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
-Dir[File.expand_path("../{support,shared}/**/*.rb", __FILE__)].each {|f| require f}
+Dir[File.expand_path("../{support,shared}/**/*.rb", __FILE__)].sort.each {|f| require f}
 
 $stderr.sync = true
 
@@ -22,11 +20,28 @@ RSpec.configure do |config|
   config.mock_with :flexmock
   config.include(FlexMock::ArgumentTypes)
 
-  config.include(WaitWatchers)
-  config.extend(WaitWatchers)
+  [WaitWatchers, SpecGlobalLogger, Pendings].each do |mod|
+    config.include(mod)
+    config.extend(mod)
+  end
+
+  if ZK.spawn_zookeeper?
+    require 'zk-server'
 
-  config.include(SpecGlobalLogger)
-  config.extend(SpecGlobalLogger)
+    config.before(:suite) do 
+      ZK.logger.debug { "Starting zookeeper service" }
+      ZK::Server.run do |c|
+        c.client_port = ZK.test_port
+        c.force_sync  = false
+        c.snap_count  = 1_000_000
+      end
+    end
+
+    config.after(:suite) do
+      ZK.logger.debug { "stopping zookeeper service" }
+      ZK::Server.shutdown
+    end
+  end
 end
 
 class ::Thread

data/spec/support/00_test_port_attr.rb

@@ -0,0 +1,20 @@
+module ZK
+  def self.spawn_zookeeper?
+    !!ENV['SPAWN_ZOOKEEPER']
+  end
+
+  def self.travis?
+    !!ENV['TRAVIS']
+  end
+
+  @test_port ||= spawn_zookeeper? ? 21811 : 2181
+
+  class << self
+    attr_accessor :test_port
+  end
+
+  # argh, blah, this affects ZK.new everywhere (which is kind of the point, but
+  # still gross)
+  self.default_port = self.test_port
+end
+

data/spec/support/event_catcher.rb

@@ -1,11 +1,79 @@
-class EventCatcher < Struct.new(:created, :changed, :deleted, :child, :all)
+class EventCatcher
+  extend Forwardable
+  include ZK::Logging
+
+  def_delegators :@mutex, :synchronize
+
+  MEMBERS = [:created, :changed, :deleted, :child, :all]
+
+  attr_reader :events
     
   def initialize(*args)
-    super
+    @mutex = Monitor.new
+    @conds = {}
+    @events = {}
+
+    MEMBERS.each do |k|
+      @conds[k] = @mutex.new_cond
+      @events[k] = []
+    end
+  end
+
+  def cond(name)
+    @conds.fetch(name)
+  end
+
+  def clear_all
+    synchronize do
+      @events.values.each(&:clear)
+    end
+  end
+
+  def add(sym,obj)
+    synchronize do
+      logger.debug { "adding #{sym.inspect} #{obj.inspect}" }
+      events[sym] << obj
+      cond(sym).broadcast
 
-    [:created, :changed, :deleted, :child, :all].each do |k|
-      self.__send__(:"#{k}=", []) if self.__send__(:"#{k}").nil?
+      events[:all] << obj
+      cond(:all).broadcast
     end
   end
-end
 
+  def wait_for(ev_name, timeout=5)
+    cond(ev_name).wait(timeout)
+  end
+
+  def wait_while(ev_name)
+    cond(ev_name).wait_while { yield @events.fetch(ev_name) }
+  end
+
+  def wait_until(ev_name)
+    cond(ev_name).wait_until { yield @events.fetch(ev_name) }
+  end
+
+  MEMBERS.each do |name|
+    class_eval <<-EOS, __FILE__, __LINE__+1
+      def #{name}
+        events[:#{name}]
+      end
+
+      def cond_#{name}
+        cond(:#{name})
+      end
+
+      # waits for an event group to not be empty (up to timeout sec)
+      def wait_for_#{name}(timeout=5)
+        cond(:#{name}).wait(timeout)
+      end
+
+      def wait_while_#{name}
+        cond(:#{name}).wait_while { yield __send__(:#{name}) }
+      end
+
+      def wait_until_#{name}
+        cond(:#{name}).wait_until { yield __send__(:#{name}) }
+      end
+    EOS
+  end
+end 

data/spec/support/logging.rb

@@ -4,7 +4,9 @@ module ZK
 #   LOG_FILE = $stderr
 end
 
-ZK.logger = Logger.new(ZK::LOG_FILE).tap { |log| log.level = Logger::DEBUG }
+# ZK.logger = ENV['TRAVIS'] ? Logger.new($stderr) : Logger.new(ZK::LOG_FILE)
+
+ZK.logger = Logger.new(ZK::LOG_FILE).tap { |l| l.level = Logger::DEBUG }
 
 # Zookeeper.logger = ZK.logger
 # Zookeeper.set_debug_level(4)

data/spec/support/pendings.rb

@@ -0,0 +1,52 @@
+module Pendings
+  def pending_192(msg)
+    if ZK.mri_19x?
+      if block_given?
+        pending(msg) { yield }
+      else
+        pending(msg)
+      end
+    else
+      yield if block_given?
+    end
+  end
+
+  def pending_187(msg)
+    if ZK.mri_187?
+      if block_given?
+        pending(msg) { yield }
+      else
+        pending(msg)
+      end
+    else
+      yield if block_given?
+    end
+  end
+
+  def pending_jruby19(msg)
+    if ZK.ruby_19? and ZK.jruby?
+      if block_given?
+        pending(msg) { yield }
+      else
+        pending(msg)
+      end
+    else
+      yield if block_given?
+    end
+  end
+
+
+  def pending_in_travis(msg)
+    # defined in the somewhat ill-named 00_test_port_attr.rb
+    if ZK.travis?
+      if block_given?
+        pending("TRAVIS: #{msg}") { yield }
+      else
+        pending("TRAVIS: #{msg}")
+      end
+    else
+      yield if block_given?
+    end
+  end
+end
+

data/spec/support/wait_watchers.rb

@@ -18,6 +18,11 @@ module WaitWatchers
   #   wait_until(2) { @a }.should == :fudge
   #
   def wait_until(timeout=2)
+    if ZK.travis? and timeout and timeout < 5
+      ZK.logger.debug { "TRAVIS: adjusting wait_until timeout from #{timeout} to 5 sec" }
+      timeout = 5
+    end
+
     time_to_stop = Time.now + timeout
     while true
       rval = yield
@@ -29,6 +34,11 @@ module WaitWatchers
 
   # inverse of wait_until
   def wait_while(timeout=2)
+    if ZK.travis? and timeout and timeout < 5
+      ZK.logger.debug { "TRAVIS: adjusting wait_while timeout from #{timeout} to 5 sec" }
+      timeout = 5
+    end
+
     time_to_stop = Time.now + timeout
     while true
       rval = yield
@@ -45,4 +55,3 @@ module WaitWatchers
   end
 end
 
-

data/spec/watch_spec.rb

@@ -1,11 +1,12 @@
 require 'spec_helper'
 
 describe ZK do
+  include_context 'connection opts'
+
   describe 'watchers' do
     before do
       mute_logger do
-        @cnx_str = "localhost:#{ZK_TEST_PORT}"
-        @zk = ZK.new(@cnx_str)
+        @zk = ZK.new(*connection_args)
 
         @path = "/_testWatch"
         wait_until { @zk.connected? }
@@ -20,7 +21,7 @@ describe ZK do
           wait_until { !@zk.connected? }
         end
 
-        ZK.open(@cnx_str) { |zk| zk.rm_rf(@path) }
+        ZK.open(*connection_args) { |zk| zk.rm_rf(@path) }
       end
     end
 
@@ -42,78 +43,84 @@ describe ZK do
       callback_called.should be_true
     end
 
-    # this is stupid, and a bad test, but we have to check that events 
-    # don't get re-delivered to a single registered callback just because 
-    # :watch => true was called twice
-    #
-    # again, we're testing a negative here, so consider this a regression check
-    #
-    def wait_for_events_to_not_be_delivered(events)
-      lambda { wait_until(0.2) { events.length >= 2 } }.should raise_error(WaitWatchers::TimeoutError)
-    end
-
-    it %[should only deliver an event once to each watcher registered for exists?] do
-      events = []
-
-      sub = @zk.register(@path) do |ev|
-        logger.debug "got event #{ev}"
-        events << ev
+    describe :regression do
+      before do
+        pending_in_travis("these tests take too long or time out")
       end
 
-      2.times do
-        @zk.exists?(@path, :watch => true).should_not be_true
+      # this is stupid, and a bad test, but we have to check that events 
+      # don't get re-delivered to a single registered callback just because 
+      # :watch => true was called twice
+      #
+      # again, we're testing a negative here, so consider this a regression check
+      #
+      def wait_for_events_to_not_be_delivered(events)
+        lambda { wait_until(0.2) { events.length >= 2 } }.should raise_error(WaitWatchers::TimeoutError)
       end
 
-      @zk.create(@path, '', :mode => :ephemeral)
-
-      wait_for_events_to_not_be_delivered(events)
+      it %[should only deliver an event once to each watcher registered for exists?] do
+        events = []
 
-      events.length.should == 1
-    end
+        sub = @zk.register(@path) do |ev|
+          logger.debug "got event #{ev}"
+          events << ev
+        end
 
-    it %[should only deliver an event once to each watcher registered for get] do
-      events = []
+        2.times do
+          @zk.exists?(@path, :watch => true).should_not be_true
+        end
 
-      @zk.create(@path, 'one', :mode => :ephemeral)
+        @zk.create(@path, '', :mode => :ephemeral)
 
-      sub = @zk.register(@path) do |ev|
-        logger.debug "got event #{ev}"
-        events << ev
-      end
+        wait_for_events_to_not_be_delivered(events)
 
-      2.times do
-        data, stat = @zk.get(@path, :watch => true)
-        data.should == 'one'
+        events.length.should == 1
       end
 
-      @zk.set(@path, 'two')
+      it %[should only deliver an event once to each watcher registered for get] do
+        events = []
 
-      wait_for_events_to_not_be_delivered(events)
+        @zk.create(@path, 'one', :mode => :ephemeral)
 
-      events.length.should == 1
-    end
+        sub = @zk.register(@path) do |ev|
+          logger.debug "got event #{ev}"
+          events << ev
+        end
 
+        2.times do
+          data, stat = @zk.get(@path, :watch => true)
+          data.should == 'one'
+        end
 
-    it %[should only deliver an event once to each watcher registered for children] do
-      events = []
+        @zk.set(@path, 'two')
 
-      @zk.create(@path, '')
+        wait_for_events_to_not_be_delivered(events)
 
-      sub = @zk.register(@path) do |ev|
-        logger.debug "got event #{ev}"
-        events << ev
+        events.length.should == 1
       end
 
-      2.times do
-        children = @zk.children(@path, :watch => true)
-        children.should be_empty
-      end
 
-      @zk.create("#{@path}/pfx", '', :mode => :ephemeral_sequential)
+      it %[should only deliver an event once to each watcher registered for children] do
+        events = []
+
+        @zk.create(@path, '')
 
-      wait_for_events_to_not_be_delivered(events)
+        sub = @zk.register(@path) do |ev|
+          logger.debug "got event #{ev}"
+          events << ev
+        end
+
+        2.times do
+          children = @zk.children(@path, :watch => true)
+          children.should be_empty
+        end
+
+        @zk.create("#{@path}/pfx", '', :mode => :ephemeral_sequential)
 
-      events.length.should == 1
+        wait_for_events_to_not_be_delivered(events)
+
+        events.length.should == 1
+      end
     end
 
     it %[should restrict_new_watches_for? if a successul watch has been set] do
@@ -186,113 +193,120 @@ describe ZK do
         before do
           @events = EventCatcher.new
 
-          @zk.register(@path, :created) do |event|
-            @events.created << event
-          end
+          [:created, :changed, :child, :deleted].each do |ev_name|
 
-          @zk.register(@path, :changed) do |event|
-            @events.changed << event
-          end
+            @zk.register(@path, :only => ev_name) do |event|
+              @events.add(ev_name, event)
+            end
 
-          @zk.register(@path, :child) do |event|
-            @events.child << event
-          end
-
-          @zk.register(@path, :deleted) do |event|
-            @events.deleted << event
-          end
-
-          # this will catch all events, that way we don't have to wait for an
-          # event to *not* be delivered to one of the other callbacks (which is
-          # kinda stupid)
-          @zk.register(@path) do |event|
-            @events.all << event
           end
         end
 
         it %[should deliver only the created event to the created block] do
-          @zk.stat(@path, :watch => true).should_not exist
+          @events.synchronize do
+            @zk.stat(@path, :watch => true).should_not exist
 
-          @zk.create(@path)
-          wait_while { @events.created.empty? }.should be_false
-          @events.created.first.should be_node_created
+            @zk.create(@path)
 
-          @zk.stat(@path, :watch => true).should exist
+            @events.wait_for_created
 
-          @events.all.length.should == 1
+            @events.created.should_not be_empty
+            @events.created.first.should be_node_created
+            @events.all.should_not be_empty
+
+            @zk.stat(@path, :watch => true).should exist
+
+            @events.all.length.should == 1
+
+            @zk.delete(@path)
 
-          @zk.delete(@path)
+            @events.wait_for_all
+          end
 
-          wait_until { @events.all.length > 1 }
+          @events.all.length.should == 2
 
           # :deleted event was delivered, make sure it didn't get delivered to the :created block
           @events.created.length.should == 1
         end
 
         it %[should deliver only the changed event to the changed block] do
-          @zk.create(@path)
+          @events.synchronize do
+            @zk.create(@path)
 
-          @zk.stat(@path, :watch => true).should exist
+            @zk.stat(@path, :watch => true).should exist
 
-          @zk.set(@path, 'data')
+            @zk.set(@path, 'data')
 
-          wait_while { @events.changed.empty? }
+            @events.wait_for_changed
+          end
 
+          @events.changed.should_not be_empty
+          @events.changed.length.should == 1
           @events.changed.first.should be_node_changed
 
-          @zk.stat(@path, :watch => true).should exist
-
           @events.all.length.should == 1
 
-          @zk.delete(@path)
+          @events.synchronize do
+            @zk.stat(@path, :watch => true).should exist
+            @zk.delete(@path)
+            @events.wait_for_all
+          end
 
-          wait_until { @events.all.length > 1 }
+          @events.all.length.should == 2
 
           # :deleted event was delivered, make sure it didn't get delivered to the :changed block
           @events.changed.length.should == 1
         end
 
         it %[should deliver only the child event to the child block] do
-          @zk.create(@path)
+          child_path = nil
 
-          @zk.children(@path, :watch => true).should be_empty
+          @events.synchronize do
+            @zk.create(@path)
+            @zk.children(@path, :watch => true).should be_empty
 
-          child_path = @zk.create("#{@path}/m", '', :sequence => true)
+            child_path = @zk.create("#{@path}/m", '', :sequence => true)
 
-          wait_while { @events.child.empty? }
+            @events.wait_for_child
 
-          @events.child.first.should be_node_child
+            @events.child.length.should == 1
+            @events.child.first.should be_node_child
 
-          @zk.stat(@path, :watch => true).should exist
+            @zk.stat(@path, :watch => true).should exist
 
-          @events.all.length.should == 1
+            @events.all.length.should == 1
 
-          @zk.set(@path, '') # equivalent to a 'touch'
+            @zk.set(@path, '') # equivalent to a 'touch'
+            @events.wait_for_all
+          end
 
-          wait_until { @events.all.length > 1 }
+          @events.all.length.should == 2
 
           # :changed event was delivered, make sure it didn't get delivered to the :child block
           @events.child.length.should == 1
         end
 
         it %[should deliver only the deleted event to the deleted block] do
-          @zk.create(@path)
-
-          @zk.stat(@path, :watch => true).should exist
+          @events.synchronize do
+            @zk.create(@path)
+            @zk.stat(@path, :watch => true).should exist
+            @zk.delete(@path)
 
-          @zk.delete(@path)
+            @events.wait_for_deleted
+            @events.wait_while_all { |all| all.empty? }
 
-          wait_while { @events.deleted.empty? }
+            @events.deleted.should_not be_empty
+            @events.deleted.first.should be_node_deleted
+            @events.all.length.should == 1
 
-          @events.deleted.first.should be_node_deleted
+            @zk.stat(@path, :watch => true).should_not exist
 
-          @zk.stat(@path, :watch => true).should_not exist
-
-          @events.all.length.should == 1
+            @zk.create(@path)
 
-          @zk.create(@path)
+            @events.wait_for_all
+          end
 
-          wait_until { @events.all.length > 1 }
+          @events.all.length.should be > 1
 
           # :deleted event was delivered, make sure it didn't get delivered to the :created block
           @events.deleted.length.should == 1
@@ -301,35 +315,40 @@ describe ZK do
 
       it %[should deliver interested events to a block registered for multiple deliveries] do
         @events = []
+        @events.extend(MonitorMixin)
+        @cond = @events.new_cond
 
-        @zk.register(@path, [:created, :changed]) do |event|
-          @events << event
+        @zk.register(@path, :only => [:created, :changed]) do |event|
+          @events.synchronize do
+            @events << event
+            @cond.broadcast
+          end
         end
 
-        @zk.stat(@path, :watch => true).should_not exist
-
-        @zk.create(@path)
-
-        wait_while { @events.empty? }
+        @events.synchronize do
+          @zk.stat(@path, :watch => true).should_not exist
 
-        @events.length.should == 1
+          @zk.create(@path)
 
-        @events.first.should be_node_created
+          @cond.wait(5)
 
-        @zk.stat(@path, :watch => true).should exist
+          @events.should_not be_empty
+          @events.length.should == 1
+          @events.first.should be_node_created
 
-        @zk.set(@path, 'blah')
+          @zk.stat(@path, :watch => true).should exist
+          @zk.set(@path, 'blah')
 
-        wait_until { @events.length > 1 }
+          @cond.wait(5)
+        end
 
         @events.length.should == 2
-
         @events.last.should be_node_changed
       end
 
       it %[should barf if an invalid event name is given] do
         lambda do
-          @zk.register(@path, :tripping) { }
+          @zk.register(@path, :only => :tripping) { }
         end.should raise_error(ArgumentError)
       end
     end # event interest
@@ -339,9 +358,9 @@ describe ZK do
     describe 'live-fire test' do
       before do
         @event = nil
-        @cnx_str = "localhost:#{ZK_TEST_PORT}"
+        @cnx_str = "localhost:#{ZK.test_port}"
 
-        @zk = ZK.new(@cnx_str) do |zk|
+        @zk = ZK.new(*connection_args) do |zk|
           @cnx_reg = zk.on_connected { |event| @event = event }
         end
       end