zk 1.5.2 → 1.5.3

data/Guardfile

@@ -26,11 +26,20 @@ guard 'rspec', :version => 2 do
     when %r{^(?:zk/locker/locker_base|spec/shared/locker)}
       Dir["spec/zk/locker/*_spec.rb"]
 
+    when %r{^zk/client/(?:base|state_mixin)}
+      Dir['spec/zk/{client,client/*,zookeeper}_spec.rb']
+
     when 'zk' # .rb
       'spec'  # run all tests
 
     else
-      "spec/#{m[1]}_spec.rb"
+      generic = "spec/#{m[1]}_spec.rb"
+      if test(?f, generic)
+        generic
+      else
+        $stderr.puts "RUNNING ALL TESTS"
+        'spec'
+      end
     end
   end
 

data/lib/zk/client/base.rb

@@ -51,7 +51,7 @@ module ZK
       # the wrapped connection object
       # @private 
       attr_reader :cnx
-      protected :cnx
+      private :cnx
 
       # maps from a symbol given as an option, to the numeric error constant that should
       # not raise an exception
@@ -1020,7 +1020,7 @@ module ZK
       def raw_event_handler(event)
       end
 
-      protected
+      private
         # does the current pid match the one that created us?
         def forked?
           Process.pid != @pid
@@ -1038,7 +1038,6 @@ module ZK
           check_rc(rv, opts)
         end
 
-        # @private
         # XXX: make this actually call the method on cnx
         def check_rc(rv_hash, inputs)
           code  = rv_hash[:rc]
@@ -1078,7 +1077,6 @@ module ZK
           Set.new(ERROR_IGNORE_MAP.values_at(*sym_array))
         end
 
-        # @private
         def setup_watcher!(watch_type, opts, &b)
           event_handler.setup_watcher!(watch_type, opts, &b)
         end

data/lib/zk/client/state_mixin.rb

@@ -78,7 +78,7 @@ module ZK
         watcher.register_state_handler(:expired_session, &block)
       end
 
-      protected
+      private
         def wrap_state_closed_error
           yield
         rescue RuntimeError => e

data/lib/zk/client/threaded.rb

@@ -45,10 +45,10 @@ module ZK
 
       # @private
       module Constants
-        CLI_RUNNING   = :running
-        CLI_PAUSED    = :paused
-        CLI_CLOSE_REQ = :close_requested
-        CLI_CLOSED    = :closed
+        RUNNING   = :running
+        PAUSED    = :paused
+        CLOSE_REQ = :close_requested
+        CLOSED    = :closed
       end
       include Constants
 
@@ -161,13 +161,12 @@ module ZK
 
         @reconnect = opts.fetch(:reconnect, true)
 
-        @mutex = Monitor.new
-        @cond = @mutex.new_cond
+        setup_locks
 
-        @cli_state = CLI_RUNNING # this is to distinguish between *our* state and the underlying connection state
+        @client_state = RUNNING # this is to distinguish between *our* state and the underlying connection state
 
         # this is the last status update we've received from the underlying connection
-        @last_cnx_state = Zookeeper::ZOO_CLOSED_STATE 
+        @last_cnx_state = nil
 
         @retry_duration = opts.fetch(:retry_duration, nil).to_i
 
@@ -183,6 +182,15 @@ module ZK
 
         connect if opts.fetch(:connect, true)
       end
+      
+      # ensure that the initializer and the reopen code set up the mutexes
+      # the same way (i.e. use a Monitor or a Mutex, no, really, I screwed 
+      # this up once) 
+      def setup_locks
+        @mutex = Monitor.new
+        @cond = @mutex.new_cond
+      end
+      private :setup_locks
 
       # @private
       def self.finalizer(hooks)
@@ -206,14 +214,18 @@ module ZK
 
           logger.debug { "reopening everything, fork detected!" }
 
-          @mutex      = Mutex.new
-          @cond       = ConditionVariable.new
+          setup_locks
+
           @pid        = Process.pid
-          @cli_state  = CLI_RUNNING                     # reset state to running if we were paused
+          @client_state  = RUNNING                     # reset state to running if we were paused
 
           old_cnx, @cnx = @cnx, nil
           old_cnx.close! if old_cnx # && !old_cnx.closed?
 
+          join_and_clear_reconnect_thread
+
+          @last_cnx_state = nil
+
           @mutex.synchronize do
             # it's important that we're holding the lock, as access to 'cnx' is
             # synchronized, and we want to avoid a race where event handlers
@@ -228,11 +240,12 @@ module ZK
           end
         else
           @mutex.synchronize do
-            if @cli_state == CLI_PAUSED
+            if @client_state == PAUSED
               # XXX: what to do in this case? does it matter?
             end
 
             logger.debug { "reopening, no fork detected" }
+            @last_cnx_state = nil
             @cnx.reopen(timeout)                # ok, we werent' forked, so just reopen
           end
         end
@@ -252,30 +265,37 @@ module ZK
       # @private
       def pause_before_fork_in_parent
         @mutex.synchronize do
-          raise InvalidStateError, "client must be running? when you call #{__method__}" unless (@cli_state == CLI_RUNNING)
-          @cli_state = CLI_PAUSED
+          raise InvalidStateError, "client must be running? when you call #{__method__}" unless (@client_state == RUNNING)
+          @client_state = PAUSED
       
           logger.debug { "#{self.class}##{__method__}" }
 
           @cond.broadcast
         end
 
+        join_and_clear_reconnect_thread
+
         # the compact is here because the @cnx *may* be nil when this callback is fired by the
         # ForkHook (in the case of ZK.open). The race is between the GC calling the finalizer
         [@event_handler, @threadpool, @cnx].compact.each(&:pause_before_fork_in_parent)
       ensure
-        logger.debug { "#{self.class}##{__method__} returning" }
+        logger.debug { "##{__method__} returning" }
       end
 
       # @private
       def resume_after_fork_in_parent
         @mutex.synchronize do
-          raise InvalidStateError, "client must be paused? when you call #{__method__}" unless (@cli_state == CLI_PAUSED)
-          @cli_state = CLI_RUNNING
+          raise InvalidStateError, "client must be paused? when you call #{__method__}" unless (@client_state == PAUSED)
+          @client_state = RUNNING
 
-          logger.debug { "#{self.class}##{__method__}" }
+          logger.debug { "##{__method__}" }
 
-          [@cnx, @event_handler, @threadpool].compact.each(&:resume_after_fork_in_parent)
+          if @cnx
+            @cnx.resume_after_fork_in_parent
+            spawn_reconnect_thread
+          end
+
+          [@event_handler, @threadpool].compact.each(&:resume_after_fork_in_parent)
 
           @cond.broadcast
         end
@@ -291,12 +311,14 @@ module ZK
       #
       def close!
         @mutex.synchronize do 
-          return if [:closed, :close_requested].include?(@cli_state)
+          return if [:closed, :close_requested].include?(@client_state)
           logger.debug { "moving to :close_requested state" }
-          @cli_state = CLI_CLOSE_REQ
+          @client_state = CLOSE_REQ
           @cond.broadcast
         end
 
+        join_and_clear_reconnect_thread
+
         on_tpool = on_threadpool?
 
         # Ok, so the threadpool will wait up to N seconds while joining each thread.
@@ -308,6 +330,7 @@ module ZK
         # and wait for it to exit
         #
         shutdown_thread = Thread.new do
+          Thread.current[:name] = 'shutdown'
           @threadpool.shutdown(10)
 
           # this will call #close
@@ -315,7 +338,8 @@ module ZK
 
           @mutex.synchronize do
             logger.debug { "moving to :closed state" }
-            @cli_state = CLI_CLOSED
+            @client_state = CLOSED
+            @last_cnx_state = nil
             @cond.broadcast
           end
         end
@@ -348,15 +372,6 @@ module ZK
         @mutex.synchronize do
           @last_cnx_state = event.state
 
-          if event.client_invalid? and @reconnect and not dead_or_dying?
-            logger.error { "Got event #{event.state_name}, calling reopen(0)! things may be messed up until this works itself out!" }
-
-            # reopen(0) means that we don't want to wait for the connection
-            # to reach the connected state before returning as we're on the
-            # event thread.
-            reopen(0)
-          end
-
           @cond.broadcast # wake anyone waiting for a connection state update
         end
       rescue Exception => e
@@ -364,26 +379,47 @@ module ZK
       end
 
       def closed?
-        return true if @mutex.synchronize { @cli_state == CLI_CLOSED }
+        return true if @mutex.synchronize { @client_state == CLOSED }
         super
       end
 
       # are we in running (not-paused) state?
+      # @private
       def running?
-        @mutex.synchronize { @cli_state == CLI_RUNNING }
+        @mutex.synchronize { @client_state == RUNNING }
       end
 
       # are we in paused state?
+      # @private
       def paused?
-        @mutex.synchronize { @cli_state == CLI_PAUSED }
+        @mutex.synchronize { @client_state == PAUSED }
       end
 
       # has shutdown time arrived?
+      # @private
       def close_requested?
-        @mutex.synchronize { @cli_state == CLI_CLOSE_REQ }
+        @mutex.synchronize { @client_state == CLOSE_REQ }
       end
 
-      protected
+      # @private
+      def wait_until_connected_or_dying(timeout)
+        time_to_stop = Time.now + timeout
+
+        @mutex.synchronize do
+          while (@last_cnx_state != Zookeeper::ZOO_CONNECTED_STATE) && (Time.now < time_to_stop) && (@client_state == RUNNING)
+            @cond.wait(timeout)
+          end
+
+          logger.debug { "@last_cnx_state: #{@last_cnx_state.inspect}, time_left? #{Time.now.to_f < time_to_stop.to_f}, @client_state: #{@client_state.inspect}" }
+        end
+      end
+
+      private
+        # this is just here so we can see it in stack traces
+        def reopen_after_session_expired
+          reopen
+        end
+        
         # in the threaded version of the client, synchronize access around cnx
         # so that callers don't wind up with a nil object when we're in the middle
         # of reopening it
@@ -391,6 +427,58 @@ module ZK
           @mutex.synchronize { @cnx }
         end
 
+        def reconnect_thread_body
+          Thread.current[:name] = 'reconnect'
+          while @reconnect  # too clever?
+            @mutex.synchronize do
+              # either we havne't seen a valid session update from this
+              # connection yet, or we're doing fine, so just wait
+              @cond.wait_while { !seen_session_state_event? or (valid_session_state? and (@client_state == RUNNING)) }
+
+              # we've entered into a non-running state, so we exit
+              # note: need to restart this thread after a fork in parent
+              if @client_state != RUNNING
+                logger.debug { "session failure watcher thread exiting, @client_state: #{@client_state}" }
+                return
+              end
+
+              # if we know that this session was valid once and it has now
+              # become invalid we call reopen
+              #
+              if seen_session_state_event? and not valid_session_state?
+                logger.debug { "session state was invalid, calling reopen" }
+
+                # reopen will reset @last_cnx_state so that
+                # seen_session_state_event? will return false until the first
+                # event has been delivered on the new connection
+                rv = reopen_after_session_expired
+
+                logger.debug { "reopen returned: #{rv.inspect}" }
+              end
+            end
+          end
+        ensure
+          logger.debug { "reconnect thread exiting" }
+        end
+
+        def join_and_clear_reconnect_thread
+          return unless @reconnect_thread
+          begin
+             # this should never time out but, just to make sure we don't hang forever
+            unless @reconnect_thread.join(30)
+              logger.error { "timed out waiting for reconnect thread to join! something is hosed!" }
+            end
+          rescue Exception => e
+            logger.error { "caught exception joining reconnect thread" }
+            logger.error { e.to_std_format }
+          end
+          @reconnect_thread = nil
+        end
+
+        def spawn_reconnect_thread
+          @reconnect_thread ||= Thread.new(&method(:reconnect_thread_body))
+        end
+
         def call_and_check_rc(meth, opts)
           if retry_duration = (opts.delete(:retry_duration) || @retry_duration)
             begin
@@ -400,7 +488,7 @@ module ZK
 
               wait_until_connected_or_dying(retry_duration)
 
-              if (@last_cnx_state != Zookeeper::ZOO_CONNECTED_STATE) || (Time.now > time_to_stop) || (@cli_state != CLI_RUNNING)
+              if (@last_cnx_state != Zookeeper::ZOO_CONNECTED_STATE) || (Time.now > time_to_stop) || (@client_state != RUNNING)
                 raise e
               else
                 retry
@@ -411,16 +499,20 @@ module ZK
           end
         end
 
-        def wait_until_connected_or_dying(timeout)
-          time_to_stop = Time.now + timeout
-
-          @mutex.synchronize do
-            while (@last_cnx_state != Zookeeper::ZOO_CONNECTED_STATE) && (Time.now < time_to_stop) && (@cli_state == CLI_RUNNING)
-              @cond.wait(timeout)
-            end
+        # have we gotten a status event for the current connection?
+        # this method is not synchronized
+        def seen_session_state_event?
+          !!@last_cnx_state
+        end
 
-            logger.debug { "@last_cnx_state: #{@last_cnx_state}, time_left? #{Time.now.to_f < time_to_stop.to_f}, @cli_state: #{@cli_state.inspect}" }
-          end
+        # we've seen a session state from the cnx, and it was not "omg we're screwed"
+        # will return false if we havne't gotten a session event yet
+        #
+        # this method is not synchronized
+        def valid_session_state?
+          # this is kind of icky, but the SESSION_INVALID and AUTH_FAILED states
+          # are both negative numbers
+          @last_cnx_state and (@last_cnx_state >= 0)
         end
 
         def create_connection(*args)
@@ -428,14 +520,14 @@ module ZK
         end
 
         def dead_or_dying?
-          (@cli_state == CLI_CLOSE_REQ) || (@cli_state == CLI_CLOSED)
+          (@client_state == CLOSE_REQ) || (@client_state == CLOSED)
         end
 
-      private
         def unlocked_connect(opts={})
           return if @cnx
           timeout = opts.fetch(:timeout, @connection_timeout)
           @cnx = create_connection(@host, timeout, @event_handler.get_default_watcher_block)
+          spawn_reconnect_thread
         end
     end
   end

data/lib/zk/election.rb

@@ -150,7 +150,7 @@ module ZK
         end
       end
 
-      protected
+      private
         def create_root_path!
           @zk.mkdir_p(root_vote_path)
         end
@@ -244,7 +244,7 @@ module ZK
         end
       end
 
-      protected
+      private
         # the inauguration, as it were
         def acknowledge_win!
           @zk.create(leader_ack_path, @data, :ephemeral => true) 
@@ -411,7 +411,7 @@ module ZK
         end
       end
 
-      protected
+      private
         def the_king_is_dead
           synchronize do
             safe_call(*@leader_death_cbs)

data/lib/zk/event_handler.rb

@@ -196,7 +196,6 @@ module ZK
       end
 
     public
-
     # used during shutdown to clear registered listeners
     # @private
     def clear! #:nodoc:
@@ -303,13 +302,11 @@ module ZK
       end
     end
 
-    protected
-      # @private
+    private
       def watcher_callback
         Zookeeper::Callbacks::WatcherCallback.create { |event| process(event) }
       end
 
-      # @private
       def state_key(arg)
         int = 
           case arg
@@ -326,7 +323,6 @@ module ZK
         raise ArgumentError, "#{arg} is not a valid zookeeper state", caller
       end
 
-      # @private
       def safe_call(callbacks, *args)
         callbacks.each do |cb|
           next unless cb.respond_to?(:call)

data/lib/zk/event_handler_subscription/base.rb

@@ -52,7 +52,7 @@ module ZK
       def resume_after_fork_in_parent
       end
 
-      protected
+      private
         def prep_interests(a)
 #           logger.debug { "prep_interests: #{a.inspect}" }
           return ALL_EVENT_SET if a.nil?

data/lib/zk/locker/exclusive_locker.rb

@@ -53,7 +53,7 @@ module ZK
         true
       end
 
-      protected
+      private
         # the node that is next-lowest in sequence number to ours, the one we
         # watch for updates to
         def next_lowest_node

data/lib/zk/locker/locker_base.rb

@@ -230,7 +230,7 @@ module ZK
         end
       end
 
-      protected 
+      private
         def synchronize
           @mutex.synchronize { yield }
         end

data/lib/zk/locker/shared_locker.rb

@@ -92,7 +92,7 @@ module ZK
       end
       alias got_lock? got_read_lock?
 
-      protected
+      private
         # TODO: make this generic, can either block or non-block
         def block_until_read_lock!
           begin

data/lib/zk/mongoid.rb

@@ -118,6 +118,7 @@ module ZK
           Thread.current.zk_mongoid_lock_registry ||= { :shared => Set.new, :exclusive => Set.new }
         end
 
+      private
         def zk_add_path_lock(opts={})
           mode, name = opts.values_at(:mode, :name)
 

data/lib/zk/pool.rb

@@ -140,7 +140,7 @@ module ZK
         @state
       end
 
-      protected
+      private
         def synchronize
           @mutex.synchronize { yield }
         end
@@ -255,7 +255,7 @@ module ZK
         @mutex.synchronize { @connections.size < @max_clients }
       end
 
-      protected
+      private
         def populate_pool!(num_cnx)
           num_cnx.times { add_connection! }
         end

data/lib/zk/subscription.rb

@@ -54,7 +54,7 @@ module ZK
         @mutex = Monitor.new
       end
 
-      protected
+      private
         def synchronize
           @mutex.synchronize { yield }
         end

data/lib/zk/threaded_callback.rb

@@ -116,7 +116,7 @@ module ZK
       end
     end
 
-    protected
+    private
       # intentionally *not* synchronized
       def spawn_dispatch_thread
         @thread = Thread.new(&method(:dispatch_thread_body))
@@ -133,7 +133,7 @@ module ZK
             @cond.wait(@mutex) while @array.empty? and @state == :running
 
             if @state != :running
-              logger.warn { "ThreadedCallback, state is #{@state.inspect}, returning" } 
+              logger.debug { "ThreadedCallback, state is #{@state.inspect}, returning" } 
               return 
             end
 

data/lib/zk/version.rb

@@ -1,3 +1,3 @@
 module ZK
-  VERSION = "1.5.2"
+  VERSION = "1.5.3"
 end

data/spec/shared/client_contexts.rb

@@ -24,16 +24,11 @@ shared_context 'threaded client connection' do
 #     raise @threadpool_exception if @threadpool_exception
 #     logger.debug { "threaded client connection - after hook" }
 
-    if @zk.closed?
-      logger.debug { "zk was closed, calling reopen" }
-      @zk.reopen 
-    end
+    @zk.close! if @zk and not @zk.closed?
 
-    wait_until(5) { @zk.connected? }
-    
-    @zk.rm_rf(@base_path)
-    @zk.close!
-    wait_until(5) { @zk.closed? }
+    ZK.open(*connection_args) do |z|
+      z.rm_rf(@base_path)
+    end
 
 #     logger.debug { "threaded client connection - end after hook" }
   end

data/spec/shared/client_examples.rb

@@ -298,15 +298,20 @@ shared_examples_for 'client' do
 
   describe 'reconnection' do
     it %[should if it receives a client_invalid? event] do
-      flexmock(@zk) do |m|
-        m.should_receive(:reopen).with(0).once
-      end
+      logger.debug { "about to cause the reconnection" }
 
       props = { :session_event? => true, :client_invalid? => true, :state_name => 'ZOO_EXPIRED_SESSION_STATE', :state => Zookeeper::ZOO_EXPIRED_SESSION_STATE }
 
       bogus_event = flexmock(:expired_session_event, props)
 
-      @zk.raw_event_handler(bogus_event)
+      th = Thread.new do
+        @zk.raw_event_handler(bogus_event)
+      end
+
+      @zk.wait_until_connected_or_dying(5)
+      @zk.should be_connected
+
+      th.join(2).should == th
     end
   end # reconnection
 

data/spec/spec_helper.rb

@@ -110,8 +110,10 @@ class ::Thread
 end
 
 if RUBY_VERSION == '1.9.3'
+  Thread.current[:name] = 'main'
+
   trap('USR1') do
-    threads = Thread.list.map { |th| { :inspect => th.inspect, :calback => th[:callback], :backtrace => th.backtrace } }
+    threads = Thread.list.map { |th| { :inspect => th.inspect, :name => th[:name], :calback => th[:callback], :backtrace => th.backtrace } }
     pp threads
   end
 end

data/spec/support/logging.rb

@@ -3,14 +3,13 @@ module ZK
 
   def self.logging_gem_setup
     layout = ::Logging.layouts.pattern(
-      :pattern => '%.1l, [%d #%p] %30.30c{2}:  %m\n',
-      :date_pattern => '%Y-%m-%d %H:%M:%S.%6N'
+      :pattern => '%.1l, [%d #%p] (%9.9T) %25.25c{2}:  %m\n',
+      :date_pattern => '%H:%M:%S.%6N'
     )
 
-
     appender = ENV['ZK_DEBUG'] ? ::Logging.appenders.stderr : ::Logging.appenders.file(ZK::TEST_LOG_PATH)
     appender.layout = layout
-    appender.immediate_at = "debug,info,warn,error,fatal"
+#     appender.immediate_at = "debug,info,warn,error,fatal"
 #     appender.auto_flushing = true
     appender.auto_flushing = 25
     appender.flush_period = 5

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: zk
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 1
   - 5
-  - 2
-  version: 1.5.2
+  - 3
+  version: 1.5.3
 platform: ruby
 authors: 
 - Jonathan D. Simms