zkruby 3.4.3 → 3.4.4.rc3

data/lib/zkruby/session.rb

@@ -1,8 +1,29 @@
 require 'set'
+require 'strand'
+require 'strand/monitor'
+#TODO Move async_op into this file
+require 'zkruby/async_op'
 module ZooKeeper
 
-    # Represents an session that may span connections
+    # Represents a session that may span connect/disconnect
+    #
+    # @note this is a private API not intended for client use
     class Session
+        # TODO this class is too big
+
+        # There are multiple threads of execution involved in a session
+        # Client threads - send requests
+        # Connection/Read thread
+        # EventLoop - callback execution 
+        # 
+        # All client activity is synchronised on this session as a monitor
+        # The read thread synchronises with client activity 
+        # only around connection/disconnection
+        # ie any changes to @session_state which always happens in
+        # conjunction with processing all entries in @pending_queue
+        #
+        # All interaction with the event loop occurs via a Queue
+        include Strand::MonitorMixin
 
         DEFAULT_TIMEOUT = 4
         DEFAULT_CONNECT_DELAY = 0.2
@@ -16,24 +37,30 @@ module ZooKeeper
         attr_reader :conn
         attr_accessor :watcher
 
-        def initialize(binding,addresses,options=nil)
-
-            @binding = binding
-
+        # @api zookeeper
+        # See {ZooKeeper.connect}
+        def initialize(addresses,options=nil)
+            super()
             @addresses = parse_addresses(addresses)
             parse_options(options)
 
-            # These are the server states
-            # :disconnected, :connected, :auth_failed, :expired
-            @keeper_state = nil
+            # our transaction id
+            @xid=0
 
-            # Client state is
-            # :ready, :closing, :closed
-            @client_state = :ready
+            # The state of the connection, nil, :disconnected, :connected, :closed, :expired
+            @session_state = nil
 
-            @xid=0
+            # The connection we'll send packets to
+            @conn = nil
+
+            # The list of pending requests
+            # When disconnected this builds up a list to send through when we are connected
+            # When connected represents the order in which we expect to see responses
             @pending_queue = []
 
+            # Client state is :ready, :closing, :closed
+            @client_state = :ready
+
             # Create the watch list
             # hash by watch type of hashes by path of set of watchers
             @watches = [ :children, :data, :exists ].inject({}) do |ws,wtype| 
@@ -41,33 +68,14 @@ module ZooKeeper
                 ws
             end
 
+            # the default watcher
             @watcher = nil
 
             @ping_logger = Slf4r::LoggerFacade.new("ZooKeeper::Session::Ping")
-
-        end
-
-        def chroot(path)
-            return @chroot + path
         end
 
-        def unchroot(path)
-            return path unless path
-            path.slice(@chroot.length..-1)
-        end
-
-        # close won't run your block if the connection is 
-        # already closed, so this is how you can check
-        def closed?
-            @client_state == :closed
-        end
-
-        # Connection API - testing whether to send a ping
-        def connected?()
-            @keeper_state == :connected
-        end
-
-        # Connection API - Injects a new connection that is ready to receive records
+        # @api connection
+        # Injects a new connection that is ready to receive records
         # @param conn that responds to #send_records(record...) and #disconnect()
         def prime_connection(conn)
             @conn = conn
@@ -76,96 +84,157 @@ module ZooKeeper
             reset_watches()
         end
 
-
-        # Connection API - called when data is available, reads and processes one packet/event
-        # @param io <IO> 
+        # @api connection
+        # called when data is available, reads and processes one packet/event
+        # @param [IO] io
         def receive_records(io)
-            case @keeper_state
+            case @session_state
             when :disconnected
                 complete_connection(io)
             when :connected
                 process_reply(io)
             else
-                logger.warn { "Receive packet for closed session #{@keeper_state}" }
+                logger.warn { "Receive packet for closed session #{@session_state}" }
             end
         end
 
-        # Connection API - called when no data has been received for #ping_interval
+        # @api connection
+        # Connection API - testing whether to send a ping
+        def connected?()
+            @session_state == :connected
+        end
+
+
+        # @api connection
+        # called when no data has been received for #ping_interval
         def ping()
-            if @keeper_state == :connected
+            if connected?
                 ping_logger.debug { "Ping send" }
                 hdr = Proto::RequestHeader.new(:xid => -2, :_type => 11)
-                conn.send_records(hdr)
+                conn.send_records(hdr) 
             end
         end
 
-        # Connection API - called when the connection has dropped from either end
+        # @api connection
+        # called when the connection has dropped from either end
         def disconnected()
-            @conn = nil
-            logger.info { "Disconnected id=#{@session_id}, keeper=:#{@keeper_state}, client=:#{@client_state}" }
+            logger.info { "Disconnected id=#{@session_id}, keeper=:#{@session_state}, client=:#{@client_state}" }
 
             # We keep trying to reconnect until the session expiration time is reached
-            @disconnect_time = Time.now if @keeper_state == :connected
+            @disconnect_time = Time.now if @session_state == :connected
             time_since_first_disconnect = (Time.now - @disconnect_time) 
 
-            if @client_state == :closed || time_since_first_disconnect > timeout
-                session_expired()
+            if @session_state == :closing
+                #We were expecting this disconnect
+                session_expired(:closed)
+            elsif time_since_first_disconnect > timeout
+                session_expired(:expired)
             else
-                # if we are connected then everything in the pending queue has been sent so
-                # we must clear
-                # if not, then we'll keep them and hope the next reconnect works
-                if @keeper_state == :connected
+                if @session_state == :connected
+                    #first disconnect
                     clear_pending_queue(:disconnected)
                     invoke_watch(@watcher,KeeperState::DISCONNECTED,nil,WatchEvent::NONE) if @watcher
+                    @conn = nil
                 end
-                @keeper_state = :disconnected
-                reconnect()
             end
         end
 
-        # Start the session - called by the ProtocolBinding
-        def start()
-            raise ProtocolError, "Already started!" unless @keeper_state.nil?
-            @keeper_state = :disconnected
+        # @api zookeeper
+        # See {ZooKeeper.connect}
+        def start(client)
+            raise ProtocolError, "Already started!" unless @session_state.nil?
+            @session_state = :disconnected
             @disconnect_time = Time.now
             logger.debug("Starting new zookeeper client session")
-            reconnect()
+            @event_loop = EventLoop.new(client)
+            Strand.new { 
+                begin
+                    reconnect()
+                    while active? 
+                        delay = rand() * @max_connect_delay
+                        Strand.sleep(delay)
+                        reconnect()
+                    end
+                rescue Exception => ex
+                    logger.error("Exception in connect loop",ex)
+                end
+            logger.debug("Session complete")
+            }
         end
 
-        def queue_request(request,op,opcode,response=nil,watch_type=nil,watcher=nil,ptype=Packet,&callback)
-            raise Error.SESSION_EXPIRED, "Session expired due to client state #{@client_state}"  unless @client_state == :ready
-            watch_type, watcher = resolve_watcher(watch_type,watcher)
 
-            xid = next_xid
+        # @api client
+        def chroot(path)
+            return @chroot if path == "/"
+            return @chroot + path
+        end
 
-            packet = ptype.new(xid,op,opcode,request,response,watch_type,watcher, callback)
+        # @api client
+        def unchroot(path)
+            return path unless path
+            path.slice(@chroot.length..-1)
+        end
 
-            queue_packet(packet)
+        # @api client
+        def request(*args,&callback)
+            AsyncOp.new(@event_loop,callback) do |op|
+                queue_request(*args) do |error,response|
+                    op.resume(error,response)
+                end
+            end
         end
 
+        # @api client
         def close(&callback)
-            case @client_state
-            when :ready
-                # we keep the requested block in a close packet
-                @close_packet = ClosePacket.new(next_xid(),:close,-11,nil,nil,nil,nil,callback)
-                close_packet = @close_packet 
-                @client_state = :closing
-
-                # If there are other requests in flight, then we wait for them to finish
-                # before sending the close packet since it immediately causes the socket
-                # to close.
-                queue_close_packet_if_necessary()
-                @close_packet
-            when :closed, :closing
-                raise ProtocolError, "Already closed"
-            else
-                raise ProtocolError, "Unexpected state #{@client_state}"
+            AsyncOp.new(@event_loop,callback) do |op|
+                close_session() do |error,response|
+                    op.resume(error,response) 
+                end
             end
         end
 
         private
+        def active?
+            [:connected,:disconnected].include?(@session_state)
+        end
+
+        def queue_request(request,op,opcode,response=nil,watch_type=nil,watcher=nil,ptype=Packet,&callback)
+            synchronize do 
+                raise ProtocolError, "Client closed #{@client_state}" unless @client_state == :ready
+                raise Error.SESSION_EXPIRED, "Session has expired #{@session_state}" unless active?
+                watch_type, watcher = resolve_watcher(watch_type,watcher)
+
+                xid = next_xid
+
+                packet = ptype.new(xid,op,opcode,request,response,watch_type,watcher, callback)
+
+                queue_packet(packet)
+            end
+        end
+
+        def close_session(&callback)
+            synchronize do
+                if @client_state == :ready
+                    if active?
+                        # we keep the requested block in a close packet but we don't send it
+                        # until we've received all pending reponses
+                        @close_packet = ClosePacket.new(next_xid(),:close,-11,nil,nil,nil,nil,callback)
+
+                        # but we can force a response by sending a ping
+                        ping()
+                        
+                    else
+                        # We've already expired put the close callback on the event loop
+                        @event_loop.invoke_close(callback,nil,true) 
+                    end
+                    @client_state = :closed
+                else
+                    raise ProtocolError, "Client already #{@client_state}"
+                end
+            end
+        end
+
         attr_reader :watches
-        attr_reader :binding
 
         def parse_addresses(addresses)
             case addresses
@@ -201,42 +270,36 @@ module ZooKeeper
         end
 
         def reconnect()
-
             #Rotate address
             host,port = @addresses.shift
             @addresses.push([host,port])
-
-            delay = rand() * @max_connect_delay
-
-            logger.debug { "Connecting id=#{@session_id} to #{host}:#{port} with delay=#{delay}, timeout=#{@connect_timeout}" } 
-            binding.connect(host,port,delay,@connect_timeout)
+            logger.debug { "Connecting id=#{@session_id} to #{host}:#{port} with timeout=#{@connect_timeout}" } 
+            connect(host,port,@connect_timeout)
         end
 
-
         def session_expired(reason=:expired)
-            clear_pending_queue(reason)
-
-            invoke_response(*@close_packet.error(reason)) if @close_packet
-
-            if @client_state == :closed
-                logger.info { "Session closed id=#{@session_id}, keeper=:#{@keeper_state}, client=:#{@client_state}" }
+            if reason == :closed
+                logger.info { "Session closed id=#{@session_id}, keeper=:#{@session_state}, client=:#{@client_state}" }
             else
-                logger.warn { "Session expired id=#{@session_id}, keeper=:#{@keeper_state}, client=:#{@client_state}" }
+                logger.warn { "Session expired reason=#{reason} id=#{@session_id}, keeper=:#{@session_state}, client=:#{@client_state}" }
             end
 
+            clear_pending_queue(reason)
+            #TODO Clients will want to distinguish between EXPIRED and CLOSED
             invoke_watch(@watcher,KeeperState::EXPIRED,nil,WatchEvent::NONE) if @watcher
-            @keeper_state = reason
-            @client_state = :closed
+            @event_loop.stop()
         end
 
         def complete_connection(response)
             result = Proto::ConnectResponse.read(response)
             if (result.time_out <= 0)
                 #We're dead!
-                session_expired()
+                session_expired() 
             else
                 @timeout = result.time_out.to_f / 1000.0
-                @keeper_state = :connected
+                @session_id = result.session_id
+                @session_passwd = result.passwd
+                logger.info { "Connected session_id=#{@session_id}, timeout=#{@time_out}, ping=#{@ping_interval}" }
 
                 # Why 2 / 7 of the timeout?. If a binding sees no server response in this period it is required to
                 # generate a ping request
@@ -244,14 +307,14 @@ module ZooKeeper
                 # so we are already more than half way through the session timeout
                 # and we need to give ourselves time to reconnect to another server
                 @ping_interval = @timeout * 2.0 / 7.0
-                @session_id = result.session_id
-                @session_passwd = result.passwd
-                logger.info { "Connected session_id=#{@session_id}, timeout=#{@time_out}, ping=#{@ping_interval}" }
 
-                logger.debug { "Sending #{@pending_queue.length} queued packets" }
-                @pending_queue.each { |p| send_packet(p) }
+                synchronize do
+                    logger.debug { "Sending #{@pending_queue.length} queued packets" }
+                    @session_state = :connected
+                    @pending_queue.each { |p| send_packet(p) }
+                    send_close_packet_if_necessary()
+                end
 
-                queue_close_packet_if_necessary()
                 invoke_watch(@watcher,KeeperState::CONNECTED,nil,WatchEvent::NONE) if @watcher
             end
         end
@@ -261,7 +324,6 @@ module ZooKeeper
             req.last_zxid_seen = @last_zxid_seen if @last_zxid_seen
             req.session_id =  @session_id if @session_id
             req.passwd = @session_passwd if @session_passwd
-
             conn.send_records(req)
         end
 
@@ -296,6 +358,7 @@ module ZooKeeper
             case header.xid.to_i
             when -2
                 ping_logger.debug { "Ping reply" }
+                send_close_packet_if_necessary()
             when -4
                 logger.debug { "Auth reply" }
                 session_expired(:auth_failed) unless header.err.to_i == 0
@@ -312,6 +375,13 @@ module ZooKeeper
                 # A normal packet reply. They should come in the order we sent them
                 # so we just match it to the packet at the front of the queue
                 packet = @pending_queue.shift
+
+                if packet == nil && @close_packet
+                    packet = @close_packet
+                    @close_packet = nil
+                    @session_state = :closing
+                end
+
                 logger.debug { "Packet reply: #{packet.inspect}" }
 
                 if (packet.xid.to_i != header.xid.to_i)
@@ -324,9 +394,8 @@ module ZooKeeper
                     invoke_response(*packet.error(:disconnected))
                     @conn.disconnect() 
                 else
-
                     @last_zxid_seen = header.zxid
-                    
+
                     callback, error, response, watch_type =  packet.result(header.err.to_i)
                     invoke_response(callback, error, response, packet_io)
 
@@ -334,11 +403,11 @@ module ZooKeeper
                         @watches[watch_type][packet.path] << packet.watcher 
                         logger.debug { "Registered #{packet.watcher} for type=#{watch_type} at #{packet.path}" }
                     end
-                    queue_close_packet_if_necessary()
+                    send_close_packet_if_necessary()
                 end
             end
-        end
 
+        end
 
         def process_watch_notification(state,path,event)
 
@@ -347,17 +416,16 @@ module ZooKeeper
 
             keeper_state = KeeperState.fetch(state)
 
-            watches = watch_types.inject(Set.new()) do | result, watch_type |
+            watches = watch_types.inject(Set.new()) do |result, watch_type|
                 more_watches = @watches[watch_type].delete(path)
-            result.merge(more_watches) if more_watches
-            result
+                result.merge(more_watches) if more_watches
+                result
             end
 
             if watches.empty?
                 logger.warn { "Received notification for unregistered watch #{state} #{path} #{event}" }
             end
             watches.each { | watch | invoke_watch(watch,keeper_state,path,watch_event) }      
-
         end
 
         def invoke_watch(watch,state,path,event)
@@ -369,39 +437,48 @@ module ZooKeeper
             else
                 raise ProtocolError("Bad watcher #{watch}")
             end
-
-            binding.invoke(callback,state,unchroot(path),event)
+            @event_loop.invoke(callback,state,unchroot(path),event)
         end
 
         def clear_pending_queue(reason)
-            @pending_queue.each  { |p| invoke_response(*p.error(reason)) }
-            @pending_queue.clear
+            synchronize do
+                @session_state = reason
+                @pending_queue.each  { |p| invoke_response(*p.error(reason)) }
+                @pending_queue.clear()
+                if @close_packet
+                    invoke_response(*@close_packet.error(reason))
+                    @close_packet = nil
+                end
+            end
         end
 
-        def queue_close_packet_if_necessary
-            if @pending_queue.empty? && @keeper_state == :connected && @close_packet
+        def send_close_packet_if_necessary
+            # We don't need to synchronize this because the creation of 
+            # the close packet was synchronized and after that all
+            # client requests are rejected
+            # we can receive watch and ping notifications after this
+            # but the server drops the connection as soon as this
+            # packet arrives
+            if @pending_queue.empty? && @session_state == :connected && @close_packet
                 logger.debug { "Sending close packet!" }
-                @client_state = :closed
-                queue_packet(@close_packet)
-                @close_packet = nil
+                send_packet(@close_packet)
             end
         end
 
         def invoke_response(callback,error,response,packet_io = nil)
             if callback
-                
                 result = if error
                              nil
-                       elsif response.respond_to?(:read) && packet_io
-                           response.read(packet_io)
-                       elsif response
-                           response
-                       else
-                           nil
-                       end
+                         elsif response.respond_to?(:read) && packet_io
+                             response.read(packet_io)
+                         elsif response
+                             response
+                         else
+                             nil
+                         end
 
                 logger.debug { "Invoking response cb=#{callback} err=#{error} resp=#{result}"  }
-                binding.invoke(callback,error,result)
+                @event_loop.invoke(callback,error,result)
             end
         end
 
@@ -420,12 +497,10 @@ module ZooKeeper
             [watch_type,watcher]
         end
 
-
         def queue_packet(packet)
+            logger.debug { "Queuing: #{packet.inspect}" }
             @pending_queue.push(packet)
-            logger.debug { "Queued: #{packet.inspect}" }
-
-            if @keeper_state == :connected
+            if @session_state == :connected
                 send_packet(packet)
             end
         end
@@ -440,6 +515,67 @@ module ZooKeeper
             conn.send_records(*records)
         end
 
+        class EventLoop 
+            include Slf4r::Logger
+
+            def initialize(client)
+                @event_queue = Strand::Queue.new()
+
+                @alive = true
+                @event_thread  = Strand.new() do
+                    logger.debug { "Starting event loop" }
+                    Strand.current[:name] = "ZooKeeper::EventLoop"
+                    Strand.current[CURRENT] = [ client ]
+                    begin
+                        pop_event_queue until dead?
+                        logger.debug { "Finished event loop" }
+                    rescue Exception => ex
+                        logger.error("Uncaught exception in event loop",ex)
+                    end
+                end
+            end
 
+            def dead?
+                !@alive
+            end
+
+            # @api async_op
+            def pop_event_queue
+                #We're alive until we get a nil result from #stop
+                queued = @event_queue.pop if @alive
+                if queued 
+                    begin
+                        callback,*args = queued
+                        callback.call(*args)
+                    rescue StandardError => ex
+                        logger.error("Uncaught error in async callback", ex)
+                    end
+                else
+                    @alive = false
+                end
+            end
+
+            # @api session
+            def invoke(*args)
+                @event_queue.push(args)
+            end
+
+            def invoke_close(callback,*args)
+                Strand.new do
+                    @event_thread.join()
+                    callback.call(*args)
+                end
+            end
+
+            # @api session
+            def stop
+                @event_queue.push(nil)
+            end
+
+            # @api async_op
+            def current?
+                Strand.current == @event_thread
+            end
+        end
     end # Session
 end