log-courier 1.1 → 1.2

data/lib/log-courier/server.rb

@@ -40,6 +40,7 @@ module LogCourier
       }.merge!(options)
 
       @logger = @options[:logger]
+      @logger['plugin'] = 'input/courier'
 
       case @options[:transport]
       when 'tcp', 'tls'
@@ -49,11 +50,12 @@ module LogCourier
         require 'log-courier/server_zmq'
         @server = ServerZmq.new(@options)
       else
-        raise '[LogCourierServer] \'transport\' must be tcp, tls, plainzmq or zmq'
+        fail 'input/courier: \'transport\' must be tcp, tls, plainzmq or zmq'
       end
 
-      # Grab the port back
+      # Grab the port back and update the logger context
       @port = @server.port
+      @logger['port'] = @port unless @logger.nil?
 
       # Load the json adapter
       @json_adapter = MultiJson.adapter.instance
@@ -75,7 +77,11 @@ module LogCourier
             when 'JDAT'
               process_jdat message, comm, event_queue
             else
-              @logger.warn("[LogCourierServer] Unknown message received from #{comm.peer}") unless @logger.nil?
+              if comm.peer.nil?
+                @logger.warn 'Unknown message received', :from => 'unknown' unless @logger.nil?
+              else
+                @logger.warn 'Unknown message received', :from => comm.peer unless @logger.nil?
+              end
               # Don't kill a client that sends a bad message
               # Just reject it and let it send it again, potentially to another server
               comm.send '????', ''
@@ -93,17 +99,21 @@ module LogCourier
           server_thread.join
         end
       end
+      return
     end
 
+    private
+
     def process_ping(message, comm)
       # Size of message should be 0
       if message.length != 0
-        raise ProtocolError, "unexpected data attached to ping message (#{message.length})"
+        fail ProtocolError, "unexpected data attached to ping message (#{message.length})"
       end
 
       # PONG!
       # NOTE: comm.send can raise a Timeout::Error of its own
       comm.send 'PONG', ''
+      return
     end
 
     def process_jdat(message, comm, event_queue)
@@ -114,11 +124,17 @@ module LogCourier
       # This allows the client to know what is being acknowledged
       # Nonce is 16 so check we have enough
       if message.length < 17
-        raise ProtocolError, "JDAT message too small (#{message.length})"
+        fail ProtocolError, "JDAT message too small (#{message.length})"
       end
 
       nonce = message[0...16]
 
+      if !@logger.nil? && @logger.debug?
+        nonce_str = nonce.each_byte.map do |b|
+          b.to_s(16).rjust(2, '0')
+        end
+      end
+
       # The remainder of the message is the compressed data block
       message = StringIO.new Zlib::Inflate.inflate(message[16...message.length])
 
@@ -136,7 +152,7 @@ module LogCourier
           # Finished!
           break
         elsif length_buf.length < 4
-          raise ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.length})"
+          fail ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.length})"
         end
 
         length = length_buf.unpack('N').first
@@ -145,7 +161,7 @@ module LogCourier
         ret = message.read length, data_buf
         if ret.nil? or data_buf.length < length
           @logger.warn()
-          raise ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.length}"
+          fail ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.length}"
         end
 
         data_buf.force_encoding('utf-8')
@@ -161,7 +177,7 @@ module LogCourier
         begin
           event = @json_adapter.load(data_buf, @json_options)
         rescue MultiJson::ParseError => e
-          @logger.warn("[LogCourierServer] JSON parse failure, falling back to plain-text: #{e}") unless @logger.nil?
+          @logger.warn e, :hint => 'JSON parse failure, falling back to plain-text' unless @logger.nil?
           event = { 'message' => data_buf }
         end
 
@@ -171,7 +187,7 @@ module LogCourier
         rescue TimeoutError
           # Full pipeline, partial ack
           # NOTE: comm.send can raise a Timeout::Error of its own
-          @logger.debug "[LogCourierServer] Partially acknowledging message #{nonce.hash} sequence #{sequence}" unless @logger.nil?
+          @logger.debug 'Partially acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
           comm.send 'ACKN', [nonce, sequence].pack('A*N')
           ack_timeout = Time.now.to_i + 5
           retry
@@ -182,8 +198,9 @@ module LogCourier
 
       # Acknowledge the full message
       # NOTE: comm.send can raise a Timeout::Error
-      @logger.debug "[LogCourierServer] Acknowledging message #{nonce.hash} sequence #{sequence}" unless @logger.nil?
+      @logger.debug 'Acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
       comm.send 'ACKN', [nonce, sequence].pack('A*N')
+      return
     end
   end
 end

data/lib/log-courier/server_tcp.rb

@@ -24,13 +24,25 @@ require 'thread'
 module LogCourier
   # Wrap around TCPServer to grab last error for use in reporting which peer had an error
   class ExtendedTCPServer < TCPServer
-    # Yield the peer
+    attr_reader :peer
+
+    def initialise
+      reset_peer
+      super
+    end
+
+    # Save the peer
     def accept
       sock = super
       peer = sock.peeraddr(:numeric)
-      Thread.current['LogCourierPeer'] = "#{peer[2]}:#{peer[1]}"
+      @peer = "#{peer[2]}:#{peer[1]}"
       return sock
     end
+
+    def reset_peer
+      @peer = 'unknown'
+      return
+    end
   end
 
   # TLS transport implementation for server
@@ -57,11 +69,11 @@ module LogCourier
 
       if @options[:transport] == 'tls'
         [:ssl_certificate, :ssl_key].each do |k|
-          raise "[LogCourierServer] '#{k}' is required" if @options[k].nil?
+          fail "input/courier: '#{k}' is required" if @options[k].nil?
         end
 
         if @options[:ssl_verify] and (!@options[:ssl_verify_default_ca] && @options[:ssl_verify_ca].nil?)
-          raise '[LogCourierServer] Either \'ssl_verify_default_ca\' or \'ssl_verify_ca\' must be specified when ssl_verify is true'
+          fail 'input/courier: Either \'ssl_verify_default_ca\' or \'ssl_verify_ca\' must be specified when ssl_verify is true'
         end
       end
 
@@ -94,16 +106,18 @@ module LogCourier
             ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
           end
 
+          # Create the OpenSSL server - set start_immediately to false so we can multithread handshake
           @server = OpenSSL::SSL::SSLServer.new(@tcp_server, ssl)
+          @server.start_immediately = false
         else
           @server = @tcp_server
         end
 
         if @options[:port] == 0
-          @logger.warn '[LogCourierServer] Transport ' + @options[:transport] + ' is listening on ephemeral port ' + @port.to_s unless @logger.nil?
+          @logger.warn 'Ephemeral port allocated', :transport => @options[:transport], :port => @port unless @logger.nil?
         end
       rescue => e
-        raise "[LogCourierServer] Failed to initialise: #{e}"
+        raise "input/courier: Failed to initialise: #{e}"
       end
     end # def initialize
 
@@ -111,20 +125,20 @@ module LogCourier
       client_threads = {}
 
       loop do
-        # This means ssl accepting is single-threaded.
+        # Because start_immediately is false, TCP accept is single thread but
+        # handshake is essentiall multithreaded as we defer it to the thread
+        @tcp_server.reset_peer
+        client = nil
         begin
           client = @server.accept
         rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
-          # Handshake failure or other issue
-          peer = Thread.current['LogCourierPeer'] || 'unknown'
-          @logger.warn "[LogCourierServer] Connection from #{peer} failed to initialise: #{e}" unless @logger.nil?
-          client.close rescue nil
+          # Accept failure or other issue
+          @logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil
+          client.close rescue nil unless client.nil?
           next
         end
 
-        peer = Thread.current['LogCourierPeer'] || 'unknown'
-
-    	  @logger.info "[LogCourierServer] New connection from #{peer}" unless @logger.nil?
+    	  @logger.info 'New connection', :peer => @tcp_server.peer unless @logger.nil?
 
         # Clear up finished threads
         client_threads.delete_if do |_, thr|
@@ -132,17 +146,16 @@ module LogCourier
         end
 
         # Start a new connection thread
-        client_threads[client] = Thread.new(client, peer) do |client_copy, peer_copy|
-          ConnectionTcp.new(@logger, client_copy, peer_copy, @options).run(&block)
+        client_threads[client] = Thread.new(client, @tcp_server.peer) do |client_copy, peer_copy|
+          run_thread client_copy, peer_copy, &block
         end
       end
+      return
     rescue ShutdownSignal
-      # Capture shutting down signal
-      0
+      return
     rescue StandardError, NativeException => e
       # Some other unknown problem
-      @logger.warn("[LogCourierServer] Unknown error: #{e}") unless @logger.nil?
-      @logger.warn("[LogCourierServer] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+      @logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
       raise e
     ensure
       # Raise shutdown in all client threads and join then
@@ -154,6 +167,24 @@ module LogCourier
 
       @tcp_server.close
     end
+
+    private
+
+    def run_thread(client, peer, &block)
+      # Perform the handshake inside the new thread so we don't block TCP accept
+      if @options[:transport] == 'tls'
+        begin
+          client.accept
+        rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
+          # Handshake failure or other issue
+          @logger.warn 'Connection failed to initialise', :error => e.message, :peer => peer unless @logger.nil?
+          client.close
+          return
+        end
+      end
+
+      ConnectionTcp.new(@logger, client, peer, @options).run(&block)
+    end
   end
 
   # Representation of a single connected client
@@ -179,7 +210,7 @@ module LogCourier
 
         # Sanity
         if length > @options[:max_packet_size]
-          raise ProtocolError, "packet too large (#{length} > #{@options[:max_packet_size]})"
+          fail ProtocolError, "packet too large (#{length} > #{@options[:max_packet_size]})"
         end
 
         # While we're processing, EOF is bad as it may occur during send
@@ -198,32 +229,61 @@ module LogCourier
         # If we EOF next it's a graceful close
         @in_progress = false
       end
+      return
     rescue TimeoutError
       # Timeout of the connection, we were idle too long without a ping/pong
-      @logger.warn("[LogCourierServer] Connection from #{@peer} timed out") unless @logger.nil?
+      @logger.warn 'Connection timed out', :peer => @peer unless @logger.nil?
+      return
     rescue EOFError
       if @in_progress
-        @logger.warn("[LogCourierServer] Premature connection close on connection from #{@peer}") unless @logger.nil?
+        @logger.warn 'Unexpected EOF', :peer => @peer unless @logger.nil?
       else
-        @logger.info("[LogCourierServer] Connection from #{@peer} closed") unless @logger.nil?
+        @logger.info 'Connection closed', :peer => @peer unless @logger.nil?
       end
+      return
     rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
       # Read errors, only action is to shutdown which we'll do in ensure
-      @logger.warn("[LogCourierServer] SSL error on connection from #{@peer}: #{e}") unless @logger.nil?
+      @logger.warn 'SSL error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
+      return
     rescue ProtocolError => e
       # Connection abort request due to a protocol error
-      @logger.warn("[LogCourierServer] Protocol error on connection from #{@peer}: #{e}") unless @logger.nil?
+      @logger.warn 'Protocol error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
+      return
     rescue ShutdownSignal
       # Shutting down
-      @logger.warn("[LogCourierServer] Closing connecting from #{@peer}: server shutting down") unless @logger.nil?
-    rescue => e
+      @logger.info 'Server shutting down, closing connection', :peer => @peer unless @logger.nil?
+      return
+    rescue StandardError, NativeException => e
       # Some other unknown problem
-      @logger.warn("[LogCourierServer] Unknown error on connection from #{@peer}: #{e}") unless @logger.nil?
-      @logger.warn("[LogCourierServer] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+      @logger.warn e, :hint => 'Unknown error, connection aborted', :peer => @peer unless @logger.nil?
+      return
     ensure
       @fd.close rescue nil
     end
 
+    def send(signature, message)
+      reset_timeout
+      data = signature + [message.length].pack('N') + message
+      done = 0
+      loop do
+        begin
+          written = @fd.write_nonblock(data[done...data.length])
+        rescue IO::WaitReadable
+          fail TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        rescue IO::WaitWritable
+          fail TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          retry
+        end
+        fail ProtocolError, "write failure (#{done}/#{data.length})" if written == 0
+        done += written
+        break if done >= data.length
+      end
+      return
+    end
+
+    private
+
     def recv(need)
       reset_timeout
       have = ''
@@ -231,16 +291,16 @@ module LogCourier
         begin
        	  buffer = @fd.read_nonblock need - have.length
         rescue IO::WaitReadable
-          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
+          fail TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
           retry
         rescue IO::WaitWritable
-          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
+          fail TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
           retry
         end
         if buffer.nil?
-          raise EOFError
+          fail EOFError
         elsif buffer.length == 0
-          raise ProtocolError, "read failure (#{have.length}/#{need})"
+          fail ProtocolError, "read failure (#{have.length}/#{need})"
         end
         if have.length == 0
           have = buffer
@@ -252,29 +312,10 @@ module LogCourier
       have
     end
 
-    def send(signature, message)
-      reset_timeout
-      data = signature + [message.length].pack('N') + message
-      done = 0
-      loop do
-        begin
-          written = @fd.write_nonblock(data[done...data.length])
-        rescue IO::WaitReadable
-          raise TimeoutError if IO.select([@fd], nil, [@fd], @timeout - Time.now.to_i).nil?
-          retry
-        rescue IO::WaitWritable
-          raise TimeoutError if IO.select(nil, [@fd], [@fd], @timeout - Time.now.to_i).nil?
-          retry
-        end
-        raise ProtocolError, "write failure (#{done}/#{data.length})" if written == 0
-        done += written
-        break if done >= data.length
-      end
-    end
-
     def reset_timeout
       # TODO: Make configurable
       @timeout = Time.now.to_i + 1_800
+      return
     end
   end
 end

data/lib/log-courier/server_zmq.rb

@@ -14,20 +14,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-begin
-  require 'ffi-rzmq-core'
-  require 'ffi-rzmq-core/version'
-  require 'ffi-rzmq'
-  require 'ffi-rzmq/version'
-rescue LoadError => e
-  raise "[LogCourierServer] Could not initialise: #{e}"
-end
+require 'thread'
+require 'log-courier/zmq_qpoll'
 
 module LogCourier
   # ZMQ transport implementation for the server
   class ServerZmq
     class ZMQError < StandardError; end
 
+    class << self
+      @print_zmq_versions = false
+
+      def print_zmq_versions(logger)
+        return if @print_zmq_versions || logger.nil?
+
+        libversion = LibZMQ.version
+        libversion = "#{libversion[:major]}.#{libversion[:minor]}.#{libversion[:patch]}"
+
+        logger.info 'libzmq', :version => libversion
+        logger.info 'ffi-rzmq-core', :version => LibZMQ::VERSION
+        logger.info 'ffi-rzmq', :version => ZMQ.version
+
+        @print_zmq_versions = true
+      end
+    end
+
     attr_reader :port
 
     def initialize(options = {})
@@ -38,19 +49,19 @@ module LogCourier
         address:          '0.0.0.0',
         curve_secret_key: nil,
         max_packet_size:  10_485_760,
+        peer_recv_queue:  10,
       }.merge!(options)
 
       @logger = @options[:logger]
 
-      libversion = LibZMQ.version
-      libversion = "#{libversion[:major]}.#{libversion[:minor]}.#{libversion[:patch]}"
+      self.class.print_zmq_versions @logger
 
       if @options[:transport] == 'zmq'
-        raise "[LogCourierServer] Transport 'zmq' requires libzmq version >= 4 (the current version is #{libversion})" unless LibZMQ.version4?
+        fail "input/courier: Transport 'zmq' requires libzmq version >= 4" unless LibZMQ.version4?
 
-        raise '[LogCourierServer] \'curve_secret_key\' is required' if @options[:curve_secret_key].nil?
+        fail 'input/courier: \'curve_secret_key\' is required' if @options[:curve_secret_key].nil?
 
-        raise '[LogCourierServer] \'curve_secret_key\' must be a valid 40 character Z85 encoded string' if @options[:curve_secret_key].length != 40 || !z85validate(@options[:curve_secret_key])
+        fail 'input/courier: \'curve_secret_key\' must be a valid 40 character Z85 encoded string' if @options[:curve_secret_key].length != 40 || !z85validate(@options[:curve_secret_key])
       end
 
       begin
@@ -60,128 +71,306 @@ module LogCourier
 
         if @options[:transport] == 'zmq'
           rc = @socket.setsockopt(ZMQ::CURVE_SERVER, 1)
-          raise ZMQError, 'setsockopt CURVE_SERVER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
+          fail 'setsockopt CURVE_SERVER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
 
           rc = @socket.setsockopt(ZMQ::CURVE_SECRETKEY, @options[:curve_secret_key])
-          raise ZMQError, 'setsockopt CURVE_SECRETKEY failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
+          fail 'setsockopt CURVE_SECRETKEY failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
         end
 
         bind = 'tcp://' + @options[:address] + (@options[:port] == 0 ? ':*' : ':' + @options[:port].to_s)
         rc = @socket.bind(bind)
-        raise ZMQError, 'failed to bind at ' + bind + ': ' + rZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
+        fail 'failed to bind at ' + bind + ': ' + rZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
 
         # Lookup port number that was allocated in case it was set to 0
         endpoint = ''
         rc = @socket.getsockopt(ZMQ::LAST_ENDPOINT, endpoint)
-        raise ZMQError, 'getsockopt LAST_ENDPOINT failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc) && %r{\Atcp://(?:.*):(?<endpoint_port>\d+)\0\z} =~ endpoint
+        fail 'getsockopt LAST_ENDPOINT failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc) && %r{\Atcp://(?:.*):(?<endpoint_port>\d+)\0\z} =~ endpoint
         @port = endpoint_port.to_i
 
-        @poller = ZMQ::Poller.new
-
         if @options[:port] == 0
-          @logger.warn '[LogCourierServer] Transport ' + @options[:transport] + ' is listening on ephemeral port ' + @port.to_s unless @logger.nil?
+          @logger.warn 'Ephemeral port allocated', :transport => @options[:transport], :port => @port unless @logger.nil?
         end
       rescue => e
-        raise "[LogCourierServer] Failed to initialise: #{e}"
+        raise "input/courier: Failed to initialise: #{e}"
       end
 
-      @logger.info "[LogCourierServer] libzmq version #{libversion}" unless @logger.nil?
-      @logger.info "[LogCourierServer] ffi-rzmq-core version #{LibZMQ::VERSION}" unless @logger.nil?
-      @logger.info "[LogCourierServer] ffi-rzmq version #{ZMQ.version}" unless @logger.nil?
-
       # TODO: Implement workers option by receiving on a ROUTER and proxying to a DEALER, with workers connecting to the DEALER
 
-      @return_route = []
+      # TODO: Make this send queue configurable?
+      @send_queue = EventQueue.new 2
+      @factory = ClientFactoryZmq.new(@options, @send_queue)
+
+      # Setup poller
+      @poller = ZMQPoll::ZMQPoll.new(@context)
+      @poller.register_socket @socket, ZMQ::POLLIN
+      @poller.register_queue_to_socket @send_queue, @socket
+
+      # Register a finaliser that sets @context to nil
+      # This allows us to detect the JRuby bug where during "exit!" finalisers
+      # are run but threads are not killed - which leaves us in a situation of
+      # a terminated @context (it has a terminate finalizer) and an IO thread
+      # looping retries
+      # JRuby will still crash and burn, but at least we don't spam STDOUT with
+      # errors
+      ObjectSpace.define_finalizer(self, Proc.new do
+        @context = nil
+      end)
+    end
+
+    def run(&block)
+      loop do
+        begin
+          @poller.poll(5_000) do |socket, r, w|
+            next if socket != @socket
+            next if !r
 
-      reset_timeout
+            receive &block
+          end
+        rescue ZMQPoll::ZMQError => e
+          # Detect JRuby bug
+          fail e if @context.nil?
+          @logger.warn e, :hint => 'ZMQ recv_string failure' unless @logger.nil?
+          next
+        rescue ZMQPoll::TimeoutError
+          # We'll let ZeroMQ manage reconnections and new connections
+          # There is no point in us doing any form of reconnect ourselves
+          next
+        end
+      end
+      return
+    rescue ShutdownSignal
+      # Shutting down
+      @logger.warn 'Server shutting down' unless @logger.nil?
+      return
+    rescue StandardError, NativeException => e
+      # Some other unknown problem
+      @logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
+      raise e
+    ensure
+      @poller.shutdown
+      @factory.shutdown
+      @socket.close
+      @context.terminate
     end
 
+    private
+
     def z85validate(z85)
       # ffi-rzmq does not implement decode - but we want to validate during startup
       decoded = FFI::MemoryPointer.from_string(' ' * (8 * z85.length / 10))
       ret = LibZMQ.zmq_z85_decode decoded, z85
       return false if ret.nil?
-
       true
     end
 
-    def run(&block)
-      loop do
-        begin
-          begin
-            # Try to receive a message
-            reset_timeout
-            data = []
-            rc = @socket.recv_strings(data, ZMQ::DONTWAIT)
-            unless ZMQ::Util.resultcode_ok?(rc)
-              raise ZMQError, 'recv_string error: ' + ZMQ::Util.error_string if ZMQ::Util.errno != ZMQ::EAGAIN
-
-              # Wait for a message to arrive, handling timeouts
-              @poller.deregister @socket, ZMQ::POLLIN | ZMQ::POLLOUT
-              @poller.register @socket, ZMQ::POLLIN
-              while @poller.poll(1_000) == 0
-                # Using this inner while triggers pollThreadEvents in JRuby which checks for Thread.raise immediately
-                raise TimeoutError while Time.now.to_i >= @timeout
-              end
-              next
-            end
-          rescue ZMQError => e
-            @logger.warn "[LogCourierServer] ZMQ recv_string failed: #{e}" unless @logger.nil?
-            next
-          end
+    def receive(&block)
+      # Try to receive a message
+      data = []
+      rc = @socket.recv_strings(data, ZMQ::DONTWAIT)
+      unless ZMQ::Util.resultcode_ok?(rc)
+        fail ZMQError, 'recv_string error: ' + ZMQ::Util.error_string if ZMQ::Util.errno != ZMQ::EAGAIN
+      end
 
-          # Save the routing information that appears before the null messages
-          @return_route = []
-          @return_route.push data.shift until data.length == 0 || data[0] == ''
+      # Save the source information that appears before the null messages
+      source = []
+      source.push data.shift until data.length == 0 || data[0] == ''
 
-          if data.length == 0
-            @logger.warn '[LogCourierServer] Invalid message: no data' unless @logger.nil?
-            next
-          elsif data.length == 1
-            @logger.warn '[LogCourierServer] Invalid message: empty data' unless @logger.nil?
-            next
+      if data.length == 0
+        @logger.warn 'Invalid message: no data', :source_length => source.length unless @logger.nil?
+        return
+      elsif data.length == 1
+        @logger.warn 'Invalid message: empty data', :source_length => source.length unless @logger.nil?
+        return
+      end
+
+      # Drop the null message separator
+      data.shift
+
+      if data.length != 1
+        @logger.warn 'Invalid message: multipart unexpected', :source_length => source.length, :data_length => data.length unless @logger.nil?
+        if !@logger.nil? && @logger.debug?
+          i = 0
+          parts = {}
+          data.each do |msg|
+            i += 1
+            parts[i] = "#{part.length}:[#{msg[0..31].gsub(/[^[:print:]]/, '.')}]"
           end
+          @logger.debug 'Data', parts
+        end
+        return
+      end
+
+      @factory.deliver source, data.first, &block
+      return
+    end
+  end
+
+  class ClientFactoryZmq
+    attr_reader :options
+    attr_reader :send_queue
+
+    def initialize(options, send_queue)
+      @options = options
+      @logger = @options[:logger]
+
+      @send_queue = send_queue
+      @index = {}
+      @client_threads = {}
+      @mutex = Mutex.new
+    end
+
+    def shutdown
+      # Stop other threads from try_drop collisions
+      client_threads = @mutex.synchronize do
+        client_threads = @client_threads
+        @client_threads = {}
+        client_threads
+      end
+
+      client_threads.each_value do |thr|
+        thr.raise ShutdownSignal
+      end
+
+      client_threads.each_value(&:join)
+      return
+    end
 
-          # Drop the null message separator
-          data.shift
-
-          if data.length != 1
-            @logger.warn "[LogCourierServer] Invalid message: multipart unexpected (#{data.length})" unless @logger.nil?
-            if !@logger.nil? && @logger.debug?
-              i = 0
-              data.each do |msg|
-                i += 1
-                part = msg[0..31].gsub(/[^[:print:]]/, '.')
-                @logger.debug "[LogCourierServer] Part #{i}: #{part.length}:[#{part}]"
-              end
+    def deliver(source, data, &block)
+      # Find the handling thread
+      # We separate each source into threads so that each thread can respond
+      # with partial ACKs if we hit a slow down
+      # If we processed in a single thread, we'd only be able to respond to
+      # a single client with partial ACKs
+      @mutex.synchronize do
+        index = @index
+        source.each do |identity|
+          index[identity] = {} if !index.key?(identity)
+          index = index[identity]
+        end
+
+        if !index.key?('')
+          source_str = source.map do |s|
+            s.each_byte.map do |b|
+              b.to_s(16).rjust(2, '0')
             end
-          else
-            recv(data.first, &block)
+          end.join
+
+          @logger.info 'New source', :source => source_str unless @logger.nil?
+
+          # Create the client and associated thread
+          client = ClientZmq.new(self, source, source_str) do
+            try_drop(source)
           end
+
+          thread = Thread.new do
+            client.run &block
+          end
+
+          @client_threads[thread] = thread
+
+          index[''] = {
+            'client' => client,
+            'thread' => thread,
+          }
+        end
+
+        # Existing thread, throw on the queue, if not enough room drop the message
+        index['']['client'].push data, 0
+      end
+      return
+    end
+
+    private
+
+    def try_drop(source, source_str)
+      # This is called when a client goes idle, to cleanup resources
+      # We may tie this into zmq monitor
+      @mutex.synchronize do
+        index = @index
+        parents = []
+        source.each do |identity|
+          if !index.key?(identity)
+            @logger.warn 'Unknown idle source failed to shutdown', :source => source_str unless @logger.nil?
+            break
+          end
+          parents.push [index, identity]
+          index = index[identity]
+        end
+
+        if !index.key?('')
+          @logger.warn 'Unknown idle source failed to shutdown', :source => source_str unless @logger.nil?
+          break
+        end
+
+        # Don't allow drop if we have messages in the queue
+        if index['']['client'].length != 0
+          @logger.warn 'Failed idle source shutdown as message queue is not empty', :source => source_str unless @logger.nil?
+          return false
+        end
+
+        @logger.info 'Idle source shutting down', :source => source_str unless @logger.nil?
+
+        # Delete the entry
+        @client_threads.delete(index['']['thread'])
+        index.delete('')
+
+        # Cleanup orphaned leafs
+        parents.reverse_each do |path|
+          path[0].delete(path[1]) if path[0][path[1]].length == 0
+        end
+      end
+
+      return true
+    end
+  end
+
+  class ClientZmq < EventQueue
+    def initialize(factory, source, source_str, &try_drop)
+      @factory = factory
+      @logger = @factory.options[:logger]
+      @send_queue = @factory.send_queue
+      @source = source
+      @source_str = source_str
+      @try_drop = try_drop
+
+      # Setup the queue for receiving events to process
+      super @factory.options[:peer_recv_queue]
+    end
+
+    def run(&block)
+      loop do
+        begin
+          # TODO: Make timeout configurable?
+          data = self.pop(30)
+          recv(data, &block)
         rescue TimeoutError
-          # We'll let ZeroMQ manage reconnections and new connections
-          # There is no point in us doing any form of reconnect ourselves
-          # We will keep this timeout in however, for shutdown checks
-          reset_timeout
-          next
+          # Try to clean up resources - if we fail, new messages have arrived
+          retry if !@try_drop.call(@source)
+          break
         end
       end
+      return
     rescue ShutdownSignal
       # Shutting down
-      @logger.warn('[LogCourierServer] Server shutting down') unless @logger.nil?
+      @logger.info 'Source shutting down', :source => @source_str unless @logger.nil?
+      return
     rescue StandardError, NativeException => e
       # Some other unknown problem
-      @logger.warn("[LogCourierServer] Unknown error: #{e}") unless @logger.nil?
-      @logger.warn("[LogCourierServer] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
+      @logger.warn e, :hint => 'Unknown error, connection aborted', :source => @source_str unless @logger.nil?
       raise e
-    ensure
-      @socket.close
-      @context.terminate
     end
 
+    def send(signature, message)
+      data = signature + [message.length].pack('N') + message
+      @send_queue.push @source + ['', data]
+      return
+    end
+
+    private
+
     def recv(data)
       if data.length < 8
-        @logger.warn '[LogCourierServer] Invalid message: not enough data' unless @logger.nil?
+        @logger.warn 'Invalid message: not enough data', :data_length => data.length, :source => @source_str unless @logger.nil?
         return
       end
 
@@ -190,52 +379,16 @@ module LogCourier
 
       # Verify length
       if data.length - 8 != length
-        @logger.warn "[LogCourierServer] Invalid message: data has invalid length (#{data.length - 8} != #{length})" unless @logger.nil?
+        @logger.warn 'Invalid message: data has invalid length', :data_length => data.length - 8, :encoded_length => length, :source => @source_str unless @logger.nil?
         return
-      elsif length > @options[:max_packet_size]
-        @logger.warn "[LogCourierServer] Invalid message: packet too large (#{length} > #{@options[:max_packet_size]})" unless @logger.nil?
+      elsif length > @factory.options[:max_packet_size]
+        @logger.warn 'Invalid message: packet too large', :size => length, :max_packet_size => @options[:max_packet_size], :source => @source_str unless @logger.nil?
         return
       end
 
       # Yield the parts
       yield signature, data[8, length], self
-    end
-
-    def send(signature, message)
-      data = signature + [message.length].pack('N') + message
-
-      # Send the return route and then the message
-      reset_timeout
-      @return_route.each do |msg|
-        send_with_poll msg, true
-      end
-      send_with_poll '', true
-      send_with_poll data
-    end
-
-    def send_with_poll(data, more = false)
-      loop do
-        # Try to send a message but never block
-        rc = @socket.send_string(data, (more ? ZMQ::SNDMORE : 0) | ZMQ::DONTWAIT)
-        break if ZMQ::Util.resultcode_ok?(rc)
-        if ZMQ::Util.errno != ZMQ::EAGAIN
-          @logger.warn "[LogCourierServer] Message send failed: #{ZMQ::Util.error_string}" unless @logger.nil?
-          raise TimeoutError
-        end
-
-        # Wait for send to become available, handling timeouts
-        @poller.deregister @socket, ZMQ::POLLIN | ZMQ::POLLOUT
-        @poller.register @socket, ZMQ::POLLOUT
-        while @poller.poll(1_000) == 0
-          # Using this inner while triggers pollThreadEvents in JRuby which checks for Thread.raise immediately
-          raise TimeoutError while Time.now.to_i >= @timeout
-        end
-      end
-    end
-
-    def reset_timeout
-      # TODO: Make configurable?
-      @timeout = Time.now.to_i + 1_800
+      return
     end
   end
 end