tipi 0.38 → 0.42

data/lib/tipi/digital_fabric/agent.rb

@@ -24,9 +24,11 @@ module DigitalFabric
     class GracefulShutdown < RuntimeError
     end
 
+    @@id = 0
+
     def run
       @fiber = Fiber.current
-      @keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      @keep_alive_timer = spin_loop("#{@fiber.tag}-keep_alive", interval: 5) { keep_alive }
       while true
         connect_and_process_incoming_requests
         return if @shutdown
@@ -119,7 +121,7 @@ module DigitalFabric
 
     def recv_df_message(msg)
       @last_recv = Time.now
-      case msg['kind']
+      case msg[Protocol::Attribute::KIND]
       when Protocol::SHUTDOWN
         recv_shutdown
       when Protocol::HTTP_REQUEST
@@ -130,7 +132,7 @@ module DigitalFabric
         recv_ws_request(msg)
       when Protocol::CONN_DATA, Protocol::CONN_CLOSE,
            Protocol::WS_DATA, Protocol::WS_CLOSE
-        fiber = @requests[msg['id']]
+        fiber = @requests[msg[Protocol::Attribute::ID]]
         fiber << msg if fiber
       end
     end
@@ -140,7 +142,7 @@ module DigitalFabric
       # messages. This is so we can correctly stop long-running requests
       # upon graceful shutdown
       if is_long_running_request_response?(msg)
-        id = msg[:id]
+        id = msg[Protocol::Attribute::ID]
         @long_running_requests[id] = @requests[id]
       end
       @last_send = Time.now
@@ -148,11 +150,11 @@ module DigitalFabric
     end
 
     def is_long_running_request_response?(msg)
-      case msg[:kind]
+      case msg[Protocol::Attribute::KIND]
       when Protocol::HTTP_UPGRADE
         true
       when Protocol::HTTP_RESPONSE
-        msg[:body] && !msg[:complete]
+        !msg[Protocol::Attribute::HttpResponse::COMPLETE]
       end
     end
 
@@ -165,13 +167,14 @@ module DigitalFabric
 
     def recv_http_request(msg)
       req = prepare_http_request(msg)
-      id = msg['id']
-      @requests[id] = spin do
+      id = msg[Protocol::Attribute::ID]
+      @requests[id] = spin("#{Fiber.current.tag}.#{id}") do
         http_request(req)
       rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
         # ignore
-      rescue Polyphony::Terminate
+      rescue Polyphony::Terminate => e
         req.respond(nil, { ':status' => Qeweney::Status::SERVICE_UNAVAILABLE }) if Fiber.current.graceful_shutdown?
+        raise e
       ensure
         @requests.delete(id)
         @long_running_requests.delete(id)
@@ -180,17 +183,19 @@ module DigitalFabric
     end
 
     def prepare_http_request(msg)
-      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
-      req.buffer_body_chunk(msg['body']) if msg['body']
-      req.complete! if msg['complete']
+      headers = msg[Protocol::Attribute::HttpRequest::HEADERS]
+      body_chunk = msg[Protocol::Attribute::HttpRequest::BODY_CHUNK]
+      complete = msg[Protocol::Attribute::HttpRequest::COMPLETE]
+      req = Qeweney::Request.new(headers, RequestAdapter.new(self, msg))
+      req.buffer_body_chunk(body_chunk) if body_chunk
       req
     end
 
     def recv_http_request_body(msg)
-      fiber = @requests[msg['id']]
+      fiber = @requests[msg[Protocol::Attribute::ID]]
       return unless fiber
 
-      fiber << msg['body']
+      fiber << msg[Protocol::Attribute::HttpRequestBody::BODY]
     end
 
     def get_http_request_body(id, limit)
@@ -199,9 +204,9 @@ module DigitalFabric
     end
 
     def recv_ws_request(msg)
-      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
-      id = msg['id']
-      @requests[id] = @long_running_requests[id] = spin do
+      req = Qeweney::Request.new(msg[Protocol::Attribute::WS::HEADERS], RequestAdapter.new(self, msg))
+      id = msg[Protocol::Attribute::ID]
+      @requests[id] = @long_running_requests[id] = spin("#{Fiber.current.tag}.#{id}-ws") do
         ws_request(req)
       rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
         # ignore

data/lib/tipi/digital_fabric/agent_proxy.rb

@@ -31,14 +31,15 @@ module DigitalFabric
     def run
       @fiber = Fiber.current
       @service.mount(route, self)
-      keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      @mounted = true
+      # keep_alive_timer = spin_loop("#{@fiber.tag}-keep_alive", interval: 5) { keep_alive }
       process_incoming_messages(false)
     rescue GracefulShutdown
       puts "Proxy got graceful shutdown, left: #{@requests.size} requests" if @requests.size > 0
-      process_incoming_messages(true)
+      move_on_after(15) { process_incoming_messages(true) }
     ensure
-      keep_alive_timer&.stop
-      @service.unmount(self)
+      # keep_alive_timer&.stop
+      unmount
     end
 
     def process_incoming_messages(shutdown = false)
@@ -48,10 +49,18 @@ module DigitalFabric
         recv_df_message(msg)
         return if shutdown && @requests.empty?
       end
-    rescue TimeoutError, IOError
+    rescue TimeoutError, IOError, SystemCallError
+      # ignore and just return in order to terminate the proxy
+    end
+
+    def unmount
+      return unless @mounted
+
+      @service.unmount(self) 
+      @mounted = nil
     end
 
-    def shutdown
+    def send_shutdown
       send_df_message(Protocol.shutdown)
       @fiber.raise GracefulShutdown.new
     end
@@ -82,9 +91,18 @@ module DigitalFabric
 
     def recv_df_message(message)
       @last_recv = Time.now
-      return if message['kind'] == Protocol::PING
+      # puts "<<< #{message.inspect}"
+
+      case message[Protocol::Attribute::KIND]
+      when Protocol::PING
+        return
+      when Protocol::UNMOUNT
+        return unmount
+      when Protocol::STATS_REQUEST
+        return handle_stats_request(message[Protocol::Attribute::ID])
+      end
 
-      handler = @requests[message['id']]
+      handler = @requests[message[Protocol::Attribute::ID]]
       if !handler
         # puts "Unknown request id in #{message}"
         return
@@ -94,6 +112,8 @@ module DigitalFabric
     end
 
     def send_df_message(message)
+      # puts ">>> #{message.inspect}" unless message[Protocol::Attribute::KIND] == Protocol::PING
+
       @last_send = Time.now
       @conn << message.to_msgpack
     end
@@ -124,57 +144,68 @@ module DigitalFabric
       t0 = Time.now
       t1 = nil
       with_request do |id|
-        send_df_message(Protocol.http_request(id, req))
+        msg = Protocol.http_request(id, req.headers, req.next_chunk(true), req.complete?)
+        send_df_message(msg)
         while (message = receive)
           unless t1
             t1 = Time.now
-            @service.record_latency_measurement(t1 - t0)
+            @service.record_latency_measurement(t1 - t0, req)
           end
-          return if http_request_message(id, req, message)
+          kind = message[Protocol::Attribute::KIND]
+          attributes = message[Protocol::Attribute::HttpRequest::HEADERS..-1]
+          return if http_request_message(id, req, kind, attributes)
         end
       end
     rescue => e
-      req.respond("Error: #{e.inspect}", ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
+      p "Internal server error: #{e.inspect}"
+      puts e.backtrace.join("\n")
+      http_request_send_error_response(e)
+    end
+
+    def http_request_send_error_response(error)
+      response = format("Error: %s\n%s", error.inspect, error.backtrace.join("\n"))
+      req.respond(response, ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
+    rescue IOError, SystemCallError
+      # ignore
     end
 
     # @return [Boolean] true if response is complete
-    def http_request_message(id, req, message)
-      case message['kind']
+    def http_request_message(id, req, kind, message)
+      case kind
       when Protocol::HTTP_UPGRADE
-        http_custom_upgrade(id, req, message)
+        http_custom_upgrade(id, req, *message)
         true
       when Protocol::HTTP_GET_REQUEST_BODY
-        http_get_request_body(id, req, message)
+        http_get_request_body(id, req, *message)
         false
       when Protocol::HTTP_RESPONSE
-        headers = message['headers']
-        body = message['body']
-        done = message['complete']
-        if !req.headers_sent? && done
-          req.respond(body, headers|| {})
-          true
-        else
-          req.send_headers(headers) if headers && !req.headers_sent?
-          req.send_chunk(body, done: done) if body or done
-          done
-        end
+        http_response(id, req, *message)
       else
         # invalid message
         true
       end
     end
 
+    def send_transfer_count(key, rx, tx)
+      send_df_message(Protocol.transfer_count(key, rx, tx))
+    end
+
+    def handle_stats_request(id)
+      stats = @service.get_stats
+      send_df_message(Protocol.stats_response(id, stats))
+    end
+
     HTTP_RESPONSE_UPGRADE_HEADERS = { ':status' => Qeweney::Status::SWITCHING_PROTOCOLS }
 
-    def http_custom_upgrade(id, req, message)
+    def http_custom_upgrade(id, req, headers)
       # send upgrade response
-      upgrade_headers = message['headers'] ?
-        message['headers'].merge(HTTP_RESPONSE_UPGRADE_HEADERS) :
+      upgrade_headers = headers ?
+       headers.merge(HTTP_RESPONSE_UPGRADE_HEADERS) :
         HTTP_RESPONSE_UPGRADE_HEADERS
       req.send_headers(upgrade_headers, true)
 
       conn = req.adapter.conn
-      reader = spin do
+      reader = spin("#{Fiber.current.tag}.#{id}") do
         conn.recv_loop do |data|
           send_df_message(Protocol.conn_data(id, data))
         end
@@ -187,9 +218,9 @@ module DigitalFabric
     end
 
     def http_custom_upgrade_message(conn, message)
-      case message['kind']
+      case message[Protocol::Attribute::KIND]
       when Protocol::CONN_DATA
-        conn << message['data']
+        conn << message[:Protocol::Attribute::ConnData::DATA]
         false
       when Protocol::CONN_CLOSE
         true
@@ -199,8 +230,30 @@ module DigitalFabric
       end
     end
 
-    def http_get_request_body(id, req, message)
-      case (limit = message['limit'])
+    def http_response(id, req, body, headers, complete, transfer_count_key)
+      if !req.headers_sent? && complete
+        req.respond(body, headers|| {})
+        if transfer_count_key
+          rx, tx = req.transfer_counts
+          send_transfer_count(transfer_count_key, rx, tx)
+        end
+        true
+      else
+        req.send_headers(headers) if headers && !req.headers_sent?
+        req.send_chunk(body, done: complete) if body or complete
+        
+        if complete && transfer_count_key
+          rx, tx = req.transfer_counts
+          send_transfer_count(transfer_count_key, rx, tx)
+        end
+        complete
+      end
+    rescue IOError, SystemCallError
+      # ignore error
+    end
+
+    def http_get_request_body(id, req, limit)
+      case limit
       when nil
         body = req.read
       else
@@ -230,9 +283,9 @@ module DigitalFabric
       with_request do |id|
         send_df_message(Protocol.ws_request(id, req.headers))
         response = receive
-        case response['kind']
+        case response[0]
         when Protocol::WS_RESPONSE
-          headers = response['headers'] || {} 
+          headers = response[2] || {} 
           status = headers[':status'] || Qeweney::Status::SWITCHING_PROTOCOLS
           if status != Qeweney::Status::SWITCHING_PROTOCOLS
             req.respond(nil, headers)
@@ -244,18 +297,20 @@ module DigitalFabric
           req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
         end
       end
+    rescue IOError, SystemCallError
+      # ignore
     end
 
     def run_websocket_connection(id, websocket)
-      reader = spin do
+      reader = spin("#{Fiber.current}.#{id}-ws") do
         websocket.recv_loop do |data|
           send_df_message(Protocol.ws_data(id, data))
         end
       end
       while (message = receive)
-        case message['kind']
+        case message[Protocol::Attribute::KIND]
         when Protocol::WS_DATA
-          websocket << message['data']
+          websocket << message[Protocol::Attribute::WS::DATA]
         when Protocol::WS_CLOSE
           return
         else

data/lib/tipi/digital_fabric/executive.rb

@@ -15,7 +15,7 @@ module DigitalFabric
       route[:executive] = true
       @service.mount(route, self)
       @current_request_count = 0
-      @updater = spin_loop(interval: 10) { update_service_stats }
+      # @updater = spin_loop(:executive_updater, interval: 10) { update_service_stats }
       update_service_stats
     end
 
@@ -33,9 +33,13 @@ module DigitalFabric
         req.respond(message.to_json, { 'Content-Type' => 'text.json' })
       when '/stream/stats'
         stream_stats(req)
+      when '/upload'
+        req.respond("body: #{req.read.inspect}")
       else
         req.respond('Invalid path', { ':status' => Qeweney::Status::NOT_FOUND })
       end
+    rescue => e
+      puts "Error: #{e.inspect}"
     ensure
       @current_request_count -= 1
     end
@@ -43,7 +47,7 @@ module DigitalFabric
     def stream_stats(req)
       req.send_headers({ 'Content-Type' => 'text/event-stream' })
 
-      @service.timer.every(10) do
+      every(10) do
         message = last_service_stats
         req.send_chunk(format_sse_event(message.to_json))
       end

data/lib/tipi/digital_fabric/protocol.rb

@@ -4,6 +4,7 @@ module DigitalFabric
   module Protocol
     PING = 'ping'
     SHUTDOWN = 'shutdown'
+    UNMOUNT = 'unmount'
 
     HTTP_REQUEST = 'http_request'
     HTTP_RESPONSE = 'http_response'
@@ -19,16 +20,75 @@ module DigitalFabric
     WS_DATA = 'ws_data'
     WS_CLOSE = 'ws_close'
 
+    TRANSFER_COUNT = 'transfer_count'
+
+    STATS_REQUEST = 'stats_request'
+    STATS_RESPONSE = 'stats_response'
+
     SEND_TIMEOUT = 15
     RECV_TIMEOUT = SEND_TIMEOUT + 5
 
+    module Attribute
+      KIND = 0
+      ID = 1
+
+      module HttpRequest
+        HEADERS = 2
+        BODY_CHUNK = 3
+        COMPLETE = 4
+      end
+
+      module HttpResponse
+        BODY = 2
+        HEADERS = 3
+        COMPLETE = 4
+        TRANSFER_COUNT_KEY = 5
+      end
+
+      module HttpUpgrade
+        HEADERS = 2
+      end
+
+      module HttpGetRequestBody
+        LIMIT = 2
+      end
+
+      module HttpRequestBody
+        BODY = 2
+        COMPLETE = 3
+      end
+
+      module ConnectionData
+        DATA = 2
+      end
+
+      module WS
+        HEADERS = 2
+        DATA = 2
+      end
+
+      module TransferCount
+        KEY = 1
+        RX = 2
+        TX = 3
+      end
+
+      module Stats
+        STATS = 2
+      end
+    end
+
     class << self
       def ping
-        { kind: PING }
+        [ PING ]
       end
 
       def shutdown
-        { kind: SHUTDOWN }
+        [ SHUTDOWN ]
+      end
+
+      def unmount
+        [ UNMOUNT ]
       end
 
       DF_UPGRADE_RESPONSE = <<~HTTP.gsub("\n", "\r\n")
@@ -42,48 +102,60 @@ module DigitalFabric
         DF_UPGRADE_RESPONSE
       end
 
-      def http_request(id, req)
-        { kind: HTTP_REQUEST, id: id, headers: req.headers, body: req.next_chunk, complete: req.complete? }
+      def http_request(id, headers, buffered_chunk, complete)
+        [ HTTP_REQUEST, id, headers, buffered_chunk, complete ]
       end
 
-      def http_response(id, body, headers, complete)
-        { kind: HTTP_RESPONSE, id: id, body: body, headers: headers, complete: complete }
+      def http_response(id, body, headers, complete, transfer_count_key = nil)
+        [ HTTP_RESPONSE, id, body, headers, complete, transfer_count_key ]
       end
 
       def http_upgrade(id, headers)
-        { kind: HTTP_UPGRADE, id: id }
+        [ HTTP_UPGRADE, id, headers ]
       end
 
       def http_get_request_body(id, limit = nil)
-        { kind: HTTP_GET_REQUEST_BODY, id: id, limit: limit }
+        [ HTTP_GET_REQUEST_BODY, id, limit ]
       end
 
       def http_request_body(id, body, complete)
-        { kind: HTTP_REQUEST_BODY, id: id, body: body, complete: complete }
+        [ HTTP_REQUEST_BODY, id, body, complete ]
       end
 
       def connection_data(id, data)
-        { kind: CONN_DATA, id: id, data: data }
+        [ CONN_DATA, id, data ]
       end
 
       def connection_close(id)
-        { kind: CONN_CLOSE, id: id }
+        [ CONN_CLOSE, id ]
       end
 
       def ws_request(id, headers)
-        { kind: WS_REQUEST, id: id, headers: headers }
+        [ WS_REQUEST, id, headers ]
       end
 
       def ws_response(id, headers)
-        { kind: WS_RESPONSE, id: id, headers: headers }
+        [ WS_RESPONSE, id, headers ]
       end
 
       def ws_data(id, data)
-        { id: id, kind: WS_DATA, data: data }
+        [ WS_DATA, id, data ]
       end
         
       def ws_close(id)
-        { id: id, kind: WS_CLOSE }
+        [ WS_CLOSE, id ]
+      end
+
+      def transfer_count(key, rx, tx)
+        [ TRANSFER_COUNT, key, rx, tx ]
+      end
+
+      def stats_request(id)
+        [ STATS_REQUEST, id ]
+      end
+
+      def stats_response(id, stats)
+        [ STATS_RESPONSE, id, stats ]
       end
     end
   end