tipi 0.31 → 0.36

data/lib/tipi/digital_fabric/agent_proxy.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require_relative './protocol'
+require 'msgpack'
+require 'tipi/websocket'
+
+module DigitalFabric
+  class AgentProxy
+    def initialize(service, req)
+      @service = service
+      @req = req
+      @conn = req.adapter.conn
+      @msgpack_reader = MessagePack::Unpacker.new
+      @requests = {}
+      @current_request_count = 0
+      @last_request_id = 0
+      @last_recv = @last_send = Time.now
+      run
+    end
+
+    def current_request_count
+      @current_request_count
+    end
+
+    class TimeoutError < RuntimeError
+    end
+
+    class GracefulShutdown < RuntimeError
+    end
+
+    def run
+      @fiber = Fiber.current
+      @service.mount(route, self)
+      keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      process_incoming_messages(false)
+    rescue GracefulShutdown
+      puts "Proxy got graceful shutdown, left: #{@requests.size} requests" if @requests.size > 0
+      process_incoming_messages(true)
+    ensure
+      keep_alive_timer&.stop
+      @service.unmount(self)
+    end
+
+    def process_incoming_messages(shutdown = false)
+      return if shutdown && @requests.empty?
+
+      @conn.feed_loop(@msgpack_reader, :feed_each) do |msg|
+        recv_df_message(msg)
+        return if shutdown && @requests.empty?
+      end
+    rescue TimeoutError, IOError
+    end
+
+    def shutdown
+      send_df_message(Protocol.shutdown)
+      @fiber.raise GracefulShutdown.new
+    end
+
+    def keep_alive
+      now = Time.now
+      if now - @last_send >= Protocol::SEND_TIMEOUT
+        send_df_message(Protocol.ping)
+      end
+      # if now - @last_recv >= Protocol::RECV_TIMEOUT
+      #   raise TimeoutError
+      # end
+    rescue TimeoutError, IOError
+    end
+
+    def route
+      case @req.headers['df-mount']
+      when /^\s*host\s*=\s*([^\s]+)/
+        { host: Regexp.last_match(1) }
+      when /^\s*path\s*=\s*([^\s]+)/
+        { path: Regexp.last_match(1) }
+      when /catch_all/
+        { catch_all: true }
+      else
+        nil
+      end
+    end
+
+    def recv_df_message(message)
+      @last_recv = Time.now
+      return if message['kind'] == Protocol::PING
+
+      handler = @requests[message['id']]
+      if !handler
+        # puts "Unknown request id in #{message}"
+        return
+      end
+
+      handler << message
+    end
+
+    def send_df_message(message)
+      @last_send = Time.now
+      @conn << message.to_msgpack
+    end
+
+    # HTTP / WebSocket
+
+    def register_request_fiber
+      id = (@last_request_id += 1)
+      @requests[id] = Fiber.current
+      id
+    end
+
+    def unregister_request_fiber(id)
+      @requests.delete(id)
+    end
+
+    def with_request
+      @current_request_count += 1
+      id = (@last_request_id += 1)
+      @requests[id] = Fiber.current
+      yield id
+    ensure
+      @current_request_count -= 1
+      @requests.delete(id)
+    end
+
+    def http_request(req)
+      t0 = Time.now
+      t1 = nil
+      with_request do |id|
+        send_df_message(Protocol.http_request(id, req))
+        while (message = receive)
+          unless t1
+            t1 = Time.now
+            @service.record_latency_measurement(t1 - t0)
+          end
+          return if http_request_message(id, req, message)
+        end
+      end
+    rescue => e
+      req.respond("Error: #{e.inspect}", ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
+    end
+
+    # @return [Boolean] true if response is complete
+    def http_request_message(id, req, message)
+      case message['kind']
+      when Protocol::HTTP_UPGRADE
+        http_custom_upgrade(id, req, message)
+        true
+      when Protocol::HTTP_GET_REQUEST_BODY
+        http_get_request_body(id, req, message)
+        false
+      when Protocol::HTTP_RESPONSE
+        headers = message['headers']
+        body = message['body']
+        done = message['complete']
+        req.send_headers(headers) if headers && !req.headers_sent?
+        req.send_chunk(body, done: done) if body or done
+        done
+      else
+        # invalid message
+        true
+      end
+    end
+
+    HTTP_RESPONSE_UPGRADE_HEADERS = { ':status' => Qeweney::Status::SWITCHING_PROTOCOLS }
+
+    def http_custom_upgrade(id, req, message)
+      # send upgrade response
+      upgrade_headers = message['headers'] ?
+        message['headers'].merge(HTTP_RESPONSE_UPGRADE_HEADERS) :
+        HTTP_RESPONSE_UPGRADE_HEADERS
+      req.send_headers(upgrade_headers, true)
+
+      conn = req.adapter.conn
+      reader = spin do
+        conn.recv_loop do |data|
+          send_df_message(Protocol.conn_data(id, data))
+        end
+      end
+      while (message = receive)
+        return if http_custom_upgrade_message(conn, message)
+      end
+    ensure
+      reader.stop
+    end
+
+    def http_custom_upgrade_message(conn, message)
+      case message['kind']
+      when Protocol::CONN_DATA
+        conn << message['data']
+        false
+      when Protocol::CONN_CLOSE
+        true
+      else
+        # invalid message
+        true
+      end
+    end
+
+    def http_get_request_body(id, req, message)
+      case (limit = message['limit'])
+      when nil
+        body = req.read
+      else
+        limit = limit.to_i
+        body = nil
+        req.each_chunk do |chunk|
+          (body ||= +'') << chunk
+          break if body.bytesize >= limit
+        end
+      end
+      send_df_message(Protocol.http_request_body(id, body, req.complete?))
+    end
+
+    def http_upgrade(req, protocol)
+      if protocol == 'websocket'
+        handle_websocket_upgrade(req)
+      else
+        # other protocol upgrades should be handled by the agent, so we just run
+        # the request normally. The agent is expected to upgrade the connection
+        # using a http_upgrade message. From that moment on, two-way
+        # communication is handled using conn_data and conn_close messages.
+        http_request(req)
+      end
+    end
+
+    def handle_websocket_upgrade(req)
+      with_request do |id|
+        send_df_message(Protocol.ws_request(id, req.headers))
+        response = receive
+        case response['kind']
+        when Protocol::WS_RESPONSE
+          headers = response['headers'] || {} 
+          status = headers[':status'] || Qeweney::Status::SWITCHING_PROTOCOLS
+          if status != Qeweney::Status::SWITCHING_PROTOCOLS
+            req.respond(nil, headers)
+            return
+          end
+          ws = Tipi::Websocket.new(req.adapter.conn, req.headers)
+          run_websocket_connection(id, ws)
+        else
+          req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
+        end
+      end
+    end
+
+    def run_websocket_connection(id, websocket)
+      reader = spin do
+        websocket.recv_loop do |data|
+          send_df_message(Protocol.ws_data(id, data))
+        end
+      end
+      while (message = receive)
+        case message['kind']
+        when Protocol::WS_DATA
+          websocket << message['data']
+        when Protocol::WS_CLOSE
+          return
+        else
+          raise "Unexpected websocket message #{message.inspect}"
+        end
+      end
+    ensure
+      reader.stop
+      websocket.close
+    end
+  end
+end

data/lib/tipi/digital_fabric/executive.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'tipi/digital_fabric'
+require 'json'
+
+module DigitalFabric
+  # agent for managing DF service
+  class Executive
+    INDEX_HTML = IO.read(File.join(__dir__, 'executive/index.html'))
+
+    attr_reader :last_service_stats
+
+    def initialize(service, route = { path: '/executive' })
+      @service = service
+      route[:executive] = true
+      @service.mount(route, self)
+      @current_request_count = 0
+      @updater = spin_loop(interval: 10) { update_service_stats }
+      update_service_stats
+    end
+
+    def current_request_count
+      @current_request_count
+    end
+
+    def http_request(req)
+      @current_request_count += 1
+      case req.path
+      when '/'
+        req.respond(INDEX_HTML, 'Content-Type' => 'text/html')
+      when '/stats'
+        message = last_service_stats
+        req.respond(message.to_json, { 'Content-Type' => 'text.json' })
+      when '/stream/stats'
+        stream_stats(req)
+      else
+        req.respond('Invalid path', { ':status' => Qeweney::Status::NOT_FOUND })
+      end
+    ensure
+      @current_request_count -= 1
+    end
+
+    def stream_stats(req)
+      req.send_headers({ 'Content-Type' => 'text/event-stream' })
+
+      @service.timer.every(10) do
+        message = last_service_stats
+        req.send_chunk(format_sse_event(message.to_json))
+      end
+    rescue IOError, SystemCallError
+      # ignore
+    ensure
+      req.send_chunk("retry: 0\n\n", true) rescue nil
+    end
+
+    def format_sse_event(data)
+      "data: #{data}\n\n"
+    end
+
+    def update_service_stats
+      @last_service_stats = {
+        service: @service.stats,
+        machine: machine_stats
+      }
+    end
+
+    TOP_CPU_REGEXP = /%Cpu(.+)/.freeze
+    TOP_CPU_IDLE_REGEXP = /([\d\.]+) id/.freeze
+    TOP_MEM_REGEXP = /MiB Mem(.+)/.freeze
+    TOP_MEM_FREE_REGEXP = /([\d\.]+) free/.freeze
+    LOADAVG_REGEXP = /^([\d\.]+)/.freeze
+
+    def machine_stats
+      top = `top -bn1 | head -n4`
+      unless top =~ TOP_CPU_REGEXP && Regexp.last_match(1) =~ TOP_CPU_IDLE_REGEXP
+        p top =~ TOP_CPU_REGEXP
+        p Regexp.last_match(1)
+        p Regexp.last_match(1) =~ TOP_CPU_IDLE_REGEXP
+        raise 'Invalid output from top (cpu)'
+      end
+      cpu_utilization = 100 - Regexp.last_match(1).to_i
+      
+      unless top =~ TOP_MEM_REGEXP && Regexp.last_match(1) =~ TOP_MEM_FREE_REGEXP
+        raise 'Invalid output from top (mem)'
+      end
+
+      mem_free = Regexp.last_match(1).to_f
+
+      stats = `cat /proc/loadavg`
+      raise 'Invalid output from /proc/loadavg' unless stats =~ LOADAVG_REGEXP
+      load_avg = Regexp.last_match(1).to_f
+
+      {
+        mem_free: mem_free,
+        cpu_utilization: cpu_utilization,
+        load_avg: load_avg
+      }
+    end
+  end
+end

data/lib/tipi/digital_fabric/executive/index.html

@@ -0,0 +1,69 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <title>Digital Fabric Executive</title>
+  <style>
+    .hidden { display: none }
+  </style>
+</head>
+<body>
+  <h1>Digital Fabric Executive</h1>
+  <script>
+    function updateStats(update) {
+      for (let k in update.service) {
+        let v = update.service[k];
+        let e = document.querySelector('#' + k);
+        if (e) e.innerText = v;
+      }
+      for (let k in update.machine) {
+        let v = update.machine[k];
+        let e = document.querySelector('#' + k);
+        if (e) e.innerText = v;
+      }
+    }
+
+    function connect() {
+      console.log("connecting...");
+      window.eventSource = new EventSource("/stream/stats");
+
+      window.eventSource.onopen = function(e) {
+        console.log("connected");
+        document.querySelector('#status').innerText = 'connected';
+        document.querySelector('#stats').className = '';
+        return false;
+      }
+
+      window.eventSource.onmessage = function(e) {
+        console.log("message", e.data);
+        updateStats(JSON.parse(e.data));
+      }
+
+      window.eventSource.onerror = function(e) {
+        console.log("error", e);
+        document.querySelector('#status').innerText = 'disconnected';
+        document.querySelector('#stats').className = 'hidden';
+        window.eventSource.close();
+        window.eventSource = null;
+        setTimeout(connect, 5000);
+      }
+    };
+
+    window.onload = connect;
+  </script>
+  <h2 id="status"></h2>
+  <div id="stats" class="hidden">
+    <h2>Service</h2>
+    <p>Request rate: <span id="http_request_rate"></span></p>
+    <p>Error rate: <span id="error_rate"></span></p>
+    <p>Average Latency: <span id="average_latency"></span>s</p>
+    <p>Connected agents: <span id="agent_count"></span></p>
+    <p>Connected clients: <span id="connection_count"></span></p>
+    <p>Concurrent requests: <span id="concurrent_requests"></span></p>
+
+    <h2>Machine</h2>
+    <p>CPU utilization: <span id="cpu_utilization"></span>%</p>
+    <p>Free memory: <span id="mem_free"></span>MB</p>
+    <p>Load average: <span id="load_avg"></span></p>
+  </div>
+</body>
+</html>

data/lib/tipi/digital_fabric/protocol.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module DigitalFabric
+  module Protocol
+    PING = 'ping'
+    SHUTDOWN = 'shutdown'
+
+    HTTP_REQUEST = 'http_request'
+    HTTP_RESPONSE = 'http_response'
+    HTTP_UPGRADE = 'http_upgrade'
+    HTTP_GET_REQUEST_BODY = 'http_get_request_body'
+    HTTP_REQUEST_BODY = 'http_request_body'
+
+    CONN_DATA = 'conn_data'
+    CONN_CLOSE = 'conn_close'
+
+    WS_REQUEST = 'ws_request'
+    WS_RESPONSE = 'ws_response'
+    WS_DATA = 'ws_data'
+    WS_CLOSE = 'ws_close'
+
+    SEND_TIMEOUT = 15
+    RECV_TIMEOUT = SEND_TIMEOUT + 5
+
+    class << self
+      def ping
+        { kind: PING }
+      end
+
+      def shutdown
+        { kind: SHUTDOWN }
+      end
+
+      DF_UPGRADE_RESPONSE = <<~HTTP.gsub("\n", "\r\n")
+        HTTP/1.1 101 Switching Protocols
+        Upgrade: df
+        Connection: Upgrade
+
+      HTTP
+
+      def df_upgrade_response
+        DF_UPGRADE_RESPONSE
+      end
+
+      def http_request(id, req)
+        { kind: HTTP_REQUEST, id: id, headers: req.headers, body: req.next_chunk, complete: req.complete? }
+      end
+
+      def http_response(id, body, headers, complete)
+        { kind: HTTP_RESPONSE, id: id, body: body, headers: headers, complete: complete }
+      end
+
+      def http_upgrade(id, headers)
+        { kind: HTTP_UPGRADE, id: id }
+      end
+
+      def http_get_request_body(id, limit = nil)
+        { kind: HTTP_GET_REQUEST_BODY, id: id, limit: limit }
+      end
+
+      def http_request_body(id, body, complete)
+        { kind: HTTP_REQUEST_BODY, id: id, body: body, complete: complete }
+      end
+
+      def connection_data(id, data)
+        { kind: CONN_DATA, id: id, data: data }
+      end
+
+      def connection_close(id)
+        { kind: CONN_CLOSE, id: id }
+      end
+
+      def ws_request(id, headers)
+        { kind: WS_REQUEST, id: id, headers: headers }
+      end
+
+      def ws_response(id, headers)
+        { kind: WS_RESPONSE, id: id, headers: headers }
+      end
+
+      def ws_data(id, data)
+        { id: id, kind: WS_DATA, data: data }
+      end
+        
+      def ws_close(id)
+        { id: id, kind: WS_CLOSE }
+      end
+    end
+  end
+end