tipi 0.40 → 0.45

data/lib/tipi/digital_fabric/agent_proxy.rb

@@ -32,13 +32,13 @@ module DigitalFabric
       @fiber = Fiber.current
       @service.mount(route, self)
       @mounted = true
-      keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      # keep_alive_timer = spin_loop("#{@fiber.tag}-keep_alive", interval: 5) { keep_alive }
       process_incoming_messages(false)
     rescue GracefulShutdown
       puts "Proxy got graceful shutdown, left: #{@requests.size} requests" if @requests.size > 0
-      process_incoming_messages(true)
+      move_on_after(15) { process_incoming_messages(true) }
     ensure
-      keep_alive_timer&.stop
+      # keep_alive_timer&.stop
       unmount
     end
 
@@ -56,11 +56,11 @@ module DigitalFabric
     def unmount
       return unless @mounted
 
-      @service.unmount(self) 
+      @service.unmount(self)
       @mounted = nil
     end
 
-    def shutdown
+    def send_shutdown
       send_df_message(Protocol.shutdown)
       @fiber.raise GracefulShutdown.new
     end
@@ -98,6 +98,8 @@ module DigitalFabric
         return
       when Protocol::UNMOUNT
         return unmount
+      when Protocol::STATS_REQUEST
+        return handle_stats_request(message[Protocol::Attribute::ID])
       end
 
       handler = @requests[message[Protocol::Attribute::ID]]
@@ -142,13 +144,20 @@ module DigitalFabric
       t0 = Time.now
       t1 = nil
       with_request do |id|
-        send_df_message(Protocol.http_request(id, req))
+        msg = Protocol.http_request(id, req.headers, req.next_chunk(true), req.complete?)
+        send_df_message(msg)
         while (message = receive)
+          kind = message[Protocol::Attribute::KIND]
           unless t1
             t1 = Time.now
-            @service.record_latency_measurement(t1 - t0)
+            if kind == Protocol::HTTP_RESPONSE
+              headers = message[Protocol::Attribute::HttpResponse::HEADERS]
+              status = (headers && headers[':status']) || 200
+              if status < Qeweney::Status::BAD_REQUEST
+                @service.record_latency_measurement(t1 - t0, req)
+              end
+            end
           end
-          kind = message[Protocol::Attribute::KIND]
           attributes = message[Protocol::Attribute::HttpRequest::HEADERS..-1]
           return if http_request_message(id, req, kind, attributes)
         end
@@ -187,6 +196,11 @@ module DigitalFabric
       send_df_message(Protocol.transfer_count(key, rx, tx))
     end
 
+    def handle_stats_request(id)
+      stats = @service.get_stats
+      send_df_message(Protocol.stats_response(id, stats))
+    end
+
     HTTP_RESPONSE_UPGRADE_HEADERS = { ':status' => Qeweney::Status::SWITCHING_PROTOCOLS }
 
     def http_custom_upgrade(id, req, headers)
@@ -197,7 +211,7 @@ module DigitalFabric
       req.send_headers(upgrade_headers, true)
 
       conn = req.adapter.conn
-      reader = spin do
+      reader = spin("#{Fiber.current.tag}.#{id}") do
         conn.recv_loop do |data|
           send_df_message(Protocol.conn_data(id, data))
         end
@@ -233,7 +247,7 @@ module DigitalFabric
       else
         req.send_headers(headers) if headers && !req.headers_sent?
         req.send_chunk(body, done: complete) if body or complete
-        
+
         if complete && transfer_count_key
           rx, tx = req.transfer_counts
           send_transfer_count(transfer_count_key, rx, tx)
@@ -277,7 +291,7 @@ module DigitalFabric
         response = receive
         case response[0]
         when Protocol::WS_RESPONSE
-          headers = response[2] || {} 
+          headers = response[2] || {}
           status = headers[':status'] || Qeweney::Status::SWITCHING_PROTOCOLS
           if status != Qeweney::Status::SWITCHING_PROTOCOLS
             req.respond(nil, headers)
@@ -294,7 +308,7 @@ module DigitalFabric
     end
 
     def run_websocket_connection(id, websocket)
-      reader = spin do
+      reader = spin("#{Fiber.current}.#{id}-ws") do
         websocket.recv_loop do |data|
           send_df_message(Protocol.ws_data(id, data))
         end

data/lib/tipi/digital_fabric/executive.rb

@@ -15,7 +15,7 @@ module DigitalFabric
       route[:executive] = true
       @service.mount(route, self)
       @current_request_count = 0
-      @updater = spin_loop(interval: 10) { update_service_stats }
+      # @updater = spin_loop(:executive_updater, interval: 10) { update_service_stats }
       update_service_stats
     end
 
@@ -33,9 +33,13 @@ module DigitalFabric
         req.respond(message.to_json, { 'Content-Type' => 'text.json' })
       when '/stream/stats'
         stream_stats(req)
+      when '/upload'
+        req.respond("body: #{req.read.inspect}")
       else
         req.respond('Invalid path', { ':status' => Qeweney::Status::NOT_FOUND })
       end
+    rescue => e
+      puts "Error: #{e.inspect}"
     ensure
       @current_request_count -= 1
     end
@@ -43,7 +47,7 @@ module DigitalFabric
     def stream_stats(req)
       req.send_headers({ 'Content-Type' => 'text/event-stream' })
 
-      @service.timer.every(10) do
+      every(10) do
         message = last_service_stats
         req.send_chunk(format_sse_event(message.to_json))
       end
@@ -79,7 +83,7 @@ module DigitalFabric
         raise 'Invalid output from top (cpu)'
       end
       cpu_utilization = 100 - Regexp.last_match(1).to_i
-      
+
       unless top =~ TOP_MEM_REGEXP && Regexp.last_match(1) =~ TOP_MEM_FREE_REGEXP
         raise 'Invalid output from top (mem)'
       end

data/lib/tipi/digital_fabric/protocol.rb

@@ -22,6 +22,9 @@ module DigitalFabric
 
     TRANSFER_COUNT = 'transfer_count'
 
+    STATS_REQUEST = 'stats_request'
+    STATS_RESPONSE = 'stats_response'
+
     SEND_TIMEOUT = 15
     RECV_TIMEOUT = SEND_TIMEOUT + 5
 
@@ -69,6 +72,10 @@ module DigitalFabric
         RX = 2
         TX = 3
       end
+
+      module Stats
+        STATS = 2
+      end
     end
 
     class << self
@@ -95,8 +102,8 @@ module DigitalFabric
         DF_UPGRADE_RESPONSE
       end
 
-      def http_request(id, req)
-        [ HTTP_REQUEST, id, req.headers, req.next_chunk, req.complete? ]
+      def http_request(id, headers, buffered_chunk, complete)
+        [ HTTP_REQUEST, id, headers, buffered_chunk, complete ]
       end
 
       def http_response(id, body, headers, complete, transfer_count_key = nil)
@@ -134,14 +141,22 @@ module DigitalFabric
       def ws_data(id, data)
         [ WS_DATA, id, data ]
       end
-        
+
       def ws_close(id)
-        [WS_CLOSE, id ]
+        [ WS_CLOSE, id ]
       end
 
       def transfer_count(key, rx, tx)
         [ TRANSFER_COUNT, key, rx, tx ]
       end
+
+      def stats_request(id)
+        [ STATS_REQUEST, id ]
+      end
+
+      def stats_response(id, stats)
+        [ STATS_RESPONSE, id, stats ]
+      end
     end
   end
 end

data/lib/tipi/digital_fabric/request_adapter.rb

@@ -17,10 +17,6 @@ module DigitalFabric
       @agent.get_http_request_body(@id, 1)
     end
 
-    def consume_request(request)
-      @agent.get_http_request_body(@id, nil)
-    end
-
     def respond(request, body, headers)
       @agent.send_df_message(
         Protocol.http_response(@id, body, headers, true)

data/lib/tipi/digital_fabric/service.rb

@@ -13,26 +13,22 @@ module DigitalFabric
       @token = token
       @agents = {}
       @routes = {}
-      @waiting_lists = {} # hash mapping routes to arrays of requests waiting for an agent to mount
       @counters = {
         connections: 0,
         http_requests: 0,
         errors: 0
       }
       @connection_count = 0
+      @current_request_count = 0
       @http_latency_accumulator = 0
       @http_latency_counter = 0
+      @http_latency_max = 0
       @last_counters = @counters.merge(stamp: Time.now.to_f - 1)
       @fiber = Fiber.current
-      @timer = Polyphony::Timer.new(resolution: 1)
-
-      stats_updater = spin { @timer.every(10) { update_stats } }
-      @stats = {}
-
-      @current_request_count = 0
+      # @timer = Polyphony::Timer.new('service_timer', resolution: 5)
     end
 
-    def update_stats
+    def calculate_stats
       now = Time.now.to_f
       elapsed = now - @last_counters[:stamp]
       connections = @counters[:connections] - @last_counters[:connections]
@@ -40,23 +36,65 @@ module DigitalFabric
       errors = @counters[:errors] - @last_counters[:errors]
       @last_counters = @counters.merge(stamp: now)
 
-      average_latency = @http_latency_counter > 0 ?
-                        @http_latency_accumulator / @http_latency_counter :
-                        0
+      average_latency = @http_latency_counter == 0 ? 0 :
+                        @http_latency_accumulator / @http_latency_counter
       @http_latency_accumulator = 0
       @http_latency_counter = 0
-
-      @stats = {
-        connection_rate: connections / elapsed,
-        http_request_rate: http_requests / elapsed,
-        error_rate: errors / elapsed,
-        average_latency: average_latency,
-        agent_count: @agents.size,
-        connection_count: @connection_count,
-        concurrent_requests: @current_request_count
+      max_latency = @http_latency_max
+      @http_latency_max = 0
+
+      cpu, rss = pid_cpu_and_rss(Process.pid)
+
+      backend_stats = Thread.backend.stats
+      op_rate = backend_stats[:op_count] / elapsed
+      switch_rate = backend_stats[:switch_count] / elapsed
+      poll_rate = backend_stats[:poll_count] / elapsed
+
+      object_space_stats = ObjectSpace.count_objects
+
+      {
+        service: {
+          agent_count: @agents.size,
+          connection_count: @connection_count,
+          connection_rate: connections / elapsed,
+          error_rate: errors / elapsed,
+          http_request_rate: http_requests / elapsed,
+          latency_avg: average_latency,
+          latency_max: max_latency,
+          pending_requests: @current_request_count,
+          },
+        backend: {
+          op_rate: op_rate,
+          pending_ops: backend_stats[:pending_ops],
+          poll_rate: poll_rate,
+          runqueue_size: backend_stats[:runqueue_size],
+          runqueue_high_watermark: backend_stats[:runqueue_max_length],
+          switch_rate: switch_rate,
+
+        },
+        process: {
+          cpu_usage: cpu,
+          rss: rss.to_f / 1024,
+          objects_total: object_space_stats[:TOTAL],
+          objects_free:  object_space_stats[:FREE]
+        }
       }
     end
 
+    def pid_cpu_and_rss(pid)
+      s = `ps -p #{pid} -o %cpu,rss`
+      cpu, rss = s.lines[1].chomp.strip.split(' ')
+      [cpu.to_f, rss.to_i]
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception
+      [nil, nil]
+    end
+
+    def get_stats
+      calculate_stats
+    end
+
     def incr_connection_count
       @connection_count += 1
     end
@@ -77,23 +115,25 @@ module DigitalFabric
       count
     end
 
-    def record_latency_measurement(latency)
+    def record_latency_measurement(latency, req)
       @http_latency_accumulator += latency
       @http_latency_counter += 1
+      @http_latency_max = latency if latency > @http_latency_max
+      return if latency < 1.0
+
+      puts format('slow request (%.1f): %p', latency, req.headers)
     end
-  
-    def http_request(req)
+
+    def http_request(req, allow_df_upgrade = false)
       @current_request_count += 1
       @counters[:http_requests] += 1
       @counters[:connections] += 1 if req.headers[':first']
 
-      return upgrade_request(req) if req.upgrade_protocol
- 
+      return upgrade_request(req, allow_df_upgrade) if req.upgrade_protocol
+
       inject_request_headers(req)
       agent = find_agent(req)
       unless agent
-        return req.respond('pong') if req.query[:q] == 'ping'
-
         @counters[:errors] += 1
         return req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
       end
@@ -119,11 +159,15 @@ module DigitalFabric
       req.headers['x-forwarded-for'] = conn.peeraddr(false)[2]
       req.headers['x-forwarded-proto'] ||= conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
     end
-  
-    def upgrade_request(req)
+
+    def upgrade_request(req, allow_df_upgrade)
       case (protocol = req.upgrade_protocol)
       when 'df'
-        df_upgrade(req)
+        if allow_df_upgrade
+          df_upgrade(req)
+        else
+          req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
+        end
       else
         agent = find_agent(req)
         unless agent
@@ -134,16 +178,20 @@ module DigitalFabric
         agent.http_upgrade(req, protocol)
       end
     end
-  
+
     def df_upgrade(req)
+      # we don't want to count connected agents
+      @current_request_count -= 1
       if req.headers['df-token'] != @token
         return req.respond(nil, ':status' => Qeweney::Status::FORBIDDEN)
       end
 
       req.adapter.conn << Protocol.df_upgrade_response
       AgentProxy.new(self, req)
+    ensure
+      @current_request_count += 1
     end
-  
+
     def mount(route, agent)
       if route[:path]
         route[:path_regexp] = path_regexp(route[:path])
@@ -151,13 +199,8 @@ module DigitalFabric
       @executive = agent if route[:executive]
       @agents[agent] = route
       @routing_changed = true
-
-      if (waiting = @waiting_lists[route])
-        waiting.each { |f| f.schedule(agent) }
-        @waiting_lists.delete(route)
-      end
     end
-  
+
     def unmount(agent)
       route = @agents[agent]
       return unless route
@@ -165,12 +208,10 @@ module DigitalFabric
       @executive = nil if route[:executive]
       @agents.delete(agent)
       @routing_changed = true
-
-      @waiting_lists[route] ||= []
     end
 
     INVALID_HOST = 'INVALID_HOST'
-    
+
     def find_agent(req)
       compile_agent_routes if @routing_changed
 
@@ -182,12 +223,6 @@ module DigitalFabric
       end
       return @routes[route] if route
 
-      # # search for a known route for an agent that recently unmounted
-      # route, wait_list = @waiting_lists.find do |route, _|
-      #   (host == route[:host]) || (path =~ route[:path_regexp])
-      # end
-      # return wait_for_agent(wait_list) if route
-
       nil
     end
 
@@ -202,13 +237,6 @@ module DigitalFabric
       @route_keys = @routes.keys
     end
 
-    def wait_for_agent(wait_list)
-      wait_list << Fiber.current
-      @timer.move_on_after(10) { suspend }
-    ensure
-      wait_list.delete(self)
-    end
-
     def path_regexp(path)
       /^#{path}/
     end
@@ -216,8 +244,8 @@ module DigitalFabric
     def graceful_shutdown
       @shutdown = true
       @agents.keys.each do |agent|
-        if agent.respond_to?(:shutdown)
-          agent.shutdown
+        if agent.respond_to?(:send_shutdown)
+          agent.send_shutdown
         else
           @agents.delete(agent)
         end

data/lib/tipi/handler.rb

@@ -20,14 +20,14 @@ module Tipi
     ensure
       socket.close
     end
-    
+
     ALPN_PROTOCOLS = %w[h2 http/1.1].freeze
     H2_PROTOCOL = 'h2'
 
     def protocol_adapter(socket, opts)
       use_http2 = socket.respond_to?(:alpn_protocol) &&
                   socket.alpn_protocol == H2_PROTOCOL
-      
+
       klass = use_http2 ? HTTP2Adapter : HTTP1Adapter
       klass.new(socket, opts)
     end