tipi 0.36 → 0.39

data/lib/tipi/digital_fabric/request_adapter.rb

@@ -6,40 +6,40 @@ module DigitalFabric
   class RequestAdapter
     def initialize(agent, msg)
       @agent = agent
-      @id = msg['id']
+      @id = msg[Protocol::Attribute::ID]
     end
 
     def protocol
       'df'
     end
 
-    def get_body_chunk
+    def get_body_chunk(request)
       @agent.get_http_request_body(@id, 1)
     end
 
-    def consume_request
+    def consume_request(request)
       @agent.get_http_request_body(@id, nil)
     end
 
-    def respond(body, headers)
+    def respond(request, body, headers)
       @agent.send_df_message(
         Protocol.http_response(@id, body, headers, true)
       )
     end
 
-    def send_headers(headers, opts = {})
+    def send_headers(request, headers, opts = {})
       @agent.send_df_message(
         Protocol.http_response(@id, nil, headers, false)
       )
   end
 
-    def send_chunk(body, done: )
+    def send_chunk(request, body, done: )
       @agent.send_df_message(
         Protocol.http_response(@id, body, nil, done)
       )
     end
 
-    def finish
+    def finish(request)
       @agent.send_df_message(
         Protocol.http_response(@id, nil, nil, true)
       )

data/lib/tipi/digital_fabric/service.rb

@@ -99,10 +99,12 @@ module DigitalFabric
       end
 
       agent.http_request(req)
-    rescue IOError, SystemCallError
+    rescue IOError, SystemCallError, HTTP2::Error::StreamClosed
       @counters[:errors] += 1
     rescue => e
       @counters[:errors] += 1
+      puts '*' * 40
+      p req
       p e
       puts e.backtrace.join("\n")
       req.respond(e.inspect, ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
@@ -115,7 +117,7 @@ module DigitalFabric
       req.headers['x-request-id'] = SecureRandom.uuid
       conn = req.adapter.conn
       req.headers['x-forwarded-for'] = conn.peeraddr(false)[2]
-      req.headers['x-forwarded-proto'] = conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
+      req.headers['x-forwarded-proto'] ||= conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
     end
   
     def upgrade_request(req)
@@ -172,7 +174,7 @@ module DigitalFabric
     def find_agent(req)
       compile_agent_routes if @routing_changed
 
-      host = req.headers['host'] || INVALID_HOST
+      host = req.headers[':authority'] || req.headers['host'] || INVALID_HOST
       path = req.headers[':path']
 
       route = @route_keys.find do |route|
@@ -180,11 +182,11 @@ module DigitalFabric
       end
       return @routes[route] if route
 
-      # search for a known route for an agent that recently unmounted
-      route, wait_list = @waiting_lists.find do |route, _|
-        (host == route[:host]) || (path =~ route[:path_regexp])
-      end
-      return wait_for_agent(wait_list) if route
+      # # search for a known route for an agent that recently unmounted
+      # route, wait_list = @waiting_lists.find do |route, _|
+      #   (host == route[:host]) || (path =~ route[:path_regexp])
+      # end
+      # return wait_for_agent(wait_list) if route
 
       nil
     end

data/lib/tipi/http1_adapter.rb

@@ -29,8 +29,10 @@ module Tipi
     
     # return [Boolean] true if client loop should stop
     def handle_incoming_data(data, &block)
+      rx = data.bytesize
       @parser << data
       while (request = @requests_head)
+        request.headers[':rx'] = rx
         if @first
           request.headers[':first'] = true
           @first = nil
@@ -53,10 +55,11 @@ module Tipi
     
     # Reads a body chunk for the current request. Transfers control to the parse
     # loop, and resumes once the parse_loop has fired the on_body callback
-    def get_body_chunk
+    def get_body_chunk(request)
       @waiting_for_body_chunk = true
       @next_chunk = nil
       while !@requests_tail.complete? && (data = @conn.readpartial(8192))
+        request.rx_incr(data.bytesize)
         @parser << data
         return @next_chunk if @next_chunk
         
@@ -70,9 +73,10 @@ module Tipi
     # Waits for the current request to complete. Transfers control to the parse
     # loop, and resumes once the parse_loop has fired the on_message_complete
     # callback
-    def consume_request
+    def consume_request(request)
       request = @requests_head
       @conn.recv_loop do |data|
+        request.rx_incr(data.bytesize)
         @parser << data
         return if request.complete?
       end
@@ -87,6 +91,9 @@ module Tipi
       headers = normalize_headers(headers)
       headers[':path'] = @parser.request_url
       headers[':method'] = @parser.http_method.downcase
+      scheme = (proto = headers['x-forwarded-proto']) ?
+                proto.downcase : scheme_from_connection
+      headers[':scheme'] = scheme
       queue_request(Qeweney::Request.new(headers, self))
     end
 
@@ -131,6 +138,14 @@ module Tipi
     # protocols, notably WebSocket, can be specified by passing a hash to the
     # :upgrade option when starting a server:
     #
+    #     def ws_handler(conn)
+    #       conn << 'hi'
+    #       msg = conn.recv
+    #       conn << "You said #{msg}"
+    #       conn << 'bye'
+    #       conn.close
+    #     end
+    #
     #     opts = {
     #       upgrade: {
     #         websocket: Tipi::Websocket.handler(&method(:ws_handler))
@@ -154,7 +169,7 @@ module Tipi
     
     def upgrade_with_handler(handler, headers)
       @parser = @requests_head = @requests_tail = nil
-      handler.(@conn, headers)
+      handler.(self, headers)
       true
     end
     
@@ -173,6 +188,14 @@ module Tipi
         ':authority' => headers['host']
       )
     end
+
+    def websocket_connection(request)
+      Tipi::Websocket.new(@conn, request.headers)
+    end
+
+    def scheme_from_connection
+      @conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
+    end
     
     # response API
 
@@ -181,53 +204,55 @@ module Tipi
 
     # Sends response including headers and body. Waits for the request to complete
     # if not yet completed. The body is sent using chunked transfer encoding.
+    # @param request [Qeweney::Request] HTTP request
     # @param body [String] response body
     # @param headers
-    def respond(body, headers)
-      consume_request if @parsing
-      data = format_headers(headers, body)
+    def respond(request, body, headers)
+      consume_request(request) if @parsing
+      formatted_headers = format_headers(headers, body, false)
+      request.tx_incr(formatted_headers.bytesize + (body ? body.bytesize : 0))
       if body
-        if @parser.http_minor == 0
-          data << body
-        else
-          data << body.bytesize.to_s(16) << CRLF << body << CRLF_ZERO_CRLF_CRLF
-        end
+        @conn.write(formatted_headers, body)
+      else
+        @conn.write(formatted_headers)
       end
-      @conn.write(data.join)
     end
       
-    DEFAULT_HEADERS_OPTS = {
-      empty_response:  false,
-      consume_request: true
-    }.freeze
-    
     # Sends response headers. If empty_response is truthy, the response status
     # code will default to 204, otherwise to 200.
+    # @param request [Qeweney::Request] HTTP request
     # @param headers [Hash] response headers
     # @param empty_response [boolean] whether a response body will be sent
+    # @param chunked [boolean] whether to use chunked transfer encoding
     # @return [void]
-    def send_headers(headers, opts = DEFAULT_HEADERS_OPTS)
-      data = format_headers(headers, true)
-      @conn.write(data.join)
+    def send_headers(request, headers, empty_response: false, chunked: true)
+      formatted_headers = format_headers(headers, !empty_response, @parser.http_minor == 1 && chunked)
+      request.tx_incr(formatted_headers.bytesize)
+      @conn.write(formatted_headers)
     end
     
     # Sends a response body chunk. If no headers were sent, default headers are
     # sent using #send_headers. if the done option is true(thy), an empty chunk
     # will be sent to signal response completion to the client.
+    # @param request [Qeweney::Request] HTTP request
     # @param chunk [String] response body chunk
     # @param done [boolean] whether the response is completed
     # @return [void]
-    def send_chunk(chunk, done: false)
-      data = []
+    def send_chunk(request, chunk, done: false)
+      data = +''
       data << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" if chunk
       data << "0\r\n\r\n" if done
-      @conn.write(data.join)
+      return if data.empty?
+
+      request.tx_incr(data.bytesize)
+      @conn.write(data)
     end
     
     # Finishes the response to the current request. If no headers were sent,
     # default headers are sent using #send_headers.
     # @return [void]
-    def finish
+    def finish(request)
+      request.tx_incr(5)
       @conn << "0\r\n\r\n"
     end
     
@@ -237,17 +262,20 @@ module Tipi
     
     private
 
+    INTERNAL_HEADER_REGEXP = /^:/.freeze
+
     # Formats response headers into an array. If empty_response is true(thy),
     # the response status code will default to 204, otherwise to 200.
     # @param headers [Hash] response headers
-    # @param empty_response [boolean] whether a response body will be sent
+    # @param body [boolean] whether a response body will be sent
+    # @param chunked [boolean] whether to use chunked transfer encoding
     # @return [String] formatted response headers
-    def format_headers(headers, body)
+    def format_headers(headers, body, chunked)
       status = headers[':status']
       status ||= (body ? Qeweney::Status::OK : Qeweney::Status::NO_CONTENT)
-      lines = [format_status_line(body, status)]
+      lines = format_status_line(body, status, chunked)
       headers.each do |k, v|
-        next if k =~ /^:/
+        next if k =~ INTERNAL_HEADER_REGEXP
         
         collect_header_lines(lines, k, v)
       end
@@ -255,11 +283,11 @@ module Tipi
       lines
     end
     
-    def format_status_line(body, status)
+    def format_status_line(body, status, chunked)
       if !body
         empty_status_line(status)
       else
-        with_body_status_line(status, body)
+        with_body_status_line(status, body, chunked)
       end
     end
     
@@ -271,17 +299,17 @@ module Tipi
       end
     end
     
-    def with_body_status_line(status, body)
-      if @parser.http_minor == 0
-        +"HTTP/1.0 #{status}\r\nContent-Length: #{body.bytesize}\r\n"
-      else
+    def with_body_status_line(status, body, chunked)
+      if chunked
         +"HTTP/1.1 #{status}\r\nTransfer-Encoding: chunked\r\n"
+      else
+        +"HTTP/1.1 #{status}\r\nContent-Length: #{body.bytesize}\r\n"
       end
     end
 
     def collect_header_lines(lines, key, value)
       if value.is_a?(Array)
-        value.inject(lines) { |lines, item| data << "#{key}: #{item}\r\n" }
+        value.inject(lines) { |_, item| lines << "#{key}: #{item}\r\n" }
       else
         lines << "#{key}: #{value}\r\n"
       end

data/lib/tipi/http2_adapter.rb

@@ -16,7 +16,9 @@ module Tipi
       @opts = opts
       @upgrade_headers = upgrade_headers
       @first = true
-      
+      @rx = (upgrade_headers && upgrade_headers[':rx']) || 0
+      @tx = (upgrade_headers && upgrade_headers[':tx']) || 0
+
       @interface = ::HTTP2::Server.new
       @connection_fiber = Fiber.current
       @interface.on(:frame, &method(:send_frame))
@@ -24,6 +26,9 @@ module Tipi
     end
     
     def send_frame(data)
+      if @transfer_count_request
+        @transfer_count_request.tx_incr(data.bytesize)
+      end
       @conn << data
     rescue Exception => e
       @connection_fiber.transfer e
@@ -38,6 +43,7 @@ module Tipi
     
     def upgrade
       @conn << UPGRADE_MESSAGE
+      @tx += UPGRADE_MESSAGE.bytesize
       settings = @upgrade_headers['http2-settings']
       Fiber.current.schedule(nil)
       @interface.upgrade(settings, @upgrade_headers, '')
@@ -49,16 +55,31 @@ module Tipi
     def each(&block)
       @interface.on(:stream) { |stream| start_stream(stream, &block) }
       upgrade if @upgrade_headers
-      
-      @conn.recv_loop(&@interface.method(:<<))
+
+      @conn.recv_loop do |data|
+        @rx += data.bytesize
+        @interface << data
+      end
     rescue SystemCallError, IOError
       # ignore
     ensure
       finalize_client_loop
     end
+
+    def get_rx_count
+      count = @rx
+      @rx = 0
+      count
+    end
+    
+    def get_tx_count
+      count = @tx
+      @tx = 0
+      count
+    end
     
     def start_stream(stream, &block)
-      stream = HTTP2StreamHandler.new(stream, @conn, @first, &block)
+      stream = HTTP2StreamHandler.new(self, stream, @conn, @first, &block)
       @first = nil if @first
       @streams[stream] = true
     end
@@ -72,5 +93,15 @@ module Tipi
     def close
       @conn.close
     end
+
+    def set_request_for_transfer_count(request)
+      @transfer_count_request = request
+    end
+
+    def unset_request_for_transfer_count(request)
+      return unless @transfer_count_request == request
+
+      @transfer_count_request = nil
+    end
   end
 end

data/lib/tipi/http2_stream.rb

@@ -9,13 +9,15 @@ module Tipi
     attr_accessor :__next__
     attr_reader :conn
     
-    def initialize(stream, conn, first, &block)
+    def initialize(adapter, stream, conn, first, &block)
+      @adapter = adapter
       @stream = stream
       @conn = conn
       @first = first
       @connection_fiber = Fiber.current
       @stream_fiber = spin { |req| handle_request(req, &block) }
-      
+      Thread.current.fiber_unschedule(@stream_fiber)
+
       # Stream callbacks occur on the connection fiber (see HTTP2Adapter#each).
       # The request handler is run on a separate fiber for each stream, allowing
       # concurrent handling of incoming requests on the same HTTP/2 connection.
@@ -47,14 +49,17 @@ module Tipi
     
     def on_headers(headers)
       @request = Qeweney::Request.new(headers.to_h, self)
+      @request.rx_incr(@adapter.get_rx_count)
+      @request.tx_incr(@adapter.get_tx_count)
       if @first
         @request.headers[':first'] = true
         @first = false
       end
       @stream_fiber.schedule @request
     end
-    
+
     def on_data(data)
+      data = data.to_s # chunks might be wrapped in a HTTP2::Buffer
       if @waiting_for_body_chunk
         @waiting_for_body_chunk = nil
         @stream_fiber.schedule data
@@ -62,7 +67,7 @@ module Tipi
         @request.buffer_body_chunk(data)
       end
     end
-    
+
     def on_half_close
       if @waiting_for_body_chunk
         @waiting_for_body_chunk = nil
@@ -78,62 +83,100 @@ module Tipi
     def protocol
       'h2'
     end
+
+    def with_transfer_count(request)
+      @adapter.set_request_for_transfer_count(request)
+      yield
+    ensure
+      @adapter.unset_request_for_transfer_count(request)
+    end
     
-    def get_body_chunk
+    def get_body_chunk(request)
       # called in the context of the stream fiber
       return nil if @request.complete?
       
-      @waiting_for_body_chunk = true
-      # the chunk (or an exception) will be returned once the stream fiber is
-      # resumed
-      suspend
+      with_transfer_count(request) do
+        @waiting_for_body_chunk = true
+        # the chunk (or an exception) will be returned once the stream fiber is
+        # resumed 
+        suspend
+      end
     ensure
       @waiting_for_body_chunk = nil
     end
     
     # Wait for request to finish
-    def consume_request
+    def consume_request(request)
       return if @request.complete?
       
-      @waiting_for_half_close = true
-      suspend
+      with_transfer_count(request) do
+        @waiting_for_half_close = true
+        suspend
+      end
     ensure
       @waiting_for_half_close = nil
     end
     
     # response API
-    def respond(chunk, headers)
+    def respond(request, chunk, headers)
       headers[':status'] ||= Qeweney::Status::OK
-      @stream.headers(headers, end_stream: false)
-      @stream.data(chunk, end_stream: true)
+      headers[':status'] = headers[':status'].to_s
+      with_transfer_count(request) do
+        @stream.headers(transform_headers(headers))
+        @stream.data(chunk || '')
+      end
       @headers_sent = true
+    rescue HTTP2::Error::StreamClosed
+      # ignore
+    end
+
+    def transform_headers(headers)
+      headers.each_with_object([]) do |(k, v), a|
+        if v.is_a?(Array)
+          v.each { |vv| a << [k, vv.to_s] }
+        else
+          a << [k, v.to_s]
+        end
+      end
     end
     
-    def send_headers(headers, empty_response = false)
+    def send_headers(request, headers, empty_response: false)
       return if @headers_sent
       
       headers[':status'] ||= (empty_response ? Qeweney::Status::NO_CONTENT : Qeweney::Status::OK).to_s
-      @stream.headers(headers, end_stream: false)
+      with_transfer_count(request) do
+        @stream.headers(transform_headers(headers), end_stream: false)
+      end
       @headers_sent = true
+    rescue HTTP2::Error::StreamClosed
+      # ignore
     end
     
-    def send_chunk(chunk, done: false)
+    def send_chunk(request, chunk, done: false)
       send_headers({}, false) unless @headers_sent
       
       if chunk
-        @stream.data(chunk, end_stream: done)
+        with_transfer_count(request) do
+          @stream.data(chunk, end_stream: done)
+        end
       elsif done
         @stream.close
       end
+    rescue HTTP2::Error::StreamClosed
+      # ignore
     end
     
-    def finish
+    def finish(request)
       if @headers_sent
         @stream.close
       else
         headers[':status'] ||= Qeweney::Status::NO_CONTENT
-        @stream.headers(headers, end_stream: true)
+        with_transfer_count(request) do
+          @stream.headers(transform_headers(headers), end_stream: true)
+        end
       end
+    rescue HTTP2::Error::StreamClosed
+      # ignore
     end
     
     def stop