tipi 0.38 → 0.42

data/security/http1.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Tipi
+  HTTP1_LIMITS = {
+    max_method_length:                       16,
+    max_path_length:                         4096,
+    max_header_key_length:                   128,
+    max_header_value_length:                 2048,
+    max_header_count:                        256,
+    max_chunked_encoding_chunk_size_length:  16,
+  }
+end

data/test/helper.rb

@@ -14,26 +14,24 @@ require 'polyphony'
 
 ::Exception.__disable_sanitized_backtrace__ = true
 
-Minitest::Reporters.use! [
-  Minitest::Reporters::SpecReporter.new
-]
-
 class MiniTest::Test
   def setup
-    # puts "* setup #{self.name}"
-    if Fiber.current.children.size > 0
-      puts "Children left: #{Fiber.current.children.inspect}"
-      exit!
-    end
+    # trace "* setup #{self.name}"
     Fiber.current.setup_main_fiber
     Fiber.current.instance_variable_set(:@auto_watcher, nil)
+    Thread.current.backend.finalize
     Thread.current.backend = Polyphony::Backend.new
     sleep 0
   end
 
   def teardown
-    # puts "* teardown #{self.name.inspect} Fiber.current: #{Fiber.current.inspect}"
+    # trace "* teardown #{self.name}"
     Fiber.current.shutdown_all_children
+    if Fiber.current.children.size > 0
+      puts "Children left after #{self.name}: #{Fiber.current.children.inspect}"
+      exit!
+    end
+    Fiber.current.instance_variable_set(:@auto_watcher, nil)
   rescue => e
     puts e
     puts e.backtrace.join("\n")
@@ -47,4 +45,55 @@ module Kernel
   rescue Exception => e
     e
   end
-end
+
+  def trace(*args)
+    STDOUT.orig_write(format_trace(args))
+  end
+
+  def format_trace(args)
+    if args.first.is_a?(String)
+      if args.size > 1
+        format("%s: %p\n", args.shift, args)
+      else
+        format("%s\n", args.first)
+      end
+    else
+      format("%p\n", args.size == 1 ? args.first : args)
+    end
+  end
+end
+
+class IO
+  # Creates two mockup sockets for simulating server-client communication
+  def self.server_client_mockup
+    server_in, client_out = IO.pipe
+    client_in, server_out = IO.pipe
+
+    server_connection = mockup_connection(server_in, server_out, client_out)
+    client_connection = mockup_connection(client_in, client_out, server_out)
+
+    [server_connection, client_connection]
+  end
+
+  def self.mockup_connection(input, output, output2)
+    eg(
+      __parser_read_method__: ->() { :readpartial },
+      read:         ->(*args) { input.read(*args) },
+      read_loop:    ->(*args, &block) { input.read_loop(*args, &block) },
+      recv_loop:    ->(*args, &block) { input.read_loop(*args, &block) },
+      readpartial:  ->(*args) { input.readpartial(*args) },
+      recv:         ->(*args) { input.readpartial(*args) },
+      '<<':         ->(*args) { output.write(*args) },
+      write:        ->(*args) { output.write(*args) },
+      close:        -> { output.close },
+      eof?:         -> { output2.closed? }
+    )
+  end
+end
+
+module Minitest::Assertions
+  def assert_in_range exp_range, act
+    msg = message(msg) { "Expected #{mu_pp(act)} to be in range #{mu_pp(exp_range)}" }
+    assert exp_range.include?(act), msg
+  end
+end

data/test/test_http1_parser.rb

@@ -0,0 +1,586 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'tipi_ext'
+require_relative '../security/http1.rb'
+
+class HTTP1ParserTest < MiniTest::Test
+  Error = Tipi::HTTP1Parser::Error
+
+  def setup
+    super
+    @i, @o = IO.pipe
+    @parser = Tipi::HTTP1Parser.new(@i)
+  end
+  alias_method :reset_parser, :setup
+
+  def test_request_line
+    msg = "GET / HTTP/1.1\r\n\r\n"
+    @o << msg
+    headers = @parser.parse_headers
+
+    assert_equal(
+      {
+        ':method' => 'get',
+        ':path' => '/',
+        ':protocol' => 'http/1.1',
+        ':rx' => msg.bytesize
+      },
+      headers
+    )
+  end
+
+  def test_request_line_whitespace
+    msg = "GET       /               HTTP/1.1\r\n\r\n"
+    @o << msg
+    headers = @parser.parse_headers
+
+    assert_equal(
+      {
+        ':method' => 'get',
+        ':path' => '/',
+        ':protocol' => 'http/1.1',
+        ':rx' => msg.bytesize
+      },
+      headers
+    )
+  end
+
+  def test_eof
+    @o << "GET / HTTP/1.1"
+    @o.close
+
+    assert_nil @parser.parse_headers
+  end
+
+  def test_method_case
+    @o << "GET / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'get', headers[':method']
+
+    reset_parser
+    @o << "post / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'post', headers[':method']
+
+    reset_parser
+    @o << "PoST / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'post', headers[':method']
+  end
+
+  def test_bad_method
+    @o << " / HTTP/1.1\r\n\r\n"
+    @o.close
+
+    assert_raises(Error) { @parser.parse_headers }
+  
+    max_length = Tipi::HTTP1_LIMITS[:max_method_length]
+
+    reset_parser
+    @o << "#{'a' * max_length} / HTTP/1.1\r\n\r\n"
+    assert_equal 'a' * max_length, @parser.parse_headers[':method']
+
+    reset_parser
+    @o << "#{'a' * (max_length + 1)} / HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_path_characters
+    @o << "GET /äBçDé¤23~{@€ HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal '/äBçDé¤23~{@€', headers[':path']
+  end
+
+  def test_bad_path
+    @o << "GET HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  
+    max_length = Tipi::HTTP1_LIMITS[:max_path_length]
+
+    reset_parser
+    @o << "get #{'a' * max_length} HTTP/1.1\r\n\r\n"
+    assert_equal 'a' * max_length, @parser.parse_headers[':path']
+
+    reset_parser
+    @o << "get #{'a' * (max_length + 1)} HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_protocol
+    @o << "GET / http/1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1', headers[':protocol']
+
+    reset_parser
+    @o << "GET / HTTP/1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1', headers[':protocol']
+
+    reset_parser
+    @o << "GET / HTTP/1.0\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1.0', headers[':protocol']
+
+    @o << "GET / HttP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1.1', headers[':protocol']
+  end
+
+    def test_bad_protocol
+    @o << "GET / blah\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/2\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/a.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_headers_eof
+    @o << "GET / HTTP/1.1\r\na"
+    @o.close
+    
+    assert_nil @parser.parse_headers
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na:"
+    @o.close
+    
+    assert_nil @parser.parse_headers
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na:      "
+    @o.close
+    
+    assert_nil @parser.parse_headers
+  end
+
+  def test_headers
+    @o << "GET / HTTP/1.1\r\nFoo: Bar\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal [':method', ':path', ':protocol', 'foo', ':rx'], headers.keys
+    assert_equal 'Bar', headers['foo']
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nFOO:    baR\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'baR', headers['foo']
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na: bbb\r\nc: ddd\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'bbb', headers['a']
+    assert_equal 'ddd', headers['c']
+  end
+
+  def test_headers_multiple_values
+    @o << "GET / HTTP/1.1\r\nFoo: Bar\r\nfoo: baz\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal ['Bar', 'baz'], headers['foo']
+  end
+
+  def test_bad_headers
+    @o << "GET / http/1.1\r\n   a: b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.1\r\na b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_key_length = Tipi::HTTP1_LIMITS[:max_header_key_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\n#{'a' * max_key_length}: b\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'b', headers['a' * max_key_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\n#{'a' * (max_key_length + 1)}: b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_value_length = Tipi::HTTP1_LIMITS[:max_header_value_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\nfoo: #{'a' * max_value_length}\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'a' * max_value_length, headers['foo']
+
+    reset_parser
+    @o << "GET / http/1.1\r\nfoo: #{'a' * (max_value_length + 1)}\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_header_count = Tipi::HTTP1_LIMITS[:max_header_count]
+
+    reset_parser
+    hdrs = (1..max_header_count).map { |i| "foo#{i}: bar\r\n" }.join
+    @o << "GET / http/1.1\r\n#{hdrs}\r\n"
+    headers = @parser.parse_headers
+    assert_equal (max_header_count + 4), headers.size
+
+    reset_parser
+    hdrs = (1..(max_header_count + 1)).map { |i| "foo#{i}: bar\r\n" }.join
+    @o << "GET / http/1.1\r\n#{hdrs}\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_request_without_cr
+    msg = "GET /foo HTTP/1.1\nBar: baz\n\n"
+    @o << msg
+    headers = @parser.parse_headers
+    assert_equal({
+      ':method'   => 'get',
+      ':path'     => '/foo',
+      ':protocol' => 'http/1.1',
+      'bar'       => 'baz',
+      ':rx'       => msg.bytesize
+    }, headers)
+  end
+
+  def test_read_body_with_content_length
+    10.times do
+      data = ' ' * rand(20..60000)
+      msg = "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+      spin do
+        @o << msg
+      end
+      headers = @parser.parse_headers
+      assert_equal data.bytesize.to_s, headers['content-length']
+
+      body = @parser.read_body
+      assert_equal data, body
+      assert_equal msg.bytesize, headers[':rx']
+    end
+  end
+
+  def test_read_body_chunk_with_content_length
+    data = 'abc' * 20000
+    msg = "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+    spin { @o << msg }
+    headers = @parser.parse_headers
+    assert_equal data.bytesize.to_s, headers['content-length']
+
+    buf = +''
+    count = 0
+    while (chunk = @parser.read_body_chunk(false))
+      count += 1
+      buf += chunk
+    end
+    assert_equal data.bytesize, data.bytesize
+    assert_equal data, buf
+    assert_in_range 1..3, count
+    assert_equal msg.bytesize, headers[':rx']
+  end
+
+  def test_read_body_with_content_length_incomplete
+    data = ' ' * rand(20..60000)
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize + 1}\r\n\r\n#{data}"
+      @o.close # !!! otherwise the parser will keep waiting
+    end
+    headers = @parser.parse_headers
+
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body }
+  end
+
+  def test_read_body_chunk_with_content_length_incomplete
+    data = 'abc' * 50
+    @o << "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize + 1}\r\n\r\n#{data}"
+    @o.close
+    headers = @parser.parse_headers
+
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body_chunk(false) }
+  end
+
+  def test_read_body_with_chunked_encoding
+    chunks = []
+    total_sent = 0
+    spin do
+      msg = "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+      rand(8..16).times do |i|
+        chunk = i.to_s * rand(200..360000)
+        msg = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+        @o << msg
+        chunks << chunk
+        total_sent += msg.bytesize
+      end
+      msg = "0\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+    end 
+    headers = @parser.parse_headers
+    assert_equal 'chunked', headers['transfer-encoding']
+
+    body = @parser.read_body
+    assert_equal chunks.join, body
+    assert_equal total_sent, headers[':rx']
+  end
+
+  def test_read_body_chunk_with_chunked_encoding
+    chunks = []
+    total_sent = 0
+    spin do
+      msg = "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+      rand(8..16).times do |i|
+        chunk = i.to_s * rand(40000..360000)
+        msg = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+        @o << msg
+        total_sent += msg.bytesize
+        chunks << chunk
+      end
+      msg = "0\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+    end 
+    headers = @parser.parse_headers
+    assert_equal 'chunked', headers['transfer-encoding']
+
+    received = []
+    while (chunk = @parser.read_body_chunk(false))
+      received << chunk
+    end
+    assert_equal chunks, received
+    assert_equal total_sent, headers[':rx']
+  end
+
+  def test_read_body_with_chunked_encoding_malformed
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n3"
+      @o << "0\r\n\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body }
+
+    reset_parser
+    # missing last empty chunk
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body }
+
+    reset_parser
+    # bad chunk size
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "-#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body }
+  end
+
+  def test_read_body_chunk_with_chunked_encoding_malformed
+    chunk = nil
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' ' * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n3"
+      @o << "0\r\n\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    read = @parser.read_body_chunk(false)
+    assert_equal chunk, read
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+    Fiber.current.shutdown_all_children
+    # missing last empty chunk
+    chunk = nil
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(20..1600)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    read = @parser.read_body_chunk(false)
+    assert_equal chunk, read
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+    Fiber.current.shutdown_all_children
+    # bad chunk size
+    spin do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(20..1600)
+      @o << "-#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+    Fiber.current.shutdown_all_children
+    # missing body
+    @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+    @o.close
+    headers = @parser.parse_headers
+    assert_raises(Tipi::HTTP1Parser::Error) { @parser.read_body_chunk(false) }
+  ensure
+    Fiber.current.shutdown_all_children
+  end
+
+  def test_complete?
+    @o << "GET / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nContent-Length: 3\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+    @o << 'foo'
+    body = @parser.read_body
+    assert_equal 'foo', body
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+    @o << "3\r\nfoo\r\n"
+    chunk = @parser.read_body_chunk(false)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << "0\r\n\r\n"
+    chunk = @parser.read_body_chunk(false)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+  end
+
+  def test_buffered_body_chunk
+    @o << "GET / HTTP/1.1\r\nContent-Length: 3\r\n\r\nfoo"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal true, @parser.complete?
+    chunk = @parser.read_body_chunk(false)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nContent-Length: 6\r\n\r\nfoo"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << 'bar'
+    chunk = @parser.read_body_chunk(false)
+    assert_equal 'bar', chunk
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nfoo\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << "0\r\n\r\n"
+    chunk = @parser.read_body_chunk(true)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+  end
+
+  def test_parser_with_tcp_socket
+    port = rand(1234..5678)
+    server = TCPServer.new('127.0.0.1', port)
+    server_fiber = spin do
+      while (socket = server.accept)
+        spin do
+          parser = Tipi::HTTP1Parser.new(socket)
+          headers = parser.parse_headers
+          socket << headers.inspect
+          socket.shutdown
+          socket.close
+        end
+      end
+    end
+
+    snooze
+    client = TCPSocket.new('127.0.0.1', port)
+    msg = "get /foo HTTP/1.1\r\nCookie: abc=def\r\n\r\n"
+    client << msg
+    reply = client.read
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/foo',
+      ':protocol' => 'http/1.1',
+      'cookie' => 'abc=def',
+      ':rx' => msg.bytesize,
+    }, eval(reply))
+  ensure
+    client.shutdown rescue nil
+    client&.close
+    server_fiber&.stop
+    server_fiber&.await
+    server&.close
+  end
+
+  def test_parser_with_callable
+    buf = []
+    request = +"GET /foo HTTP/1.1\r\nHost: bar\r\n\r\n"
+    callable = proc do |len|
+      buf << {len: len}
+      request
+    end
+
+    parser = Tipi::HTTP1Parser.new(callable)
+
+    headers = parser.parse_headers
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/foo',
+      ':protocol' => 'http/1.1',
+      'host' => 'bar',
+      ':rx' => request.bytesize,
+
+    }, headers)
+    assert_equal [{len: 4096}], buf
+
+    request = +"GET /bar HTTP/1.1\r\nHost: baz\r\n\r\n"
+    headers = parser.parse_headers
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/bar',
+      ':protocol' => 'http/1.1',
+      'host' => 'baz',
+      ':rx' => request.bytesize,
+
+    }, headers)
+    assert_equal [{len: 4096}, {len: 4096}], buf
+  end
+end