h1p 0.1

data/ext/h1p/h1p.h

@@ -0,0 +1,18 @@
+#ifndef H1P_H
+#define H1P_H
+
+#include "ruby.h"
+
+// debugging
+#define OBJ_ID(obj) (NUM2LONG(rb_funcall(obj, rb_intern("object_id"), 0)))
+#define INSPECT(str, obj) { printf(str); VALUE s = rb_funcall(obj, rb_intern("inspect"), 0); printf(": %s\n", StringValueCStr(s)); }
+#define TRACE_CALLER() { VALUE c = rb_funcall(rb_mKernel, rb_intern("caller"), 0); INSPECT("caller: ", c); }
+#define TRACE_C_STACK() { \
+  void *entries[10]; \
+  size_t size = backtrace(entries, 10); \
+  char **strings = backtrace_symbols(entries, size); \
+  for (unsigned long i = 0; i < size; i++) printf("%s\n", strings[i]); \
+  free(strings); \
+}
+
+#endif /* H1P_H */

data/ext/h1p/limits.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+H1P_LIMITS = {
+  max_method_length:                       16,
+  max_path_length:                         4096,
+  max_header_key_length:                   128,
+  max_header_value_length:                 2048,
+  max_header_count:                        256,
+  max_chunked_encoding_chunk_size_length:  16,
+}

data/h1p.gemspec

@@ -0,0 +1,25 @@
+require_relative './lib/h1p/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'h1p'
+  s.version     = H1P::VERSION
+  s.licenses    = ['MIT']
+  s.summary     = 'H1P is a blocking HTTP/1 parser for Ruby'
+  s.author      = 'Sharon Rosner'
+  s.email       = 'sharon@noteflakes.com'
+  s.files       = `git ls-files`.split
+  s.homepage    = 'http://github.com/digital-fabric/h1p'
+  s.metadata    = {
+    "source_code_uri" => "https://github.com/digital-fabric/h1p"
+  }
+  s.rdoc_options = ["--title", "h1p", "--main", "README.md"]
+  s.extra_rdoc_files = ["README.md"]
+  s.extensions = ["ext/h1p/extconf.rb"]
+  s.require_paths = ["lib"]
+  s.required_ruby_version = '>= 2.6'
+
+  s.add_development_dependency  'rake-compiler',        '1.1.1'
+  s.add_development_dependency  'rake',               '~>12.3.3'
+  s.add_development_dependency  'minitest',           '~>5.11.3'
+  s.add_development_dependency  'minitest-reporters', '~>1.4.2'
+end

data/lib/h1p.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'h1p_ext'
+
+unless Object.const_defined?('Polyphony')
+  class IO
+    def __parser_read_method__
+      :stock_readpartial
+    end
+  end
+
+  require 'socket'
+
+  class Socket
+    def __parser_read_method__
+      :stock_readpartial
+    end
+  end
+
+  class TCPSocket
+    def __parser_read_method__
+      :stock_readpartial
+    end
+  end
+
+  class UNIXSocket
+    def __parser_read_method__
+      :stock_readpartial
+    end
+  end
+end

data/lib/h1p/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module H1P
+  VERSION = '0.1'
+end

data/test/helper.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require 'fileutils'
+
+require 'minitest/autorun'
+require 'minitest/reporters'
+
+module Minitest::Assertions
+  def assert_in_range exp_range, act
+    msg = message(msg) { "Expected #{mu_pp(act)} to be in range #{mu_pp(exp_range)}" }
+    assert exp_range.include?(act), msg
+  end
+end

data/test/test_h1p.rb

@@ -0,0 +1,584 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'h1p'
+require 'socket'
+require_relative '../ext/h1p/limits'
+
+class H1PTest < MiniTest::Test
+  Error = H1P::Error
+
+  def setup
+    super
+    @i, @o = IO.pipe
+    @parser = H1P::Parser.new(@i)
+  end
+  alias_method :reset_parser, :setup
+
+  def test_request_line
+    msg = "GET / HTTP/1.1\r\n\r\n"
+    @o << msg
+    headers = @parser.parse_headers
+
+    assert_equal(
+      {
+        ':method' => 'get',
+        ':path' => '/',
+        ':protocol' => 'http/1.1',
+        ':rx' => msg.bytesize
+      },
+      headers
+    )
+  end
+
+  def test_request_line_whitespace
+    msg = "GET       /               HTTP/1.1\r\n\r\n"
+    @o << msg
+    headers = @parser.parse_headers
+
+    assert_equal(
+      {
+        ':method' => 'get',
+        ':path' => '/',
+        ':protocol' => 'http/1.1',
+        ':rx' => msg.bytesize
+      },
+      headers
+    )
+  end
+
+  def test_eof
+    @o << "GET / HTTP/1.1"
+    @o.close
+
+    assert_nil @parser.parse_headers
+  end
+
+  def test_method_case
+    @o << "GET / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'get', headers[':method']
+
+    reset_parser
+    @o << "post / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'post', headers[':method']
+
+    reset_parser
+    @o << "PoST / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'post', headers[':method']
+  end
+
+  def test_bad_method
+    @o << " / HTTP/1.1\r\n\r\n"
+    @o.close
+
+    assert_raises(Error) { @parser.parse_headers }
+  
+    max_length = H1P_LIMITS[:max_method_length]
+
+    reset_parser
+    @o << "#{'a' * max_length} / HTTP/1.1\r\n\r\n"
+    assert_equal 'a' * max_length, @parser.parse_headers[':method']
+
+    reset_parser
+    @o << "#{'a' * (max_length + 1)} / HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_path_characters
+    @o << "GET /äBçDé¤23~{@€ HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal '/äBçDé¤23~{@€', headers[':path']
+  end
+
+  def test_bad_path
+    @o << "GET HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  
+    max_length = H1P_LIMITS[:max_path_length]
+
+    reset_parser
+    @o << "get #{'a' * max_length} HTTP/1.1\r\n\r\n"
+    assert_equal 'a' * max_length, @parser.parse_headers[':path']
+
+    reset_parser
+    @o << "get #{'a' * (max_length + 1)} HTTP/1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_protocol
+    @o << "GET / http/1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1', headers[':protocol']
+
+    reset_parser
+    @o << "GET / HTTP/1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1', headers[':protocol']
+
+    reset_parser
+    @o << "GET / HTTP/1.0\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1.0', headers[':protocol']
+
+    @o << "GET / HttP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'http/1.1', headers[':protocol']
+  end
+
+    def test_bad_protocol
+    @o << "GET / blah\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/2\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/a.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.1.1\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_headers_eof
+    @o << "GET / HTTP/1.1\r\na"
+    @o.close
+    
+    assert_nil @parser.parse_headers
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na:"
+    @o.close
+    
+    assert_nil @parser.parse_headers
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na:      "
+    @o.close
+    
+    assert_nil @parser.parse_headers
+  end
+
+  def test_headers
+    @o << "GET / HTTP/1.1\r\nFoo: Bar\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal [':method', ':path', ':protocol', 'foo', ':rx'], headers.keys
+    assert_equal 'Bar', headers['foo']
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nFOO:    baR\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'baR', headers['foo']
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\na: bbb\r\nc: ddd\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'bbb', headers['a']
+    assert_equal 'ddd', headers['c']
+  end
+
+  def test_headers_multiple_values
+    @o << "GET / HTTP/1.1\r\nFoo: Bar\r\nfoo: baz\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal ['Bar', 'baz'], headers['foo']
+  end
+
+  def test_bad_headers
+    @o << "GET / http/1.1\r\n   a: b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    reset_parser
+    @o << "GET / http/1.1\r\na b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_key_length = H1P_LIMITS[:max_header_key_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\n#{'a' * max_key_length}: b\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'b', headers['a' * max_key_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\n#{'a' * (max_key_length + 1)}: b\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_value_length = H1P_LIMITS[:max_header_value_length]
+
+    reset_parser
+    @o << "GET / http/1.1\r\nfoo: #{'a' * max_value_length}\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal 'a' * max_value_length, headers['foo']
+
+    reset_parser
+    @o << "GET / http/1.1\r\nfoo: #{'a' * (max_value_length + 1)}\r\n\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+
+    max_header_count = H1P_LIMITS[:max_header_count]
+
+    reset_parser
+    hdrs = (1..max_header_count).map { |i| "foo#{i}: bar\r\n" }.join
+    @o << "GET / http/1.1\r\n#{hdrs}\r\n"
+    headers = @parser.parse_headers
+    assert_equal (max_header_count + 4), headers.size
+
+    reset_parser
+    hdrs = (1..(max_header_count + 1)).map { |i| "foo#{i}: bar\r\n" }.join
+    @o << "GET / http/1.1\r\n#{hdrs}\r\n"
+    assert_raises(Error) { @parser.parse_headers }
+  end
+
+  def test_request_without_cr
+    msg = "GET /foo HTTP/1.1\nBar: baz\n\n"
+    @o << msg
+    headers = @parser.parse_headers
+    assert_equal({
+      ':method'   => 'get',
+      ':path'     => '/foo',
+      ':protocol' => 'http/1.1',
+      'bar'       => 'baz',
+      ':rx'       => msg.bytesize
+    }, headers)
+  end
+
+  def test_read_body_with_content_length
+    10.times do
+      data = ' ' * rand(20..60000)
+      msg = "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+      Thread.new { @o << msg }
+
+      headers = @parser.parse_headers
+      assert_equal data.bytesize.to_s, headers['content-length']
+
+      body = @parser.read_body
+      assert_equal data, body
+      assert_equal msg.bytesize, headers[':rx']
+    end
+  end
+
+  def test_read_body_chunk_with_content_length
+    data = 'abc' * 20000
+    msg = "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+    Thread.new { @o << msg }
+    headers = @parser.parse_headers
+    assert_equal data.bytesize.to_s, headers['content-length']
+
+    buf = +''
+    count = 0
+    while (chunk = @parser.read_body_chunk(false))
+      count += 1
+      buf += chunk
+    end
+    assert_equal data.bytesize, data.bytesize
+    assert_equal data, buf
+    assert_in_range 1..20, count
+    assert_equal msg.bytesize, headers[':rx']
+  end
+
+  def test_read_body_with_content_length_incomplete
+    data = ' ' * rand(20..60000)
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize + 1}\r\n\r\n#{data}"
+      @o.close # !!! otherwise the parser will keep waiting
+    end
+    headers = @parser.parse_headers
+
+    assert_raises(H1P::Error) { @parser.read_body }
+  end
+
+  def test_read_body_chunk_with_content_length_incomplete
+    data = 'abc' * 50
+    @o << "POST /foo HTTP/1.1\r\nContent-Length: #{data.bytesize + 1}\r\n\r\n#{data}"
+    @o.close
+    headers = @parser.parse_headers
+
+    assert_raises(H1P::Error) { @parser.read_body_chunk(false) }
+  end
+
+  def test_read_body_with_chunked_encoding
+    chunks = []
+    total_sent = 0
+    Thread.new do
+      msg = "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+      rand(8..16).times do |i|
+        chunk = i.to_s * rand(200..360000)
+        msg = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+        @o << msg
+        chunks << chunk
+        total_sent += msg.bytesize
+      end
+      msg = "0\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+    end 
+    headers = @parser.parse_headers
+    assert_equal 'chunked', headers['transfer-encoding']
+
+    body = @parser.read_body
+    assert_equal chunks.join, body
+    assert_equal total_sent, headers[':rx']
+  end
+
+  def test_read_body_chunk_with_chunked_encoding
+    chunks = []
+    total_sent = 0
+    Thread.new do
+      msg = "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+      rand(8..16).times do |i|
+        chunk = i.to_s * rand(40000..360000)
+        msg = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+        @o << msg
+        total_sent += msg.bytesize
+        chunks << chunk
+      end
+      msg = "0\r\n\r\n"
+      @o << msg
+      total_sent += msg.bytesize
+    end 
+    headers = @parser.parse_headers
+    assert_equal 'chunked', headers['transfer-encoding']
+
+    received = []
+    while (chunk = @parser.read_body_chunk(false))
+      received << chunk
+    end
+    assert_equal chunks, received
+    assert_equal total_sent, headers[':rx']
+  end
+
+  def test_read_body_with_chunked_encoding_malformed
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n3"
+      @o << "0\r\n\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    assert_raises(H1P::Error) { @parser.read_body }
+
+    reset_parser
+    # missing last empty chunk
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(H1P::Error) { @parser.read_body }
+
+    reset_parser
+    # bad chunk size
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(40000..360000)
+      @o << "-#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(H1P::Error) { @parser.read_body }
+  end
+
+  def test_read_body_chunk_with_chunked_encoding_malformed
+    chunk = nil
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' ' * rand(40000..360000)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n3"
+      @o << "0\r\n\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    read = @parser.read_body_chunk(false)
+    assert_equal chunk, read
+    assert_raises(H1P::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+
+    # missing last empty chunk
+    chunk = nil
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(20..1600)
+      @o << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end
+    headers = @parser.parse_headers
+    read = @parser.read_body_chunk(false)
+    assert_equal chunk, read
+    assert_raises(H1P::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+
+    # bad chunk size
+    Thread.new do
+      @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+      chunk = ' '.to_s * rand(20..1600)
+      @o << "-#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+      @o.close
+    end 
+    headers = @parser.parse_headers
+    assert_raises(H1P::Error) { @parser.read_body_chunk(false) }
+
+    reset_parser
+
+    # missing body
+    @o << "POST /foo HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+    @o.close
+    headers = @parser.parse_headers
+    assert_raises(H1P::Error) { @parser.read_body_chunk(false) }
+  end
+
+  def test_complete?
+    @o << "GET / HTTP/1.1\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nContent-Length: 3\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+    @o << 'foo'
+    body = @parser.read_body
+    assert_equal 'foo', body
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+    @o << "3\r\nfoo\r\n"
+    chunk = @parser.read_body_chunk(false)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << "0\r\n\r\n"
+    chunk = @parser.read_body_chunk(false)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+  end
+
+  def test_buffered_body_chunk
+    @o << "GET / HTTP/1.1\r\nContent-Length: 3\r\n\r\nfoo"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal true, @parser.complete?
+    chunk = @parser.read_body_chunk(false)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nContent-Length: 6\r\n\r\nfoo"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << 'bar'
+    chunk = @parser.read_body_chunk(false)
+    assert_equal 'bar', chunk
+    assert_equal true, @parser.complete?
+
+    reset_parser
+    @o << "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nfoo\r\n"
+    headers = @parser.parse_headers
+    assert_equal false, @parser.complete?
+
+    chunk = @parser.read_body_chunk(true)
+    assert_equal 'foo', chunk
+    assert_equal false, @parser.complete?
+    @o << "0\r\n\r\n"
+    chunk = @parser.read_body_chunk(true)
+    assert_nil chunk
+    assert_equal true, @parser.complete?
+  end
+
+  def test_parser_with_tcp_socket
+    port = rand(1234..5678)
+    server = TCPServer.new('127.0.0.1', port)
+    server_thread = Thread.new do
+      while (socket = server.accept)
+        Thread.new do
+          parser = H1P::Parser.new(socket)
+          headers = parser.parse_headers
+          socket << headers.inspect
+          socket.shutdown
+          socket.close
+        end
+      end
+    end
+
+    sleep 0.001
+    client = TCPSocket.new('127.0.0.1', port)
+    msg = "get /foo HTTP/1.1\r\nCookie: abc=def\r\n\r\n"
+    client << msg
+    reply = client.read
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/foo',
+      ':protocol' => 'http/1.1',
+      'cookie' => 'abc=def',
+      ':rx' => msg.bytesize,
+    }, eval(reply))
+  ensure
+    client.shutdown rescue nil
+    client&.close
+    server_thread&.kill
+    server_thread&.join
+    server&.close
+  end
+
+  def test_parser_with_callable
+    buf = []
+    request = +"GET /foo HTTP/1.1\r\nHost: bar\r\n\r\n"
+    callable = proc do |len|
+      buf << {len: len}
+      request
+    end
+
+    parser = H1P::Parser.new(callable)
+
+    headers = parser.parse_headers
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/foo',
+      ':protocol' => 'http/1.1',
+      'host' => 'bar',
+      ':rx' => request.bytesize,
+
+    }, headers)
+    assert_equal [{len: 4096}], buf
+
+    request = +"GET /bar HTTP/1.1\r\nHost: baz\r\n\r\n"
+    headers = parser.parse_headers
+    assert_equal({
+      ':method' => 'get',
+      ':path' => '/bar',
+      ':protocol' => 'http/1.1',
+      'host' => 'baz',
+      ':rx' => request.bytesize,
+
+    }, headers)
+    assert_equal [{len: 4096}, {len: 4096}], buf
+  end
+end