tipi 0.41 → 0.42

data/examples/full_service.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+
+::Exception.__disable_sanitized_backtrace__ = true
+
+certificate_db_path = File.expand_path('certificate_store.db', __dir__)
+certificate_store = Tipi::ACME::SQLiteCertificateStore.new(certificate_db_path)
+
+Tipi.full_service(
+  certificate_store: certificate_store
+) { |req| req.respond('Hello, world!') }

data/examples/http1_parser.rb

@@ -31,23 +31,25 @@ f = spin do
     break unless headers
     trace headers
 
-    body = parser.read_body(headers)
+    body = parser.read_body
     trace "body: #{body ? body.bytesize : 0} bytes"
+    trace body if body && body.bytesize < 80
   end
 end
 
 o << "GET /a HTTP/1.1\r\n\r\n"
-o << "GET /b HTTP/1.1\r\n\r\n"
+
+# o << "GET /a HTTP/1.1\r\nContent-Length: 0\r\n\r\n"
 
 # o << "GET / HTTP/1.1\r\nHost: localhost:10080\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n"
 
-# o << "post /?q=time&blah=blah HTTP/1\r\nHost: dev.realiteq.net\r\n\r\n"
+o << "post /?q=time&blah=blah HTTP/1\r\nTransfer-Encoding: chunked\r\n\r\na\r\nabcdefghij\r\n0\r\n\r\n"
+
+data = " " * 4000000
+o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
 
-# data = " " * 4000000
-# o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
 
-# o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
+o.close
 
-# o.close
- 
 f.await

data/examples/http_server.rb

@@ -14,7 +14,7 @@ puts 'Listening on port 10080...'
 # GC.disable
 # Thread.current.backend.idle_gc_period = 60
 
-spin_loop(interval: 10) { p Thread.current.fiber_scheduling_stats }
+spin_loop(interval: 10) { p Thread.backend.stats }
 
 spin_loop(interval: 10) do
   GC.compact
@@ -29,6 +29,9 @@ spin do
       sleep 1
       req.send_chunk("bar\n")
       req.finish
+    elsif req.path == '/upload'
+      body = req.read
+      req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
     else
       req.respond("Hello world!\n")
     end

data/examples/https_server.rb

@@ -23,6 +23,9 @@ Tipi.serve('0.0.0.0', 1234, opts) do |req|
     req.send_chunk("foo\n")
     sleep 0.5
     req.send_chunk("bar\n", done: true)
+  elsif req.path == '/upload'
+    body = req.read
+    req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
   else
     req.respond("Hello world!\n")
   end

data/examples/servername_cb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'fiber'
+
+ctx = OpenSSL::SSL::SSLContext.new
+
+f = Fiber.new { |peer| loop { p peer: peer; _name, peer = peer.transfer nil } }
+ctx.servername_cb = proc { |_socket, name|
+  p servername_cb: name
+  f.transfer([name, Fiber.current]).tap { |r| p result: r }
+}
+
+socket = Socket.new(:INET, :STREAM).tap do |s|
+  s.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
+  s.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
+  s.bind(Socket.sockaddr_in(12345, '0.0.0.0'))
+  s.listen(Socket::SOMAXCONN)
+end
+server = OpenSSL::SSL::SSLServer.new(socket, ctx)
+
+Thread.new do
+  sleep 0.5
+  socket = TCPSocket.new('127.0.0.1', 12345)
+  client = OpenSSL::SSL::SSLSocket.new(socket)
+  client.hostname = 'example.com'
+  p client: client
+  client.connect
+rescue => e
+  p client_error: e
+end
+
+while true
+  conn = server.accept
+  p accepted: conn
+  break
+end

data/ext/tipi/extconf.rb

@@ -3,10 +3,11 @@
 require 'rubygems'
 require 'mkmf'
 
-$CFLAGS << " -Wno-pointer-arith"
+require_relative '../../security/http1'
 
+$CFLAGS << " -Wno-format-security"
 CONFIG['optflags'] << ' -fno-strict-aliasing' unless RUBY_PLATFORM =~ /mswin/
-
+Tipi::HTTP1_LIMITS.each { |k, v| $defs << "-D#{k.upcase}=#{v}" }
 
 dir_config 'tipi_ext'
 create_makefile 'tipi_ext'

data/ext/tipi/http1_parser.c

@@ -1,36 +1,66 @@
 #include "ruby.h"
 #include "http1_parser.h"
 
-#define str_downcase(str) (rb_funcall((str), ID_downcase, 0))
+// Security-related limits are defined in security/http1.rb and injected as
+// defines in extconf.rb
+
+#define INITIAL_BUFFER_SIZE     4096
+#define BUFFER_TRIM_MIN_LEN     4096
+#define BUFFER_TRIM_MIN_POS     2048
+#define MAX_HEADERS_READ_LENGTH 4096
+#define MAX_BODY_READ_LENGTH    (1 << 20) // 1MB
 
-const int MAX_METHOD_LENGTH       = 16;
-const int MAX_PATH_LENGTH         = 1024;
-const int MAX_HEADER_KEY_LENGTH   = 128;
-const int MAX_HEADER_VALUE_LENGTH = 2048;
+#define BODY_READ_MODE_UNKNOWN  -2
+#define BODY_READ_MODE_CHUNKED  -1
 
+ID ID_arity;
 ID ID_backend_read;
+ID ID_backend_recv;
+ID ID_call;
 ID ID_downcase;
+ID ID_eq;
+ID ID_parser_read_method;
 ID ID_read;
 ID ID_readpartial;
 ID ID_to_i;
 
-VALUE mPolyphony;
-VALUE cError;
+static VALUE mPolyphony = Qnil;
+static VALUE cError;
 
-VALUE MAX_READ_LENGTH;
-VALUE BUFFER_END;
+VALUE NUM_max_headers_read_length;
+VALUE NUM_buffer_start;
+VALUE NUM_buffer_end;
 
 VALUE STR_pseudo_method;
 VALUE STR_pseudo_path;
 VALUE STR_pseudo_protocol;
+VALUE STR_pseudo_rx;
 
+VALUE STR_chunked;
 VALUE STR_content_length;
 VALUE STR_transfer_encoding;
 
+VALUE SYM_backend_read;
+VALUE SYM_backend_recv;
+
+enum read_method {
+  method_readpartial,   // receiver.readpartial (Polyphony-specific)
+  method_backend_read,  // Polyphony.backend_read (Polyphony-specific)
+  method_backend_recv,  // Polyphony.backend_recv (Polyphony-specific)
+  method_call           // receiver.call(len) (Universal)
+};
+
 typedef struct parser {
   VALUE io;
   VALUE buffer;
-  int pos;
+  VALUE headers;
+  int   pos;
+  int   current_request_rx;
+
+  enum  read_method read_method;
+  int   body_read_mode;
+  int   body_left;
+  int   request_completed;
 } Parser_t;
 
 VALUE cParser = Qnil;
@@ -39,10 +69,10 @@ static void Parser_mark(void *ptr) {
   Parser_t *parser = ptr;
   rb_gc_mark(parser->io);
   rb_gc_mark(parser->buffer);
+  rb_gc_mark(parser->headers);
 }
 
 static void Parser_free(void *ptr) {
-  Parser_t *parser = ptr;
   xfree(ptr);
 }
 
@@ -66,40 +96,43 @@ static VALUE Parser_allocate(VALUE klass) {
 #define GetParser(obj, parser) \
   TypedData_Get_Struct((obj), Parser_t, &Parser_type, (parser))
 
+enum read_method detect_read_method(VALUE io) {
+  if (rb_respond_to(io, ID_parser_read_method)) {
+    if (mPolyphony == Qnil)
+      mPolyphony = rb_const_get(rb_cObject, rb_intern("Polyphony"));
+    VALUE method = rb_funcall(io, ID_parser_read_method, 0);
+    if (method == SYM_backend_read) return method_backend_read;
+    if (method == SYM_backend_recv) return method_backend_recv;
+    return method_readpartial;
+  }
+  else if (rb_respond_to(io, ID_call)) {
+    return method_call;
+  }
+  else
+    rb_raise(rb_eRuntimeError, "Provided reader should be a callable or respond to #__parser_read_method__");
+}
+
 VALUE Parser_initialize(VALUE self, VALUE io) {
   Parser_t *parser;
   GetParser(self, parser);
 
   parser->io = io;
   parser->buffer = rb_str_new_literal("");
+  parser->headers = Qnil;
   parser->pos = 0;
 
+  // pre-allocate the buffer
+  rb_str_modify_expand(parser->buffer, INITIAL_BUFFER_SIZE);
+
+  parser->read_method = detect_read_method(io);
+  parser->body_read_mode = BODY_READ_MODE_UNKNOWN;
+  parser->body_left = 0;
   return self;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////
-
-struct parser_state {
-  struct parser *parser;
-  char *ptr;
-  int len;
-};
 
-#define READ_CONCAT(io, buffer, len) \
-  rb_funcall(io, ID_readpartial, 4, len, buffer, BUFFER_END, Qfalse)
-
-static inline int fill_buffer(struct parser_state *state) {
-  READ_CONCAT(state->parser->io, state->parser->buffer, MAX_READ_LENGTH);
-  int len = RSTRING_LEN(state->parser->buffer);
-  int read_bytes = len - state->len;
-  if (!read_bytes) return 0;
-  
-  state->ptr = RSTRING_PTR(state->parser->buffer);
-  state->len = len;
-  return read_bytes;
-}
+#define str_downcase(str) (rb_funcall((str), ID_downcase, 0))
 
 #define FILL_BUFFER_OR_GOTO_EOF(state) { if (!fill_buffer(state)) goto eof; }
 
@@ -107,6 +140,7 @@ static inline int fill_buffer(struct parser_state *state) {
 #define BUFFER_LEN(state) ((state)->len)
 #define BUFFER_CUR(state) ((state)->ptr[(state)->parser->pos])
 #define BUFFER_AT(state, pos) ((state)->ptr[pos])
+#define BUFFER_PTR(state, pos) ((state)->ptr + pos)
 #define BUFFER_STR(state, pos, len) (rb_utf8_str_new((state)->ptr + pos, len))
 
 #define INC_BUFFER_POS(state) { \
@@ -114,7 +148,7 @@ static inline int fill_buffer(struct parser_state *state) {
   if (BUFFER_POS(state) == BUFFER_LEN(state)) FILL_BUFFER_OR_GOTO_EOF(state); \
 }
 
-#define INC_BUFFER_POS_NO_READ(state) BUFFER_POS(state)++;
+#define INC_BUFFER_POS_NO_FILL(state) BUFFER_POS(state)++;
 
 #define INC_BUFFER_POS_UTF8(state, len) { \
   unsigned char c = BUFFER_CUR(state); \
@@ -162,8 +196,66 @@ static inline int fill_buffer(struct parser_state *state) {
   RB_GC_GUARD(value); \
 }
 
-#define BUFFER_TRIM_MIN_LEN 4096
-#define BUFFER_TRIM_MIN_POS 2048
+#define CONSUME_CRLF(state) { \
+  INC_BUFFER_POS(state); \
+  if (BUFFER_CUR(state) != '\n') goto bad_request; \
+  INC_BUFFER_POS(state); \
+}
+
+#define CONSUME_CRLF_NO_FILL(state) { \
+  INC_BUFFER_POS(state); \
+  if (BUFFER_CUR(state) != '\n') goto bad_request; \
+  INC_BUFFER_POS_NO_FILL(state); \
+}
+
+#define GLOBAL_STR(v, s) v = rb_str_new_literal(s); rb_global_variable(&v)
+
+struct parser_state {
+  struct parser *parser;
+  char *ptr;
+  int len;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline VALUE io_read_call(VALUE io, VALUE maxlen, VALUE buf, VALUE buf_pos) {
+  VALUE result = rb_funcall(io, ID_call, 1, maxlen);
+  if (result == Qnil) return Qnil;
+
+  if (buf_pos == NUM_buffer_start) rb_str_set_len(buf, 0);
+  rb_str_append(buf, result);
+  RB_GC_GUARD(result);
+  return buf;
+}
+
+static inline VALUE parser_io_read(Parser_t *parser, VALUE maxlen, VALUE buf, VALUE buf_pos) {
+  switch (parser->read_method) {
+    case method_backend_read:
+      return rb_funcall(mPolyphony, ID_backend_read, 5, parser->io, buf, maxlen, Qfalse, buf_pos);
+    case method_backend_recv:
+      return rb_funcall(mPolyphony, ID_backend_recv, 4, parser->io, buf, maxlen, buf_pos);
+    case method_readpartial:
+      return rb_funcall(parser->io, ID_readpartial, 4, maxlen, buf, buf_pos, Qfalse);
+    case method_call:
+      return io_read_call(parser->io, maxlen, buf, buf_pos);
+    default:
+      return Qnil;
+  }
+}
+
+static inline int fill_buffer(struct parser_state *state) {
+  VALUE ret = parser_io_read(state->parser, NUM_max_headers_read_length, state->parser->buffer, NUM_buffer_end);
+  if (ret == Qnil) return 0;
+
+  state->parser->buffer = ret;
+  int len = RSTRING_LEN(state->parser->buffer);
+  int read_bytes = len - state->len;
+  if (!read_bytes) return 0;
+  
+  state->ptr = RSTRING_PTR(state->parser->buffer);
+  state->len = len;
+  return read_bytes;
+}
 
 static inline void buffer_trim(struct parser_state *state) {
   int len = RSTRING_LEN(state->parser->buffer);
@@ -184,27 +276,34 @@ static inline void buffer_trim(struct parser_state *state) {
   state->parser->pos = 0;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////
+static inline void str_append_from_buffer(VALUE str, char *ptr, int len) {
+  int str_len = RSTRING_LEN(str);
+  rb_str_modify_expand(str, len);
+  memcpy(RSTRING_PTR(str) + str_len, ptr, len);
+  rb_str_set_len(str, str_len + len);
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 static inline int parse_method(struct parser_state *state, VALUE headers) {
   int pos = BUFFER_POS(state);
   int len = 0;
 
-loop:
-  switch (BUFFER_CUR(state)) {
-    case ' ':
-      if (len < 1 || len > MAX_METHOD_LENGTH) goto bad_request;
-      INC_BUFFER_POS(state);
-      goto done;
-    case '\r':
-    case '\n':
-      goto bad_request;
-    default:
-      INC_BUFFER_POS_UTF8(state, len);
-      if (len >= MAX_METHOD_LENGTH) goto bad_request;
-      goto loop;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
+        if (len < 1 || len > MAX_METHOD_LENGTH) goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+      case '\n':
+        goto bad_request;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_METHOD_LENGTH) goto bad_request;
+    }
   }
 done:
   SET_HEADER_DOWNCASE_VALUE_FROM_BUFFER(state, headers, STR_pseudo_method, pos, len);
@@ -215,29 +314,31 @@ eof:
   return 0;
 }
 
-static int parse_path(struct parser_state *state, VALUE headers) {
+static int parse_request_target(struct parser_state *state, VALUE headers) {
   while (BUFFER_CUR(state) == ' ') INC_BUFFER_POS(state);
   int pos = BUFFER_POS(state);
   int len = 0;
-loop:
-  switch (BUFFER_CUR(state)) {
-    case ' ':
-      if (len < 1 || len > MAX_PATH_LENGTH) goto bad_request;
-      INC_BUFFER_POS(state);
-      goto done;
-    case '\r':
-    case '\n':
-      goto bad_request;
-    default:
-      INC_BUFFER_POS_UTF8(state, len);
-      if (len >= MAX_PATH_LENGTH) goto bad_request;
-      goto loop;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
+        if (len < 1 || len > MAX_PATH_LENGTH) goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+      case '\n':
+        goto bad_request;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_PATH_LENGTH) goto bad_request;
+    }
   }
 done:
   SET_HEADER_VALUE_FROM_BUFFER(state, headers, STR_pseudo_path, pos, len);
   return 1;
 bad_request:
-  RAISE_BAD_REQUEST("Invalid path");
+  RAISE_BAD_REQUEST("Invalid request target");
 eof:
   return 0;
 }
@@ -257,26 +358,27 @@ static int parse_protocol(struct parser_state *state, VALUE headers) {
   if (BUFFER_CUR(state) == '/') INC_BUFFER_POS(state) else goto bad_request;
   if (BUFFER_CUR(state) == '1') INC_BUFFER_POS(state) else goto bad_request;
   len = 6;
-loop:
-  switch (BUFFER_CUR(state)) {
-    case '\r':
-      INC_BUFFER_POS(state);
-      goto eol;
-    case '\n':
-      INC_BUFFER_POS(state);
-      goto done;
-    case '.':
-    case '1':
-      len++;
-      if (len > 8) goto bad_request;
-      INC_BUFFER_POS(state);
-      goto loop;
-    default:
-      goto bad_request;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case '\r':
+        CONSUME_CRLF(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS(state);
+        goto done;
+      case '.':
+        INC_BUFFER_POS(state);
+        char c = BUFFER_CUR(state);
+        if (c == '0' || c == '1') {
+          INC_BUFFER_POS(state);
+          len += 2;
+          continue;
+        }
+        goto bad_request;
+      default:
+        goto bad_request;
+    }
   }
-eol:
-  if (BUFFER_CUR(state) != '\n') goto bad_request;
-  INC_BUFFER_POS(state);
 done:
   if (len < 6 || len > 8) goto bad_request;
   SET_HEADER_DOWNCASE_VALUE_FROM_BUFFER(state, headers, STR_pseudo_protocol, pos, len);
@@ -289,7 +391,7 @@ eof:
 
 int parse_request_line(struct parser_state *state, VALUE headers) {
   if (!parse_method(state, headers)) goto eof;
-  if (!parse_path(state, headers)) goto eof;
+  if (!parse_request_target(state, headers)) goto eof;
   if (!parse_protocol(state, headers)) goto eof;
 
   return 1;
@@ -301,31 +403,31 @@ static inline int parse_header_key(struct parser_state *state, VALUE *key) {
   int pos = BUFFER_POS(state);
   int len = 0;
 
-loop:
-  switch (BUFFER_CUR(state)) {
-    case ':':
-      if (len < 1 || len > MAX_HEADER_KEY_LENGTH)
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
         goto bad_request;
-      INC_BUFFER_POS(state);
-      goto done;
-    case '\r':
-      if (BUFFER_POS(state) > pos) goto bad_request;
-
-      INC_BUFFER_POS(state);
-      goto eol;
-    case '\n':
-      if (BUFFER_POS(state) > pos) goto bad_request;
-
-      INC_BUFFER_POS_NO_READ(state);
-      goto done;
-    default:
-      INC_BUFFER_POS_UTF8(state, len);
-      if (len >= MAX_HEADER_KEY_LENGTH) goto bad_request;
-      goto loop;
+      case ':':
+        if (len < 1 || len > MAX_HEADER_KEY_LENGTH)
+          goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+        if (BUFFER_POS(state) > pos) goto bad_request;
+        CONSUME_CRLF_NO_FILL(state);
+        goto done;
+      case '\n':
+        if (BUFFER_POS(state) > pos) goto bad_request;
+
+        INC_BUFFER_POS_NO_FILL(state);
+        goto done;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_HEADER_KEY_LENGTH) goto bad_request;
+    }
   }
-eol:
-  if (BUFFER_CUR(state) != '\n') goto bad_request;
-  INC_BUFFER_POS_NO_READ(state);
 done:
   if (len == 0) return -1;
   (*key) = str_downcase(BUFFER_STR(state, pos, len));
@@ -342,21 +444,19 @@ static inline int parse_header_value(struct parser_state *state, VALUE *value) {
   int pos = BUFFER_POS(state);
   int len = 0;
 
-loop:
-  switch (BUFFER_CUR(state)) {
-    case '\r':
-      INC_BUFFER_POS(state);
-      goto eol;
-    case '\n':
-      goto done;
-    default:
-      INC_BUFFER_POS_UTF8(state, len);
-      if (len >= MAX_HEADER_VALUE_LENGTH) goto bad_request;
-      goto loop;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case '\r':
+        CONSUME_CRLF(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS(state);
+        goto done;
+      default:
+        INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_HEADER_VALUE_LENGTH) goto bad_request;
+    }
   }
-eol:
-  if (BUFFER_CUR(state) != '\n') goto bad_request;
-  INC_BUFFER_POS(state);
 done:
   if (len < 1 || len > MAX_HEADER_VALUE_LENGTH) goto bad_request;
   (*value) = BUFFER_STR(state, pos, len);
@@ -400,111 +500,285 @@ eof:
 VALUE Parser_parse_headers(VALUE self) {
   struct parser_state state;
   GetParser(self, state.parser);
-  VALUE headers = rb_hash_new();
+  state.parser->headers = rb_hash_new();
 
   buffer_trim(&state);
+  int initial_pos = state.parser->pos;
   INIT_PARSER_STATE(&state);
+  state.parser->current_request_rx = 0;
 
-  if (!parse_request_line(&state, headers)) goto eof;
+  if (!parse_request_line(&state, state.parser->headers)) goto eof;
 
-loop:
-  switch (parse_header(&state, headers)) {
-    case -1: goto done; // empty header => end of headers
-    case 0: goto eof;
-    default: goto loop;
+  int header_count = 0;
+  while (1) {
+    if (header_count > MAX_HEADER_COUNT) RAISE_BAD_REQUEST("Too many headers");
+    switch (parse_header(&state, state.parser->headers)) {
+      case -1: goto done; // empty header => end of headers
+      case 0: goto eof;
+    }
+    header_count++;
   }
-
-done:
-  RB_GC_GUARD(headers);
-  return headers;
 eof:
-  return Qnil;
+  state.parser->headers = Qnil;
+done:
+  state.parser->body_read_mode = BODY_READ_MODE_UNKNOWN;
+  int read_bytes = BUFFER_POS(&state) - initial_pos;
+
+  state.parser->current_request_rx += read_bytes;
+  if (state.parser->headers != Qnil)
+    rb_hash_aset(state.parser->headers, STR_pseudo_rx, INT2NUM(read_bytes));
+  return state.parser->headers;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
-const int READ_MAX_LEN = 1 << 20;
+static inline int str_to_int(VALUE value, const char *error_msg) {
+  char *ptr = RSTRING_PTR(value);
+  int len = RSTRING_LEN(value);
+  int int_value = 0;
 
-static inline int parse_content_length(VALUE value) {
-  VALUE to_i = rb_funcall(value, ID_to_i, 0);
-  return NUM2INT(to_i);
-}
+  while (len) {
+    char c = *ptr;
+    if ((c >= '0') && (c <= '9'))
+      int_value = int_value * 10 + (c - '0');
+    else
+      RAISE_BAD_REQUEST(error_msg);
+    len--;
+    ptr++;
+  }
 
-VALUE read_body_with_content_length(VALUE self, int content_length) {
-  if (content_length < 0) return Qnil;
+  return int_value;
+}
 
-  Parser_t *parser;
-  GetParser(self, parser);
+VALUE read_body_with_content_length(Parser_t *parser, int read_entire_body, int buffered_only) {
+  if (parser->body_left <= 0) return Qnil;
 
-  VALUE body;
+  VALUE body = Qnil;
 
   int len = RSTRING_LEN(parser->buffer);
   int pos = parser->pos;
-  int left = content_length;
 
   if (pos < len) {
     int available = len - pos;
-    if (available > content_length) available = content_length;
+    if (available > parser->body_left) available = parser->body_left;
     body = rb_str_new(RSTRING_PTR(parser->buffer) + pos, available);
     parser->pos += available;
-    left -= available;
-    len = available;
+    parser->current_request_rx += available;
+    parser->body_left -= available;
+    if (!parser->body_left) parser->request_completed = 1;
   }
   else {
     body = Qnil;
     len = 0;
   }
+  if (buffered_only) return body;
   
-  while (left) {
-    int maxlen = left <= READ_MAX_LEN ? left : READ_MAX_LEN;
-
-    if (body != Qnil) {
-      VALUE tmp_buf = rb_funcall(
-        mPolyphony, ID_backend_read, 5, parser->io, Qnil, INT2NUM(maxlen), Qfalse, INT2NUM(0)
-      );
+  while (parser->body_left) {
+    int maxlen = parser->body_left <= MAX_BODY_READ_LENGTH ? parser->body_left : MAX_BODY_READ_LENGTH;
+    VALUE tmp_buf = parser_io_read(parser, INT2NUM(maxlen), Qnil, NUM_buffer_start);
+    if (tmp_buf == Qnil) goto eof;
+    if (body != Qnil)
       rb_str_append(body, tmp_buf);
-      RB_GC_GUARD(tmp_buf);
-    }
-    else {
-      body = rb_funcall(
-        mPolyphony, ID_backend_read, 5, parser->io, Qnil, INT2NUM(maxlen), Qfalse, INT2NUM(0)
-      );
+    else
+      body = tmp_buf;
+    int read_bytes = RSTRING_LEN(tmp_buf);
+    parser->current_request_rx += read_bytes;
+    parser->body_left -= read_bytes;
+    if (!parser->body_left) parser->request_completed = 1;
+    RB_GC_GUARD(tmp_buf);
+    if (!read_entire_body) goto done;
+  }
+done:
+  rb_hash_aset(parser->headers, STR_pseudo_rx, INT2NUM(parser->current_request_rx));
+  RB_GC_GUARD(body);
+  return body;
+eof:
+  RAISE_BAD_REQUEST("Incomplete body");
+}
+
+int chunked_encoding_p(VALUE transfer_encoding) {
+  if (transfer_encoding == Qnil) return 0;
+  return rb_funcall(str_downcase(transfer_encoding), ID_eq, 1, STR_chunked) == Qtrue;
+}
+
+int parse_chunk_size(struct parser_state *state, int *chunk_size) {
+  int len = 0;
+  int value = 0;
+  int initial_pos = BUFFER_POS(state);
+
+  while (1) {
+    char c = BUFFER_CUR(state);
+    if ((c >= '0') && (c <= '9'))       value = (value << 4) + (c - '0');
+    else if ((c >= 'a') && (c <= 'f'))  value = (value << 4) + (c - 'a' + 10);
+    else if ((c >= 'A') && (c <= 'F'))  value = (value << 4) + (c - 'A' + 10);
+    else switch (c) {
+      case '\r':
+        CONSUME_CRLF_NO_FILL(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS_NO_FILL(state);
+        goto done;
+      default:
+        goto bad_request;
     }
-    int new_len = RSTRING_LEN(body);
-    int read_bytes = new_len - len;
-    if (!read_bytes) RAISE_BAD_REQUEST("Incomplete request body");
+    INC_BUFFER_POS(state);
+    len++;
+    if (len >= MAX_CHUNKED_ENCODING_CHUNK_SIZE_LENGTH) goto bad_request;
+  }
+done:
+  if (len == 0) goto bad_request;
+  (*chunk_size) = value;
+  state->parser->current_request_rx += BUFFER_POS(state) - initial_pos;
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid chunk size");
+eof:
+  return 0;
+}
 
-    len = new_len;
+int read_body_chunk_with_chunked_encoding(struct parser_state *state, VALUE *body, int chunk_size, int buffered_only) {
+  int len = RSTRING_LEN(state->parser->buffer);
+  int pos = state->parser->pos;
+  int left = chunk_size;
+
+  if (pos < len) {
+    int available = len - pos;
+    if (available > left) available = left;
+    if (*body != Qnil)
+      str_append_from_buffer(*body, RSTRING_PTR(state->parser->buffer) + pos, available);
+    else
+      *body = rb_str_new(RSTRING_PTR(state->parser->buffer) + pos, available);
+    state->parser->pos += available;
+    state->parser->current_request_rx += available;
+    left -= available;
+  }
+  if (buffered_only) return 1;
+
+  while (left) {
+    int maxlen = left <= MAX_BODY_READ_LENGTH ? left : MAX_BODY_READ_LENGTH;
+
+    VALUE tmp_buf = parser_io_read(state->parser, INT2NUM(maxlen), Qnil, NUM_buffer_start);
+    if (tmp_buf == Qnil) goto eof;
+    if (*body != Qnil)
+      rb_str_append(*body, tmp_buf);
+    else
+      *body = tmp_buf;
+    int read_bytes = RSTRING_LEN(tmp_buf);
+    state->parser->current_request_rx += read_bytes;
     left -= read_bytes;
+    RB_GC_GUARD(tmp_buf);
   }
+  return 1;
+eof:
+  return 0;
+}
+
+static inline int parse_chunk_postfix(struct parser_state *state) {
+  int initial_pos = BUFFER_POS(state);
+  if (initial_pos == BUFFER_LEN(state)) FILL_BUFFER_OR_GOTO_EOF(state);
+  switch (BUFFER_CUR(state)) {
+    case '\r':
+      CONSUME_CRLF_NO_FILL(state);
+      goto done;
+    case '\n':
+      INC_BUFFER_POS_NO_FILL(state);
+      goto done;
+    default:
+      goto bad_request;
+  }
+done:
+  state->parser->current_request_rx += BUFFER_POS(state) - initial_pos;
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid protocol");
+eof:
+  return 0;
+}
 
+VALUE read_body_with_chunked_encoding(Parser_t *parser, int read_entire_body, int buffered_only) {
+  struct parser_state state;
+  state.parser = parser;
+  buffer_trim(&state);
+  INIT_PARSER_STATE(&state);
+  VALUE body = Qnil;
+
+  while (1) {
+    int chunk_size = 0;
+    if (BUFFER_POS(&state) == BUFFER_LEN(&state)) FILL_BUFFER_OR_GOTO_EOF(&state);
+    if (!parse_chunk_size(&state, &chunk_size)) goto bad_request;
+    
+    if (chunk_size) {
+      if (!read_body_chunk_with_chunked_encoding(&state, &body, chunk_size, buffered_only)) goto bad_request;
+    }
+    else parser->request_completed = 1;
+
+    if (!parse_chunk_postfix(&state)) goto bad_request;
+    if (!chunk_size || !read_entire_body) goto done;
+  }
+bad_request:
+  RAISE_BAD_REQUEST("Malformed request body");
+eof:
+  RAISE_BAD_REQUEST("Incomplete request body");
+done:
+  rb_hash_aset(parser->headers, STR_pseudo_rx, INT2NUM(state.parser->current_request_rx));
   RB_GC_GUARD(body);
   return body;
 }
 
-VALUE Parser_read_body(VALUE self, VALUE headers) {
-  VALUE content_length = rb_hash_aref(headers, STR_content_length);
-  if (content_length != Qnil)
-    return read_body_with_content_length(self, parse_content_length(content_length));
+static inline void detect_body_read_mode(Parser_t *parser) {
+  VALUE content_length = rb_hash_aref(parser->headers, STR_content_length);
+  if (content_length != Qnil) {
+    int int_content_length = str_to_int(content_length, "Invalid content length");
+    if (int_content_length < 0) RAISE_BAD_REQUEST("Invalid body content length");
+    parser->body_read_mode = parser->body_left = int_content_length;
+    parser->request_completed = 0;
+    return;
+  }
   
-  // VALUE transfer_encoding = rb_hash_aref(headers, STR_transfer_encoding);
-  // if (chunked_encoding_p(transfer_encoding)) {
-  //   return read_body_with_chunked_encoding(self);
-  // }
+  VALUE transfer_encoding = rb_hash_aref(parser->headers, STR_transfer_encoding);
+  if (chunked_encoding_p(transfer_encoding)) {
+    parser->body_read_mode = BODY_READ_MODE_CHUNKED;
+    parser->request_completed = 0;
+    return;
+  }
+  parser->request_completed = 1;
 
-  return Qnil;
 }
 
-#define GLOBAL_STR(v, s) v = rb_str_new_literal(s); rb_global_variable(&v)
+static inline VALUE read_body(VALUE self, int read_entire_body, int buffered_only) {
+  Parser_t *parser;
+  GetParser(self, parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_UNKNOWN)
+    detect_body_read_mode(parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_CHUNKED)
+    return read_body_with_chunked_encoding(parser, read_entire_body, buffered_only);
+  return read_body_with_content_length(parser, read_entire_body, buffered_only);
+}
+
+VALUE Parser_read_body(VALUE self) {
+  return read_body(self, 1, 0);
+}
+
+VALUE Parser_read_body_chunk(VALUE self, VALUE buffered_only) {
+  return read_body(self, 0, buffered_only == Qtrue);
+}
+
+VALUE Parser_complete_p(VALUE self) {
+  Parser_t *parser;
+  GetParser(self, parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_UNKNOWN)
+    detect_body_read_mode(parser);
+
+  return parser->request_completed ? Qtrue : Qfalse;
+}
 
 void Init_HTTP1_Parser() {
   VALUE mTipi;
   VALUE cHTTP1Parser;
 
-  mPolyphony = rb_const_get(rb_cObject, rb_intern("Polyphony"));
-
   mTipi = rb_define_module("Tipi");
   cHTTP1Parser = rb_define_class_under(mTipi, "HTTP1Parser", rb_cObject);
   rb_define_alloc_func(cHTTP1Parser, Parser_allocate);
@@ -514,21 +788,36 @@ void Init_HTTP1_Parser() {
   // backend methods
   rb_define_method(cHTTP1Parser, "initialize", Parser_initialize, 1);
   rb_define_method(cHTTP1Parser, "parse_headers", Parser_parse_headers, 0);
-  rb_define_method(cHTTP1Parser, "read_body", Parser_read_body, 1);
-
-  ID_backend_read = rb_intern("backend_read");
-  ID_downcase     = rb_intern("downcase");
-  ID_read         = rb_intern("read");
-  ID_readpartial  = rb_intern("readpartial");
-  ID_to_i         = rb_intern("to_i");
-
-  MAX_READ_LENGTH = INT2NUM(4096);
-  BUFFER_END = INT2NUM(-1);
+  rb_define_method(cHTTP1Parser, "read_body", Parser_read_body, 0);
+  rb_define_method(cHTTP1Parser, "read_body_chunk", Parser_read_body_chunk, 1);
+  rb_define_method(cHTTP1Parser, "complete?", Parser_complete_p, 0);
+
+  ID_arity                  = rb_intern("arity");
+  ID_backend_read           = rb_intern("backend_read");
+  ID_backend_recv           = rb_intern("backend_recv");
+  ID_call                   = rb_intern("call");
+  ID_downcase               = rb_intern("downcase");
+  ID_eq                     = rb_intern("==");
+  ID_parser_read_method     = rb_intern("__parser_read_method__");
+  ID_read                   = rb_intern("read");
+  ID_readpartial            = rb_intern("readpartial");
+  ID_to_i                   = rb_intern("to_i");
+
+  NUM_max_headers_read_length = INT2NUM(MAX_HEADERS_READ_LENGTH);
+  NUM_buffer_start = INT2NUM(0);
+  NUM_buffer_end = INT2NUM(-1);
 
   GLOBAL_STR(STR_pseudo_method,       ":method");
   GLOBAL_STR(STR_pseudo_path,         ":path");
   GLOBAL_STR(STR_pseudo_protocol,     ":protocol");
+  GLOBAL_STR(STR_pseudo_rx,           ":rx");
 
+  GLOBAL_STR(STR_chunked,             "chunked");
   GLOBAL_STR(STR_content_length,      "content-length");
   GLOBAL_STR(STR_transfer_encoding,   "transfer-encoding");
+
+  SYM_backend_read = ID2SYM(ID_backend_read);
+  SYM_backend_recv = ID2SYM(ID_backend_recv);
+
+  rb_global_variable(&mTipi);
 }