h1p 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4fb3573e65df46350986981759a454c8bc2c4166901a46dd827fdc17a27b08b5
+  data.tar.gz: afc10ead82e3286f61fa2bee1739499f275b75a386ed3e3401988805470c9b3a
+SHA512:
+  metadata.gz: 72ba304cda888fcfb1c8afaee1a03b94ddb15fde0110c303886e4036125fb5b464d214ebb731ea91dac792070c9cc3d5eccb789f667c8a590fae01afb88d5665
+  data.tar.gz: 31cd6c6d7bb6695d338596b56be808cd1fed99cc76f11ee5f38fbdf7ff5d1206960a7134932ac82e9ca9c19116a20ee99db8f2a002fb76c1997134ac4d432dae

data/.github/workflows/test.yml

@@ -0,0 +1,31 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        ruby: [2.6, 2.7, 3.0]
+
+    name: >-
+      ${{matrix.os}}, ${{matrix.ruby}}
+
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: ${{matrix.ruby}}
+    - name: Install dependencies
+      run: |
+        gem install bundler
+        bundle install
+    - name: Show Linux kernel version
+      run: uname -r
+    - name: Compile C-extension
+      run: bundle exec rake compile
+    - name: Run tests
+      run:  bundle exec rake test

data/.gitignore

@@ -0,0 +1,57 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*
+lib/*.so

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 0.42 2021-08-16
+
+## 0.1 2021-08-19
+
+- Import code from Tipi

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,33 @@
+PATH
+  remote: .
+  specs:
+    h1p (0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.5.0)
+    builder (3.2.4)
+    minitest (5.11.3)
+    minitest-reporters (1.4.3)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
+    rake (12.3.3)
+    rake-compiler (1.1.1)
+      rake
+    ruby-progressbar (1.11.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  h1p!
+  minitest (~> 5.11.3)
+  minitest-reporters (~> 1.4.2)
+  rake (~> 12.3.3)
+  rake-compiler (= 1.1.1)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Digital Fabric
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,5 @@
+# h1p
+
+h1p 
+
+HTTP 1.1 parser for Ruby

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/clean"
+
+require "rake/extensiontask"
+Rake::ExtensionTask.new("h1p_ext") do |ext|
+  ext.ext_dir = "ext/h1p"
+end
+
+task :recompile => [:clean, :compile]
+task :default => [:compile, :test]
+
+task :test do
+  exec 'ruby test/test_h1p.rb'
+end

data/TODO.md

@@ -0,0 +1,106 @@
+## Add an API for reading a request body chunk into an IO (pipe)
+
+      ```ruby
+      # currently
+      chunk = req.next_chunk
+      # or
+      req.each_chunk { |c| do_something(c) }
+
+      # what we'd like to do
+      r, w = IO.pipe
+      len = req.splice_chunk(w)
+      sock << "Here comes a chunk of #{len} bytes\n"
+      sock.splice(r, len)
+
+      # or:
+      r, w = IO.pipe
+      req.splice_each_chunk(w) do |len|
+        sock << "Here comes a chunk of #{len} bytes\n"
+        sock.splice(r, len)
+      end
+      ```
+
+# HTTP/1.1 parser
+
+- httparser.rb is not actively updated
+- the httparser.rb C parser code comes originally from https://github.com/nodejs/llhttp
+- there's a Ruby gem https://github.com/metabahn/llhttp, but its API is too low-level
+  (lots of callbacks, headers need to be retained across callbacks)
+- the basic idea is to import the C-code, then build a parser object with the following
+  callbacks:
+
+  ```ruby
+  on_headers_complete(headers)
+  on_body_chunk(chunk)
+  on_message_complete
+  ```
+
+- The llhttp gem's C-code is here: https://github.com/metabahn/llhttp/tree/main/mri
+
+- Actually, if you do a C extension, instead of a callback-based API, we can
+  design a blocking API:
+
+  ```ruby
+  parser = Tipi::HTTP1::Parser.new
+  parser.each_request(socket) do |headers|
+    request = Request.new(normalize_headers(headers))
+    handle_request(request)
+  end
+  ```
+
+# What about HTTP/2?
+
+It would be a nice exercise in converting a callback-based API to a blocking
+one:
+
+```ruby
+parser = Tipi::HTTP2::Parser.new(socket)
+parser.each_stream(socket) do |stream|
+  spin { handle_stream(stream) }
+end
+```
+
+
+
+# DF
+
+- Add attack protection for IP-address HTTP host:
+
+  ```ruby
+  IPV4_REGEXP = /^\d+\.\d+\.\d+\.\d+$/.freeze
+
+  def is_attack_request?(req)
+    return true if req.host =~ IPV4_REGEXP && req.query[:q] != 'ping'
+  end
+  ```
+
+- Add attack route to Qeweney routing API
+
+
+
+# Roadmap
+
+- Update README (get rid of non-http stuff)
+- Improve Rack spec compliance, add tests
+- Homogenize HTTP 1 and HTTP 2 headers - downcase symbols
+
+- Use `http-2-next` instead of `http-2` for http/2
+  - https://gitlab.com/honeyryderchuck/http-2-next
+  - Open an issue there, ask what's the difference between the two gems?
+
+## 0.38
+
+- Add more poly CLI commands and options:
+
+  - serve static files from given directory
+  - serve from rack up file
+  - serve both http and https
+  - use custom certificate files for SSL
+  - set host address to bind to
+  - set port to bind to
+  - set forking process count
+
+## 0.39 Working Sinatra application
+
+- app with database access (postgresql)
+- benchmarks!

data/benchmarks/bm_http1_parser.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
+
+def measure_time_and_allocs
+  4.times { GC.start }
+  GC.disable
+
+  t0 = Time.now
+  a0 = object_count
+  yield
+  t1 = Time.now
+  a1 = object_count
+  [t1 - t0, a1 - a0]
+ensure
+  GC.enable
+end
+
+def object_count
+  count = ObjectSpace.count_objects
+  count[:TOTAL] - count[:FREE]
+end
+
+def benchmark_other_http1_parser(iterations)
+  STDOUT << "http_parser.rb: "
+  require 'http_parser.rb'
+  
+  i, o = IO.pipe
+  parser = Http::Parser.new
+  done = false
+  headers = nil
+  parser.on_headers_complete = proc do |h|
+    headers = h
+    headers[':method'] = parser.http_method
+    headers[':path'] = parser.request_url
+  end
+  parser.on_message_complete = proc { done = true }
+
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      done = false
+      while !done
+        msg = i.readpartial(4096)
+        parser << msg
+      end
+    end
+  end
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def benchmark_tipi_http1_parser(iterations)
+  STDOUT << "tipi parser: "
+  require_relative '../lib/tipi_ext'
+  i, o = IO.pipe
+  reader = proc { |len| i.readpartial(len) }
+  parser = Tipi::HTTP1Parser.new(reader)
+
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      headers = parser.parse_headers
+    end
+  end
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def fork_benchmark(method, iterations)
+  pid = fork do
+    send(method, iterations)
+  rescue Exception => e
+    p e
+    p e.backtrace
+    exit!
+  end
+  Process.wait(pid)
+end
+
+x = 500000
+# fork_benchmark(:benchmark_other_http1_parser, x)
+# fork_benchmark(:benchmark_tipi_http1_parser, x)
+
+benchmark_tipi_http1_parser(x)

data/examples/http_server.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+
+opts = {
+  reuse_addr:  true,
+  dont_linger: true
+}
+
+puts "pid: #{Process.pid}"
+puts 'Listening on port 10080...'
+
+# GC.disable
+# Thread.current.backend.idle_gc_period = 60
+
+spin_loop(interval: 10) { p Thread.backend.stats }
+
+spin_loop(interval: 10) do
+  GC.compact
+end
+
+spin do
+  Tipi.serve('0.0.0.0', 10080, opts) do |req|
+    if req.path == '/stream'
+      req.send_headers('Foo' => 'Bar')
+      sleep 1
+      req.send_chunk("foo\n")
+      sleep 1
+      req.send_chunk("bar\n")
+      req.finish
+    elsif req.path == '/upload'
+      body = req.read
+      req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
+    else
+      req.respond("Hello world!\n")
+    end
+#    p req.transfer_counts
+  end
+  p 'done...'
+end.await

data/ext/h1p/extconf.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'mkmf'
+
+$CFLAGS << " -Wno-format-security"
+CONFIG['optflags'] << ' -fno-strict-aliasing' unless RUBY_PLATFORM =~ /mswin/
+
+require_relative './limits'
+H1P_LIMITS.each { |k, v| $defs << "-D#{k.upcase}=#{v}" }
+
+dir_config 'h1p_ext'
+create_makefile 'h1p_ext'

data/ext/h1p/h1p.c

@@ -0,0 +1,860 @@
+#include "ruby.h"
+#include "h1p.h"
+
+// Security-related limits are defined in limits.rb and injected as
+// defines in extconf.rb
+
+#define INITIAL_BUFFER_SIZE     4096
+#define BUFFER_TRIM_MIN_LEN     4096
+#define BUFFER_TRIM_MIN_POS     2048
+#define MAX_HEADERS_READ_LENGTH 4096
+#define MAX_BODY_READ_LENGTH    (1 << 20) // 1MB
+
+#define BODY_READ_MODE_UNKNOWN  -2
+#define BODY_READ_MODE_CHUNKED  -1
+
+ID ID_arity;
+ID ID_backend_read;
+ID ID_backend_recv;
+ID ID_call;
+ID ID_downcase;
+ID ID_eof_p;
+ID ID_eq;
+ID ID_parser_read_method;
+ID ID_read;
+ID ID_readpartial;
+ID ID_to_i;
+
+static VALUE mPolyphony = Qnil;
+static VALUE cError;
+
+VALUE NUM_max_headers_read_length;
+VALUE NUM_buffer_start;
+VALUE NUM_buffer_end;
+
+VALUE STR_pseudo_method;
+VALUE STR_pseudo_path;
+VALUE STR_pseudo_protocol;
+VALUE STR_pseudo_rx;
+
+VALUE STR_chunked;
+VALUE STR_content_length;
+VALUE STR_transfer_encoding;
+
+VALUE SYM_backend_read;
+VALUE SYM_backend_recv;
+VALUE SYM_stock_readpartial;
+
+enum read_method {
+  method_readpartial,       // receiver.readpartial(len, buf, pos, raise_on_eof: false) (Polyphony-specific)
+  method_backend_read,      // Polyphony.backend_read (Polyphony-specific)
+  method_backend_recv,      // Polyphony.backend_recv (Polyphony-specific)
+  method_call,              // receiver.call(len) (Universal)
+  method_stock_readpartial  // receiver.readpartial(len)
+};
+
+typedef struct parser {
+  VALUE io;
+  VALUE buffer;
+  VALUE headers;
+  int   pos;
+  int   current_request_rx;
+
+  enum  read_method read_method;
+  int   body_read_mode;
+  int   body_left;
+  int   request_completed;
+} Parser_t;
+
+VALUE cParser = Qnil;
+
+static void Parser_mark(void *ptr) {
+  Parser_t *parser = ptr;
+  rb_gc_mark(parser->io);
+  rb_gc_mark(parser->buffer);
+  rb_gc_mark(parser->headers);
+}
+
+static void Parser_free(void *ptr) {
+  xfree(ptr);
+}
+
+static size_t Parser_size(const void *ptr) {
+  return sizeof(Parser_t);
+}
+
+static const rb_data_type_t Parser_type = {
+  "Parser",
+  {Parser_mark, Parser_free, Parser_size,},
+  0, 0, 0
+};
+
+static VALUE Parser_allocate(VALUE klass) {
+  Parser_t *parser;
+
+  parser = ALLOC(Parser_t);
+  return TypedData_Wrap_Struct(klass, &Parser_type, parser);
+}
+
+#define GetParser(obj, parser) \
+  TypedData_Get_Struct((obj), Parser_t, &Parser_type, (parser))
+
+static inline void get_polyphony() {
+  if (mPolyphony != Qnil) return;
+
+  mPolyphony = rb_const_get(rb_cObject, rb_intern("Polyphony"));
+  rb_gc_register_mark_object(mPolyphony);
+}
+
+enum read_method detect_read_method(VALUE io) {
+  if (rb_respond_to(io, ID_parser_read_method)) {
+    VALUE method = rb_funcall(io, ID_parser_read_method, 0);
+    if (method == SYM_stock_readpartial) return method_stock_readpartial;
+
+    get_polyphony();
+    if (method == SYM_backend_read)      return method_backend_read;
+    if (method == SYM_backend_recv)      return method_backend_recv;
+
+    return method_readpartial;
+  }
+  else if (rb_respond_to(io, ID_call)) {
+    return method_call;
+  }
+  else
+    rb_raise(rb_eRuntimeError, "Provided reader should be a callable or respond to #__parser_read_method__");
+}
+
+VALUE Parser_initialize(VALUE self, VALUE io) {
+  Parser_t *parser;
+  GetParser(self, parser);
+
+  parser->io = io;
+  parser->buffer = rb_str_new_literal("");
+  parser->headers = Qnil;
+  parser->pos = 0;
+
+  // pre-allocate the buffer
+  rb_str_modify_expand(parser->buffer, INITIAL_BUFFER_SIZE);
+
+  parser->read_method = detect_read_method(io);
+  parser->body_read_mode = BODY_READ_MODE_UNKNOWN;
+  parser->body_left = 0;
+  return self;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define str_downcase(str) (rb_funcall((str), ID_downcase, 0))
+
+#define FILL_BUFFER_OR_GOTO_EOF(state) { if (!fill_buffer(state)) goto eof; }
+
+#define BUFFER_POS(state) ((state)->parser->pos)
+#define BUFFER_LEN(state) ((state)->len)
+#define BUFFER_CUR(state) ((state)->ptr[(state)->parser->pos])
+#define BUFFER_AT(state, pos) ((state)->ptr[pos])
+#define BUFFER_PTR(state, pos) ((state)->ptr + pos)
+#define BUFFER_STR(state, pos, len) (rb_obj_freeze(rb_utf8_str_new((state)->ptr + pos, len)))
+#define BUFFER_STR_DOWNCASE(state, pos, len) (rb_obj_freeze(str_downcase(rb_utf8_str_new((state)->ptr + pos, len))))
+
+#define INC_BUFFER_POS(state) { \
+  BUFFER_POS(state)++; \
+  if (BUFFER_POS(state) == BUFFER_LEN(state)) FILL_BUFFER_OR_GOTO_EOF(state); \
+}
+
+#define INC_BUFFER_POS_NO_FILL(state) BUFFER_POS(state)++;
+
+#define INC_BUFFER_POS_UTF8(state, len) { \
+  unsigned char c = BUFFER_CUR(state); \
+  if ((c & 0xf0) == 0xf0) { \
+    while (BUFFER_LEN(state) - BUFFER_POS(state) < 4) FILL_BUFFER_OR_GOTO_EOF(state); \
+    BUFFER_POS(state) += 4; \
+    len += 4; \
+  } \
+  else if ((c & 0xe0) == 0xe0) { \
+    while (BUFFER_LEN(state) - BUFFER_POS(state) < 3) FILL_BUFFER_OR_GOTO_EOF(state); \
+    BUFFER_POS(state) += 3; \
+    len += 3; \
+  } \
+  else if ((c & 0xc0) == 0xc0) { \
+    while (BUFFER_LEN(state) - BUFFER_POS(state) < 2) FILL_BUFFER_OR_GOTO_EOF(state); \
+    BUFFER_POS(state) += 2; \
+    len += 2; \
+  } \
+  else { \
+    BUFFER_POS(state)++; \
+    len ++; \
+    if (BUFFER_POS(state) == BUFFER_LEN(state)) FILL_BUFFER_OR_GOTO_EOF(state); \
+  } \
+}
+
+#define INIT_PARSER_STATE(state) { \
+  (state)->len = RSTRING_LEN((state)->parser->buffer); \
+  if (BUFFER_POS(state) == BUFFER_LEN(state)) \
+    FILL_BUFFER_OR_GOTO_EOF(state) \
+  else \
+    (state)->ptr = RSTRING_PTR((state)->parser->buffer); \
+}
+
+#define RAISE_BAD_REQUEST(msg) rb_raise(cError, msg)
+
+#define SET_HEADER_VALUE_FROM_BUFFER(state, headers, key, pos, len) { \
+  VALUE value = BUFFER_STR(state, pos, len); \
+  rb_hash_aset(headers, key, value); \
+  RB_GC_GUARD(value); \
+}
+
+#define SET_HEADER_DOWNCASE_VALUE_FROM_BUFFER(state, headers, key, pos, len) { \
+  VALUE value = BUFFER_STR_DOWNCASE(state, pos, len); \
+  rb_hash_aset(headers, key, value); \
+  RB_GC_GUARD(value); \
+}
+
+#define CONSUME_CRLF(state) { \
+  INC_BUFFER_POS(state); \
+  if (BUFFER_CUR(state) != '\n') goto bad_request; \
+  INC_BUFFER_POS(state); \
+}
+
+#define CONSUME_CRLF_NO_FILL(state) { \
+  INC_BUFFER_POS(state); \
+  if (BUFFER_CUR(state) != '\n') goto bad_request; \
+  INC_BUFFER_POS_NO_FILL(state); \
+}
+
+#define GLOBAL_STR(v, s) v = rb_str_new_literal(s); rb_global_variable(&v); rb_obj_freeze(v)
+
+struct parser_state {
+  struct parser *parser;
+  char *ptr;
+  int len;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline VALUE io_call(VALUE io, VALUE maxlen, VALUE buf, VALUE buf_pos) {
+  VALUE result = rb_funcall(io, ID_call, 1, maxlen);
+  if (result == Qnil) return Qnil;
+
+  if (buf_pos == NUM_buffer_start) rb_str_set_len(buf, 0);
+  rb_str_append(buf, result);
+  RB_GC_GUARD(result);
+  return buf;
+}
+
+static inline VALUE io_stock_readpartial(VALUE io, VALUE maxlen, VALUE buf, VALUE buf_pos) {
+  VALUE eof = rb_funcall(io, ID_eof_p, 0);
+  if (RTEST(eof)) return Qnil;
+
+  VALUE result = rb_funcall(io, ID_readpartial, 1, maxlen);
+  if (result == Qnil) return Qnil;
+  if (buf == Qnil) return result;
+
+  if (buf_pos == NUM_buffer_start) rb_str_set_len(buf, 0);
+  rb_str_append(buf, result);
+  RB_GC_GUARD(result);
+  return buf;
+}
+
+static inline VALUE parser_io_read(Parser_t *parser, VALUE maxlen, VALUE buf, VALUE buf_pos) {
+  switch (parser->read_method) {
+    case method_backend_read:
+      return rb_funcall(mPolyphony, ID_backend_read, 5, parser->io, buf, maxlen, Qfalse, buf_pos);
+    case method_backend_recv:
+      return rb_funcall(mPolyphony, ID_backend_recv, 4, parser->io, buf, maxlen, buf_pos);
+    case method_readpartial:    
+      return rb_funcall(parser->    io, ID_readpartial, 4, maxlen, buf, buf_pos, Qfalse);
+    case method_call:   
+      return io_call(parser    ->io, maxlen, buf, buf_pos);
+    case method_stock_readpartial:
+      return io_stock_readpartial(parser->io, maxlen, buf, buf_pos);
+    default:
+      return Qnil;
+  }
+}
+
+static inline int fill_buffer(struct parser_state *state) {
+  VALUE ret = parser_io_read(state->parser, NUM_max_headers_read_length, state->parser->buffer, NUM_buffer_end);
+  if (ret == Qnil) return 0;
+
+  state->parser->buffer = ret;
+  int len = RSTRING_LEN(state->parser->buffer);
+  int read_bytes = len - state->len;
+  if (!read_bytes) return 0;
+  
+  state->ptr = RSTRING_PTR(state->parser->buffer);
+  state->len = len;
+  return read_bytes;
+}
+
+static inline void buffer_trim(struct parser_state *state) {
+  int len = RSTRING_LEN(state->parser->buffer);
+  int pos = state->parser->pos;
+  int left = len - pos;
+
+  // The buffer is trimmed only if length and position thresholds are passed,
+  // *and* position is past the halfway point. 
+  if (len < BUFFER_TRIM_MIN_LEN || 
+      pos < BUFFER_TRIM_MIN_POS ||
+      left >= pos) return;
+
+  if (left > 0) {
+    char *ptr = RSTRING_PTR(state->parser->buffer);
+    memcpy(ptr, ptr + pos, left);
+  }
+  rb_str_set_len(state->parser->buffer, left);
+  state->parser->pos = 0;
+}
+
+static inline void str_append_from_buffer(VALUE str, char *ptr, int len) {
+  int str_len = RSTRING_LEN(str);
+  rb_str_modify_expand(str, len);
+  memcpy(RSTRING_PTR(str) + str_len, ptr, len);
+  rb_str_set_len(str, str_len + len);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline int parse_method(struct parser_state *state, VALUE headers) {
+  int pos = BUFFER_POS(state);
+  int len = 0;
+
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
+        if (len < 1 || len > MAX_METHOD_LENGTH) goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+      case '\n':
+        goto bad_request;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_METHOD_LENGTH) goto bad_request;
+    }
+  }
+done:
+  SET_HEADER_DOWNCASE_VALUE_FROM_BUFFER(state, headers, STR_pseudo_method, pos, len);
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid method");
+eof:
+  return 0;
+}
+
+static int parse_request_target(struct parser_state *state, VALUE headers) {
+  while (BUFFER_CUR(state) == ' ') INC_BUFFER_POS(state);
+  int pos = BUFFER_POS(state);
+  int len = 0;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
+        if (len < 1 || len > MAX_PATH_LENGTH) goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+      case '\n':
+        goto bad_request;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_PATH_LENGTH) goto bad_request;
+    }
+  }
+done:
+  SET_HEADER_VALUE_FROM_BUFFER(state, headers, STR_pseudo_path, pos, len);
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid request target");
+eof:
+  return 0;
+}
+
+// case-insensitive compare
+#define CMP_CI(state, down, up) ((BUFFER_CUR(state) == down) || (BUFFER_CUR(state) == up))
+
+static int parse_protocol(struct parser_state *state, VALUE headers) {
+  while (BUFFER_CUR(state) == ' ') INC_BUFFER_POS(state);
+  int pos = BUFFER_POS(state);
+  int len = 0;
+
+  if (CMP_CI(state, 'H', 'h')) INC_BUFFER_POS(state) else goto bad_request;
+  if (CMP_CI(state, 'T', 't')) INC_BUFFER_POS(state) else goto bad_request;
+  if (CMP_CI(state, 'T', 't')) INC_BUFFER_POS(state) else goto bad_request;
+  if (CMP_CI(state, 'P', 'p')) INC_BUFFER_POS(state) else goto bad_request;
+  if (BUFFER_CUR(state) == '/') INC_BUFFER_POS(state) else goto bad_request;
+  if (BUFFER_CUR(state) == '1') INC_BUFFER_POS(state) else goto bad_request;
+  len = 6;
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case '\r':
+        CONSUME_CRLF(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS(state);
+        goto done;
+      case '.':
+        INC_BUFFER_POS(state);
+        char c = BUFFER_CUR(state);
+        if (c == '0' || c == '1') {
+          INC_BUFFER_POS(state);
+          len += 2;
+          continue;
+        }
+        goto bad_request;
+      default:
+        goto bad_request;
+    }
+  }
+done:
+  if (len < 6 || len > 8) goto bad_request;
+  SET_HEADER_DOWNCASE_VALUE_FROM_BUFFER(state, headers, STR_pseudo_protocol, pos, len);
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid protocol");
+eof:
+  return 0;
+}
+
+int parse_request_line(struct parser_state *state, VALUE headers) {
+  if (!parse_method(state, headers)) goto eof;
+  if (!parse_request_target(state, headers)) goto eof;
+  if (!parse_protocol(state, headers)) goto eof;
+
+  return 1;
+eof:
+  return 0;
+}
+
+static inline int parse_header_key(struct parser_state *state, VALUE *key) {
+  int pos = BUFFER_POS(state);
+  int len = 0;
+
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case ' ':
+        goto bad_request;
+      case ':':
+        if (len < 1 || len > MAX_HEADER_KEY_LENGTH)
+          goto bad_request;
+        INC_BUFFER_POS(state);
+        goto done;
+      case '\r':
+        if (BUFFER_POS(state) > pos) goto bad_request;
+        CONSUME_CRLF_NO_FILL(state);
+        goto done;
+      case '\n':
+        if (BUFFER_POS(state) > pos) goto bad_request;
+
+        INC_BUFFER_POS_NO_FILL(state);
+        goto done;
+      default:
+        INC_BUFFER_POS(state);
+        len++;
+        // INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_HEADER_KEY_LENGTH) goto bad_request;
+    }
+  }
+done:
+  if (len == 0) return -1;
+  (*key) = BUFFER_STR_DOWNCASE(state, pos, len);
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid header key");
+eof:
+  return 0;
+}
+
+static inline int parse_header_value(struct parser_state *state, VALUE *value) {
+  while (BUFFER_CUR(state) == ' ') INC_BUFFER_POS(state);
+
+  int pos = BUFFER_POS(state);
+  int len = 0;
+
+  while (1) {
+    switch (BUFFER_CUR(state)) {
+      case '\r':
+        CONSUME_CRLF(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS(state);
+        goto done;
+      default:
+        INC_BUFFER_POS_UTF8(state, len);
+        if (len > MAX_HEADER_VALUE_LENGTH) goto bad_request;
+    }
+  }
+done:
+  if (len < 1 || len > MAX_HEADER_VALUE_LENGTH) goto bad_request;
+  (*value) = BUFFER_STR(state, pos, len);
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid header value");
+eof:
+  return 0;
+}
+
+static inline int parse_header(struct parser_state *state, VALUE headers) {
+  VALUE key, value;
+
+  switch (parse_header_key(state, &key)) {
+    case -1: return -1;
+    case 0: goto eof;
+  }
+
+  if (!parse_header_value(state, &value)) goto eof;
+
+  VALUE existing = rb_hash_aref(headers, key);
+  if (existing != Qnil) {
+    if (TYPE(existing) != T_ARRAY) {
+      existing = rb_ary_new3(2, existing, value);
+      rb_hash_aset(headers, key, existing);
+    }
+    else
+      rb_ary_push(existing, value);
+  }
+  else
+    rb_hash_aset(headers, key, value);
+
+  RB_GC_GUARD(existing);
+  RB_GC_GUARD(key);
+  RB_GC_GUARD(value);
+  return 1;
+eof:
+  return 0;
+}
+
+VALUE Parser_parse_headers(VALUE self) {
+  struct parser_state state;
+  GetParser(self, state.parser);
+  state.parser->headers = rb_hash_new();
+
+  buffer_trim(&state);
+  int initial_pos = state.parser->pos;
+  INIT_PARSER_STATE(&state);
+  state.parser->current_request_rx = 0;
+
+  if (!parse_request_line(&state, state.parser->headers)) goto eof;
+
+  int header_count = 0;
+  while (1) {
+    if (header_count > MAX_HEADER_COUNT) RAISE_BAD_REQUEST("Too many headers");
+    switch (parse_header(&state, state.parser->headers)) {
+      case -1: goto done; // empty header => end of headers
+      case 0: goto eof;
+    }
+    header_count++;
+  }
+eof:
+  state.parser->headers = Qnil;
+done:
+  state.parser->body_read_mode = BODY_READ_MODE_UNKNOWN;
+  int read_bytes = BUFFER_POS(&state) - initial_pos;
+
+  state.parser->current_request_rx += read_bytes;
+  if (state.parser->headers != Qnil)
+    rb_hash_aset(state.parser->headers, STR_pseudo_rx, INT2NUM(read_bytes));
+  return state.parser->headers;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static inline int str_to_int(VALUE value, const char *error_msg) {
+  char *ptr = RSTRING_PTR(value);
+  int len = RSTRING_LEN(value);
+  int int_value = 0;
+
+  while (len) {
+    char c = *ptr;
+    if ((c >= '0') && (c <= '9'))
+      int_value = int_value * 10 + (c - '0');
+    else
+      RAISE_BAD_REQUEST(error_msg);
+    len--;
+    ptr++;
+  }
+
+  return int_value;
+}
+
+VALUE read_body_with_content_length(Parser_t *parser, int read_entire_body, int buffered_only) {
+  if (parser->body_left <= 0) return Qnil;
+
+  VALUE body = Qnil;
+
+  int len = RSTRING_LEN(parser->buffer);
+  int pos = parser->pos;
+
+  if (pos < len) {
+    int available = len - pos;
+    if (available > parser->body_left) available = parser->body_left;
+    body = rb_str_new(RSTRING_PTR(parser->buffer) + pos, available);
+    parser->pos += available;
+    parser->current_request_rx += available;
+    parser->body_left -= available;
+    if (!parser->body_left) parser->request_completed = 1;
+  }
+  else {
+    body = Qnil;
+    len = 0;
+  }
+  if (buffered_only) return body;
+  
+  while (parser->body_left) {
+    int maxlen = parser->body_left <= MAX_BODY_READ_LENGTH ? parser->body_left : MAX_BODY_READ_LENGTH;
+    VALUE tmp_buf = parser_io_read(parser, INT2NUM(maxlen), Qnil, NUM_buffer_start);
+    if (tmp_buf == Qnil) goto eof;
+    if (body != Qnil)
+      rb_str_append(body, tmp_buf);
+    else
+      body = tmp_buf;
+    int read_bytes = RSTRING_LEN(tmp_buf);
+    parser->current_request_rx += read_bytes;
+    parser->body_left -= read_bytes;
+    if (!parser->body_left) parser->request_completed = 1;
+    RB_GC_GUARD(tmp_buf);
+    if (!read_entire_body) goto done;
+  }
+done:
+  rb_hash_aset(parser->headers, STR_pseudo_rx, INT2NUM(parser->current_request_rx));
+  RB_GC_GUARD(body);
+  return body;
+eof:
+  RAISE_BAD_REQUEST("Incomplete body");
+}
+
+int chunked_encoding_p(VALUE transfer_encoding) {
+  if (transfer_encoding == Qnil) return 0;
+  return rb_funcall(str_downcase(transfer_encoding), ID_eq, 1, STR_chunked) == Qtrue;
+}
+
+int parse_chunk_size(struct parser_state *state, int *chunk_size) {
+  int len = 0;
+  int value = 0;
+  int initial_pos = BUFFER_POS(state);
+
+  while (1) {
+    char c = BUFFER_CUR(state);
+    if ((c >= '0') && (c <= '9'))       value = (value << 4) + (c - '0');
+    else if ((c >= 'a') && (c <= 'f'))  value = (value << 4) + (c - 'a' + 10);
+    else if ((c >= 'A') && (c <= 'F'))  value = (value << 4) + (c - 'A' + 10);
+    else switch (c) {
+      case '\r':
+        CONSUME_CRLF_NO_FILL(state);
+        goto done;
+      case '\n':
+        INC_BUFFER_POS_NO_FILL(state);
+        goto done;
+      default:
+        goto bad_request;
+    }
+    INC_BUFFER_POS(state);
+    len++;
+    if (len >= MAX_CHUNKED_ENCODING_CHUNK_SIZE_LENGTH) goto bad_request;
+  }
+done:
+  if (len == 0) goto bad_request;
+  (*chunk_size) = value;
+  state->parser->current_request_rx += BUFFER_POS(state) - initial_pos;
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid chunk size");
+eof:
+  return 0;
+}
+
+int read_body_chunk_with_chunked_encoding(struct parser_state *state, VALUE *body, int chunk_size, int buffered_only) {
+  int len = RSTRING_LEN(state->parser->buffer);
+  int pos = state->parser->pos;
+  int left = chunk_size;
+
+  if (pos < len) {
+    int available = len - pos;
+    if (available > left) available = left;
+    if (*body != Qnil)
+      str_append_from_buffer(*body, RSTRING_PTR(state->parser->buffer) + pos, available);
+    else
+      *body = rb_str_new(RSTRING_PTR(state->parser->buffer) + pos, available);
+    state->parser->pos += available;
+    state->parser->current_request_rx += available;
+    left -= available;
+  }
+  if (buffered_only) return 1;
+
+  while (left) {
+    int maxlen = left <= MAX_BODY_READ_LENGTH ? left : MAX_BODY_READ_LENGTH;
+
+    VALUE tmp_buf = parser_io_read(state->parser, INT2NUM(maxlen), Qnil, NUM_buffer_start);
+    if (tmp_buf == Qnil) goto eof;
+    if (*body != Qnil)
+      rb_str_append(*body, tmp_buf);
+    else
+      *body = tmp_buf;
+    int read_bytes = RSTRING_LEN(tmp_buf);
+    state->parser->current_request_rx += read_bytes;
+    left -= read_bytes;
+    RB_GC_GUARD(tmp_buf);
+  }
+  return 1;
+eof:
+  return 0;
+}
+
+static inline int parse_chunk_postfix(struct parser_state *state) {
+  int initial_pos = BUFFER_POS(state);
+  if (initial_pos == BUFFER_LEN(state)) FILL_BUFFER_OR_GOTO_EOF(state);
+  switch (BUFFER_CUR(state)) {
+    case '\r':
+      CONSUME_CRLF_NO_FILL(state);
+      goto done;
+    case '\n':
+      INC_BUFFER_POS_NO_FILL(state);
+      goto done;
+    default:
+      goto bad_request;
+  }
+done:
+  state->parser->current_request_rx += BUFFER_POS(state) - initial_pos;
+  return 1;
+bad_request:
+  RAISE_BAD_REQUEST("Invalid protocol");
+eof:
+  return 0;
+}
+
+VALUE read_body_with_chunked_encoding(Parser_t *parser, int read_entire_body, int buffered_only) {
+  struct parser_state state;
+  state.parser = parser;
+  buffer_trim(&state);
+  INIT_PARSER_STATE(&state);
+  VALUE body = Qnil;
+
+  while (1) {
+    int chunk_size = 0;
+    if (BUFFER_POS(&state) == BUFFER_LEN(&state)) FILL_BUFFER_OR_GOTO_EOF(&state);
+    if (!parse_chunk_size(&state, &chunk_size)) goto bad_request;
+    
+    if (chunk_size) {
+      if (!read_body_chunk_with_chunked_encoding(&state, &body, chunk_size, buffered_only)) goto bad_request;
+    }
+    else parser->request_completed = 1;
+
+    if (!parse_chunk_postfix(&state)) goto bad_request;
+    if (!chunk_size || !read_entire_body) goto done;
+  }
+bad_request:
+  RAISE_BAD_REQUEST("Malformed request body");
+eof:
+  RAISE_BAD_REQUEST("Incomplete request body");
+done:
+  rb_hash_aset(parser->headers, STR_pseudo_rx, INT2NUM(state.parser->current_request_rx));
+  RB_GC_GUARD(body);
+  return body;
+}
+
+static inline void detect_body_read_mode(Parser_t *parser) {
+  VALUE content_length = rb_hash_aref(parser->headers, STR_content_length);
+  if (content_length != Qnil) {
+    int int_content_length = str_to_int(content_length, "Invalid content length");
+    if (int_content_length < 0) RAISE_BAD_REQUEST("Invalid body content length");
+    parser->body_read_mode = parser->body_left = int_content_length;
+    parser->request_completed = 0;
+    return;
+  }
+  
+  VALUE transfer_encoding = rb_hash_aref(parser->headers, STR_transfer_encoding);
+  if (chunked_encoding_p(transfer_encoding)) {
+    parser->body_read_mode = BODY_READ_MODE_CHUNKED;
+    parser->request_completed = 0;
+    return;
+  }
+  parser->request_completed = 1;
+
+}
+
+static inline VALUE read_body(VALUE self, int read_entire_body, int buffered_only) {
+  Parser_t *parser;
+  GetParser(self, parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_UNKNOWN)
+    detect_body_read_mode(parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_CHUNKED)
+    return read_body_with_chunked_encoding(parser, read_entire_body, buffered_only);
+  return read_body_with_content_length(parser, read_entire_body, buffered_only);
+}
+
+VALUE Parser_read_body(VALUE self) {
+  return read_body(self, 1, 0);
+}
+
+VALUE Parser_read_body_chunk(VALUE self, VALUE buffered_only) {
+  return read_body(self, 0, buffered_only == Qtrue);
+}
+
+VALUE Parser_complete_p(VALUE self) {
+  Parser_t *parser;
+  GetParser(self, parser);
+
+  if (parser->body_read_mode == BODY_READ_MODE_UNKNOWN)
+    detect_body_read_mode(parser);
+
+  return parser->request_completed ? Qtrue : Qfalse;
+}
+
+void Init_H1P() {
+  VALUE mH1P;
+  VALUE cParser;
+
+  mH1P = rb_define_module("H1P");
+  rb_gc_register_mark_object(mH1P);
+  cParser = rb_define_class_under(mH1P, "Parser", rb_cObject);
+  rb_define_alloc_func(cParser, Parser_allocate);
+
+  cError = rb_define_class_under(mH1P, "Error", rb_eRuntimeError);
+  rb_gc_register_mark_object(cError);
+
+  // backend methods
+  rb_define_method(cParser, "initialize", Parser_initialize, 1);
+  rb_define_method(cParser, "parse_headers", Parser_parse_headers, 0);
+  rb_define_method(cParser, "read_body", Parser_read_body, 0);
+  rb_define_method(cParser, "read_body_chunk", Parser_read_body_chunk, 1);
+  rb_define_method(cParser, "complete?", Parser_complete_p, 0);
+
+  ID_arity                  = rb_intern("arity");
+  ID_backend_read           = rb_intern("backend_read");
+  ID_backend_recv           = rb_intern("backend_recv");
+  ID_call                   = rb_intern("call");
+  ID_downcase               = rb_intern("downcase");
+  ID_eof_p                  = rb_intern("eof?");
+  ID_eq                     = rb_intern("==");
+  ID_parser_read_method     = rb_intern("__parser_read_method__");
+  ID_read                   = rb_intern("read");
+  ID_readpartial            = rb_intern("readpartial");
+  ID_to_i                   = rb_intern("to_i");
+
+  NUM_max_headers_read_length = INT2NUM(MAX_HEADERS_READ_LENGTH);
+  NUM_buffer_start = INT2NUM(0);
+  NUM_buffer_end = INT2NUM(-1);
+
+  GLOBAL_STR(STR_pseudo_method,       ":method");
+  GLOBAL_STR(STR_pseudo_path,         ":path");
+  GLOBAL_STR(STR_pseudo_protocol,     ":protocol");
+  GLOBAL_STR(STR_pseudo_rx,           ":rx");
+
+  GLOBAL_STR(STR_chunked,             "chunked");
+  GLOBAL_STR(STR_content_length,      "content-length");
+  GLOBAL_STR(STR_transfer_encoding,   "transfer-encoding");
+
+  SYM_backend_read = ID2SYM(ID_backend_read);
+  SYM_backend_recv = ID2SYM(ID_backend_recv);
+  SYM_stock_readpartial = ID2SYM(rb_intern("stock_readpartial"));
+
+  rb_global_variable(&mH1P);
+}
+
+void Init_h1p_ext() {
+  Init_H1P();
+}