h1p 0.1 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fb3573e65df46350986981759a454c8bc2c4166901a46dd827fdc17a27b08b5
-  data.tar.gz: afc10ead82e3286f61fa2bee1739499f275b75a386ed3e3401988805470c9b3a
+  metadata.gz: a27a3323482c8a0cec845e60962516b755338ccc54d0ba76bf6716bbb449a8ff
+  data.tar.gz: 22613ade68e261e3fca56cafcab410b1bed4587efe8c1da4ef52d1d335bc7efd
 SHA512:
-  metadata.gz: 72ba304cda888fcfb1c8afaee1a03b94ddb15fde0110c303886e4036125fb5b464d214ebb731ea91dac792070c9cc3d5eccb789f667c8a590fae01afb88d5665
-  data.tar.gz: 31cd6c6d7bb6695d338596b56be808cd1fed99cc76f11ee5f38fbdf7ff5d1206960a7134932ac82e9ca9c19116a20ee99db8f2a002fb76c1997134ac4d432dae
+  metadata.gz: 36cb9020745437627ad3dca5f83de6f81f8ad6e04075909db192a509b9f022960c5a7768c1f1ab8fdb807ae0d7b54e6febbc76581e68475c5f27057ff5b7aeb5
+  data.tar.gz: 20943cfead5e0236e60cb3a0d70057c69a91bbde45973f0f4ff87f38a1866523c1a02132254eebc13b8c5e2c4f75cd8e5791246257a792f63391cff88a6e1011

data/CHANGELOG.md

@@ -1,4 +1,15 @@
-## 0.42 2021-08-16
+## 0.4 2022-02-28
+
+- Rename `__parser_read_method__` to `__read_method__`
+
+## 0.3 2022-02-03
+
+- Add support for parsing HTTP responses (#1)
+- Put state directly in parser struct
+
+## 0.2 2021-08-20
+
+- Use uppercase for HTTP method
 
 ## 0.1 2021-08-19
 

data/Gemfile.lock

@@ -1,33 +1,24 @@
 PATH
   remote: .
   specs:
-    h1p (0.1)
+    h1p (0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ansi (1.5.0)
-    builder (3.2.4)
-    minitest (5.11.3)
-    minitest-reporters (1.4.3)
-      ansi
-      builder
-      minitest (>= 5.0)
-      ruby-progressbar
-    rake (12.3.3)
+    minitest (5.14.4)
+    rake (13.0.6)
     rake-compiler (1.1.1)
       rake
-    ruby-progressbar (1.11.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   h1p!
-  minitest (~> 5.11.3)
-  minitest-reporters (~> 1.4.2)
-  rake (~> 12.3.3)
+  minitest (~> 5.14.4)
+  rake (~> 13.0.6)
   rake-compiler (= 1.1.1)
 
 BUNDLED WITH
-   2.1.4
+   2.2.26

data/README.md

@@ -1,5 +1,236 @@
-# h1p
+# H1P - a blocking HTTP/1 parser for Ruby
 
-h1p 
+[![Gem Version](https://badge.fury.io/rb/h1p.svg)](http://rubygems.org/gems/h1p)
+[![H1P Test](https://github.com/digital-fabric/h1p/workflows/Tests/badge.svg)](https://github.com/digital-fabric/h1p/actions?query=workflow%3ATests)
+[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/digital-fabric/h1p/blob/master/LICENSE)
 
-HTTP 1.1 parser for Ruby
+H1P is a blocking/synchronous HTTP/1 parser for Ruby with a simple and intuitive
+API. Its design lends itself to writing HTTP servers in a sequential style. As
+such, it might prove useful in conjunction with the new fiber scheduler
+introduced in Ruby 3.0, but is also useful with a normal thread-based server
+(see
+[example](https://github.com/digital-fabric/h1p/blob/main/examples/http_server.rb).)
+The H1P was originally written as part of
+[Tipi](https://github.com/digital-fabric/tipi), a web server running on top of
+[Polyphony](https://github.com/digital-fabric/polyphony).
+
+> H1P is still a very young project and as such should be used with caution. It
+> has not undergone any significant conformance or security testing, and its API
+> is not yet stable.
+
+## Features
+
+- Simple, blocking/synchronous API
+- Zero dependencies
+- Transport-agnostic
+- Parses both HTTP request and HTTP response
+- Support for chunked encoding
+- Support for both `LF` and `CRLF` line breaks
+- Track total incoming traffic
+
+## Installing
+
+If you're using bundler just add it to your `Gemfile`:
+
+```ruby
+source 'https://rubygems.org'
+
+gem 'h1p'
+```
+
+You can then run `bundle install` to install it. Otherwise, just run `gem install h1p`.
+
+## Usage
+
+Start by creating an instance of `H1P::Parser`, passing a connection instance and the parsing mode:
+
+```ruby
+require 'h1p'
+
+parser = H1P::Parser.new(conn, :server)
+```
+
+In order to parse HTTP responses, change the mode to `:client`:
+
+```ruby
+parser = H1P::Parser.new(conn, :client)
+```
+
+To read the next message from the connection, call `#parse_headers`:
+
+```ruby
+loop do
+  headers = parser.parse_headers
+  break unless headers
+
+  handle_request(headers)
+end
+```
+
+The `#parse_headers` method returns a single hash containing the different HTTP
+headers. In case the client has closed the connection, `#parse_headers` will
+return `nil` (see the guard clause above).
+
+In addition to the header keys and values, the resulting hash also contains the
+following "pseudo-headers" (in server mode):
+
+- `:method`: the HTTP method (in upper case)
+- `:path`: the request target
+- `:protocol`: the protocol used (either `'http/1.0'` or `'http/1.1'`)
+- `:rx`: the total bytes read by the parser
+
+In client mode, the following pseudo-headers will be present:
+
+- `:protocol`: the protocol used (either `'http/1.0'` or `'http/1.1'`)
+- `:status': the HTTP status as an integer
+- `:status_message`: the HTTP status message
+- `:rx`: the total bytes read by the parser
+
+
+The header keys are always lower-cased. Consider the following HTTP request:
+
+```
+GET /foo HTTP/1.1
+Host: example.com
+User-Agent: curl/7.74.0
+Accept: */*
+
+```
+
+The request will be parsed into the following Ruby hash:
+
+```ruby
+{
+  ":method"     => "get",
+  ":path"       => "/foo",
+  ":protocol"   => "http/1.1",
+  "host"        => "example.com",
+  "user-agent"  => "curl/7.74.0",
+  "accept"      => "*/*",
+  ":rx"         => 78
+}
+```
+
+Multiple headers with the same key will be coalesced into a single key-value
+where the value is an array containing the corresponding values. For example,
+multiple `Cookie` headers will appear in the hash as a single `"cookie"` entry,
+e.g. `{ "cookie" => ['a=1', 'b=2'] }`
+
+### Handling of invalid message
+
+When an invalid message is encountered, the parser will raise a `H1P::Error`
+exception. An incoming message may be considered invalid if an invalid character
+has been encountered at any point in parsing the message, or if any of the
+tokens have an invalid length. You can consult the limits used by the parser
+[here](https://github.com/digital-fabric/h1p/blob/main/ext/h1p/limits.rb).
+
+### Reading the message body
+
+To read the message body use `#read_body`:
+
+```ruby
+# read entire body
+body = parser.read_body
+```
+
+The H1P parser knows how to read both message bodies with a specified
+`Content-Length` and request bodies in chunked encoding. The method call will
+return when the entire body has been read. If the body is incomplete or has
+invalid formatting, the parser will raise a `H1P::Error` exception.
+
+You can also read a single chunk of the body by calling `#read_body_chunk`:
+
+```ruby
+# read a body chunk
+chunk = parser.read_body_chunk(false)
+
+# read chunk only from buffer:
+chunk = parser.read_body_chunk(true)
+```
+
+If no more chunks are availble, `#read_body_chunk` will return nil. To test
+whether the request is complete, you can call `#complete?`:
+
+```ruby
+headers = parser.parse_headers
+unless parser.complete?
+  body = parser.read_body
+end
+```
+
+The `#read_body` and `#read_body_chunk` methods will return `nil` if no body is
+expected (based on the received headers).
+
+## Parsing from arbitrary transports
+
+The H1P parser was built to read from any arbitrary transport or source, as long
+as they conform to one of two alternative interfaces:
+
+- An object implementing a `__read_method__` method, which returns any of
+  the following values:
+
+  - `:stock_readpartial` - to be used for instances of `IO`, `Socket`,
+    `TCPSocket`, `SSLSocket` etc.
+  - `:backend_read` - for use in Polyphony-based servers.
+  - `:backend_recv` - for use in Polyphony-based servers.
+  - `:readpartial` - for use in Polyphony-based servers.
+
+- An object implementing a `call` method, such as a `Proc` or any other. The
+  call is given a single argument signifying the maximum number of bytes to
+  read, and is expected to return either a string with the read data, or `nil`
+  if no more data is available. The callable can be passed as an argument or as
+  a block. Here's an example for parsing from a callable:
+
+  ```ruby
+  data = ['GET ', '/foo', " HTTP/1.1\r\n", "\r\n"]
+  data = ['GET ', '/foo', " HTTP/1.1\r\n", "\r\n"]
+  parser = H1P::Parser.new { data.shift }
+  parser.parse_headers
+  #=> {":method"=>"get", ":path"=>"/foo", ":protocol"=>"http/1.1", ":rx"=>21}
+  ```
+
+## Design
+
+The H1P parser design is based on the following principles:
+
+- Implement a blocking API for use with a sequential programming style.
+- Minimize copying of data between buffers.
+- Parse each piece of data only once.
+- Minimize object and buffer allocations.
+- Minimize the API surface area.
+
+One of the unique aspects of H1P is that instead of the server needing to feed
+data to the parser, the parser itself reads data from its source whenever it
+needs more of it. If no data is yet available, the parser blocks until more data
+is received.
+
+The different parts of the request are parsed one byte at a time, and once each
+token is considered complete, it is copied from the buffer into a new string, to
+be stored in the headers hash.
+
+## Performance
+
+The included benchmark (against
+[http_parser.rb](https://github.com/tmm1/http_parser.rb), based on the *old*
+[node.js HTTP parser](https://github.com/nodejs/http-parser)) shows the H1P
+parser to be about 10-20% slower than http_parser.rb.
+
+However, in a fiber-based environment such as
+[Polyphony](https://github.com/digital-fabric/polyphony), H1P is slightly
+faster, as the overhead of dealing with pipelined requests (which will cause
+`http_parser.rb` to emit callbacks multiple times) significantly affects its
+performance.
+
+## Roadmap
+
+Here are some of the features and enhancements planned for H1P:
+
+- Add conformance and security tests
+- Add ability to splice the message body into an arbitrary fd
+  (Polyphony-specific)
+- Improve performance
+
+## Contributing
+
+Issues and pull requests will be gladly accepted. If you have found this gem
+useful, please let me know.

data/Rakefile

@@ -12,5 +12,5 @@ task :recompile => [:clean, :compile]
 task :default => [:compile, :test]
 
 task :test do
-  exec 'ruby test/test_h1p.rb'
+  exec 'ruby test/run.rb'
 end

data/benchmarks/bm_http1_parser.rb

@@ -1,8 +1,18 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\nUser-Agent: foobar\r\n\r\n"
 
-HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
+# HTTP_REQUEST =
+#   "GET /wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg HTTP/1.1\r\n" +
+#   "Host: www.kittyhell.com\r\n" +
+#   "User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9\r\n" +
+#   "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +
+#   "Accept-Language: ja,en-us;q=0.7,en;q=0.3\r\n" +
+#   "Accept-Encoding: gzip,deflate\r\n" +
+#   "Accept-Charset: Shift_JIS,utf-8;q=0.7,*;q=0.7\r\n" +
+#   "Keep-Alive: 115\r\n" +
+#   "Connection: keep-alive\r\n" +
+#   "Cookie: wp_ozh_wsa_visits=2; wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral\r\n\r\n"
 
 def measure_time_and_allocs
   4.times { GC.start }
@@ -26,15 +36,17 @@ end
 def benchmark_other_http1_parser(iterations)
   STDOUT << "http_parser.rb: "
   require 'http_parser.rb'
-  
+
   i, o = IO.pipe
   parser = Http::Parser.new
   done = false
   headers = nil
+  rx = 0
   parser.on_headers_complete = proc do |h|
     headers = h
     headers[':method'] = parser.http_method
     headers[':path'] = parser.request_url
+    headers[':rx'] = rx
   end
   parser.on_message_complete = proc { done = true }
 
@@ -42,8 +54,10 @@ def benchmark_other_http1_parser(iterations)
     iterations.times do
       o << HTTP_REQUEST
       done = false
+      rx = 0
       while !done
         msg = i.readpartial(4096)
+        rx += msg.bytesize
         parser << msg
       end
     end
@@ -52,11 +66,10 @@ def benchmark_other_http1_parser(iterations)
 end
 
 def benchmark_tipi_http1_parser(iterations)
-  STDOUT << "tipi parser: "
-  require_relative '../lib/tipi_ext'
+  STDOUT << "H1P parser: "
+  require_relative '../lib/h1p'
   i, o = IO.pipe
-  reader = proc { |len| i.readpartial(len) }
-  parser = Tipi::HTTP1Parser.new(reader)
+  parser = H1P::Parser.new(i)
 
   elapsed, allocated = measure_time_and_allocs do
     iterations.times do
@@ -78,8 +91,8 @@ def fork_benchmark(method, iterations)
   Process.wait(pid)
 end
 
-x = 500000
-# fork_benchmark(:benchmark_other_http1_parser, x)
-# fork_benchmark(:benchmark_tipi_http1_parser, x)
+x = 100000
+fork_benchmark(:benchmark_other_http1_parser, x)
+fork_benchmark(:benchmark_tipi_http1_parser, x)
 
-benchmark_tipi_http1_parser(x)
+# benchmark_tipi_http1_parser(x)

data/benchmarks/pipelined.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\nUser-Agent: foobar\r\n\r\n" +
+               "GET /bar HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\nUser-Agent: foobar\r\n\r\n"
+
+def measure_time_and_allocs
+  4.times { GC.start }
+  GC.disable
+
+  t0 = Time.now
+  a0 = object_count
+  yield
+  t1 = Time.now
+  a1 = object_count
+  [t1 - t0, a1 - a0]
+ensure
+  GC.enable
+end
+
+def object_count
+  count = ObjectSpace.count_objects
+  count[:TOTAL] - count[:FREE]
+end
+
+def benchmark_other_http1_parser(iterations)
+  STDOUT << "http_parser.rb: "
+  require 'http_parser.rb'
+
+  i, o = IO.pipe
+  parser = Http::Parser.new
+  done = false
+  queue = nil
+  rx = 0
+  req_count = 0
+  parser.on_headers_complete = proc do |h|
+    h[':method'] = parser.http_method
+    h[':path'] = parser.request_url
+    h[':rx'] = rx
+    queue << h
+  end
+  parser.on_message_complete = proc { done = true }
+
+  writer = Thread.new do
+    iterations.times { o << HTTP_REQUEST }
+    o.close
+  end
+
+  elapsed, allocated = measure_time_and_allocs do
+    queue = []
+    done = false
+    rx = 0
+    loop do
+      data = i.readpartial(4096) rescue nil
+      break unless data
+
+      rx += data.bytesize
+      parser << data
+      while (req = queue.shift)
+        req_count += 1
+      end
+    end
+  end
+  puts(format('count: %d, elapsed: %f, allocated: %d (%f/req), rate: %f ips', req_count, elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def benchmark_h1p_parser(iterations)
+  STDOUT << "H1P parser: "
+  require_relative '../lib/h1p'
+  i, o = IO.pipe
+  parser = H1P::Parser.new(i)
+  req_count = 0
+
+  writer = Thread.new do
+    iterations.times { o << HTTP_REQUEST }
+    o.close
+  end
+
+  elapsed, allocated = measure_time_and_allocs do
+    while (headers = parser.parse_headers)
+      req_count += 1
+    end
+  end
+  puts(format('count: %d, elapsed: %f, allocated: %d (%f/req), rate: %f ips', req_count, elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def fork_benchmark(method, iterations)
+  pid = fork do
+    send(method, iterations)
+  rescue Exception => e
+    p e
+    p e.backtrace
+    exit!
+  end
+  Process.wait(pid)
+end
+
+x = 100000
+fork_benchmark(:benchmark_other_http1_parser, x)
+fork_benchmark(:benchmark_h1p_parser, x)
+
+# benchmark_h1p_parser(x)

data/examples/callable.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'h1p'
+
+data = ['GET ', '/foo', " HTTP/1.1\r\n", "\r\n"]
+parser = H1P::Parser.new(proc { data.shift }, :server)
+
+headers = parser.parse_headers
+p headers

data/examples/http_server.rb

@@ -1,41 +1,39 @@
 # frozen_string_literal: true
 
 require 'bundler/setup'
-require 'tipi'
-
-opts = {
-  reuse_addr:  true,
-  dont_linger: true
-}
+require 'h1p'
 
 puts "pid: #{Process.pid}"
-puts 'Listening on port 10080...'
+puts 'Listening on port 1234...'
 
-# GC.disable
-# Thread.current.backend.idle_gc_period = 60
+trap('SIGINT') { exit! }
 
-spin_loop(interval: 10) { p Thread.backend.stats }
+def handle_client(conn)
+  Thread.new do
+    parser = H1P::Parser.new(conn, :server)
+    loop do
+      headers = parser.parse_headers
+      break unless headers
 
-spin_loop(interval: 10) do
-  GC.compact
-end
+      req_body = parser.read_body
+
+      p headers: headers
+      p body: req_body
 
-spin do
-  Tipi.serve('0.0.0.0', 10080, opts) do |req|
-    if req.path == '/stream'
-      req.send_headers('Foo' => 'Bar')
-      sleep 1
-      req.send_chunk("foo\n")
-      sleep 1
-      req.send_chunk("bar\n")
-      req.finish
-    elsif req.path == '/upload'
-      body = req.read
-      req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
-    else
-      req.respond("Hello world!\n")
+      resp = 'Hello, world!'
+      conn << "HTTP/1.1 200 OK\r\nContent-Length: #{resp.bytesize}\r\n\r\n#{resp}"
+    rescue H1P::Error => e
+      puts "Invalid request: #{e.message}"
+    ensure
+      conn.close
+      break
     end
-#    p req.transfer_counts
   end
-  p 'done...'
-end.await
+end
+
+require 'socket'
+server = TCPServer.new('0.0.0.0', 1234)
+loop do
+  conn = server.accept
+  handle_client(conn)
+end