pico_http_parser 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77d9ce494785aca207b88dd3bdc5340d94cac9ad
-  data.tar.gz: 9ee980d7dd5d1c5f5c2467603dffc0444513fb71
+  metadata.gz: 417f57e6c7289be7606e9fdda1b65346751f3c10
+  data.tar.gz: 420957b9c7d8ed9338c29d20030109ea0f6c17c0
 SHA512:
-  metadata.gz: 7fb958f767c35d5d71234f729ad74e346685329072122b612766787ff000e4ff21c2d12b42b5e73f6096ed63682f570b56fb7ae9f1defed42b8c1fb87d9935d5
-  data.tar.gz: 7dfa7423fda76a94ea59df99a0e6d15b59ba61620d3e4575269508a1a3411de1e064bb0b0d99bd7d03f2e95f02e65eb0c45783cbf8c24ac570f8f3ae3d118cf7
+  metadata.gz: 1cd7184656ea42efb5f22aa27a42d30ed381f241c665859fbc9e94937195248db6216a4be25903799841148c51842fd2fa93bf0ddf39b699cb48192eb44d0e1a
+  data.tar.gz: 8656caa1f569d72acc097392e67c9316e26c7aa5546d165d1d600566e80a6e1057660696e8096fb00db9a6b7a88860fa890d8a801dbdbedf9f877c4f06e51f62

data/README.md

@@ -54,16 +54,17 @@ given request is incomplete
 On my Macbook Air
 
 ```
+$ ruby benchmark.rb
 Calculating -------------------------------------
-      PicoHTTPParser    13.988k i/100ms
-Unicorn's HttpParser    12.250k i/100ms
+      PicoHTTPParser    13.655k i/100ms
+Unicorn's HttpParser    11.554k i/100ms
 -------------------------------------------------
-      PicoHTTPParser    160.563k (± 4.3%) i/s -    811.304k
-Unicorn's HttpParser    135.976k (± 4.1%) i/s -    686.000k
+      PicoHTTPParser    155.331k (�� 3.7%) i/s -    778.335k
+Unicorn's HttpParser    127.455k (�� 3.5%) i/s -    647.024k
 
 Comparison:
-      PicoHTTPParser:   160562.7 i/s
-Unicorn's HttpParser:   135975.8 i/s - 1.18x slower
+      PicoHTTPParser:   155331.2 i/s
+Unicorn's HttpParser:   127455.1 i/s - 1.22x slower
 ```
 
 ## SEE ALSO

data/benchmark/benchmark.rb

@@ -2,15 +2,9 @@ require File.expand_path(File.dirname(__FILE__) + '/bench_helper')
 
 require 'pico_http_parser'
 
-request_body = <<REQ
-GET /blakjsdfkas HTTP/1.1\r
-Host: blooperblorp\r
-Cookie: blah=woop\r
-\r
-REQ
-loop = 300000
-
-#File.read(File.expand_path(File.dirname(__FILE__) + '/sample_request.http'))
+# request_body = "GET /foo/bar/baz.html?key=value HTTP/1.0\r\nHost: blooperblorp\r\n\r\n"
+request_body = "GET /foo/bar/baz.html?key=value HTTP/1.0\r\nHost: blooperblorp\r\nCookie: foobar\r\nX-Forwared-For: 127.0.0.1\r\n\r\n"
+# request_body = "GET /foo/bar/baz.html?key=value HTTP/1.0\r\n\r\n"
 
 Benchmark.ips do |x|
   x.time = 5

data/ext/pico_http_parser/pico_http_parser.c

@@ -8,6 +8,58 @@
 
 VALUE cPicoHTTPParser;
 
+static VALUE request_method_key;
+static VALUE request_uri_key;
+static VALUE script_name_key;
+static VALUE server_protocol_key;
+static VALUE query_string_key;
+
+struct common_header {
+  const char * name;
+  size_t name_len;
+  VALUE key;
+};
+static int common_headers_num = 0;
+static struct common_header common_headers[20];
+
+static
+void set_common_header(const char * key, int key_len, const int raw)
+{
+  char tmp[MAX_HEADER_NAME_LEN + sizeof("HTTP_") - 1];
+  const char* name;
+  size_t name_len;
+  const char * s;
+  char* d;
+  size_t n;
+  VALUE env_key;
+
+  if ( raw == 1) {
+    for (s = key, n = key_len, d = tmp;
+      n != 0;
+      s++, --n, d++) {
+      *d = *s == '-' ? '_' : TOU(*s);
+      name = tmp;
+      name_len = key_len;
+    }
+  } else {
+    strcpy(tmp, "HTTP_");
+    for (s = key, n = key_len, d = tmp + 5;
+      n != 0;
+      s++, --n, d++) {
+      *d = *s == '-' ? '_' : TOU(*s);
+      name = tmp;
+      name_len = key_len + 5;
+    }
+  }
+  env_key = rb_obj_freeze(rb_str_new(name,name_len));
+  common_headers[common_headers_num].name = key;
+  common_headers[common_headers_num].name_len = key_len;
+  common_headers[common_headers_num].key = env_key;
+  rb_gc_register_address(&common_headers[common_headers_num].key);
+  common_headers_num++;
+}
+
+
 static
 size_t find_ch(const char* s, size_t len, char ch)
 {
@@ -31,6 +83,16 @@ int header_is(const struct phr_header* header, const char* name,
   return 1;
 }
 
+static
+VALUE find_common_header(const struct phr_header* header) {
+  int i;
+  for ( i = 0; i < common_headers_num; i++ ) {
+    if ( header_is(header, common_headers[i].name, common_headers[i].name_len) ) {
+      return common_headers[i].key;
+    }
+  }
+  return Qnil;
+}
 
 static
 int store_path_info(VALUE envref, const char* src, size_t src_len) {
@@ -93,11 +155,12 @@ VALUE phr_parse_http_request(VALUE self, VALUE buf, VALUE envref)
   if (ret < 0)
     goto done;
 
-  rb_hash_aset(envref, rb_str_new2("REQUEST_METHOD"), rb_str_new(method,method_len));
-  rb_hash_aset(envref, rb_str_new2("REQUEST_URI"), rb_str_new(path, path_len));
-  rb_hash_aset(envref, rb_str_new2("SCRIPT_NAME"), rb_str_new2(""));
-  i = sprintf(tmp,"HTTP/1.%d",minor_version);
-  rb_hash_aset(envref, rb_str_new2("SERVER_PROTOCOL"), rb_str_new(tmp, i));
+  rb_hash_aset(envref, request_method_key, rb_str_new(method,method_len));
+  rb_hash_aset(envref, request_uri_key, rb_str_new(path, path_len));
+  rb_hash_aset(envref, script_name_key, rb_str_new2(""));
+  strcpy(tmp, "HTTP/1.");
+  tmp[7] = 48 + ((minor_version > 1 || minor_version < 0 ) ? 0 : minor_version);
+  rb_hash_aset(envref, server_protocol_key, rb_str_new(tmp, sizeof("HTTP/1.0") - 1));
 
   /* PATH_INFO QUERY_STRING */
   path_len = find_ch(path, path_len, '#'); /* strip off all text after # after storing request_uri */
@@ -108,7 +171,7 @@ VALUE phr_parse_http_request(VALUE self, VALUE buf, VALUE envref)
     goto done;
   }
   if (question_at != path_len) ++question_at;
-  rb_hash_aset(envref, rb_str_new2("QUERY_STRING"), rb_str_new(path + question_at, path_len - question_at));
+  rb_hash_aset(envref, query_string_key, rb_str_new(path + question_at, path_len - question_at));
 
   last_value = Qnil;
   for (i = 0; i < num_headers; ++i) {
@@ -116,13 +179,9 @@ VALUE phr_parse_http_request(VALUE self, VALUE buf, VALUE envref)
       const char* name;
       size_t name_len;
       VALUE slot;
-      if (header_is(headers + i, "CONTENT-TYPE", sizeof("CONTENT-TYPE") - 1)) {
-        name = "CONTENT_TYPE";
-        name_len = sizeof("CONTENT_TYPE") - 1;
-      } else if (header_is(headers + i, "CONTENT-LENGTH", sizeof("CONTENT-LENGTH") - 1)) {
-        name = "CONTENT_LENGTH";
-        name_len = sizeof("CONTENT_LENGTH") - 1;
-      } else {
+      VALUE env_key;
+      env_key = find_common_header(headers + i);
+      if ( env_key == Qnil ) {
         const char* s;
         char* d;
         size_t n;
@@ -138,20 +197,22 @@ VALUE phr_parse_http_request(VALUE self, VALUE buf, VALUE envref)
             *d = *s == '-' ? '_' : TOU(*s);
             name = tmp;
             name_len = headers[i].name_len + 5;
+            env_key = rb_str_new(name, name_len);
         }
       }
-      slot = rb_hash_aref(envref, rb_str_new(name, name_len));
+      slot = rb_hash_aref(envref, env_key);
       if ( slot != Qnil ) {
         rb_str_cat2(slot, ", ");
         rb_str_cat(slot, headers[i].value, headers[i].value_len);
       } else {
         slot = rb_str_new(headers[i].value, headers[i].value_len);
-        rb_hash_aset(envref, rb_str_new(name, name_len), slot);
+        rb_hash_aset(envref, env_key, slot);
         last_value = slot;
       }
     } else {
       /* continuing lines of a mulitiline header */
-      rb_str_cat(last_value, headers[i].value, headers[i].value_len);
+      if ( last_value != Qnil )
+        rb_str_cat(last_value, headers[i].value, headers[i].value_len);
     }
   }
 
@@ -161,6 +222,30 @@ VALUE phr_parse_http_request(VALUE self, VALUE buf, VALUE envref)
 
 void Init_pico_http_parser()
 {
+  request_method_key = rb_obj_freeze(rb_str_new2("REQUEST_METHOD"));
+  rb_gc_register_address(&request_method_key);
+  request_uri_key = rb_obj_freeze(rb_str_new2("REQUEST_URI"));
+  rb_gc_register_address(&request_uri_key);
+  script_name_key = rb_obj_freeze(rb_str_new2("SCRIPT_NAME"));
+  rb_gc_register_address(&script_name_key);
+  server_protocol_key = rb_obj_freeze(rb_str_new2("SERVER_PROTOCOL"));
+  rb_gc_register_address(&server_protocol_key);
+  query_string_key = rb_obj_freeze(rb_str_new2("QUERY_STRING"));
+  rb_gc_register_address(&query_string_key);
+
+  set_common_header("ACCEPT",sizeof("ACCEPT") - 1, 0);
+  set_common_header("ACCEPT-ENCODING",sizeof("ACCEPT-ENCODING") - 1, 0);
+  set_common_header("ACCEPT-LANGUAGE",sizeof("ACCEPT-LANGUAGE") - 1, 0);
+  set_common_header("CACHE-CONTROL",sizeof("CACHE-CONTROL") - 1, 0);
+  set_common_header("CONNECTION",sizeof("CONNECTION") - 1, 0);
+  set_common_header("CONTENT-LENGTH",sizeof("CONTENT-LENGTH") - 1, 1);
+  set_common_header("CONTENT-TYPE",sizeof("CONTENT-TYPE") - 1, 1);
+  set_common_header("COOKIE",sizeof("COOKIE") - 1, 0);
+  set_common_header("HOST",sizeof("HOST") - 1, 0);
+  set_common_header("IF-MODIFIED-SINCE",sizeof("IF-MODIFIED-SINCE") - 1, 0);
+  set_common_header("REFERER",sizeof("REFERER") - 1, 0);
+  set_common_header("USER-AGENT",sizeof("USER-AGENT") - 1, 0);
+  set_common_header("X-FORWARDED-FOR",sizeof("X-FORWARDED-FOR") - 1, 0);
 
   cPicoHTTPParser = rb_const_get(rb_cObject, rb_intern("PicoHTTPParser"));
   rb_define_module_function(cPicoHTTPParser, "parse_http_request", phr_parse_http_request, 2);

data/ext/pico_http_parser/picohttpparser/Makefile

@@ -1,5 +1,6 @@
 #
-# Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase
+# Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,
+#                         Shigeo Mitsunari
 #
 # The software is licensed under either the MIT License (below) or the Perl
 # license.

data/ext/pico_http_parser/picohttpparser/README.md

@@ -1,7 +1,7 @@
 PicoHTTPParser
 =============
 
-Copyright (c) 2009-2014 [Kazuho Oku](https://github.com/kazuho), [Tokuhiro Matsuno](https://github.com/tokuhirom), [Daisuke Murase](https://github.com/typester)
+Copyright (c) 2009-2014 [Kazuho Oku](https://github.com/kazuho), [Tokuhiro Matsuno](https://github.com/tokuhirom), [Daisuke Murase](https://github.com/typester), [Shigeo Mitsunari](https://github.com/herumi)
 
 PicoHTTPParser is a tiny, primitive, fast HTTP request/response parser.
 
@@ -17,8 +17,8 @@ The software is dual-licensed under the Perl License or the MIT License.
 Benchmark
 ---------
 
-![benchmark results](http://i.gyazo.com/7e098703c29128d69d02c9a216bfb6fb.png)
+![benchmark results](http://i.gyazo.com/a85c18d3162dfb46b485bb41e0ad443a.png)
 
-The benchmark code is from [fukamachi/fast-http](https://github.com/fukamachi/fast-http/).
+The benchmark code is from [fukamachi/fast-http@6b91103](https://github.com/fukamachi/fast-http/tree/6b9110347c7a3407310c08979aefd65078518478).
 
 The internals of picohttpparser has been described to some extent in [my blog entry]( http://blog.kazuhooku.com/2014/11/the-internals-h2o-or-how-to-write-fast.html).

data/ext/pico_http_parser/picohttpparser/bench.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase
+ * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,
+ *                         Shigeo Mitsunari
  *
  * The software is licensed under either the MIT License (below) or the Perl
  * license.
@@ -40,7 +41,7 @@ int main(void)
   size_t num_headers;
   int i, ret;
   
-  for (i = 0; i < 1000000; i++) {
+  for (i = 0; i < 10000000; i++) {
     num_headers = sizeof(headers) / sizeof(headers[0]);
     ret = phr_parse_request(REQ, sizeof(REQ) - 1, &method, &method_len, &path,
                             &path_len, &minor_version, headers, &num_headers,

data/ext/pico_http_parser/picohttpparser/picohttpparser.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase
+ * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,
+ *                         Shigeo Mitsunari
  *
  * The software is licensed under either the MIT License (below) or the Perl
  * license.
@@ -24,6 +25,9 @@
  */
 
 #include <stddef.h>
+#ifdef __SSE4_2__
+# include <x86intrin.h>
+#endif
 #include "picohttpparser.h"
 
 /* $Id$ */
@@ -53,8 +57,13 @@
 
 #define ADVANCE_TOKEN(tok, toklen) do { \
     const char* tok_start = buf; \
-    for (; ; ++buf) { \
+    static const char ranges2[] __attribute__((aligned(16))) = "\000\040\177\177"; \
+    int found2; \
+    buf = findchar_fast(buf, buf_end, ranges2, sizeof(ranges2) - 1, &found2); \
+    if (! found2) { \
       CHECK_EOF(); \
+    } \
+    while (1) { \
       if (*buf == ' ') { \
         break; \
       } else if (unlikely(! IS_PRINTABLE_ASCII(*buf))) { \
@@ -63,6 +72,8 @@
           return NULL; \
         } \
       } \
+      ++buf; \
+      CHECK_EOF(); \
     } \
     tok = tok_start; \
     toklen = buf - tok_start; \
@@ -78,12 +89,50 @@ static const char* token_char_map =
   "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
   "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
+static const char* findchar_fast(const char* buf, const char* buf_end, const char *ranges, size_t ranges_size, int* found)
+{
+  *found = 0;
+#if __SSE4_2__
+  if (likely(buf_end - buf >= 16)) {
+    __m128i ranges16 = _mm_loadu_si128((const __m128i*)ranges);
+
+    size_t left = (buf_end - buf) & ~15;
+    do {
+      __m128i b16 = _mm_loadu_si128((void*)buf);
+      int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS);
+      if (unlikely(r != 16)) {
+        buf += r;
+        *found = 1;
+        break;
+      }
+      buf += 16;
+      left -= 16;
+    } while (likely(left != 0));
+  }
+#endif
+  return buf;
+}
+
 static const char* get_token_to_eol(const char* buf, const char* buf_end,
                                     const char** token, size_t* token_len,
                                     int* ret)
 {
   const char* token_start = buf;
   
+#ifdef __SSE4_2__
+  static const char ranges1[] =
+    "\0\010"
+    /* allow HT */
+    "\012\037"
+    /* allow SP and up to but not including DEL */
+    "\177\177"
+    /* allow chars w. MSB set */
+    ;
+  int found;
+  buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found);
+  if (found)
+    goto FOUND_CTL;
+#else
   /* find non-printable char within the next 8 bytes, this is the hottest code; manually inlined */
   while (likely(buf_end - buf >= 8)) {
 #define DOIT() if (unlikely(! IS_PRINTABLE_ASCII(*buf))) goto NonPrintable; ++buf
@@ -97,6 +146,7 @@ static const char* get_token_to_eol(const char* buf, const char* buf_end,
     }
     ++buf;
   }
+#endif
   for (; ; ++buf) {
     CHECK_EOF();
     if (unlikely(! IS_PRINTABLE_ASCII(*buf))) {
@@ -211,14 +261,21 @@ static const char* parse_headers(const char* buf, const char* buf_end,
       /* parsing name, but do not discard SP before colon, see
        * http://www.mozilla.org/security/announce/2006/mfsa2006-33.html */
       headers[*num_headers].name = buf;
-      for (; ; ++buf) {
+      static const char ranges1[] __attribute__((aligned(16))) = "::\x00\037";
+      int found;
+      buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found);
+      if (! found) {
         CHECK_EOF();
+      }
+      while (1) {
         if (*buf == ':') {
           break;
         } else if (*buf < ' ') {
           *ret = -1;
           return NULL;
         }
+        ++buf;
+        CHECK_EOF();
       }
       headers[*num_headers].name_len = buf - headers[*num_headers].name;
       ++buf;

data/ext/pico_http_parser/picohttpparser/picohttpparser.h

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase
+ * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,
+ *                         Shigeo Mitsunari
  *
  * The software is licensed under either the MIT License (below) or the Perl
  * license.

data/ext/pico_http_parser/picohttpparser/test.c

@@ -1,6 +1,7 @@
 /* use `make test` to run the test */
 /*
- * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase
+ * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase,
+ *                         Shigeo Mitsunari
  *
  * The software is licensed under either the MIT License (below) or the Perl
  * license.
@@ -97,6 +98,15 @@ static void test_request(void)
   ok(headers[2].name == NULL);
   ok(bufis(headers[2].value, headers[2].value_len, "  \tc"));
   
+  PARSE("GET / HTTP/1.0\r\nfoo : ab\r\n\r\n", 0, 0,
+        "parse header name with trailing space");
+  ok(num_headers == 1);
+  ok(bufis(method, method_len, "GET"));
+  ok(bufis(path, path_len, "/"));
+  ok(minor_version == 0);
+  ok(bufis(headers[0].name, headers[0].name_len, "foo "));
+  ok(bufis(headers[0].value, headers[0].value_len, "ab"));
+
   PARSE("GET", 0, -2, "incomplete 1");
   ok(method == NULL);
   PARSE("GET ", 0, -2, "incomplete 2");
@@ -125,6 +135,8 @@ static void test_request(void)
   PARSE("GET /\x7fhello HTTP/1.0\r\n\r\n", 0, -1, "DEL in uri-path");
   PARSE("GET / HTTP/1.0\r\na\0b: c\r\n\r\n", 0, -1, "NUL in header name");
   PARSE("GET / HTTP/1.0\r\nab: c\0d\r\n\r\n", 0, -1, "NUL in header value");
+  PARSE("GET / HTTP/1.0\r\na\033b: c\r\n\r\n", 0, -1, "CTL in header name");
+  PARSE("GET / HTTP/1.0\r\nab: c\033\r\n\r\n", 0, -1, "CTL in header value");
   PARSE("GET /\xa0 HTTP/1.0\r\nh: c\xa2y\r\n\r\n", 0, 0, "accept MSB chars");
   ok(num_headers == 1);
   ok(bufis(method, method_len, "GET"));

data/lib/pico_http_parser/version.rb

@@ -1,3 +1,3 @@
 class PicoHTTPParser
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pico_http_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Masahiro Nagano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-21 00:00:00.000000000 Z
+date: 2014-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler