http_parser.rb 0.5.0-x86-mingw32 → 0.5.1-x86-mingw32

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Gemfile.lock

@@ -0,0 +1,32 @@
+PATH
+  remote: .
+  specs:
+    http_parser.rb (0.5.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    json (1.4.6)
+    rake (0.8.7)
+    rake-compiler (0.7.0)
+      rake (>= 0.8.3, < 0.9)
+    rspec (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
+      rspec-mocks (~> 2.0.1)
+    rspec-core (2.0.1)
+    rspec-expectations (2.0.1)
+      diff-lcs (>= 1.1.2)
+    rspec-mocks (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  http_parser.rb!
+  json (>= 1.4.6)
+  rake-compiler (>= 0.7.0)
+  rspec (>= 2.0.1)

data/README.md

@@ -20,14 +20,15 @@ This gem aims to work on all major Ruby platforms, including:
 
     parser = Http::Parser.new
 
-    parser.on_headers_complete = proc do |headers|
-      p parser.http_method
+    parser.on_headers_complete = proc do
       p parser.http_version
 
-      p parser.request_url # for requests
+      p parser.http_method # for requests
+      p parser.request_url
+
       p parser.status_code # for responses
 
-      p headers
+      p parser.headers
     end
 
     parser.on_body = proc do |chunk|
@@ -43,3 +44,42 @@ This gem aims to work on all major Ruby platforms, including:
     # Feed raw data from the socket to the parser
     parser << raw_data
 
+## Advanced Usage
+
+### Accept callbacks on an object
+
+    module MyHttpConnection
+      def connection_completed
+        @parser = Http::Parser.new(self)
+      end
+
+      def receive_data(data)
+        @parser << data
+      end
+
+      def on_message_begin
+        @headers = nil
+        @body = ''
+      end
+
+      def on_headers_complete
+        @headers = @parser.headers
+      end
+
+      def on_body(chunk)
+        @body << chunk
+      end
+
+      def on_message_complete
+        p [@headers, @body]
+      end
+    end
+
+### Stop parsing after headers
+
+    parser = Http::Parser.new
+    parser.on_headers_complete = proc{ :stop }
+
+    offset = parser << request_data
+    body = request_data[offset..-1]
+

data/ext/ruby_http_parser/extconf.rb

@@ -1,5 +1,13 @@
 require 'mkmf'
 
+# check out code if it hasn't been already
+if Dir[File.expand_path('../vendor/http-parser/*', __FILE__)].empty?
+  Dir.chdir(File.expand_path('../../../', __FILE__)) do
+    xsystem 'git submodule init'
+    xsystem 'git submodule update'
+  end
+end
+
 # mongrel and http-parser both define http_parser_(init|execute), so we
 # rename functions in http-parser before using them.
 vendor_dir = File.expand_path('../vendor/http-parser/', __FILE__)

data/ext/ruby_http_parser/org/ruby_http_parser/RubyHttpParser.java

@@ -120,7 +120,7 @@ public class RubyHttpParser extends RubyObject {
         if (_current_header == null)
           _current_header = new String(data);
         else
-          _current_header.concat(new String(data));
+          _current_header = _current_header.concat(new String(data));
 
         return 0;
       }

data/ext/ruby_http_parser/ruby_http_parser.c

@@ -30,10 +30,10 @@ typedef struct ParserWrapper {
 
   VALUE callback_object;
   VALUE stopped;
+  VALUE completed;
 
   VALUE last_field_name;
-  const char *last_field_name_at;
-  size_t last_field_name_length;
+  VALUE curr_field_name;
 
   enum ryah_http_parser_type type;
 } ParserWrapper;
@@ -50,10 +50,10 @@ void ParserWrapper_init(ParserWrapper *wrapper) {
   wrapper->fragment = Qnil;
 
   wrapper->headers = Qnil;
+  wrapper->completed = Qfalse;
 
   wrapper->last_field_name = Qnil;
-  wrapper->last_field_name_at = NULL;
-  wrapper->last_field_name_length = 0;
+  wrapper->curr_field_name = Qnil;
 }
 
 void ParserWrapper_mark(void *data) {
@@ -70,6 +70,7 @@ void ParserWrapper_mark(void *data) {
     rb_gc_mark_maybe(wrapper->on_message_complete);
     rb_gc_mark_maybe(wrapper->callback_object);
     rb_gc_mark_maybe(wrapper->last_field_name);
+    rb_gc_mark_maybe(wrapper->curr_field_name);
   }
 }
 
@@ -147,13 +148,11 @@ int on_fragment(ryah_http_parser *parser, const char *at, size_t length) {
 int on_header_field(ryah_http_parser *parser, const char *at, size_t length) {
   GET_WRAPPER(wrapper, parser);
 
-  wrapper->last_field_name = Qnil;
-
-  if (wrapper->last_field_name_at == NULL) {
-    wrapper->last_field_name_at = at;
-    wrapper->last_field_name_length = length;
+  if (wrapper->curr_field_name == Qnil) {
+    wrapper->last_field_name = Qnil;
+    wrapper->curr_field_name = rb_str_new(at, length);
   } else {
-    wrapper->last_field_name_length += length;
+    rb_str_cat(wrapper->curr_field_name, at, length);
   }
 
   return 0;
@@ -163,15 +162,13 @@ int on_header_value(ryah_http_parser *parser, const char *at, size_t length) {
   GET_WRAPPER(wrapper, parser);
 
   if (wrapper->last_field_name == Qnil) {
-    wrapper->last_field_name = rb_str_new(wrapper->last_field_name_at, wrapper->last_field_name_length);
+    wrapper->last_field_name = wrapper->curr_field_name;
+    wrapper->curr_field_name = Qnil;
 
     VALUE val = rb_hash_aref(wrapper->headers, wrapper->last_field_name);
     if (val != Qnil) {
       rb_str_cat(val, ", ", 2);
     }
-
-    wrapper->last_field_name_at = NULL;
-    wrapper->last_field_name_length = 0;
   }
 
   HASH_CAT(wrapper->headers, wrapper->last_field_name, at, length);
@@ -221,6 +218,7 @@ int on_message_complete(ryah_http_parser *parser) {
   GET_WRAPPER(wrapper, parser);
 
   VALUE ret = Qnil;
+  wrapper->completed = Qtrue;
 
   if (wrapper->callback_object != Qnil && rb_respond_to(wrapper->callback_object, Ion_message_complete)) {
     ret = rb_funcall(wrapper->callback_object, Ion_message_complete, 0);
@@ -290,6 +288,8 @@ VALUE Parser_initialize(int argc, VALUE *argv, VALUE self) {
 
 VALUE Parser_execute(VALUE self, VALUE data) {
   ParserWrapper *wrapper = NULL;
+
+  Check_Type(data, T_STRING);
   char *ptr = RSTRING_PTR(data);
   long len = RSTRING_LEN(data);
 
@@ -301,7 +301,7 @@ VALUE Parser_execute(VALUE self, VALUE data) {
   if (wrapper->parser.upgrade) {
     // upgrade request
   } else if (nparsed != len) {
-    if (!RTEST(wrapper->stopped))
+    if (!RTEST(wrapper->stopped) && !RTEST(wrapper->completed))
       rb_raise(eParserError, "Could not parse data entirely");
     else
       nparsed += 1; // error states fail on the current character

data/ext/ruby_http_parser/vendor/http-parser/http_parser.c

@@ -19,16 +19,6 @@
  * IN THE SOFTWARE.
  */
 #include <http_parser.h>
-#if defined(_WIN32) && !defined(__MINGW32__)
-typedef __int8 int8_t;
-typedef unsigned __int8 uint8_t;
-typedef __int16 int16_t;
-typedef unsigned __int16 uint16_t;
-typedef __int16 int32_t;
-typedef unsigned __int32 uint32_t;
-#else
-#include <stdint.h>
-#endif
 #include <assert.h>
 #include <stddef.h>
 
@@ -103,12 +93,21 @@ static const char *method_strings[] =
   , "MKACTIVITY"
   , "CHECKOUT"
   , "MERGE"
+  , "M-SEARCH"
+  , "NOTIFY"
+  , "SUBSCRIBE"
+  , "UNSUBSCRIBE"
   };
 
 
-/* ' ', '_', '-' and all alpha-numeric ascii characters are accepted by acceptable_header.
-   The 'A'-'Z' are lower-cased.  */
-static const char acceptable_header[256] = {
+/* Tokens as defined by rfc 2616. Also lowercases them.
+ *        token       = 1*<any CHAR except CTLs or separators>
+ *     separators     = "(" | ")" | "<" | ">" | "@"
+ *                    | "," | ";" | ":" | "\" | <">
+ *                    | "/" | "[" | "]" | "?" | "="
+ *                    | "{" | "}" | SP | HT
+ */
+static const char tokens[256] = {
 /*   0 nul    1 soh    2 stx    3 etx    4 eot    5 enq    6 ack    7 bel  */
         0,       0,       0,       0,       0,       0,       0,       0,
 /*   8 bs     9 ht    10 nl    11 vt    12 np    13 cr    14 so    15 si   */
@@ -118,9 +117,9 @@ static const char acceptable_header[256] = {
 /*  24 can   25 em    26 sub   27 esc   28 fs    29 gs    30 rs    31 us  */
         0,       0,       0,       0,       0,       0,       0,       0,
 /*  32 sp    33  !    34  "    35  #    36  $    37  %    38  &    39  '  */
-       ' ',      0,       0,       0,       0,       0,       0,       0,
+       ' ',      '!',     '"',     '#',     '$',     '%',     '&',    '\'',
 /*  40  (    41  )    42  *    43  +    44  ,    45  -    46  .    47  /  */
-        0,       0,       0,       0,       0,      '-',      0,       0,
+        0,       0,      '*',     '+',      0,      '-',     '.',     '/',
 /*  48  0    49  1    50  2    51  3    52  4    53  5    54  6    55  7  */
        '0',     '1',     '2',     '3',     '4',     '5',     '6',     '7',
 /*  56  8    57  9    58  :    59  ;    60  <    61  =    62  >    63  ?  */
@@ -132,15 +131,15 @@ static const char acceptable_header[256] = {
 /*  80  P    81  Q    82  R    83  S    84  T    85  U    86  V    87  W  */
        'p',     'q',     'r',     's',     't',     'u',     'v',     'w',
 /*  88  X    89  Y    90  Z    91  [    92  \    93  ]    94  ^    95  _  */
-       'x',     'y',     'z',      0,       0,       0,       0,      '_',
+       'x',     'y',     'z',      0,       0,       0,      '^',     '_',
 /*  96  `    97  a    98  b    99  c   100  d   101  e   102  f   103  g  */
-        0,      'a',     'b',     'c',     'd',     'e',     'f',     'g',
+       '`',     'a',     'b',     'c',     'd',     'e',     'f',     'g',
 /* 104  h   105  i   106  j   107  k   108  l   109  m   110  n   111  o  */
        'h',     'i',     'j',     'k',     'l',     'm',     'n',     'o',
 /* 112  p   113  q   114  r   115  s   116  t   117  u   118  v   119  w  */
        'p',     'q',     'r',     's',     't',     'u',     'v',     'w',
 /* 120  x   121  y   122  z   123  {   124  |   125  }   126  ~   127 del */
-       'x',     'y',     'z',      0,       0,       0,       0,       0 };
+       'x',     'y',     'z',      0,      '|',     '}',     '~',       0 };
 
 
 static const int8_t unhex[256] =
@@ -302,6 +301,7 @@ enum flags
 #define CR '\r'
 #define LF '\n'
 #define LOWER(c) (unsigned char)(c | 0x20)
+#define TOKEN(c) tokens[(unsigned char)c]
 
 
 #define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res)
@@ -579,12 +579,14 @@ size_t http_parser_execute (http_parser *parser,
           case 'G': parser->method = HTTP_GET; break;
           case 'H': parser->method = HTTP_HEAD; break;
           case 'L': parser->method = HTTP_LOCK; break;
-          case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE */ break;
+          case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break;
+          case 'N': parser->method = HTTP_NOTIFY; break;
           case 'O': parser->method = HTTP_OPTIONS; break;
           case 'P': parser->method = HTTP_POST; /* or PROPFIND or PROPPATCH or PUT */ break;
           case 'R': parser->method = HTTP_REPORT; break;
+          case 'S': parser->method = HTTP_SUBSCRIBE; break;
           case 'T': parser->method = HTTP_TRACE; break;
-          case 'U': parser->method = HTTP_UNLOCK; break;
+          case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break;
           default: goto error;
         }
         state = s_req_method;
@@ -612,6 +614,8 @@ size_t http_parser_execute (http_parser *parser,
             parser->method = HTTP_MOVE;
           } else if (index == 1 && ch == 'E') {
             parser->method = HTTP_MERGE;
+          } else if (index == 1 && ch == '-') {
+            parser->method = HTTP_MSEARCH;
           } else if (index == 2 && ch == 'A') {
             parser->method = HTTP_MKACTIVITY;
           }
@@ -619,6 +623,8 @@ size_t http_parser_execute (http_parser *parser,
           parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */
         } else if (index == 1 && parser->method == HTTP_POST && ch == 'U') {
           parser->method = HTTP_PUT;
+        } else if (index == 2 && parser->method == HTTP_UNLOCK && ch == 'S') {
+          parser->method = HTTP_UNSUBSCRIBE;
         } else if (index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') {
           parser->method = HTTP_PROPPATCH;
         } else {
@@ -632,7 +638,7 @@ size_t http_parser_execute (http_parser *parser,
       {
         if (ch == ' ') break;
 
-        if (ch == '/') {
+        if (ch == '/' || ch == '*') {
           MARK(url);
           MARK(path);
           state = s_req_path;
@@ -662,6 +668,9 @@ size_t http_parser_execute (http_parser *parser,
         } else if (ch == '.') {
           state = s_req_host;
           break;
+        } else if ('0' <= ch && ch <= '9') {
+          state = s_req_host;
+          break;
         }
 
         goto error;
@@ -739,12 +748,14 @@ size_t http_parser_execute (http_parser *parser,
           case CR:
             CALLBACK(url);
             CALLBACK(path);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_req_line_almost_done;
             break;
           case LF:
             CALLBACK(url);
             CALLBACK(path);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_header_field_start;
             break;
@@ -779,11 +790,13 @@ size_t http_parser_execute (http_parser *parser,
             break;
           case CR:
             CALLBACK(url);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_req_line_almost_done;
             break;
           case LF:
             CALLBACK(url);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_header_field_start;
             break;
@@ -812,12 +825,14 @@ size_t http_parser_execute (http_parser *parser,
           case CR:
             CALLBACK(url);
             CALLBACK(query_string);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_req_line_almost_done;
             break;
           case LF:
             CALLBACK(url);
             CALLBACK(query_string);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_header_field_start;
             break;
@@ -846,11 +861,13 @@ size_t http_parser_execute (http_parser *parser,
             break;
           case CR:
             CALLBACK(url);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_req_line_almost_done;
             break;
           case LF:
             CALLBACK(url);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_header_field_start;
             break;
@@ -879,12 +896,14 @@ size_t http_parser_execute (http_parser *parser,
           case CR:
             CALLBACK(url);
             CALLBACK(fragment);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_req_line_almost_done;
             break;
           case LF:
             CALLBACK(url);
             CALLBACK(fragment);
+            parser->http_major = 0;
             parser->http_minor = 9;
             state = s_header_field_start;
             break;
@@ -1006,9 +1025,9 @@ size_t http_parser_execute (http_parser *parser,
           goto headers_almost_done;
         }
 
-        c = LOWER(ch);
+        c = TOKEN(ch);
 
-        if (c < 'a' || 'z' < c) goto error;
+        if (!c) goto error;
 
         MARK(header_field);
 
@@ -1041,7 +1060,7 @@ size_t http_parser_execute (http_parser *parser,
 
       case s_header_field:
       {
-        c = acceptable_header[(unsigned char)ch];
+        c = TOKEN(ch);
 
         if (c) {
           switch (header_state) {
@@ -1177,23 +1196,18 @@ size_t http_parser_execute (http_parser *parser,
         state = s_header_value;
         index = 0;
 
-        c = acceptable_header[(unsigned char)ch];
-
-        if (!c) {
-          if (ch == CR) {
-            CALLBACK(header_value);
-            header_state = h_general;
-            state = s_header_almost_done;
-            break;
-          }
-
-          if (ch == LF) {
-            CALLBACK(header_value);
-            state = s_header_field_start;
-            break;
-          }
+        c = LOWER(ch);
 
+        if (ch == CR) {
+          CALLBACK(header_value);
           header_state = h_general;
+          state = s_header_almost_done;
+          break;
+        }
+
+        if (ch == LF) {
+          CALLBACK(header_value);
+          state = s_header_field_start;
           break;
         }
 
@@ -1238,22 +1252,19 @@ size_t http_parser_execute (http_parser *parser,
 
       case s_header_value:
       {
-        c = acceptable_header[(unsigned char)ch];
-
-        if (!c) {
-          if (ch == CR) {
-            CALLBACK(header_value);
-            state = s_header_almost_done;
-            break;
-          }
+        c = LOWER(ch);
 
-          if (ch == LF) {
-            CALLBACK(header_value);
-            goto header_almost_done;
-          }
+        if (ch == CR) {
+          CALLBACK(header_value);
+          state = s_header_almost_done;
           break;
         }
 
+        if (ch == LF) {
+          CALLBACK(header_value);
+          goto header_almost_done;
+        }
+
         switch (header_state) {
           case h_general:
             break;
@@ -1373,6 +1384,7 @@ size_t http_parser_execute (http_parser *parser,
               break;
 
             default:
+              parser->state = state;
               return p - data; /* Error */
           }
         }

data/ext/ruby_http_parser/vendor/http-parser/http_parser.h

@@ -26,11 +26,20 @@ extern "C" {
 
 
 #include <sys/types.h>
-#include <stdint.h>
-
 #if defined(_WIN32) && !defined(__MINGW32__)
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+
 typedef unsigned int size_t;
 typedef int ssize_t;
+#else
+#include <stdint.h>
 #endif
 
 /* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
@@ -92,6 +101,11 @@ enum http_method
   , HTTP_MKACTIVITY
   , HTTP_CHECKOUT
   , HTTP_MERGE
+  /* upnp */
+  , HTTP_MSEARCH
+  , HTTP_NOTIFY
+  , HTTP_SUBSCRIBE
+  , HTTP_UNSUBSCRIBE
   };
 
 

data/ext/ruby_http_parser/vendor/http-parser/test.c

@@ -31,7 +31,7 @@
 #undef FALSE
 #define FALSE 0
 
-#define MAX_HEADERS 10
+#define MAX_HEADERS 13
 #define MAX_ELEMENT_SIZE 500
 
 #define MIN(a,b) ((a) < (b) ? (a) : (b))
@@ -498,7 +498,7 @@ const struct message requests[] =
 #define CONNECT_REQUEST 17
 , {.name = "connect request"
   ,.type= HTTP_REQUEST
-  ,.raw= "CONNECT home.netscape.com:443 HTTP/1.0\r\n"
+  ,.raw= "CONNECT home0.netscape.com:443 HTTP/1.0\r\n"
          "User-agent: Mozilla/1.1N\r\n"
          "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
          "\r\n"
@@ -510,7 +510,7 @@ const struct message requests[] =
   ,.query_string= ""
   ,.fragment= ""
   ,.request_path= ""
-  ,.request_url= "home.netscape.com:443"
+  ,.request_url= "home0.netscape.com:443"
   ,.num_headers= 2
   ,.upgrade=1
   ,.headers= { { "User-agent", "Mozilla/1.1N" }
@@ -538,6 +538,50 @@ const struct message requests[] =
   ,.body= ""
   }
 
+#define NO_HTTP_VERSION 19
+, {.name= "request with no http version"
+  ,.type= HTTP_REQUEST
+  ,.raw= "GET /\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 0
+  ,.http_minor= 9
+  ,.method= HTTP_GET
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/"
+  ,.request_url= "/"
+  ,.num_headers= 0
+  ,.headers= {}
+  ,.body= ""
+  }
+
+#define MSEARCH_REQ 19
+, {.name= "m-search request"
+  ,.type= HTTP_REQUEST
+  ,.raw= "M-SEARCH * HTTP/1.1\r\n"
+         "HOST: 239.255.255.250:1900\r\n"
+         "MAN: \"ssdp:discover\"\r\n"
+         "ST: \"ssdp:all\"\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_MSEARCH
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "*"
+  ,.request_url= "*"
+  ,.num_headers= 3
+  ,.headers= { { "HOST", "239.255.255.250:1900" }
+             , { "MAN", "\"ssdp:discover\"" }
+             , { "ST", "\"ssdp:all\"" }
+             }
+  ,.body= ""
+  }
+
 , {.name= NULL } /* sentinel */
 };
 
@@ -551,9 +595,10 @@ const struct message responses[] =
          "Content-Type: text/html; charset=UTF-8\r\n"
          "Date: Sun, 26 Apr 2009 11:11:49 GMT\r\n"
          "Expires: Tue, 26 May 2009 11:11:49 GMT\r\n"
+         "X-$PrototypeBI-Version: 1.6.0.3\r\n" /* $ char in header field */
          "Cache-Control: public, max-age=2592000\r\n"
          "Server: gws\r\n"
-         "Content-Length: 219\r\n"
+         "Content-Length:  219  \r\n"
          "\r\n"
          "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n"
          "<TITLE>301 Moved</TITLE></HEAD><BODY>\n"
@@ -566,15 +611,16 @@ const struct message responses[] =
   ,.http_major= 1
   ,.http_minor= 1
   ,.status_code= 301
-  ,.num_headers= 7
+  ,.num_headers= 8
   ,.headers=
     { { "Location", "http://www.google.com/" }
     , { "Content-Type", "text/html; charset=UTF-8" }
     , { "Date", "Sun, 26 Apr 2009 11:11:49 GMT" }
     , { "Expires", "Tue, 26 May 2009 11:11:49 GMT" }
+    , { "X-$PrototypeBI-Version", "1.6.0.3" }
     , { "Cache-Control", "public, max-age=2592000" }
     , { "Server", "gws" }
-    , { "Content-Length", "219" }
+    , { "Content-Length", "219  " }
     }
   ,.body= "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n"
           "<TITLE>301 Moved</TITLE></HEAD><BODY>\n"
@@ -833,6 +879,71 @@ const struct message responses[] =
   ,.body= "<xml>hello</xml>"
   }
 
+
+#define RES_FIELD_UNDERSCORE 10
+/* Should handle spaces in header fields */
+, {.name= "field underscore"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 200 OK\r\n"
+         "Date: Tue, 28 Sep 2010 01:14:13 GMT\r\n"
+         "Server: Apache\r\n"
+         "Cache-Control: no-cache, must-revalidate\r\n"
+         "Expires: Mon, 26 Jul 1997 05:00:00 GMT\r\n"
+         ".et-Cookie: PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com\r\n"
+         "Vary: Accept-Encoding\r\n"
+         "_eep-Alive: timeout=45\r\n" /* semantic value ignored */
+         "_onnection: Keep-Alive\r\n" /* semantic value ignored */
+         "Transfer-Encoding: chunked\r\n"
+         "Content-Type: text/html\r\n"
+         "Connection: close\r\n"
+         "\r\n"
+         "0\r\n\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 200
+  ,.num_headers= 11
+  ,.headers=
+    { { "Date", "Tue, 28 Sep 2010 01:14:13 GMT" }
+    , { "Server", "Apache" }
+    , { "Cache-Control", "no-cache, must-revalidate" }
+    , { "Expires", "Mon, 26 Jul 1997 05:00:00 GMT" }
+    , { ".et-Cookie", "PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com" }
+    , { "Vary", "Accept-Encoding" }
+    , { "_eep-Alive", "timeout=45" }
+    , { "_onnection", "Keep-Alive" }
+    , { "Transfer-Encoding", "chunked" }
+    , { "Content-Type", "text/html" }
+    , { "Connection", "close" }
+    }
+  ,.body= ""
+  }
+
+#define NON_ASCII_IN_STATUS_LINE 11
+/* Should handle non-ASCII in status line */
+, {.name= "non-ASCII in status line"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 500 Oriëntatieprobleem\r\n"
+         "Date: Fri, 5 Nov 2010 23:07:12 GMT+2\r\n"
+         "Content-Length: 0\r\n"
+         "Connection: close\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 500
+  ,.num_headers= 3
+  ,.headers=
+    { { "Date", "Fri, 5 Nov 2010 23:07:12 GMT+2" }
+    , { "Content-Length", "0" }
+    , { "Connection", "close" }
+    }
+  ,.body= ""
+  }
+
+
 , {.name= NULL } /* sentinel */
 };
 

data/http_parser.rb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "http_parser.rb"
-  s.version = "0.5.0"
+  s.version = "0.5.1"
   s.summary = "Simple callback-based HTTP request/response parser"
   s.description = "Ruby bindings to http://github.com/ry/http-parser and http://github.com/a2800276/http-parser.java"
 
@@ -12,4 +12,8 @@ Gem::Specification.new do |s|
 
   s.require_paths = ["lib"]
   s.extensions    = ["ext/ruby_http_parser/extconf.rb"]
+
+  s.add_development_dependency 'rake-compiler', '>= 0.7.5'
+  s.add_development_dependency 'rspec', '>= 2.0.1'
+  s.add_development_dependency 'json', '>= 1.4.6'
 end

data/spec/parser_spec.rb

@@ -115,6 +115,23 @@ describe HTTP::Parser do
     @done.should be_true
   end
 
+  it "should parse headers incrementally" do
+    request =
+      "GET / HTTP/1.0\r\n" +
+      "Header1: value 1\r\n" +
+      "Header2: value 2\r\n" +
+      "\r\n"
+
+    while chunk = request.slice!(0,2) and !chunk.empty?
+      @parser << chunk
+    end
+
+    @parser.headers.should == {
+      'Header1' => 'value 1',
+      'Header2' => 'value 2'
+    }
+  end
+
   it "should handle multiple headers" do
     @parser <<
       "GET / HTTP/1.0\r\n" +
@@ -143,6 +160,18 @@ describe HTTP::Parser do
     @done.should be_true
   end
 
+  it "should ignore extra content beyond specified length" do
+    @parser <<
+      "GET / HTTP/1.0\r\n" +
+      "Content-Length: 5\r\n" +
+      "\r\n" +
+      "hello" +
+      "  \n"
+
+    @body.should == 'hello'
+    @done.should be_true
+  end
+
   %w[ request response ].each do |type|
     JSON.parse(File.read(File.expand_path("../support/#{type}s.json", __FILE__))).each do |test|
       test['headers'] ||= {}

data/spec/spec_helper.rb

@@ -1,2 +1 @@
-require "rubygems"
 require "http_parser"

data/spec/support/requests.json

@@ -341,7 +341,7 @@
   {
     "name": "connect request",
     "type": "HTTP_REQUEST",
-    "raw": "CONNECT home.netscape.com:443 HTTP/1.0\r\nUser-agent: Mozilla/1.1N\r\nProxy-authorization: basic aGVsbG86d29ybGQ=\r\n\r\n",
+    "raw": "CONNECT home0.netscape.com:443 HTTP/1.0\r\nUser-agent: Mozilla/1.1N\r\nProxy-authorization: basic aGVsbG86d29ybGQ=\r\n\r\n",
     "should_keep_alive": false,
     "message_complete_on_eof": false,
     "http_major": 1,
@@ -350,7 +350,7 @@
     "query_string": "",
     "fragment": "",
     "request_path": "",
-    "request_url": "home.netscape.com:443",
+    "request_url": "home0.netscape.com:443",
     "num_headers": 2,
     "upgrade": 1,
     "headers": {
@@ -375,6 +375,25 @@
     "num_headers": 0,
     "headers": {
 
+    },
+    "body": ""
+  },
+  {
+    "name": "request with no http version",
+    "type": "HTTP_REQUEST",
+    "raw": "GET /\r\n\r\n",
+    "should_keep_alive": false,
+    "message_complete_on_eof": false,
+    "http_major": 0,
+    "http_minor": 9,
+    "method": "GET",
+    "query_string": "",
+    "fragment": "",
+    "request_path": "/",
+    "request_url": "/",
+    "num_headers": 0,
+    "headers": {
+
     },
     "body": ""
   }

data/spec/support/responses.json

@@ -2,21 +2,22 @@
   {
     "name": "google 301",
     "type": "HTTP_RESPONSE",
-    "raw": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.google.com/\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Sun, 26 Apr 2009 11:11:49 GMT\r\nExpires: Tue, 26 May 2009 11:11:49 GMT\r\nCache-Control: public, max-age=2592000\r\nServer: gws\r\nContent-Length: 219\r\n\r\n<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n",
+    "raw": "HTTP/1.1 301 Moved Permanently\r\nLocation: http://www.google.com/\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Sun, 26 Apr 2009 11:11:49 GMT\r\nExpires: Tue, 26 May 2009 11:11:49 GMT\r\nX-$PrototypeBI-Version: 1.6.0.3\r\nCache-Control: public, max-age=2592000\r\nServer: gws\r\nContent-Length:  219  \r\n\r\n<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n",
     "should_keep_alive": true,
     "message_complete_on_eof": false,
     "http_major": 1,
     "http_minor": 1,
     "status_code": 301,
-    "num_headers": 7,
+    "num_headers": 8,
     "headers": {
       "Location": "http://www.google.com/",
       "Content-Type": "text/html; charset=UTF-8",
       "Date": "Sun, 26 Apr 2009 11:11:49 GMT",
       "Expires": "Tue, 26 May 2009 11:11:49 GMT",
+      "X-$PrototypeBI-Version": "1.6.0.3",
       "Cache-Control": "public, max-age=2592000",
       "Server": "gws",
-      "Content-Length": "219"
+      "Content-Length": "219  "
     },
     "body": "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n"
   },
@@ -182,5 +183,47 @@
       "Connection": "keep-alive"
     },
     "body": "<xml>hello</xml>"
+  },
+  {
+    "name": "field underscore",
+    "type": "HTTP_RESPONSE",
+    "raw": "HTTP/1.1 200 OK\r\nDate: Tue, 28 Sep 2010 01:14:13 GMT\r\nServer: Apache\r\nCache-Control: no-cache, must-revalidate\r\nExpires: Mon, 26 Jul 1997 05:00:00 GMT\r\n.et-Cookie: PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com\r\nVary: Accept-Encoding\r\n_eep-Alive: timeout=45\r\n_onnection: Keep-Alive\r\nTransfer-Encoding: chunked\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n0\r\n\r\n",
+    "should_keep_alive": false,
+    "message_complete_on_eof": false,
+    "http_major": 1,
+    "http_minor": 1,
+    "status_code": 200,
+    "num_headers": 11,
+    "headers": {
+      "Date": "Tue, 28 Sep 2010 01:14:13 GMT",
+      "Server": "Apache",
+      "Cache-Control": "no-cache, must-revalidate",
+      "Expires": "Mon, 26 Jul 1997 05:00:00 GMT",
+      ".et-Cookie": "PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com",
+      "Vary": "Accept-Encoding",
+      "_eep-Alive": "timeout=45",
+      "_onnection": "Keep-Alive",
+      "Transfer-Encoding": "chunked",
+      "Content-Type": "text/html",
+      "Connection": "close"
+    },
+    "body": ""
+  },
+  {
+    "name": "non-ASCII in status line",
+    "type": "HTTP_RESPONSE",
+    "raw": "HTTP/1.1 500 Oriëntatieprobleem\r\nDate: Fri, 5 Nov 2010 23:07:12 GMT+2\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
+    "should_keep_alive": false,
+    "message_complete_on_eof": false,
+    "http_major": 1,
+    "http_minor": 1,
+    "status_code": 500,
+    "num_headers": 3,
+    "headers": {
+      "Date": "Fri, 5 Nov 2010 23:07:12 GMT+2",
+      "Content-Length": "0",
+      "Connection": "close"
+    },
+    "body": ""
   }
 ]

data/tasks/compile.rake

@@ -32,6 +32,7 @@ RUBY_VERSION =~ /(\\d+.\\d+)/
 require "\#{$1}/ruby_http_parser"
     eoruby
   end
+  at_exit{ FileUtils.rm t.name if File.exists?(t.name) }
 end
 
 if Rake::Task.task_defined?(:cross)

data/tasks/fixtures.rake

@@ -0,0 +1,65 @@
+desc "Generate test fixtures"
+task :fixtures => :submodules do
+  require 'yajl'
+  data = File.read File.expand_path('../../ext/ruby_http_parser/vendor/http-parser/test.c', __FILE__)
+
+  %w[ requests responses ].each do |type|
+    # find test definitions in between requests/responses[]= and .name=NULL
+    tmp = data[/#{type}\[\]\s*=(.+?),\s*\{\s*\.name=\s*NULL/m, 1]
+
+    # replace first { with a [ (parsing an array of test cases)
+    tmp.sub!('{','[')
+
+    # replace booleans
+    tmp.gsub!('TRUE', 'true')
+    tmp.gsub!('FALSE', 'false')
+
+    # remove macros and comments
+    tmp.gsub!(/^#define.+$/,'')
+    tmp.gsub!(/\/\*(.+?)\*\/$/,'')
+
+    # HTTP_* enums become strings
+    tmp.gsub!(/(= )(HTTP_\w+)/){
+      "#{$1}#{$2.dump}"
+    }
+
+    # join multiline strings for body and raw data
+    tmp.gsub!(/((body|raw)\s*=)(.+?)(\n\s+[\},])/m){
+      before, after = $1, $4
+      raw = $3.split("\n").map{ |l| l.strip[1..-2] }.join('')
+      "#{before} \"#{raw}\" #{after}"
+    }
+
+    # make headers an array of array tuples
+    tmp.gsub!(/(\.headers\s*=)(.+?)(\s*,\.)/m){
+      before, after = $1, $3
+      raw = $2.gsub('{', '[').gsub('}', ']')
+      "#{before} #{raw} #{after}"
+    }
+
+    # .name= becomes "name":
+    tmp.gsub!(/^(.{2,5})\.(\w+)\s*=/){
+      "#{$1}#{$2.dump}: "
+    }
+
+    # evaluate addition expressions
+    tmp.gsub!(/(body_size\":\s*)(\d+)\+(\d+)/){
+      "#{$1}#{$2.to_i+$3.to_i}"
+    }
+
+    # end result array
+    tmp << ']'
+
+    # normalize data
+    results = Yajl.load(tmp, :symbolize_keys => true)
+    results.map{ |res|
+      res[:headers] and res[:headers] = Hash[*res[:headers].flatten]
+      res[:method]  and res[:method].gsub!(/^HTTP_/, '')
+    }
+
+    # write to a file
+    File.open("spec/support/#{type}.json", 'w'){ |f|
+      f.write Yajl.dump(results, :pretty => true)
+    }
+  end
+end

metadata

@@ -1,12 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: http_parser.rb
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 5
-  - 0
-  version: 0.5.0
+  version: 0.5.1
 platform: x86-mingw32
 authors: 
 - Marc-Andre Cournoyer
@@ -15,10 +10,39 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-28 00:00:00 -07:00
+date: 2011-01-25 00:00:00 -08:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake-compiler
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.7.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.6
+    version: 
 description: Ruby bindings to http://github.com/ry/http-parser and http://github.com/a2800276/http-parser.java
 email: 
 - macournoyer@gmail.com
@@ -32,6 +56,8 @@ extra_rdoc_files: []
 files: 
 - .gitignore
 - .gitmodules
+- Gemfile
+- Gemfile.lock
 - README.md
 - Rakefile
 - bench/thin.rb
@@ -50,6 +76,7 @@ files:
 - spec/support/requests.json
 - spec/support/responses.json
 - tasks/compile.rake
+- tasks/fixtures.rake
 - tasks/spec.rake
 - tasks/submodules.rake
 - ext/ruby_http_parser/vendor/http-parser/CONTRIBUTIONS
@@ -103,20 +130,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
+  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
+  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Simple callback-based HTTP request/response parser