http_parser.rb 0.5.3 → 0.6.0.beta.1

data/ext/ruby_http_parser/vendor/http-parser/http_parser.gyp

@@ -0,0 +1,79 @@
+# This file is used with the GYP meta build system.
+# http://code.google.com/p/gyp/
+# To build try this:
+#   svn co http://gyp.googlecode.com/svn/trunk gyp
+#   ./gyp/gyp -f make --depth=`pwd` http_parser.gyp 
+#   ./out/Debug/test 
+{
+  'target_defaults': {
+    'default_configuration': 'Debug',
+    'configurations': {
+      # TODO: hoist these out and put them somewhere common, because
+      #       RuntimeLibrary MUST MATCH across the entire project
+      'Debug': {
+        'defines': [ 'DEBUG', '_DEBUG' ],
+        'msvs_settings': {
+          'VCCLCompilerTool': {
+            'RuntimeLibrary': 1, # static debug
+          },
+        },
+      },
+      'Release': {
+        'defines': [ 'NDEBUG' ],
+        'msvs_settings': {
+          'VCCLCompilerTool': {
+            'RuntimeLibrary': 0, # static release
+          },
+        },
+      }
+    },
+    'msvs_settings': {
+      'VCCLCompilerTool': {
+      },
+      'VCLibrarianTool': {
+      },
+      'VCLinkerTool': {
+        'GenerateDebugInformation': 'true',
+      },
+    },
+    'conditions': [
+      ['OS == "win"', {
+        'defines': [
+          'WIN32'
+        ],
+      }]
+    ],
+  },
+
+  'targets': [
+    {
+      'target_name': 'http_parser',
+      'type': 'static_library',
+      'include_dirs': [ '.' ],
+      'direct_dependent_settings': {
+        'include_dirs': [ '.' ],
+      },
+      'defines': [ 'HTTP_PARSER_STRICT=0' ],
+      'sources': [ './http_parser.c', ],
+      'conditions': [
+        ['OS=="win"', {
+          'msvs_settings': {
+            'VCCLCompilerTool': {
+              # Compile as C++. http_parser.c is actually C99, but C++ is
+              # close enough in this case.
+              'CompileAs': 2,
+            },
+          },
+        }]
+      ],
+    },
+
+    {
+      'target_name': 'test',
+      'type': 'executable',
+      'dependencies': [ 'http_parser' ],
+      'sources': [ 'test.c' ]
+    }
+  ]
+}
+

data/ext/ruby_http_parser/vendor/http-parser/http_parser.h

@@ -28,7 +28,7 @@ extern "C" {
 #define HTTP_PARSER_VERSION_MINOR 0
 
 #include <sys/types.h>
-#if defined(_WIN32) && !defined(__MINGW32__)
+#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
 typedef __int8 int8_t;
 typedef unsigned __int8 uint8_t;
 typedef __int16 int16_t;
@@ -49,8 +49,13 @@ typedef int ssize_t;
  */
 #ifndef HTTP_PARSER_STRICT
 # define HTTP_PARSER_STRICT 1
-#else
-# define HTTP_PARSER_STRICT 0
+#endif
+
+/* Compile with -DHTTP_PARSER_DEBUG=1 to add extra debugging information to
+ * the error reporting facility.
+ */
+#ifndef HTTP_PARSER_DEBUG
+# define HTTP_PARSER_DEBUG 0
 #endif
 
 
@@ -60,6 +65,7 @@ typedef int ssize_t;
 
 typedef struct http_parser http_parser;
 typedef struct http_parser_settings http_parser_settings;
+typedef struct http_parser_result http_parser_result;
 
 
 /* Callbacks should return non-zero to indicate an error. The parser will
@@ -108,35 +114,118 @@ enum http_method
   , HTTP_NOTIFY
   , HTTP_SUBSCRIBE
   , HTTP_UNSUBSCRIBE
+  /* RFC-5789 */
+  , HTTP_PATCH
+  , HTTP_PURGE
   };
 
 
 enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
 
 
+/* Flag values for http_parser.flags field */
+enum flags
+  { F_CHUNKED               = 1 << 0
+  , F_CONNECTION_KEEP_ALIVE = 1 << 1
+  , F_CONNECTION_CLOSE      = 1 << 2
+  , F_TRAILING              = 1 << 3
+  , F_UPGRADE               = 1 << 4
+  , F_SKIPBODY              = 1 << 5
+  };
+
+
+/* Map for errno-related constants
+ * 
+ * The provided argument should be a macro that takes 2 arguments.
+ */
+#define HTTP_ERRNO_MAP(XX)                                           \
+  /* No error */                                                     \
+  XX(OK, "success")                                                  \
+                                                                     \
+  /* Callback-related errors */                                      \
+  XX(CB_message_begin, "the on_message_begin callback failed")       \
+  XX(CB_url, "the on_url callback failed")                           \
+  XX(CB_header_field, "the on_header_field callback failed")         \
+  XX(CB_header_value, "the on_header_value callback failed")         \
+  XX(CB_headers_complete, "the on_headers_complete callback failed") \
+  XX(CB_body, "the on_body callback failed")                         \
+  XX(CB_message_complete, "the on_message_complete callback failed") \
+                                                                     \
+  /* Parsing-related errors */                                       \
+  XX(INVALID_EOF_STATE, "stream ended at an unexpected time")        \
+  XX(HEADER_OVERFLOW,                                                \
+     "too many header bytes seen; overflow detected")                \
+  XX(CLOSED_CONNECTION,                                              \
+     "data received after completed connection: close message")      \
+  XX(INVALID_VERSION, "invalid HTTP version")                        \
+  XX(INVALID_STATUS, "invalid HTTP status code")                     \
+  XX(INVALID_METHOD, "invalid HTTP method")                          \
+  XX(INVALID_URL, "invalid URL")                                     \
+  XX(INVALID_HOST, "invalid host")                                   \
+  XX(INVALID_PORT, "invalid port")                                   \
+  XX(INVALID_PATH, "invalid path")                                   \
+  XX(INVALID_QUERY_STRING, "invalid query string")                   \
+  XX(INVALID_FRAGMENT, "invalid fragment")                           \
+  XX(LF_EXPECTED, "LF character expected")                           \
+  XX(INVALID_HEADER_TOKEN, "invalid character in header")            \
+  XX(INVALID_CONTENT_LENGTH,                                         \
+     "invalid character in content-length header")                   \
+  XX(INVALID_CHUNK_SIZE,                                             \
+     "invalid character in chunk size header")                       \
+  XX(INVALID_CONSTANT, "invalid constant string")                    \
+  XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
+  XX(STRICT, "strict mode assertion failed")                         \
+  XX(PAUSED, "parser is paused")                                     \
+  XX(UNKNOWN, "an unknown error occurred")
+
+
+/* Define HPE_* values for each errno value above */
+#define HTTP_ERRNO_GEN(n, s) HPE_##n,
+enum http_errno {
+  HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
+};
+#undef HTTP_ERRNO_GEN
+
+
+/* Get an http_errno value from an http_parser */
+#define HTTP_PARSER_ERRNO(p)            ((enum http_errno) (p)->http_errno)
+
+/* Get the line number that generated the current error */
+#if HTTP_PARSER_DEBUG
+#define HTTP_PARSER_ERRNO_LINE(p)       ((p)->error_lineno)
+#else
+#define HTTP_PARSER_ERRNO_LINE(p)       0
+#endif
+
+
 struct http_parser {
   /** PRIVATE **/
-  unsigned char type : 2;
-  unsigned char flags : 6;
-  unsigned char state;
-  unsigned char header_state;
-  unsigned char index;
+  unsigned char type : 2;     /* enum http_parser_type */
+  unsigned char flags : 6;    /* F_* values from 'flags' enum; semi-public */
+  unsigned char state;        /* enum state from http_parser.c */
+  unsigned char header_state; /* enum header_state from http_parser.c */
+  unsigned char index;        /* index into current matcher */
 
-  uint32_t nread;
-  int64_t content_length;
+  uint32_t nread;          /* # bytes read in various scenarios */
+  uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
 
   /** READ-ONLY **/
   unsigned short http_major;
   unsigned short http_minor;
   unsigned short status_code; /* responses only */
-  unsigned char method;    /* requests only */
+  unsigned char method;       /* requests only */
+  unsigned char http_errno : 7;
 
   /* 1 = Upgrade header was present and the parser has exited because of that.
    * 0 = No upgrade header present.
    * Should be checked when http_parser_execute() returns in addition to
    * error checking.
    */
-  char upgrade;
+  unsigned char upgrade : 1;
+
+#if HTTP_PARSER_DEBUG
+  uint32_t error_lineno;
+#endif
 
   /** PUBLIC **/
   void *data; /* A pointer to get hook to the "connection" or "socket" object */
@@ -145,10 +234,7 @@ struct http_parser {
 
 struct http_parser_settings {
   http_cb      on_message_begin;
-  http_data_cb on_path;
-  http_data_cb on_query_string;
   http_data_cb on_url;
-  http_data_cb on_fragment;
   http_data_cb on_header_field;
   http_data_cb on_header_value;
   http_cb      on_headers_complete;
@@ -157,6 +243,35 @@ struct http_parser_settings {
 };
 
 
+enum http_parser_url_fields
+  { UF_SCHEMA           = 0
+  , UF_HOST             = 1
+  , UF_PORT             = 2
+  , UF_PATH             = 3
+  , UF_QUERY            = 4
+  , UF_FRAGMENT         = 5
+  , UF_MAX              = 6
+  };
+
+
+/* Result structure for http_parser_parse_url().
+ *
+ * Callers should index into field_data[] with UF_* values iff field_set
+ * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
+ * because we probably have padding left over), we convert any port to
+ * a uint16_t.
+ */
+struct http_parser_url {
+  uint16_t field_set;           /* Bitmask of (1 << UF_*) values */
+  uint16_t port;                /* Converted UF_PORT string */
+
+  struct {
+    uint16_t off;               /* Offset into buffer in which field starts */
+    uint16_t len;               /* Length of run in buffer */
+  } field_data[UF_MAX];
+};
+
+
 void http_parser_init(http_parser *parser, enum http_parser_type type);
 
 
@@ -175,7 +290,21 @@ size_t http_parser_execute(http_parser *parser,
 int http_should_keep_alive(http_parser *parser);
 
 /* Returns a string version of the HTTP method. */
-const char *http_method_str(enum http_method);
+const char *http_method_str(enum http_method m);
+
+/* Return a string name of the given error */
+const char *http_errno_name(enum http_errno err);
+
+/* Return a string description of the given error */
+const char *http_errno_description(enum http_errno err);
+
+/* Parse a URL; return nonzero on failure */
+int http_parser_parse_url(const char *buf, size_t buflen,
+                          int is_connect,
+                          struct http_parser_url *u);
+
+/* Pause or un-pause the parser; a nonzero value pauses */
+void http_parser_pause(http_parser *parser, int paused);
 
 #ifdef __cplusplus
 }

data/ext/ruby_http_parser/vendor/http-parser/test.c

@@ -50,12 +50,13 @@ struct message {
   char query_string[MAX_ELEMENT_SIZE];
   char body[MAX_ELEMENT_SIZE];
   size_t body_size;
+  uint16_t port;
   int num_headers;
   enum { NONE=0, FIELD, VALUE } last_header_element;
   char headers [MAX_HEADERS][2][MAX_ELEMENT_SIZE];
   int should_keep_alive;
 
-  int upgrade;
+  const char *upgrade; // upgraded body
 
   unsigned short http_major;
   unsigned short http_minor;
@@ -70,6 +71,7 @@ static int currently_parsing_eof;
 
 static struct message messages[5];
 static int num_messages;
+static http_parser_settings *current_pause_parser;
 
 /* * R E Q U E S T S * */
 const struct message requests[] =
@@ -473,6 +475,7 @@ const struct message requests[] =
          "Sec-WebSocket-Key1: 4 @1  46546xW%0l 1 5\r\n"
          "Origin: http://example.com\r\n"
          "\r\n"
+         "Hot diggity dogg"
   ,.should_keep_alive= TRUE
   ,.message_complete_on_eof= FALSE
   ,.http_major= 1
@@ -483,7 +486,7 @@ const struct message requests[] =
   ,.request_path= "/demo"
   ,.request_url= "/demo"
   ,.num_headers= 7
-  ,.upgrade=1
+  ,.upgrade="Hot diggity dogg"
   ,.headers= { { "Host", "example.com" }
              , { "Connection", "Upgrade" }
              , { "Sec-WebSocket-Key2", "12998 5 Y3 1  .P00" }
@@ -498,10 +501,12 @@ const struct message requests[] =
 #define CONNECT_REQUEST 17
 , {.name = "connect request"
   ,.type= HTTP_REQUEST
-  ,.raw= "CONNECT home0.netscape.com:443 HTTP/1.0\r\n"
+  ,.raw= "CONNECT 0-home0.netscape.com:443 HTTP/1.0\r\n"
          "User-agent: Mozilla/1.1N\r\n"
          "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
          "\r\n"
+         "some data\r\n"
+         "and yet even more data"
   ,.should_keep_alive= FALSE
   ,.message_complete_on_eof= FALSE
   ,.http_major= 1
@@ -510,9 +515,9 @@ const struct message requests[] =
   ,.query_string= ""
   ,.fragment= ""
   ,.request_path= ""
-  ,.request_url= "home0.netscape.com:443"
+  ,.request_url= "0-home0.netscape.com:443"
   ,.num_headers= 2
-  ,.upgrade=1
+  ,.upgrade="some data\r\nand yet even more data"
   ,.headers= { { "User-agent", "Mozilla/1.1N" }
              , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
              }
@@ -582,28 +587,36 @@ const struct message requests[] =
   ,.body= ""
   }
 
-#define UTF8_PATH_REQ 21
-, {.name= "utf-8 path request"
+#define LINE_FOLDING_IN_HEADER 20
+, {.name= "line folding in header value"
   ,.type= HTTP_REQUEST
-  ,.raw= "GET /δ¶/δt/pope?q=1#narf HTTP/1.1\r\n"
-         "Host: github.com\r\n"
+  ,.raw= "GET / HTTP/1.1\r\n"
+         "Line1:   abc\r\n"
+         "\tdef\r\n"
+         " ghi\r\n"
+         "\t\tjkl\r\n"
+         "  mno \r\n"
+         "\t \tqrs\r\n"
+         "Line2: \t line2\t\r\n"
          "\r\n"
   ,.should_keep_alive= TRUE
   ,.message_complete_on_eof= FALSE
   ,.http_major= 1
   ,.http_minor= 1
   ,.method= HTTP_GET
-  ,.query_string= "q=1"
-  ,.fragment= "narf"
-  ,.request_path= "/δ¶/δt/pope"
-  ,.request_url= "/δ¶/δt/pope?q=1#narf"
-  ,.num_headers= 1
-  ,.headers= { {"Host", "github.com" }
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/"
+  ,.request_url= "/"
+  ,.num_headers= 2
+  ,.headers= { { "Line1", "abcdefghijklmno qrs" }
+             , { "Line2", "line2\t" }
              }
   ,.body= ""
   }
 
-#define QUERY_TERMINATED_HOST 22
+
+#define QUERY_TERMINATED_HOST 21
 , {.name= "host terminated by a query string"
   ,.type= HTTP_REQUEST
   ,.raw= "GET http://hypnotoad.org?hail=all HTTP/1.1\r\n"
@@ -622,7 +635,7 @@ const struct message requests[] =
   ,.body= ""
   }
 
-#define QUERY_TERMINATED_HOSTPORT 23
+#define QUERY_TERMINATED_HOSTPORT 22
 , {.name= "host:port terminated by a query string"
   ,.type= HTTP_REQUEST
   ,.raw= "GET http://hypnotoad.org:1234?hail=all HTTP/1.1\r\n"
@@ -636,12 +649,13 @@ const struct message requests[] =
   ,.fragment= ""
   ,.request_path= ""
   ,.request_url= "http://hypnotoad.org:1234?hail=all"
+  ,.port= 1234
   ,.num_headers= 0
   ,.headers= { }
   ,.body= ""
   }
 
-#define SPACE_TERMINATED_HOSTPORT 24
+#define SPACE_TERMINATED_HOSTPORT 23
 , {.name= "host:port terminated by a space"
   ,.type= HTTP_REQUEST
   ,.raw= "GET http://hypnotoad.org:1234 HTTP/1.1\r\n"
@@ -655,11 +669,187 @@ const struct message requests[] =
   ,.fragment= ""
   ,.request_path= ""
   ,.request_url= "http://hypnotoad.org:1234"
+  ,.port= 1234
   ,.num_headers= 0
   ,.headers= { }
   ,.body= ""
   }
 
+#define PATCH_REQ 24
+, {.name = "PATCH request"
+  ,.type= HTTP_REQUEST
+  ,.raw= "PATCH /file.txt HTTP/1.1\r\n"
+         "Host: www.example.com\r\n"
+         "Content-Type: application/example\r\n"
+         "If-Match: \"e0023aa4e\"\r\n"
+         "Content-Length: 10\r\n"
+         "\r\n"
+         "cccccccccc"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_PATCH
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/file.txt"
+  ,.request_url= "/file.txt"
+  ,.num_headers= 4
+  ,.headers= { { "Host", "www.example.com" }
+             , { "Content-Type", "application/example" }
+             , { "If-Match", "\"e0023aa4e\"" }
+             , { "Content-Length", "10" }
+             }
+  ,.body= "cccccccccc"
+  }
+
+#define CONNECT_CAPS_REQUEST 25
+, {.name = "connect caps request"
+  ,.type= HTTP_REQUEST
+  ,.raw= "CONNECT HOME0.NETSCAPE.COM:443 HTTP/1.0\r\n"
+         "User-agent: Mozilla/1.1N\r\n"
+         "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 0
+  ,.method= HTTP_CONNECT
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= ""
+  ,.request_url= "HOME0.NETSCAPE.COM:443"
+  ,.num_headers= 2
+  ,.upgrade=""
+  ,.headers= { { "User-agent", "Mozilla/1.1N" }
+             , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
+             }
+  ,.body= ""
+  }
+
+#if !HTTP_PARSER_STRICT
+#define UTF8_PATH_REQ 26
+, {.name= "utf-8 path request"
+  ,.type= HTTP_REQUEST
+  ,.raw= "GET /δ¶/δt/pope?q=1#narf HTTP/1.1\r\n"
+         "Host: github.com\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_GET
+  ,.query_string= "q=1"
+  ,.fragment= "narf"
+  ,.request_path= "/δ¶/δt/pope"
+  ,.request_url= "/δ¶/δt/pope?q=1#narf"
+  ,.num_headers= 1
+  ,.headers= { {"Host", "github.com" }
+             }
+  ,.body= ""
+  }
+
+#define HOSTNAME_UNDERSCORE 27
+, {.name = "hostname underscore"
+  ,.type= HTTP_REQUEST
+  ,.raw= "CONNECT home_0.netscape.com:443 HTTP/1.0\r\n"
+         "User-agent: Mozilla/1.1N\r\n"
+         "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 0
+  ,.method= HTTP_CONNECT
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= ""
+  ,.request_url= "home_0.netscape.com:443"
+  ,.num_headers= 2
+  ,.upgrade=""
+  ,.headers= { { "User-agent", "Mozilla/1.1N" }
+             , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
+             }
+  ,.body= ""
+  }
+#endif  /* !HTTP_PARSER_STRICT */
+
+/* see https://github.com/ry/http-parser/issues/47 */
+#define EAT_TRAILING_CRLF_NO_CONNECTION_CLOSE 28
+, {.name = "eat CRLF between requests, no \"Connection: close\" header"
+  ,.raw= "POST / HTTP/1.1\r\n"
+         "Host: www.example.com\r\n"
+         "Content-Type: application/x-www-form-urlencoded\r\n"
+         "Content-Length: 4\r\n"
+         "\r\n"
+         "q=42\r\n" /* note the trailing CRLF */
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_POST
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/"
+  ,.request_url= "/"
+  ,.num_headers= 3
+  ,.upgrade= 0
+  ,.headers= { { "Host", "www.example.com" }
+             , { "Content-Type", "application/x-www-form-urlencoded" }
+             , { "Content-Length", "4" }
+             }
+  ,.body= "q=42"
+  }
+
+/* see https://github.com/ry/http-parser/issues/47 */
+#define EAT_TRAILING_CRLF_WITH_CONNECTION_CLOSE 29
+, {.name = "eat CRLF between requests even if \"Connection: close\" is set"
+  ,.raw= "POST / HTTP/1.1\r\n"
+         "Host: www.example.com\r\n"
+         "Content-Type: application/x-www-form-urlencoded\r\n"
+         "Content-Length: 4\r\n"
+         "Connection: close\r\n"
+         "\r\n"
+         "q=42\r\n" /* note the trailing CRLF */
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE /* input buffer isn't empty when on_message_complete is called */
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_POST
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/"
+  ,.request_url= "/"
+  ,.num_headers= 4
+  ,.upgrade= 0
+  ,.headers= { { "Host", "www.example.com" }
+             , { "Content-Type", "application/x-www-form-urlencoded" }
+             , { "Content-Length", "4" }
+             , { "Connection", "close" }
+             }
+  ,.body= "q=42"
+  }
+
+#define PURGE_REQ 30
+, {.name = "PURGE request"
+  ,.type= HTTP_REQUEST
+  ,.raw= "PURGE /file.txt HTTP/1.1\r\n"
+         "Host: www.example.com\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.method= HTTP_PURGE
+  ,.query_string= ""
+  ,.fragment= ""
+  ,.request_path= "/file.txt"
+  ,.request_url= "/file.txt"
+  ,.num_headers= 1
+  ,.headers= { { "Host", "www.example.com" } }
+  ,.body= ""
+  }
+
 , {.name= NULL } /* sentinel */
 };
 
@@ -760,8 +950,8 @@ const struct message responses[] =
 , {.name= "404 no headers no body"
   ,.type= HTTP_RESPONSE
   ,.raw= "HTTP/1.1 404 Not Found\r\n\r\n"
-  ,.should_keep_alive= TRUE
-  ,.message_complete_on_eof= FALSE
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= TRUE
   ,.http_major= 1
   ,.http_minor= 1
   ,.status_code= 404
@@ -775,8 +965,8 @@ const struct message responses[] =
 , {.name= "301 no response phrase"
   ,.type= HTTP_RESPONSE
   ,.raw= "HTTP/1.1 301\r\n\r\n"
-  ,.should_keep_alive = TRUE
-  ,.message_complete_on_eof= FALSE
+  ,.should_keep_alive = FALSE
+  ,.message_complete_on_eof= TRUE
   ,.http_major= 1
   ,.http_minor= 1
   ,.status_code= 301
@@ -925,40 +1115,7 @@ const struct message responses[] =
   ,.body= ""
   }
 
-#define SPACE_IN_FIELD_RES 9
-/* Should handle spaces in header fields */
-, {.name= "field space"
-  ,.type= HTTP_RESPONSE
-  ,.raw= "HTTP/1.1 200 OK\r\n"
-         "Server: Microsoft-IIS/6.0\r\n"
-         "X-Powered-By: ASP.NET\r\n"
-         "en-US Content-Type: text/xml\r\n" /* this is the problem */
-         "Content-Type: text/xml\r\n"
-         "Content-Length: 16\r\n"
-         "Date: Fri, 23 Jul 2010 18:45:38 GMT\r\n"
-         "Connection: keep-alive\r\n"
-         "\r\n"
-         "<xml>hello</xml>" /* fake body */
-  ,.should_keep_alive= TRUE
-  ,.message_complete_on_eof= FALSE
-  ,.http_major= 1
-  ,.http_minor= 1
-  ,.status_code= 200
-  ,.num_headers= 7
-  ,.headers=
-    { { "Server",  "Microsoft-IIS/6.0" }
-    , { "X-Powered-By", "ASP.NET" }
-    , { "en-US Content-Type", "text/xml" }
-    , { "Content-Type", "text/xml" }
-    , { "Content-Length", "16" }
-    , { "Date", "Fri, 23 Jul 2010 18:45:38 GMT" }
-    , { "Connection", "keep-alive" }
-    }
-  ,.body= "<xml>hello</xml>"
-  }
-
-
-#define RES_FIELD_UNDERSCORE 10
+#define RES_FIELD_UNDERSCORE 9
 /* Should handle spaces in header fields */
 , {.name= "field underscore"
   ,.type= HTTP_RESPONSE
@@ -998,7 +1155,7 @@ const struct message responses[] =
   ,.body= ""
   }
 
-#define NON_ASCII_IN_STATUS_LINE 11
+#define NON_ASCII_IN_STATUS_LINE 10
 /* Should handle non-ASCII in status line */
 , {.name= "non-ASCII in status line"
   ,.type= HTTP_RESPONSE
@@ -1018,21 +1175,196 @@ const struct message responses[] =
     , { "Content-Length", "0" }
     , { "Connection", "close" }
     }
-  ,.body= ""
+  ,.body= ""
+  }
+
+#define HTTP_VERSION_0_9 11
+/* Should handle HTTP/0.9 */
+, {.name= "http version 0.9"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/0.9 200 OK\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= TRUE
+  ,.http_major= 0
+  ,.http_minor= 9
+  ,.status_code= 200
+  ,.num_headers= 0
+  ,.headers=
+    {}
+  ,.body= ""
+  }
+
+#define NO_CONTENT_LENGTH_NO_TRANSFER_ENCODING_RESPONSE 12
+/* The client should wait for the server's EOF. That is, when neither
+ * content-length nor transfer-encoding is specified, the end of body
+ * is specified by the EOF.
+ */
+, {.name= "neither content-length nor transfer-encoding response"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 200 OK\r\n"
+         "Content-Type: text/plain\r\n"
+         "\r\n"
+         "hello world"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= TRUE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 200
+  ,.num_headers= 1
+  ,.headers=
+    { { "Content-Type", "text/plain" }
+    }
+  ,.body= "hello world"
+  }
+
+#define NO_BODY_HTTP10_KA_200 13
+, {.name= "HTTP/1.0 with keep-alive and EOF-terminated 200 status"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.0 200 OK\r\n"
+         "Connection: keep-alive\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= TRUE
+  ,.http_major= 1
+  ,.http_minor= 0
+  ,.status_code= 200
+  ,.num_headers= 1
+  ,.headers=
+    { { "Connection", "keep-alive" }
+    }
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#define NO_BODY_HTTP10_KA_204 14
+, {.name= "HTTP/1.0 with keep-alive and a 204 status"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.0 204 No content\r\n"
+         "Connection: keep-alive\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 0
+  ,.status_code= 204
+  ,.num_headers= 1
+  ,.headers=
+    { { "Connection", "keep-alive" }
+    }
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#define NO_BODY_HTTP11_KA_200 15
+, {.name= "HTTP/1.1 with an EOF-terminated 200 status"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 200 OK\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= TRUE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 200
+  ,.num_headers= 0
+  ,.headers={}
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#define NO_BODY_HTTP11_KA_204 16
+, {.name= "HTTP/1.1 with a 204 status"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 204 No content\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 204
+  ,.num_headers= 0
+  ,.headers={}
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#define NO_BODY_HTTP11_NOKA_204 17
+, {.name= "HTTP/1.1 with a 204 status and keep-alive disabled"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 204 No content\r\n"
+         "Connection: close\r\n"
+         "\r\n"
+  ,.should_keep_alive= FALSE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 204
+  ,.num_headers= 1
+  ,.headers=
+    { { "Connection", "close" }
+    }
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#define NO_BODY_HTTP11_KA_CHUNKED_200 18
+, {.name= "HTTP/1.1 with chunked endocing and a 200 response"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 200 OK\r\n"
+         "Transfer-Encoding: chunked\r\n"
+         "\r\n"
+         "0\r\n"
+         "\r\n"
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 200
+  ,.num_headers= 1
+  ,.headers=
+    { { "Transfer-Encoding", "chunked" }
+    }
+  ,.body_size= 0
+  ,.body= ""
+  }
+
+#if !HTTP_PARSER_STRICT
+#define SPACE_IN_FIELD_RES 19
+/* Should handle spaces in header fields */
+, {.name= "field space"
+  ,.type= HTTP_RESPONSE
+  ,.raw= "HTTP/1.1 200 OK\r\n"
+         "Server: Microsoft-IIS/6.0\r\n"
+         "X-Powered-By: ASP.NET\r\n"
+         "en-US Content-Type: text/xml\r\n" /* this is the problem */
+         "Content-Type: text/xml\r\n"
+         "Content-Length: 16\r\n"
+         "Date: Fri, 23 Jul 2010 18:45:38 GMT\r\n"
+         "Connection: keep-alive\r\n"
+         "\r\n"
+         "<xml>hello</xml>" /* fake body */
+  ,.should_keep_alive= TRUE
+  ,.message_complete_on_eof= FALSE
+  ,.http_major= 1
+  ,.http_minor= 1
+  ,.status_code= 200
+  ,.num_headers= 7
+  ,.headers=
+    { { "Server",  "Microsoft-IIS/6.0" }
+    , { "X-Powered-By", "ASP.NET" }
+    , { "en-US Content-Type", "text/xml" }
+    , { "Content-Type", "text/xml" }
+    , { "Content-Length", "16" }
+    , { "Date", "Fri, 23 Jul 2010 18:45:38 GMT" }
+    , { "Connection", "keep-alive" }
+    }
+  ,.body= "<xml>hello</xml>"
   }
-
+#endif /* !HTTP_PARSER_STRICT */
 
 , {.name= NULL } /* sentinel */
 };
 
-int
-request_path_cb (http_parser *p, const char *buf, size_t len)
-{
-  assert(p == parser);
-  strncat(messages[num_messages].request_path, buf, len);
-  return 0;
-}
-
 int
 request_url_cb (http_parser *p, const char *buf, size_t len)
 {
@@ -1041,22 +1373,6 @@ request_url_cb (http_parser *p, const char *buf, size_t len)
   return 0;
 }
 
-int
-query_string_cb (http_parser *p, const char *buf, size_t len)
-{
-  assert(p == parser);
-  strncat(messages[num_messages].query_string, buf, len);
-  return 0;
-}
-
-int
-fragment_cb (http_parser *p, const char *buf, size_t len)
-{
-  assert(p == parser);
-  strncat(messages[num_messages].fragment, buf, len);
-  return 0;
-}
-
 int
 header_field_cb (http_parser *p, const char *buf, size_t len)
 {
@@ -1146,14 +1462,151 @@ message_complete_cb (http_parser *p)
   return 0;
 }
 
+/* These dontcall_* callbacks exist so that we can verify that when we're
+ * paused, no additional callbacks are invoked */
+int
+dontcall_message_begin_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_message_begin() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_header_field_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_header_field() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_header_value_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_header_value() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_request_url_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_request_url() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_body_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_body_cb() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_headers_complete_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_headers_complete() called on paused "
+                  "parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_message_complete_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_message_complete() called on paused "
+                  "parser ***\n\n");
+  exit(1);
+}
+
+static http_parser_settings settings_dontcall =
+  {.on_message_begin = dontcall_message_begin_cb
+  ,.on_header_field = dontcall_header_field_cb
+  ,.on_header_value = dontcall_header_value_cb
+  ,.on_url = dontcall_request_url_cb
+  ,.on_body = dontcall_body_cb
+  ,.on_headers_complete = dontcall_headers_complete_cb
+  ,.on_message_complete = dontcall_message_complete_cb
+  };
+
+/* These pause_* callbacks always pause the parser and just invoke the regular
+ * callback that tracks content. Before returning, we overwrite the parser
+ * settings to point to the _dontcall variety so that we can verify that
+ * the pause actually did, you know, pause. */
+int
+pause_message_begin_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return message_begin_cb(p);
+}
+
+int
+pause_header_field_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return header_field_cb(p, buf, len);
+}
+
+int
+pause_header_value_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return header_value_cb(p, buf, len);
+}
+
+int
+pause_request_url_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return request_url_cb(p, buf, len);
+}
+
+int
+pause_body_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return body_cb(p, buf, len);
+}
+
+int
+pause_headers_complete_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return headers_complete_cb(p);
+}
+
+int
+pause_message_complete_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return message_complete_cb(p);
+}
+
+static http_parser_settings settings_pause =
+  {.on_message_begin = pause_message_begin_cb
+  ,.on_header_field = pause_header_field_cb
+  ,.on_header_value = pause_header_value_cb
+  ,.on_url = pause_request_url_cb
+  ,.on_body = pause_body_cb
+  ,.on_headers_complete = pause_headers_complete_cb
+  ,.on_message_complete = pause_message_complete_cb
+  };
+
 static http_parser_settings settings =
   {.on_message_begin = message_begin_cb
   ,.on_header_field = header_field_cb
   ,.on_header_value = header_value_cb
-  ,.on_path = request_path_cb
   ,.on_url = request_url_cb
-  ,.on_fragment = fragment_cb
-  ,.on_query_string = query_string_cb
   ,.on_body = body_cb
   ,.on_headers_complete = headers_complete_cb
   ,.on_message_complete = message_complete_cb
@@ -1163,10 +1616,7 @@ static http_parser_settings settings_count_body =
   {.on_message_begin = message_begin_cb
   ,.on_header_field = header_field_cb
   ,.on_header_value = header_value_cb
-  ,.on_path = request_path_cb
   ,.on_url = request_url_cb
-  ,.on_fragment = fragment_cb
-  ,.on_query_string = query_string_cb
   ,.on_body = count_body_cb
   ,.on_headers_complete = headers_complete_cb
   ,.on_message_complete = message_complete_cb
@@ -1176,10 +1626,7 @@ static http_parser_settings settings_null =
   {.on_message_begin = 0
   ,.on_header_field = 0
   ,.on_header_value = 0
-  ,.on_path = 0
   ,.on_url = 0
-  ,.on_fragment = 0
-  ,.on_query_string = 0
   ,.on_body = 0
   ,.on_headers_complete = 0
   ,.on_message_complete = 0
@@ -1224,12 +1671,29 @@ size_t parse_count_body (const char *buf, size_t len)
   return nparsed;
 }
 
+size_t parse_pause (const char *buf, size_t len)
+{
+  size_t nparsed;
+  http_parser_settings s = settings_pause;
+
+  currently_parsing_eof = (len == 0);
+  current_pause_parser = &s;
+  nparsed = http_parser_execute(parser, current_pause_parser, buf, len);
+  return nparsed;
+}
+
 static inline int
 check_str_eq (const struct message *m,
               const char *prop,
               const char *expected,
               const char *found) {
-  if (0 != strcmp(expected, found)) {
+  if ((expected == NULL) != (found == NULL)) {
+    printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name);
+    printf("expected %s\n", (expected == NULL) ? "NULL" : expected);
+    printf("   found %s\n", (found == NULL) ? "NULL" : found);
+    return 0;
+  }
+  if (expected != NULL && 0 != strcmp(expected, found)) {
     printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name);
     printf("expected '%s'\n", expected);
     printf("   found '%s'\n", found);
@@ -1258,6 +1722,20 @@ check_num_eq (const struct message *m,
 #define MESSAGE_CHECK_NUM_EQ(expected, found, prop) \
   if (!check_num_eq(expected, #prop, expected->prop, found->prop)) return 0
 
+#define MESSAGE_CHECK_URL_EQ(u, expected, found, prop, fn)           \
+do {                                                                 \
+  char ubuf[256];                                                    \
+                                                                     \
+  if ((u)->field_set & (1 << (fn))) {                                \
+    memcpy(ubuf, (found)->request_url + (u)->field_data[(fn)].off,   \
+      (u)->field_data[(fn)].len);                                    \
+    ubuf[(u)->field_data[(fn)].len] = '\0';                          \
+  } else {                                                           \
+    ubuf[0] = '\0';                                                  \
+  }                                                                  \
+                                                                     \
+  check_str_eq(expected, #prop, expected->prop, ubuf);               \
+} while(0)
 
 int
 message_eq (int index, const struct message *expected)
@@ -1282,10 +1760,29 @@ message_eq (int index, const struct message *expected)
   assert(m->message_complete_cb_called);
 
 
-  MESSAGE_CHECK_STR_EQ(expected, m, request_path);
-  MESSAGE_CHECK_STR_EQ(expected, m, query_string);
-  MESSAGE_CHECK_STR_EQ(expected, m, fragment);
   MESSAGE_CHECK_STR_EQ(expected, m, request_url);
+
+  /* Check URL components; we can't do this w/ CONNECT since it doesn't
+   * send us a well-formed URL.
+   */
+  if (*m->request_url && m->method != HTTP_CONNECT) {
+    struct http_parser_url u;
+
+    if (http_parser_parse_url(m->request_url, strlen(m->request_url), 0, &u)) {
+      fprintf(stderr, "\n\n*** failed to parse URL %s ***\n\n",
+        m->request_url);
+      exit(1);
+    }
+
+    m->port = (u.field_set & (1 << UF_PORT)) ?
+      u.port : 0;
+
+    MESSAGE_CHECK_URL_EQ(&u, expected, m, query_string, UF_QUERY);
+    MESSAGE_CHECK_URL_EQ(&u, expected, m, fragment, UF_FRAGMENT);
+    MESSAGE_CHECK_URL_EQ(&u, expected, m, request_path, UF_PATH);
+    MESSAGE_CHECK_NUM_EQ(expected, m, port);
+  }
+
   if (expected->body_size) {
     MESSAGE_CHECK_NUM_EQ(expected, m, body_size);
   } else {
@@ -1302,13 +1799,81 @@ message_eq (int index, const struct message *expected)
     if (!r) return 0;
   }
 
+  MESSAGE_CHECK_STR_EQ(expected, m, upgrade);
+
   return 1;
 }
 
+/* Given a sequence of varargs messages, return the number of them that the
+ * parser should successfully parse, taking into account that upgraded
+ * messages prevent all subsequent messages from being parsed.
+ */
+size_t
+count_parsed_messages(const size_t nmsgs, ...) {
+  size_t i;
+  va_list ap;
+
+  va_start(ap, nmsgs);
+
+  for (i = 0; i < nmsgs; i++) {
+    struct message *m = va_arg(ap, struct message *);
+
+    if (m->upgrade) {
+      va_end(ap);
+      return i + 1;
+    }
+  }
+
+  va_end(ap);
+  return nmsgs;
+}
+
+/* Given a sequence of bytes and the number of these that we were able to
+ * parse, verify that upgrade bodies are correct.
+ */
+void
+upgrade_message_fix(char *body, const size_t nread, const size_t nmsgs, ...) {
+  va_list ap;
+  size_t i;
+  size_t off = 0;
+ 
+  va_start(ap, nmsgs);
+
+  for (i = 0; i < nmsgs; i++) {
+    struct message *m = va_arg(ap, struct message *);
+
+    off += strlen(m->raw);
+
+    if (m->upgrade) {
+      off -= strlen(m->upgrade);
+
+      /* Check the portion of the response after its specified upgrade */
+      if (!check_str_eq(m, "upgrade", body + off, body + nread)) {
+        exit(1);
+      }
+
+      /* Fix up the response so that message_eq() will verify the beginning
+       * of the upgrade */
+      *(body + nread + strlen(m->upgrade)) = '\0';
+      messages[num_messages -1 ].upgrade = body + nread;
+
+      va_end(ap);
+      return;
+    }
+  }
+
+  va_end(ap);
+  printf("\n\n*** Error: expected a message with upgrade ***\n");
+
+  exit(1);
+}
+
 static void
 print_error (const char *raw, size_t error_location)
 {
-  fprintf(stderr, "\n*** parse error ***\n\n");
+  fprintf(stderr, "\n*** %s:%d -- %s ***\n\n",
+          "http_parser.c", HTTP_PARSER_ERRNO_LINE(parser),
+          http_errno_description(HTTP_PARSER_ERRNO(parser)));
 
   int this_line = 0, char_len = 0;
   size_t i, j, len = strlen(raw), error_location_line = 0;
@@ -1346,6 +1911,218 @@ print_error (const char *raw, size_t error_location)
   fprintf(stderr, "^\n\nerror location: %u\n", (unsigned int)error_location);
 }
 
+void
+test_preserve_data (void)
+{
+  char my_data[] = "application-specific data";
+  http_parser parser;
+  parser.data = my_data;
+  http_parser_init(&parser, HTTP_REQUEST);
+  if (parser.data != my_data) {
+    printf("\n*** parser.data not preserved accross http_parser_init ***\n\n");
+    exit(1);
+  }
+}
+
+struct url_test {
+  const char *name;
+  const char *url;
+  int is_connect;
+  struct http_parser_url u;
+  int rv;
+};
+
+const struct url_test url_tests[] =
+{ {.name="proxy request"
+  ,.url="http://hostname/"
+  ,.is_connect=0
+  ,.u=
+    {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH)
+    ,.port=0
+    ,.field_data=
+      {{  0,  4 } /* UF_SCHEMA */
+      ,{  7,  8 } /* UF_HOST */
+      ,{  0,  0 } /* UF_PORT */
+      ,{ 15,  1 } /* UF_PATH */
+      ,{  0,  0 } /* UF_QUERY */
+      ,{  0,  0 } /* UF_FRAGMENT */
+      }
+    }
+  ,.rv=0
+  }
+
+, {.name="CONNECT request"
+  ,.url="hostname:443"
+  ,.is_connect=1
+  ,.u=
+    {.field_set=(1 << UF_HOST) | (1 << UF_PORT)
+    ,.port=443
+    ,.field_data=
+      {{  0,  0 } /* UF_SCHEMA */
+      ,{  0,  8 } /* UF_HOST */
+      ,{  9,  3 } /* UF_PORT */
+      ,{  0,  0 } /* UF_PATH */
+      ,{  0,  0 } /* UF_QUERY */
+      ,{  0,  0 } /* UF_FRAGMENT */
+      }
+    }
+  ,.rv=0
+  }
+
+, {.name="proxy ipv6 request"
+  ,.url="http://[1:2::3:4]/"
+  ,.is_connect=0
+  ,.u=
+    {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH)
+    ,.port=0
+    ,.field_data=
+      {{  0,  4 } /* UF_SCHEMA */
+      ,{  8,  8 } /* UF_HOST */
+      ,{  0,  0 } /* UF_PORT */
+      ,{ 17,  1 } /* UF_PATH */
+      ,{  0,  0 } /* UF_QUERY */
+      ,{  0,  0 } /* UF_FRAGMENT */
+      }
+    }
+  ,.rv=0
+  }
+
+, {.name="CONNECT ipv6 address"
+  ,.url="[1:2::3:4]:443"
+  ,.is_connect=1
+  ,.u=
+    {.field_set=(1 << UF_HOST) | (1 << UF_PORT)
+    ,.port=443
+    ,.field_data=
+      {{  0,  0 } /* UF_SCHEMA */
+      ,{  1,  8 } /* UF_HOST */
+      ,{ 11,  3 } /* UF_PORT */
+      ,{  0,  0 } /* UF_PATH */
+      ,{  0,  0 } /* UF_QUERY */
+      ,{  0,  0 } /* UF_FRAGMENT */
+      }
+    }
+  ,.rv=0
+  }
+
+, {.name="extra ? in query string"
+  ,.url="http://a.tbcdn.cn/p/fp/2010c/??fp-header-min.css,fp-base-min.css,fp-channel-min.css,fp-product-min.css,fp-mall-min.css,fp-category-min.css,fp-sub-min.css,fp-gdp4p-min.css,fp-css3-min.css,fp-misc-min.css?t=20101022.css"
+  ,.is_connect=0
+  ,.u=
+    {.field_set=(1<<UF_SCHEMA) | (1<<UF_HOST) | (1<<UF_PATH) | (1<<UF_QUERY)
+    ,.port=0
+    ,.field_data=
+      {{  0,  4 } /* UF_SCHEMA */
+      ,{  7, 10 } /* UF_HOST */
+      ,{  0,  0 } /* UF_PORT */
+      ,{ 17, 12 } /* UF_PATH */
+      ,{ 30,187 } /* UF_QUERY */
+      ,{  0,  0 } /* UF_FRAGMENT */
+      }
+    }
+  ,.rv=0
+  }
+
+, {.name="proxy empty host"
+  ,.url="http://:443/"
+  ,.is_connect=0
+  ,.rv=1
+  }
+
+, {.name="proxy empty port"
+  ,.url="http://hostname:/"
+  ,.is_connect=0
+  ,.rv=1
+  }
+
+, {.name="CONNECT empty host"
+  ,.url=":443"
+  ,.is_connect=1
+  ,.rv=1
+  }
+
+, {.name="CONNECT empty port"
+  ,.url="hostname:"
+  ,.is_connect=1
+  ,.rv=1
+  }
+
+, {.name="CONNECT with extra bits"
+  ,.url="hostname:443/"
+  ,.is_connect=1
+  ,.rv=1
+  }
+};
+
+void
+dump_url (const char *url, const struct http_parser_url *u)
+{
+  char part[512];
+  unsigned int i;
+
+  printf("\tfield_set: 0x%x, port: %u\n", u->field_set, u->port);
+  for (i = 0; i < UF_MAX; i++) {
+    if ((u->field_set & (1 << i)) == 0) {
+      printf("\tfield_data[%u]: unset\n", i);
+      continue;
+    }
+
+    memcpy(part, url + u->field_data[i].off, u->field_data[i].len);
+    part[u->field_data[i].len] = '\0';
+
+    printf("\tfield_data[%u]: off: %u len: %u part: \"%s\"\n",
+           i,
+           u->field_data[i].off,
+           u->field_data[i].len,
+           part);
+  }
+}
+
+void
+test_parse_url (void)
+{
+  struct http_parser_url u;
+  const struct url_test *test;
+  unsigned int i;
+  int rv;
+
+  for (i = 0; i < (sizeof(url_tests) / sizeof(url_tests[0])); i++) {
+    test = &url_tests[i];
+    memset(&u, 0, sizeof(u));
+
+    rv = http_parser_parse_url(test->url,
+                               strlen(test->url),
+                               test->is_connect,
+                               &u);
+
+    if (test->rv == 0) {
+      if (rv != 0) {
+        printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, "
+               "unexpected rv %d ***\n\n", test->url, test->name, rv);
+        exit(1);
+      }
+
+      if (memcmp(&u, &test->u, sizeof(u)) != 0) {
+        printf("\n*** http_parser_parse_url(\"%s\") \"%s\" failed ***\n",
+               test->url, test->name);
+
+        printf("target http_parser_url:\n");
+        dump_url(test->url, &test->u);
+        printf("result http_parser_url:\n");
+        dump_url(test->url, &u);
+
+        exit(1);
+      }
+    } else {
+      /* test->rv != 0 */
+      if (rv == 0) {
+        printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, "
+               "unexpected rv %d ***\n\n", test->url, test->name, rv);
+        exit(1);
+      }
+    }
+  }
+}
 
 void
 test_message (const struct message *message)
@@ -1363,7 +2140,10 @@ test_message (const struct message *message)
     if (msg1len) {
       read = parse(msg1, msg1len);
 
-      if (message->upgrade && parser->upgrade) goto test;
+      if (message->upgrade && parser->upgrade) {
+        messages[num_messages - 1].upgrade = msg1 + read;
+        goto test;
+      }
 
       if (read != msg1len) {
         print_error(msg1, read);
@@ -1374,7 +2154,10 @@ test_message (const struct message *message)
 
     read = parse(msg2, msg2len);
 
-    if (message->upgrade && parser->upgrade) goto test;
+    if (message->upgrade && parser->upgrade) {
+      messages[num_messages - 1].upgrade = msg2 + read;
+      goto test;
+    }
 
     if (read != msg2len) {
       print_error(msg2, read);
@@ -1383,8 +2166,6 @@ test_message (const struct message *message)
 
     read = parse(NULL, 0);
 
-    if (message->upgrade && parser->upgrade) goto test;
-
     if (read != 0) {
       print_error(message->raw, read);
       exit(1);
@@ -1440,21 +2221,32 @@ test_message_count_body (const struct message *message)
 }
 
 void
-test_simple (const char *buf, int should_pass)
+test_simple (const char *buf, enum http_errno err_expected)
 {
   parser_init(HTTP_REQUEST);
 
   size_t parsed;
   int pass;
+  enum http_errno err;
+
   parsed = parse(buf, strlen(buf));
   pass = (parsed == strlen(buf));
+  err = HTTP_PARSER_ERRNO(parser);
   parsed = parse(NULL, 0);
   pass &= (parsed == 0);
 
   parser_free();
 
-  if (pass != should_pass) {
-    fprintf(stderr, "\n*** test_simple expected %s ***\n\n%s", should_pass ? "success" : "error", buf);
+  /* In strict mode, allow us to pass with an unexpected HPE_STRICT as
+   * long as the caller isn't expecting success.
+   */
+#if HTTP_PARSER_STRICT
+  if (err_expected != err && err_expected != HPE_OK && err != HPE_STRICT) {
+#else
+  if (err_expected != err) {
+#endif
+    fprintf(stderr, "\n*** test_simple expected %s, but saw %s ***\n\n%s\n",
+        http_errno_name(err_expected), http_errno_name(err), buf);
     exit(1);
   }
 }
@@ -1471,10 +2263,14 @@ test_header_overflow_error (int req)
   assert(parsed == strlen(buf));
 
   buf = "header-key: header-value\r\n";
+  size_t buflen = strlen(buf);
+
   int i;
   for (i = 0; i < 10000; i++) {
-    if (http_parser_execute(&parser, &settings_null, buf, strlen(buf)) != strlen(buf)) {
+    parsed = http_parser_execute(&parser, &settings_null, buf, buflen);
+    if (parsed != buflen) {
       //fprintf(stderr, "error found on iter %d\n", i);
+      assert(HTTP_PARSER_ERRNO(&parser) == HPE_HEADER_OVERFLOW);
       return;
     }
   }
@@ -1483,6 +2279,53 @@ test_header_overflow_error (int req)
   exit(1);
 }
 
+static void
+test_content_length_overflow (const char *buf, size_t buflen, int expect_ok)
+{
+  http_parser parser;
+  http_parser_init(&parser, HTTP_RESPONSE);
+  http_parser_execute(&parser, &settings_null, buf, buflen);
+
+  if (expect_ok)
+    assert(HTTP_PARSER_ERRNO(&parser) == HPE_OK);
+  else
+    assert(HTTP_PARSER_ERRNO(&parser) == HPE_INVALID_CONTENT_LENGTH);
+}
+
+void
+test_header_content_length_overflow_error (void)
+{
+#define X(size)                                                               \
+  "HTTP/1.1 200 OK\r\n"                                                       \
+  "Content-Length: " #size "\r\n"                                             \
+  "\r\n"
+  const char a[] = X(18446744073709551614); /* 2^64-2 */
+  const char b[] = X(18446744073709551615); /* 2^64-1 */
+  const char c[] = X(18446744073709551616); /* 2^64   */
+#undef X
+  test_content_length_overflow(a, sizeof(a) - 1, 1); /* expect ok      */
+  test_content_length_overflow(b, sizeof(b) - 1, 0); /* expect failure */
+  test_content_length_overflow(c, sizeof(c) - 1, 0); /* expect failure */
+}
+
+void
+test_chunk_content_length_overflow_error (void)
+{
+#define X(size)                                                               \
+    "HTTP/1.1 200 OK\r\n"                                                     \
+    "Transfer-Encoding: chunked\r\n"                                          \
+    "\r\n"                                                                    \
+    #size "\r\n"                                                              \
+    "..."
+  const char a[] = X(FFFFFFFFFFFFFFFE);  /* 2^64-2 */
+  const char b[] = X(FFFFFFFFFFFFFFFF);  /* 2^64-1 */
+  const char c[] = X(10000000000000000); /* 2^64   */
+#undef X
+  test_content_length_overflow(a, sizeof(a) - 1, 1); /* expect ok      */
+  test_content_length_overflow(b, sizeof(b) - 1, 0); /* expect failure */
+  test_content_length_overflow(c, sizeof(c) - 1, 0); /* expect failure */
+}
+
 void
 test_no_overflow_long_body (int req, size_t length)
 {
@@ -1519,12 +2362,7 @@ test_no_overflow_long_body (int req, size_t length)
 void
 test_multiple3 (const struct message *r1, const struct message *r2, const struct message *r3)
 {
-  int message_count = 1;
-  if (!r1->upgrade) {
-    message_count++;
-    if (!r2->upgrade) message_count++;
-  }
-  int has_upgrade = (message_count < 3 || r3->upgrade);
+  int message_count = count_parsed_messages(3, r1, r2, r3);
 
   char total[ strlen(r1->raw)
             + strlen(r2->raw)
@@ -1543,7 +2381,10 @@ test_multiple3 (const struct message *r1, const struct message *r2, const struct
 
   read = parse(total, strlen(total));
 
-  if (has_upgrade && parser->upgrade) goto test;
+  if (parser->upgrade) {
+    upgrade_message_fix(total, read, 3, r1, r2, r3);
+    goto test;
+  }
 
   if (read != strlen(total)) {
     print_error(total, read);
@@ -1552,8 +2393,6 @@ test_multiple3 (const struct message *r1, const struct message *r2, const struct
 
   read = parse(NULL, 0);
 
-  if (has_upgrade && parser->upgrade) goto test;
-
   if (read != 0) {
     print_error(total, read);
     exit(1);
@@ -1567,12 +2406,8 @@ test:
   }
 
   if (!message_eq(0, r1)) exit(1);
-  if (message_count > 1) {
-    if (!message_eq(1, r2)) exit(1);
-    if (message_count > 2) {
-      if (!message_eq(2, r3)) exit(1);
-    }
-  }
+  if (message_count > 1 && !message_eq(1, r2)) exit(1);
+  if (message_count > 2 && !message_eq(2, r3)) exit(1);
 
   parser_free();
 }
@@ -1601,6 +2436,7 @@ test_scan (const struct message *r1, const struct message *r2, const struct mess
   int ops = 0 ;
 
   size_t buf1_len, buf2_len, buf3_len;
+  int message_count = count_parsed_messages(3, r1, r2, r3);
 
   int i,j,type_both;
   for (type_both = 0; type_both < 2; type_both ++ ) {
@@ -1629,27 +2465,27 @@ test_scan (const struct message *r1, const struct message *r2, const struct mess
 
         read = parse(buf1, buf1_len);
 
-        if (r3->upgrade && parser->upgrade) goto test;
+        if (parser->upgrade) goto test;
 
         if (read != buf1_len) {
           print_error(buf1, read);
           goto error;
         }
 
-        read = parse(buf2, buf2_len);
+        read += parse(buf2, buf2_len);
 
-        if (r3->upgrade && parser->upgrade) goto test;
+        if (parser->upgrade) goto test;
 
-        if (read != buf2_len) {
+        if (read != buf1_len + buf2_len) {
           print_error(buf2, read);
           goto error;
         }
 
-        read = parse(buf3, buf3_len);
+        read += parse(buf3, buf3_len);
 
-        if (r3->upgrade && parser->upgrade) goto test;
+        if (parser->upgrade) goto test;
 
-        if (read != buf3_len) {
+        if (read != buf1_len + buf2_len + buf3_len) {
           print_error(buf3, read);
           goto error;
         }
@@ -1657,9 +2493,13 @@ test_scan (const struct message *r1, const struct message *r2, const struct mess
         parse(NULL, 0);
 
 test:
+        if (parser->upgrade) {
+          upgrade_message_fix(total, read, 3, r1, r2, r3);
+        }
 
-        if (3 != num_messages) {
-          fprintf(stderr, "\n\nParser didn't see 3 messages only %d\n", num_messages);
+        if (message_count != num_messages) {
+          fprintf(stderr, "\n\nParser didn't see %d messages only %d\n",
+            message_count, num_messages);
           goto error;
         }
 
@@ -1668,12 +2508,12 @@ test:
           goto error;
         }
 
-        if (!message_eq(1, r2)) {
+        if (message_count > 1 && !message_eq(1, r2)) {
           fprintf(stderr, "\n\nError matching messages[1] in test_scan.\n");
           goto error;
         }
 
-        if (!message_eq(2, r3)) {
+        if (message_count > 2 && !message_eq(2, r3)) {
           fprintf(stderr, "\n\nError matching messages[2] in test_scan.\n");
           goto error;
         }
@@ -1724,6 +2564,58 @@ create_large_chunked_message (int body_size_in_kb, const char* headers)
   return buf;
 }
 
+/* Verify that we can pause parsing at any of the bytes in the
+ * message and still get the result that we're expecting. */
+void
+test_message_pause (const struct message *msg)
+{
+  char *buf = (char*) msg->raw;
+  size_t buflen = strlen(msg->raw);
+  size_t nread;
+
+  parser_init(msg->type);
+
+  do {
+    nread = parse_pause(buf, buflen);
+
+    // We can only set the upgrade buffer once we've gotten our message
+    // completion callback.
+    if (messages[0].message_complete_cb_called &&
+        msg->upgrade &&
+        parser->upgrade) {
+      messages[0].upgrade = buf + nread;
+      goto test;
+    }
+
+    if (nread < buflen) {
+
+      // Not much do to if we failed a strict-mode check
+      if (HTTP_PARSER_ERRNO(parser) == HPE_STRICT) {
+        parser_free();
+        return;
+      }
+
+      assert (HTTP_PARSER_ERRNO(parser) == HPE_PAUSED);
+    }
+
+    buf += nread;
+    buflen -= nread;
+    http_parser_pause(parser, 0);
+  } while (buflen > 0);
+
+  nread = parse_pause(NULL, 0);
+  assert (nread == 0);
+
+test:
+  if (num_messages != 1) {
+    printf("\n*** num_messages != 1 after testing '%s' ***\n\n", msg->name);
+    exit(1);
+  }
+
+  if(!message_eq(0, msg)) exit(1);
+
+  parser_free();
+}
 
 int
 main (void)
@@ -1738,6 +2630,10 @@ main (void)
   for (request_count = 0; requests[request_count].name; request_count++);
   for (response_count = 0; responses[response_count].name; response_count++);
 
+  //// API
+  test_preserve_data();
+  test_parse_url();
+
   //// OVERFLOW CONDITIONS
 
   test_header_overflow_error(HTTP_REQUEST);
@@ -1748,12 +2644,19 @@ main (void)
   test_no_overflow_long_body(HTTP_RESPONSE, 1000);
   test_no_overflow_long_body(HTTP_RESPONSE, 100000);
 
+  test_header_content_length_overflow_error();
+  test_chunk_content_length_overflow_error();
+
   //// RESPONSES
 
   for (i = 0; i < response_count; i++) {
     test_message(&responses[i]);
   }
 
+  for (i = 0; i < response_count; i++) {
+    test_message_pause(&responses[i]);
+  }
+
   for (i = 0; i < response_count; i++) {
     if (!responses[i].should_keep_alive) continue;
     for (j = 0; j < response_count; j++) {
@@ -1798,7 +2701,7 @@ main (void)
 
   printf("response scan 1/2      ");
   test_scan( &responses[TRAILING_SPACE_ON_CHUNKED_BODY]
-           , &responses[NO_HEADERS_NO_BODY_404]
+           , &responses[NO_BODY_HTTP10_KA_204]
            , &responses[NO_REASON_PHRASE]
            );
 
@@ -1813,13 +2716,13 @@ main (void)
 
   /// REQUESTS
 
-  test_simple("hello world", 0);
-  test_simple("GET / HTP/1.1\r\n\r\n", 0);
+  test_simple("hello world", HPE_INVALID_METHOD);
+  test_simple("GET / HTP/1.1\r\n\r\n", HPE_INVALID_VERSION);
 
 
-  test_simple("ASDF / HTTP/1.1\r\n\r\n", 0);
-  test_simple("PROPPATCHA / HTTP/1.1\r\n\r\n", 0);
-  test_simple("GETA / HTTP/1.1\r\n\r\n", 0);
+  test_simple("ASDF / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
+  test_simple("PROPPATCHA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
+  test_simple("GETA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
 
   // Well-formed but incomplete
   test_simple("GET / HTTP/1.1\r\n"
@@ -1827,7 +2730,7 @@ main (void)
               "Content-Length: 6\r\n"
               "\r\n"
               "fooba",
-              0);
+              HPE_OK);
 
   static const char *all_methods[] = {
     "DELETE",
@@ -1845,12 +2748,31 @@ main (void)
     "PROPFIND",
     "PROPPATCH",
     "UNLOCK",
+    "REPORT",
+    "MKACTIVITY",
+    "CHECKOUT",
+    "MERGE",
+    "M-SEARCH",
+    "NOTIFY",
+    "SUBSCRIBE",
+    "UNSUBSCRIBE",
+    "PATCH",
     0 };
   const char **this_method;
   for (this_method = all_methods; *this_method; this_method++) {
     char buf[200];
     sprintf(buf, "%s / HTTP/1.1\r\n\r\n", *this_method);
-    test_simple(buf, 1);
+    test_simple(buf, HPE_OK);
+  }
+
+  static const char *bad_methods[] = {
+      "C******",
+      "M****",
+      0 };
+  for (this_method = bad_methods; *this_method; this_method++) {
+    char buf[200];
+    sprintf(buf, "%s / HTTP/1.1\r\n\r\n", *this_method);
+    test_simple(buf, HPE_UNKNOWN);
   }
 
   const char *dumbfuck2 =
@@ -1888,7 +2810,7 @@ main (void)
     "\tRA==\r\n"
     "\t-----END CERTIFICATE-----\r\n"
     "\r\n";
-  test_simple(dumbfuck2, 0);
+  test_simple(dumbfuck2, HPE_OK);
 
 #if 0
   // NOTE(Wed Nov 18 11:57:27 CET 2009) this seems okay. we just read body
@@ -1910,7 +2832,9 @@ main (void)
     test_message(&requests[i]);
   }
 
-
+  for (i = 0; i < request_count; i++) {
+    test_message_pause(&requests[i]);
+  }
 
   for (i = 0; i < request_count; i++) {
     if (!requests[i].should_keep_alive) continue;