http_parser.rb 0.5.1 → 0.6.0.beta.1

data/ext/ruby_http_parser/ruby_http_parser.c

@@ -17,12 +17,11 @@ typedef struct ParserWrapper {
   ryah_http_parser parser;
 
   VALUE request_url;
-  VALUE request_path;
-  VALUE query_string;
-  VALUE fragment;
 
   VALUE headers;
 
+  VALUE upgrade_data;
+
   VALUE on_message_begin;
   VALUE on_headers_complete;
   VALUE on_body;
@@ -32,6 +31,8 @@ typedef struct ParserWrapper {
   VALUE stopped;
   VALUE completed;
 
+  VALUE header_value_type;
+
   VALUE last_field_name;
   VALUE curr_field_name;
 
@@ -45,9 +46,8 @@ void ParserWrapper_init(ParserWrapper *wrapper) {
   wrapper->parser.http_minor = 0;
 
   wrapper->request_url = Qnil;
-  wrapper->request_path = Qnil;
-  wrapper->query_string = Qnil;
-  wrapper->fragment = Qnil;
+
+  wrapper->upgrade_data = Qnil;
 
   wrapper->headers = Qnil;
   wrapper->completed = Qfalse;
@@ -60,9 +60,7 @@ void ParserWrapper_mark(void *data) {
   if(data) {
     ParserWrapper *wrapper = (ParserWrapper *) data;
     rb_gc_mark_maybe(wrapper->request_url);
-    rb_gc_mark_maybe(wrapper->request_path);
-    rb_gc_mark_maybe(wrapper->query_string);
-    rb_gc_mark_maybe(wrapper->fragment);
+    rb_gc_mark_maybe(wrapper->upgrade_data);
     rb_gc_mark_maybe(wrapper->headers);
     rb_gc_mark_maybe(wrapper->on_message_begin);
     rb_gc_mark_maybe(wrapper->on_headers_complete);
@@ -93,6 +91,10 @@ static ID Ion_body;
 static ID Ion_message_complete;
 
 static VALUE Sstop;
+static VALUE Sreset;
+static VALUE Sarrays;
+static VALUE Sstrings;
+static VALUE Smixed;
 
 /** Callbacks **/
 
@@ -100,10 +102,8 @@ int on_message_begin(ryah_http_parser *parser) {
   GET_WRAPPER(wrapper, parser);
 
   wrapper->request_url = rb_str_new2("");
-  wrapper->request_path = rb_str_new2("");
-  wrapper->query_string = rb_str_new2("");
-  wrapper->fragment = rb_str_new2("");
   wrapper->headers = rb_hash_new();
+  wrapper->upgrade_data = rb_str_new2("");
 
   VALUE ret = Qnil;
 
@@ -127,24 +127,6 @@ int on_url(ryah_http_parser *parser, const char *at, size_t length) {
   return 0;
 }
 
-int on_path(ryah_http_parser *parser, const char *at, size_t length) {
-  GET_WRAPPER(wrapper, parser);
-  rb_str_cat(wrapper->request_path, at, length);
-  return 0;
-}
-
-int on_query_string(ryah_http_parser *parser, const char *at, size_t length) {
-  GET_WRAPPER(wrapper, parser);
-  rb_str_cat(wrapper->query_string, at, length);
-  return 0;
-}
-
-int on_fragment(ryah_http_parser *parser, const char *at, size_t length) {
-  GET_WRAPPER(wrapper, parser);
-  rb_str_cat(wrapper->fragment, at, length);
-  return 0;
-}
-
 int on_header_field(ryah_http_parser *parser, const char *at, size_t length) {
   GET_WRAPPER(wrapper, parser);
 
@@ -161,17 +143,45 @@ int on_header_field(ryah_http_parser *parser, const char *at, size_t length) {
 int on_header_value(ryah_http_parser *parser, const char *at, size_t length) {
   GET_WRAPPER(wrapper, parser);
 
+  int new_field = 0;
+  VALUE current_value;
+
   if (wrapper->last_field_name == Qnil) {
+    new_field = 1;
     wrapper->last_field_name = wrapper->curr_field_name;
     wrapper->curr_field_name = Qnil;
+  }
 
-    VALUE val = rb_hash_aref(wrapper->headers, wrapper->last_field_name);
-    if (val != Qnil) {
-      rb_str_cat(val, ", ", 2);
+  current_value = rb_hash_aref(wrapper->headers, wrapper->last_field_name);
+
+  if (new_field == 1) {
+    if (current_value == Qnil) {
+      if (wrapper->header_value_type == Sarrays) {
+        rb_hash_aset(wrapper->headers, wrapper->last_field_name, rb_ary_new3(1, rb_str_new2("")));
+      } else {
+        rb_hash_aset(wrapper->headers, wrapper->last_field_name, rb_str_new2(""));
+      }
+    } else {
+      if (wrapper->header_value_type == Smixed) {
+        if (TYPE(current_value) == T_STRING) {
+          rb_hash_aset(wrapper->headers, wrapper->last_field_name, rb_ary_new3(2, current_value, rb_str_new2("")));
+        } else {
+          rb_ary_push(current_value, rb_str_new2(""));
+        }
+      } else if (wrapper->header_value_type == Sarrays) {
+        rb_ary_push(current_value, rb_str_new2(""));
+      } else {
+        rb_str_cat(current_value, ", ", 2);
+      }
     }
+    current_value = rb_hash_aref(wrapper->headers, wrapper->last_field_name);
   }
 
-  HASH_CAT(wrapper->headers, wrapper->last_field_name, at, length);
+  if (TYPE(current_value) == T_ARRAY) {
+    current_value = rb_ary_entry(current_value, -1);
+  }
+
+  rb_str_cat(current_value, at, length);
 
   return 0;
 }
@@ -190,6 +200,8 @@ int on_headers_complete(ryah_http_parser *parser) {
   if (ret == Sstop) {
     wrapper->stopped = Qtrue;
     return -1;
+  } else if (ret == Sreset){
+    return 1;
   } else {
     return 0;
   }
@@ -236,10 +248,7 @@ int on_message_complete(ryah_http_parser *parser) {
 
 static ryah_http_parser_settings settings = {
   .on_message_begin = on_message_begin,
-  .on_path = on_path,
-  .on_query_string = on_query_string,
   .on_url = on_url,
-  .on_fragment = on_fragment,
   .on_header_field = on_header_field,
   .on_header_value = on_header_value,
   .on_headers_complete = on_headers_complete,
@@ -276,12 +285,24 @@ VALUE ResponseParser_alloc(VALUE klass) {
   return Parser_alloc_by_type(klass, HTTP_RESPONSE);
 }
 
+VALUE Parser_strict_p(VALUE klass) {
+  return HTTP_PARSER_STRICT == 1 ? Qtrue : Qfalse;
+}
+
 VALUE Parser_initialize(int argc, VALUE *argv, VALUE self) {
   ParserWrapper *wrapper = NULL;
   DATA_GET(self, ParserWrapper, wrapper);
 
-  if (argc == 1)
+  wrapper->header_value_type = rb_iv_get(CLASS_OF(self), "@default_header_value_type");
+
+  if (argc == 1) {
     wrapper->callback_object = argv[0];
+  }
+
+  if (argc == 2) {
+    wrapper->callback_object = argv[0];
+    wrapper->header_value_type = argv[1];
+  }
 
   return self;
 }
@@ -299,10 +320,14 @@ VALUE Parser_execute(VALUE self, VALUE data) {
   size_t nparsed = ryah_http_parser_execute(&wrapper->parser, &settings, ptr, len);
 
   if (wrapper->parser.upgrade) {
-    // upgrade request
-  } else if (nparsed != len) {
+    if (RTEST(wrapper->stopped))
+      nparsed += 1;
+
+    rb_str_cat(wrapper->upgrade_data, ptr + nparsed, len - nparsed);
+
+  } else if (nparsed != (size_t)len) {
     if (!RTEST(wrapper->stopped) && !RTEST(wrapper->completed))
-      rb_raise(eParserError, "Could not parse data entirely");
+      rb_raise(eParserError, "Could not parse data entirely (%zu != %zu)", nparsed, len);
     else
       nparsed += 1; // error states fail on the current character
   }
@@ -353,7 +378,7 @@ VALUE Parser_upgrade_p(VALUE self) {
   ParserWrapper *wrapper = NULL;
   DATA_GET(self, ParserWrapper, wrapper);
 
-  return wrapper->parser.upgrade == 1 ? Qtrue : Qfalse;
+  return wrapper->parser.upgrade ? Qtrue : Qfalse;
 }
 
 VALUE Parser_http_version(VALUE self) {
@@ -414,10 +439,20 @@ VALUE Parser_status_code(VALUE self) {
   }
 
 DEFINE_GETTER(request_url);
-DEFINE_GETTER(request_path);
-DEFINE_GETTER(query_string);
-DEFINE_GETTER(fragment);
 DEFINE_GETTER(headers);
+DEFINE_GETTER(upgrade_data);
+DEFINE_GETTER(header_value_type);
+
+VALUE Parser_set_header_value_type(VALUE self, VALUE val) {
+  if (val != Sarrays && val != Sstrings && val != Smixed) {
+    rb_raise(rb_eArgError, "Invalid header value type");
+  }
+
+  ParserWrapper *wrapper = NULL;
+  DATA_GET(self, ParserWrapper, wrapper);
+  wrapper->header_value_type = val;
+  return wrapper->header_value_type;
+}
 
 VALUE Parser_reset(VALUE self) {
   ParserWrapper *wrapper = NULL;
@@ -441,11 +476,17 @@ void Init_ruby_http_parser() {
   Ion_body = rb_intern("on_body");
   Ion_message_complete = rb_intern("on_message_complete");
   Sstop = ID2SYM(rb_intern("stop"));
+  Sreset = ID2SYM(rb_intern("reset"));
+
+  Sarrays = ID2SYM(rb_intern("arrays"));
+  Sstrings = ID2SYM(rb_intern("strings"));
+  Smixed = ID2SYM(rb_intern("mixed"));
 
   rb_define_alloc_func(cParser, Parser_alloc);
   rb_define_alloc_func(cRequestParser, RequestParser_alloc);
   rb_define_alloc_func(cResponseParser, ResponseParser_alloc);
 
+  rb_define_singleton_method(cParser, "strict?", Parser_strict_p, 0);
   rb_define_method(cParser, "initialize", Parser_initialize, -1);
 
   rb_define_method(cParser, "on_message_begin=", Parser_set_on_message_begin, 1);
@@ -465,10 +506,10 @@ void Init_ruby_http_parser() {
   rb_define_method(cParser, "status_code", Parser_status_code, 0);
 
   rb_define_method(cParser, "request_url", Parser_request_url, 0);
-  rb_define_method(cParser, "request_path", Parser_request_path, 0);
-  rb_define_method(cParser, "query_string", Parser_query_string, 0);
-  rb_define_method(cParser, "fragment", Parser_fragment, 0);
   rb_define_method(cParser, "headers", Parser_headers, 0);
+  rb_define_method(cParser, "upgrade_data", Parser_upgrade_data, 0);
+  rb_define_method(cParser, "header_value_type", Parser_header_value_type, 0);
+  rb_define_method(cParser, "header_value_type=", Parser_set_header_value_type, 1);
 
   rb_define_method(cParser, "reset!", Parser_reset, 0);
 }

data/ext/ruby_http_parser/vendor/http-parser/AUTHORS

@@ -0,0 +1,32 @@
+# Authors ordered by first contribution.
+Ryan Dahl <ry@tinyclouds.org>
+Jeremy Hinegardner <jeremy@hinegardner.org>
+Sergey Shepelev <temotor@gmail.com>
+Joe Damato <ice799@gmail.com>
+tomika <tomika_nospam@freemail.hu>
+Phoenix Sol <phoenix@burninglabs.com>
+Cliff Frey <cliff@meraki.com>
+Ewen Cheslack-Postava <ewencp@cs.stanford.edu>
+Santiago Gala <sgala@apache.org>
+Tim Becker <tim.becker@syngenio.de>
+Jeff Terrace <jterrace@gmail.com>
+Ben Noordhuis <info@bnoordhuis.nl>
+Nathan Rajlich <nathan@tootallnate.net>
+Mark Nottingham <mnot@mnot.net>
+Aman Gupta <aman@tmm1.net>
+Tim Becker <tim.becker@kuriositaet.de>
+Sean Cunningham <sean.cunningham@mandiant.com>
+Peter Griess <pg@std.in>
+Salman Haq <salman.haq@asti-usa.com>
+Cliff Frey <clifffrey@gmail.com>
+Jon Kolb <jon@b0g.us>
+Fouad Mardini <f.mardini@gmail.com>
+Paul Querna <pquerna@apache.org>
+Felix Geisendörfer <felix@debuggable.com>
+koichik <koichik@improvement.jp>
+Andre Caron <andre.l.caron@gmail.com>
+Ivo Raisr <ivosh@ivosh.net>
+James McLaughlin <jamie@lacewing-project.org>
+David Gwynne <loki@animata.net>
+LE ROUX Thomas <thomas@procheo.fr>
+Randy Rizun <rrizun@ortivawireless.com>

data/ext/ruby_http_parser/vendor/http-parser/LICENSE-MIT

@@ -1,4 +1,8 @@
-Copyright 2009,2010 Ryan Dahl <ry@tinyclouds.org>
+http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright
+Igor Sysoev.
+
+Additional changes are licensed under the same terms as NGINX and
+copyright Joyent, Inc. and other Node contributors. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to

data/ext/ruby_http_parser/vendor/http-parser/Makefile

@@ -1,41 +1,58 @@
-OPT_DEBUG=-O0 -g -Wall -Wextra -Werror -I.
-OPT_FAST=-O3 -DHTTP_PARSER_STRICT=0 -I.
-
 CC?=gcc
+AR?=ar
+
+CPPFLAGS += -I.
+CPPFLAGS_DEBUG = $(CPPFLAGS) -DHTTP_PARSER_STRICT=1 -DHTTP_PARSER_DEBUG=1
+CPPFLAGS_DEBUG += $(CPPFLAGS_DEBUG_EXTRA)
+CPPFLAGS_FAST = $(CPPFLAGS) -DHTTP_PARSER_STRICT=0 -DHTTP_PARSER_DEBUG=0
+CPPFLAGS_FAST += $(CPPFLAGS_FAST_EXTRA)
 
+CFLAGS += -Wall -Wextra -Werror
+CFLAGS_DEBUG = $(CFLAGS) -O0 -g $(CFLAGS_DEBUG_EXTRA)
+CFLAGS_FAST = $(CFLAGS) -O3 $(CFLAGS_FAST_EXTRA)
+CFLAGS_LIB = $(CFLAGS_FAST) -fPIC
 
-test: test_g
+test: test_g test_fast
 	./test_g
+	./test_fast
 
 test_g: http_parser_g.o test_g.o
-	$(CC) $(OPT_DEBUG) http_parser_g.o test_g.o -o $@
+	$(CC) $(CFLAGS_DEBUG) $(LDFLAGS) http_parser_g.o test_g.o -o $@
 
 test_g.o: test.c http_parser.h Makefile
-	$(CC) $(OPT_DEBUG) -c test.c -o $@
-
-test.o: test.c http_parser.h Makefile
-	$(CC) $(OPT_FAST) -c test.c -o $@
+	$(CC) $(CPPFLAGS_DEBUG) $(CFLAGS_DEBUG) -c test.c -o $@
 
 http_parser_g.o: http_parser.c http_parser.h Makefile
-	$(CC) $(OPT_DEBUG) -c http_parser.c -o $@
+	$(CC) $(CPPFLAGS_DEBUG) $(CFLAGS_DEBUG) -c http_parser.c -o $@
 
-test-valgrind: test_g
-	valgrind ./test_g
+test_fast: http_parser.o test.o http_parser.h
+	$(CC) $(CFLAGS_FAST) $(LDFLAGS) http_parser.o test.o -o $@
 
-http_parser.o: http_parser.c http_parser.h Makefile
-	$(CC) $(OPT_FAST) -c http_parser.c
+test.o: test.c http_parser.h Makefile
+	$(CC) $(CPPFLAGS_FAST) $(CFLAGS_FAST) -c test.c -o $@
 
-test_fast: http_parser.o test.c http_parser.h
-	$(CC) $(OPT_FAST) http_parser.o test.c -o $@
+http_parser.o: http_parser.c http_parser.h Makefile
+	$(CC) $(CPPFLAGS_FAST) $(CFLAGS_FAST) -c http_parser.c
 
 test-run-timed: test_fast
 	while(true) do time ./test_fast > /dev/null; done
 
+test-valgrind: test_g
+	valgrind ./test_g
+
+libhttp_parser.o: http_parser.c http_parser.h Makefile
+	$(CC) $(CPPFLAGS_FAST) $(CFLAGS_LIB) -c http_parser.c -o libhttp_parser.o
+
+library: libhttp_parser.o
+	$(CC) -shared -o libhttp_parser.so libhttp_parser.o
+
+package: http_parser.o
+	$(AR) rcs libhttp_parser.a http_parser.o
 
 tags: http_parser.c http_parser.h test.c
 	ctags $^
 
 clean:
-	rm -f *.o test test_fast test_g http_parser.tar tags
+	rm -f *.o *.a test test_fast test_g http_parser.tar tags libhttp_parser.so libhttp_parser.o
 
 .PHONY: clean package test-run test-run-timed test-valgrind

data/ext/ruby_http_parser/vendor/http-parser/README.md

@@ -24,7 +24,7 @@ The parser extracts the following information from HTTP messages:
   * Response status code
   * Transfer-Encoding
   * HTTP version
-  * Request path, query string, fragment
+  * Request URL
   * Message body
 
 
@@ -126,7 +126,7 @@ There are two types of callbacks:
 * notification `typedef int (*http_cb) (http_parser*);`
     Callbacks: on_message_begin, on_headers_complete, on_message_complete.
 * data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);`
-    Callbacks: (requests only) on_path, on_query_string, on_uri, on_fragment,
+    Callbacks: (requests only) on_uri,
                (common) on_header_field, on_header_value, on_body;
 
 Callbacks must return 0 on success. Returning a non-zero value indicates
@@ -164,6 +164,13 @@ and apply following logic:
      ------------------------ ------------ --------------------------------------------
 
 
+Parsing URLs
+------------
+
+A simplistic zero-copy URL parser is provided as `http_parser_parse_url()`.
+Users of this library may wish to use it to parse URLs constructed from
+consecutive `on_url` callbacks.
+
 See examples of reading in headers:
 
 * [partial example](http://gist.github.com/155877) in C