yakischloba-http-parser 0.0.2

data/Rakefile

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'ffi'
+
+task :default => [:clean, :build, :test]
+
+task :clean do
+  chdir "src/" do
+    sh "make clean"
+    sh "rm -rf http_parser.#{FFI::Platform::LIBSUFFIX}"
+  end
+end
+
+task :build do
+  dynamicflag = (FFI::Platform::OS == "darwin" ? "-dynamiclib" : "-fPIC -shared")
+  chdir "src/" do
+    sh "make"
+    sh "gcc #{dynamicflag} -o http_parser.#{FFI::Platform::LIBSUFFIX} http_parser.o"
+  end
+end
+
+task :test do
+  chdir "src/" do
+    sh "make test"
+  end
+  require 'lib/http-parser'
+  require 'bacon'
+  Bacon.summary_at_exit
+  require 'test/request.rb'
+  require 'test/response.rb'
+end

data/http-parser.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |s|
+  s.name = "http-parser"
+  s.version = "0.0.2"
+  s.date = "2009-08-07"
+  s.authors = ["Ryan Dahl", "Zed Shaw", "Jake Douglas"]
+  s.email = "jakecdouglas@gmail.com"
+  s.has_rdoc = true
+  # We have rake as a dependency because of a JRuby thing that
+  # requires the actual rake gem to be installed.
+  s.add_dependency('rake')
+  s.add_dependency('ffi')
+  s.add_dependency('bacon')
+  s.summary = "FFI binding to ry's http-parser"
+  s.homepage = "http://www.github.com/yakischloba/http-parser-ffi"
+  s.description = "FFI binding to ry's http-parser"
+  s.extensions = ["Rakefile"]
+  s.files =
+    ["http-parser.gemspec",
+    #"README.rdoc",
+    "Rakefile",
+    "lib/http-parser.rb",
+    "src/http_parser.h",
+    "src/http_parser.rl",
+    "src/LICENSE",
+    "src/Makefile",
+    "src/README.md",
+    "src/test.c",
+    "test/request.rb",
+    "test/response.rb"]
+end

data/lib/http-parser.rb

@@ -0,0 +1,193 @@
+require 'ffi'
+
+class HttpParser
+  extend FFI::Library
+  ffi_lib File.dirname(File.expand_path(__FILE__)) + "/../src/" + "http_parser.#{FFI::Platform::LIBSUFFIX}"
+
+  callback :http_data_cb, [:pointer, :pointer, :size_t], :int
+  callback :http_cb, [:pointer], :int
+
+  # Request methods
+  HTTP_COPY       = 0x0001
+  HTTP_DELETE     = 0x0002
+  HTTP_GET        = 0x0004
+  HTTP_HEAD       = 0x0008
+  HTTP_LOCK       = 0x0010
+  HTTP_MKCOL      = 0x0020
+  HTTP_MOVE       = 0x0040
+  HTTP_OPTIONS    = 0x0080
+  HTTP_POST       = 0x0100
+  HTTP_PROPFIND   = 0x0200
+  HTTP_PROPPATCH  = 0x0400
+  HTTP_PUT        = 0x0800
+  HTTP_TRACE      = 0x1000
+  HTTP_UNLOCK     = 0x2000
+
+  # Transfer encodings
+  HTTP_IDENTITY   = 0x01
+  HTTP_CHUNKED    = 0x02
+
+  enum :http_parser_type, [:HTTP_REQUEST, :HTTP_RESPONSE]
+
+  class HttpParserStruct < FFI::Struct
+    # PRIVATE
+    layout :cs, :int,
+    :http_parser_type,    :int,
+    :chunk_size,          :size_t,
+    :eating,              :int,
+    :error,               :int,
+    :body_read,           :size_t,
+    :header_field_mark,   :string,
+    :header_field_size,   :size_t,
+    :header_value_mark,   :string,
+    :header_value_size,   :size_t,
+    :query_string_mark,   :string,
+    :query_string_size,   :size_t,
+    :path_mark,           :string,
+    :path_size,           :size_t,
+    :uri_mark,            :string,
+    :uri_size,            :size_t,
+    :fragment_mark,       :string,
+    :fragment_size,       :size_t,
+    # READ-ONLY           
+    :status_code,         :ushort, # responses only
+    :method,              :ushort, # requests only
+    :transfer_encoding,   :short,
+    :version_major,       :ushort,
+    :version_minor,       :ushort,
+    :keep_alive,          :short,
+    :content_length,      :size_t,
+    # PUBLIC              
+    :data,                :pointer, # A pointer to get hook to the "connection" or "socket" object
+    :on_message_begin,    :http_cb,
+    :on_path,             :http_data_cb,
+    :on_query_string,     :http_data_cb,
+    :on_uri,              :http_data_cb,
+    :on_fragment,         :http_data_cb,
+    :on_header_field,     :http_data_cb,
+    :on_header_value,     :http_data_cb,
+    :on_headers_complete, :http_cb,
+    :on_body,             :http_data_cb,
+    :on_message_complete, :http_cb
+  end
+
+  attach_function :http_parser_init, [:pointer, :http_parser_type], :void
+  attach_function :http_parser_execute, [:pointer, :string, :size_t], :size_t
+  attach_function :http_parser_has_error, [:pointer], :int
+  attach_function :http_parser_should_keep_alive, [:pointer], :int
+
+  def initialize(type)
+    @type = type
+    @parser = HttpParserStruct.new
+    HttpParser.http_parser_init(@parser, HttpParser.enum_value(@type))
+  end
+
+  def execute(data)
+    @abort = false
+    HttpParser.http_parser_execute(@parser, data, data.length)
+  end
+
+  def has_error?
+    HttpParser.http_parser_has_error(@parser).zero? ? false : true
+  end
+
+  def should_keep_alive?
+    HttpParser.http_parser_should_keep_alive(@parser).zero? ? false : true
+  end
+
+  # You can call abort! from inside your callbacks which will halt the processing.
+  def abort!
+    @abort = true
+  end
+
+  def on_message_begin(proc=nil, &blk)
+    proc ||= blk
+    @on_message_begin = Proc.new do |ptr|
+      proc.call
+      @abort ? 1 : 0
+    end
+    @parser[:on_message_begin] = @on_message_begin
+  end
+
+  def on_path(proc=nil, &blk)
+    proc ||= blk
+    @on_path = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_path] = @on_path
+  end
+
+  def on_query_string(proc=nil, &blk)
+    proc ||= blk
+    @on_query_string = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_query_string] = @on_query_string
+  end
+
+  def on_uri(proc=nil, &blk)
+    proc ||= blk
+    @on_uri = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_uri] = @on_uri
+  end
+
+  def on_fragment(proc=nil, &blk)
+    proc ||= blk
+    @on_fragment = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_fragment] = @on_fragment
+  end
+
+  def on_header_field(proc=nil, &blk)
+    proc ||= blk
+    @on_header_field = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_header_field] = @on_header_field
+  end
+
+  def on_header_value(proc=nil, &blk)
+    proc ||= blk
+    @on_header_value= Proc.new do |ptr,data,len|      
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_header_value] = @on_header_value
+  end
+
+  def on_headers_complete(proc=nil, &blk)
+    proc ||= blk
+    @on_headers_complete = Proc.new do |ptr|
+      proc.call
+      @abort ? 1 : 0
+    end
+    @parser[:on_headers_complete] = @on_headers_complete
+  end
+
+  def on_body(proc=nil, &blk)
+    proc ||= blk
+    @on_body = Proc.new do |ptr,data,len|
+      proc.call(data.get_string(0, len))
+      @abort ? 1 : 0
+    end
+    @parser[:on_body] = @on_body
+  end
+
+  def on_message_complete(proc=nil, &blk)
+    proc ||= blk
+    @on_message_complete = Proc.new do |ptr|
+      proc.call
+      @abort ? 1 : 0
+    end
+    @parser[:on_message_complete] = @on_message_complete
+  end
+
+end

data/src/LICENSE

@@ -0,0 +1,79 @@
+Copyright 2009, Ryan Lienhart Dahl. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE. 
+
+
+
+
+http_parser is based on Zed Shaw's Mongrel. Mongrel's license is as follows.
+
+-- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT --
+Mongrel Web Server (Mongrel) is copyrighted free software by Zed A. Shaw
+<zedshaw at zedshaw dot com> and contributors. You can redistribute it 
+and/or modify it under either the terms of the GPL2 or the conditions below:
+
+1. You may make and give away verbatim copies of the source form of the
+   software without restriction, provided that you duplicate all of the
+   original copyright notices and associated disclaimers.
+
+2. You may modify your copy of the software in any way, provided that
+   you do at least ONE of the following:
+
+     a) place your modifications in the Public Domain or otherwise make them
+     Freely Available, such as by posting said modifications to Usenet or an
+     equivalent medium, or by allowing the author to include your
+     modifications in the software.
+
+     b) use the modified software only within your corporation or
+        organization.
+
+     c) rename any non-standard executables so the names do not conflict with
+     standard executables, which must also be provided.
+
+     d) make other distribution arrangements with the author.
+
+3. You may distribute the software in object code or executable
+   form, provided that you do at least ONE of the following:
+
+     a) distribute the executables and library files of the software,
+     together with instructions (in the manual page or equivalent) on where
+     to get the original distribution.
+
+     b) accompany the distribution with the machine-readable source of the
+     software.
+
+     c) give non-standard executables non-standard names, with
+        instructions on where to get the original software distribution.
+
+     d) make other distribution arrangements with the author.
+
+4. You may modify and include the part of the software into any other
+   software (possibly commercial).  But some files in the distribution
+   are not written by the author, so that they are not under this terms.
+
+5. The scripts and library files supplied as input to or produced as 
+   output from the software do not automatically fall under the
+   copyright of the software, but belong to whomever generated them, 
+   and may be sold commercially, and may be aggregated with this
+   software.
+
+6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+   IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+   PURPOSE.
+-- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT ---- CUT --

data/src/Makefile

@@ -0,0 +1,27 @@
+#OPT=-O0 -g -Wall -Wextra -Werror
+OPT=-O2
+
+test: http_parser.o test.c 
+	gcc $(OPT) http_parser.o test.c -o $@ 
+
+http_parser.o: http_parser.c http_parser.h Makefile
+	gcc $(OPT) -c http_parser.c
+
+http_parser.c: http_parser.rl Makefile
+	ragel -s -G2 http_parser.rl -o $@
+
+tags: http_parser.rl http_parser.h test.c
+	ctags $^
+
+clean:
+	rm -f *.o http_parser.c test http_parser.tar
+
+package: http_parser.c
+	@rm -rf /tmp/http_parser && mkdir /tmp/http_parser && \
+	cp LICENSE README.md Makefile http_parser.c http_parser.rl \
+		http_parser.h test.c /tmp/http_parser && \
+	cd /tmp && \
+	tar -cf http_parser.tar http_parser/
+	@echo /tmp/http_parser.tar
+
+.PHONY: clean package

data/src/README.md

@@ -0,0 +1,129 @@
+HTTP Parser
+===========
+
+This is a parser for HTTP messages written in C. It parses both requests
+and responses. The parser is designed to be used in performance HTTP
+applications. It does not make any allocations, it does not buffer data, and
+it can be interrupted at anytime. It only requires about 128 bytes of data
+per message stream (in a web server that is per connection). 
+
+Features:
+
+  * No dependencies 
+  * Parses both requests and responses.
+  * Handles keep-alive streams.
+  * Decodes chunked encoding.
+  * Extracts the following data from a message
+    * header fields and values
+    * content-length
+    * request method
+    * response status code
+    * transfer-encoding
+    * http version
+    * request path, query string, fragment
+    * message body
+  * Defends against buffer overflow attacks.
+
+Usage
+-----
+
+One `http_parser` object is used per TCP connection. Initialize the struct
+using `http_parser_init()` and set the callbacks. That might look something
+like this:
+
+    http_parser *parser = malloc(sizeof(http_parser));
+    http_parser_init(parser, HTTP_REQUEST);
+    parser->on_path = my_path_callback;
+    parser->on_header_field = my_header_field_callback;
+    parser->data = my_socket;
+
+When data is received on the socket execute the parser and check for errors.
+
+    size_t len = 80*1024;
+    char buf[len];
+    ssize_t recved;
+
+    recved = read(fd, buf, len);
+    if (recved != 0) // handle error
+
+    http_parser_execute(parser, buf, recved);
+
+    if (http_parser_has_error(parser)) {
+      // handle error. usually just close the connection
+    }
+
+Scalar valued message information such as `status_code`, `method`, and the
+HTTP version are stored in the parser structure. This data is only
+temporarlly stored in `http_parser` and gets reset on each new message. If
+this information is needed later, copy it out of the structure during the
+`headers_complete` callback.
+  
+The parser decodes the transfer-encoding for both requests and responses
+transparently. That is, a chunked encoding is decoded before being sent to
+the on_body callback.
+
+It does not decode the content-encoding (gzip). Not all HTTP applications
+need to inspect the body. Decoding gzip is non-neglagable amount of
+processing (and requires making allocations). HTTP proxies using this
+parser, for example, would not want such a feature.
+
+Callbacks
+---------
+
+During the `http_parser_execute()` call, the callbacks set in `http_parser`
+will be executed. The parser maintains state and never looks behind, so
+buffering the data is not necessary. If you need to save certain data for
+later usage, you can do that from the callbacks.
+
+There are two types of callbacks:
+
+* notification `typedef int (*http_cb) (http_parser*);`
+    Callbacks: on_message_begin, on_headers_complete, on_message_complete.
+* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);`
+    Callbacks: (requests only) on_path, on_query_string, on_uri, on_fragment,
+               (common) on_header_field, on_header_value, on_body;
+
+In case you parse HTTP message in chunks (i.e. `read()` request line
+from socket, parse, read half headers, parse, etc) your data callbacks
+may be called more than once. Http-parser guarantees that data pointer is only
+valid for the lifetime of callback. You can also `read()` into a heap allocated
+buffer to avoid copying memory around if this fits your application.
+
+Reading headers may be a tricky task if you read/parse headers partially.
+Basically, you need to remember whether last header callback was field or value
+and apply following logic:
+
+    /* on_header_field and on_header_value shortened to on_h_*
+     ------------------------ ------------ --------------------------------------------
+    | State (prev. callback) | Callback   | Description/action                         |
+     ------------------------ ------------ --------------------------------------------
+    | nothing (first call)   | on_h_field | Allocate new buffer and copy callback data |
+    |                        |            | into it                                    |
+     ------------------------ ------------ --------------------------------------------
+    | value                  | on_h_field | New header started.                        |
+    |                        |            | Copy current name,value buffers to headers |
+    |                        |            | list and allocate new buffer for new name  |
+     ------------------------ ------------ --------------------------------------------
+    | field                  | on_h_field | Previous name continues. Reallocate name   |
+    |                        |            | buffer and append callback data to it      |
+     ------------------------ ------------ --------------------------------------------
+    | field                  | on_h_value | Value for current header started. Allocate |
+    |                        |            | new buffer and copy callback data to it    |
+     ------------------------ ------------ --------------------------------------------
+    | value                  | on_h_value | Value continues. Reallocate value buffer   |
+    |                        |            | and append callback data to it             |
+     ------------------------ ------------ --------------------------------------------
+    */
+
+See examples of reading in headers:
+
+* [partial example](http://gist.github.com/155877) in C
+* [from http-parser tests](http://github.com/ry/http-parser/blob/37a0ff8928fb0d83cec0d0d8909c5a4abcd221af/test.c#L403) in C
+* [from Node library](http://github.com/ry/node/blob/842eaf446d2fdcb33b296c67c911c32a0dabc747/src/http.js#L284) in Javascript
+
+Releases
+--------
+
+  * [0.1](http://s3.amazonaws.com/four.livejournal/20090427/http_parser-0.1.tar.gz)
+
+The source repo is at [github](http://github.com/ry/http-parser).

data/src/http_parser.h

@@ -0,0 +1,133 @@
+/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
+ * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
+ *
+ * All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
+ */
+#ifndef http_parser_h
+#define http_parser_h
+#ifdef __cplusplus
+extern "C" {
+#endif 
+
+#include <sys/types.h> 
+
+typedef struct http_parser http_parser;
+
+/* Callbacks should return non-zero to indicate an error. The parse will
+ * then halt execution. 
+ * 
+ * http_data_cb does not return data chunks. It will be call arbitrarally
+ * many times for each string. E.G. you might get 10 callbacks for "on_path"
+ * each providing just a few characters more data.
+ */
+typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
+typedef int (*http_cb) (http_parser*);
+
+/* Request Methods */
+#define HTTP_COPY       0x0001
+#define HTTP_DELETE     0x0002
+#define HTTP_GET        0x0004
+#define HTTP_HEAD       0x0008
+#define HTTP_LOCK       0x0010
+#define HTTP_MKCOL      0x0020
+#define HTTP_MOVE       0x0040
+#define HTTP_OPTIONS    0x0080
+#define HTTP_POST       0x0100
+#define HTTP_PROPFIND   0x0200
+#define HTTP_PROPPATCH  0x0400
+#define HTTP_PUT        0x0800
+#define HTTP_TRACE      0x1000
+#define HTTP_UNLOCK     0x2000
+
+/* Transfer Encodings */
+#define HTTP_IDENTITY   0x01
+#define HTTP_CHUNKED    0x02
+
+enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE };
+
+struct http_parser {
+  /** PRIVATE **/
+  int cs;
+  enum http_parser_type type;
+
+  size_t chunk_size;
+  int eating;
+  int error;
+  size_t body_read;
+
+  const char *header_field_mark; 
+  size_t      header_field_size; 
+  const char *header_value_mark; 
+  size_t      header_value_size; 
+  const char *query_string_mark; 
+  size_t      query_string_size; 
+  const char *path_mark; 
+  size_t      path_size; 
+  const char *uri_mark; 
+  size_t      uri_size; 
+  const char *fragment_mark; 
+  size_t      fragment_size; 
+
+  /** READ-ONLY **/
+  unsigned short status_code; /* responses only */
+  unsigned short method;      /* requests only */
+  short transfer_encoding;
+  unsigned short version_major;
+  unsigned short version_minor;
+  short keep_alive;
+  size_t content_length;
+
+  /** PUBLIC **/
+  void *data; /* A pointer to get hook to the "connection" or "socket" object */
+
+  /* an ordered list of callbacks */
+
+  http_cb      on_message_begin;
+
+  /* requests only */
+  http_data_cb on_path;
+  http_data_cb on_query_string;
+  http_data_cb on_uri;
+  http_data_cb on_fragment;
+
+  http_data_cb on_header_field;
+  http_data_cb on_header_value;
+  http_cb      on_headers_complete;
+  http_data_cb on_body;
+  http_cb      on_message_complete;
+};
+
+/* Initializes an http_parser structure.  The second argument specifies if
+ * it will be parsing requests or responses. 
+ */
+void http_parser_init (http_parser *parser, enum http_parser_type);
+
+size_t http_parser_execute (http_parser *parser, const char *data, size_t len);
+
+int http_parser_has_error (http_parser *parser);
+
+int http_parser_should_keep_alive (http_parser *parser);
+
+#ifdef __cplusplus
+}
+#endif 
+#endif