http_parser.rb 0.5.0

data/.gitignore

@@ -0,0 +1,11 @@
+tmp
+*.bundle
+*.gem
+*.o
+*.so
+*.bundle
+*.jar
+*.swp
+Makefile
+tags
+*.rbc

data/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "ext/ruby_http_parser/vendor/http-parser"]
+	path = ext/ruby_http_parser/vendor/http-parser
+	url = git://github.com/ry/http-parser.git
+[submodule "ext/ruby_http_parser/vendor/http-parser-java"]
+	path = ext/ruby_http_parser/vendor/http-parser-java
+	url = git://github.com/a2800276/http-parser.java.git

data/README.md

@@ -0,0 +1,45 @@
+# http_parser.rb
+
+A simple callback-based HTTP request/response parser for writing http
+servers, clients and proxies.
+
+This gem is built on top of [ry/http-parser](http://github.com/ry/http-parser) and its java port [a2800276/http-parser.java](http://github.com/a2800276/http-parser.java).
+
+## Supported Platforms
+
+This gem aims to work on all major Ruby platforms, including:
+
+- MRI 1.8 and 1.9
+- Rubinius
+- JRuby
+- win32
+
+## Usage
+
+    require "http/parser"
+
+    parser = Http::Parser.new
+
+    parser.on_headers_complete = proc do |headers|
+      p parser.http_method
+      p parser.http_version
+
+      p parser.request_url # for requests
+      p parser.status_code # for responses
+
+      p headers
+    end
+
+    parser.on_body = proc do |chunk|
+      # One chunk of the body
+      p chunk
+    end
+
+    parser.on_message_complete = proc do |env|
+      # Headers and body is all parsed
+      puts "Done!"
+    end
+
+    # Feed raw data from the socket to the parser
+    parser << raw_data
+

data/Rakefile

@@ -0,0 +1,6 @@
+# load tasks
+Dir['tasks/*.rake'].sort.each { |f| load f }
+
+# default task
+task :compile => :submodules
+task :default => [:compile, :spec]

data/bench/thin.rb

@@ -0,0 +1,57 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require "rubygems"
+require "thin_parser"
+require "http_parser"
+require "benchmark"
+
+data = "POST /postit HTTP/1.1\r\n" +
+       "Host: localhost:3000\r\n" +
+       "User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.9) Gecko/20071025 Firefox/2.0.0.9\r\n" +
+       "Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5\r\n" +
+       "Accept-Language: en-us,en;q=0.5\r\n" +
+       "Accept-Encoding: gzip,deflate\r\n" +
+       "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" +
+       "Keep-Alive: 300\r\n" +
+       "Connection: keep-alive\r\n" +
+       "Content-Type: text/html\r\n" +
+       "Content-Length: 37\r\n" +
+       "\r\n" +
+       "name=marc&email=macournoyer@gmail.com"
+
+def thin(data)
+  env = {"rack.input" => StringIO.new}
+  Thin::HttpParser.new.execute(env, data, 0)
+  env
+end
+
+def http_parser(data)
+  body = StringIO.new
+  env = nil
+
+  parser = HTTP::RequestParser.new
+  parser.on_headers_complete = proc { |e| env = e }
+  parser.on_body = proc { |c| body << c }
+  parser << data
+
+  env["rack-input"] = body
+  env
+end
+
+# p thin(data)
+# p http_parser(data)
+
+TESTS = 30_000
+Benchmark.bmbm do |results|
+  results.report("thin:") { TESTS.times { thin data } }
+  results.report("http-parser:") { TESTS.times { http_parser data } }
+end
+
+# On my MBP core duo 2.2Ghz
+# Rehearsal ------------------------------------------------
+# thin:          1.470000   0.000000   1.470000 (  1.474737)
+# http-parser:   1.270000   0.020000   1.290000 (  1.292758)
+# --------------------------------------- total: 2.760000sec
+#
+#                    user     system      total        real
+# thin:          1.150000   0.030000   1.180000 (  1.173767)
+# http-parser:   1.250000   0.010000   1.260000 (  1.263796)

data/ext/ruby_http_parser/.gitignore

@@ -0,0 +1 @@
+ryah_http_parser.*

data/ext/ruby_http_parser/RubyHttpParserService.java

@@ -0,0 +1,18 @@
+import java.io.IOException;
+
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyModule;
+import org.jruby.runtime.load.BasicLibraryService;
+
+import org.ruby_http_parser.*;
+
+public class RubyHttpParserService implements BasicLibraryService {
+  public boolean basicLoad(final Ruby runtime) throws IOException {
+    RubyModule mHTTP = runtime.defineModule("HTTP");
+    RubyClass cParser = mHTTP.defineClassUnder("Parser", runtime.getObject(), RubyHttpParser.ALLOCATOR);
+    cParser.defineAnnotatedMethods(RubyHttpParser.class);
+    cParser.defineClassUnder("Error", runtime.getClass("IOError"),runtime.getClass("IOError").getAllocator());
+    return true;
+  }
+}

data/ext/ruby_http_parser/ext_help.h

@@ -0,0 +1,18 @@
+#ifndef ext_help_h
+#define ext_help_h
+
+#define RAISE_NOT_NULL(T) if(T == NULL) rb_raise(rb_eArgError, "NULL found for " # T " when shouldn't be.");
+#define DATA_GET(from,type,name) Data_Get_Struct(from,type,name); RAISE_NOT_NULL(name);
+#define REQUIRE_TYPE(V, T) if(TYPE(V) != T) rb_raise(rb_eTypeError, "Wrong argument type for " # V " required " # T);
+
+/* for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_PTR */
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+
+/* for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_LEN */
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+
+#endif

data/ext/ruby_http_parser/extconf.rb

@@ -0,0 +1,16 @@
+require 'mkmf'
+
+# mongrel and http-parser both define http_parser_(init|execute), so we
+# rename functions in http-parser before using them.
+vendor_dir = File.expand_path('../vendor/http-parser/', __FILE__)
+src_dir = File.expand_path('../', __FILE__)
+%w[ http_parser.c http_parser.h ].each do |file|
+  File.open(File.join(src_dir, "ryah_#{file}"), 'w'){ |f|
+    f.write File.read(File.join(vendor_dir, file)).gsub('http_parser', 'ryah_http_parser')
+  }
+end
+
+$CFLAGS << " -I#{src_dir}"
+
+dir_config("ruby_http_parser")
+create_makefile("ruby_http_parser")

data/ext/ruby_http_parser/org/ruby_http_parser/RubyHttpParser.java

@@ -0,0 +1,403 @@
+package org.ruby_http_parser;
+
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyHash;
+import org.jruby.RubyModule;
+import org.jruby.RubyNumeric;
+import org.jruby.RubyObject;
+import org.jruby.RubyString;
+
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+import org.jruby.anno.JRubyMethod;
+import org.jruby.exceptions.RaiseException;
+
+import java.nio.ByteBuffer;
+import http_parser.*;
+import http_parser.lolevel.ParserSettings;
+import http_parser.lolevel.HTTPCallback;
+import http_parser.lolevel.HTTPDataCallback;
+
+public class RubyHttpParser extends RubyObject {
+
+  public static ObjectAllocator ALLOCATOR = new ObjectAllocator() {
+    public IRubyObject allocate(Ruby runtime, RubyClass klass) {
+      return new RubyHttpParser(runtime, klass);
+    }
+  };
+
+  byte[] fetchBytes (ByteBuffer b, int pos, int len) {
+    byte[] by = new byte[len];
+    int saved = b.position();
+    b.position(pos);
+    b.get(by);
+    b.position(saved);
+    return by;
+  }
+
+  public class StopException extends RuntimeException {
+  }
+
+  private Ruby runtime;
+  private HTTPParser parser;
+  private ParserSettings settings;
+
+  private RubyClass eParserError;
+
+  private RubyHash headers;
+
+  private IRubyObject on_message_begin;
+  private IRubyObject on_headers_complete;
+  private IRubyObject on_body;
+  private IRubyObject on_message_complete;
+
+  private IRubyObject requestUrl;
+  private IRubyObject requestPath;
+  private IRubyObject queryString;
+  private IRubyObject fragment;
+
+  private IRubyObject callback_object;
+
+  private String _current_header;
+  private String _last_header;
+
+  public RubyHttpParser(final Ruby runtime, RubyClass clazz) {
+    super(runtime,clazz);
+
+    this.runtime = runtime;
+    this.eParserError = (RubyClass)runtime.getModule("HTTP").getClass("Parser").getConstant("Error");
+
+    this.on_message_begin = null;
+    this.on_headers_complete = null;
+    this.on_body = null;
+    this.on_message_complete = null;
+
+    this.callback_object = null;
+
+    initSettings();
+    init();
+  }
+
+  private void initSettings() {
+    this.settings = new ParserSettings();
+
+    this.settings.on_url = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+        ((RubyString)requestUrl).concat(runtime.newString(new String(data)));
+        return 0;
+      }
+    };
+    this.settings.on_path = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+        ((RubyString)requestPath).concat(runtime.newString(new String(data)));
+        return 0;
+      }
+    };
+    this.settings.on_query_string = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+        ((RubyString)queryString).concat(runtime.newString(new String(data)));
+        return 0;
+      }
+    };
+    this.settings.on_fragment = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+        ((RubyString)fragment).concat(runtime.newString(new String(data)));
+        return 0;
+      }
+    };
+
+    this.settings.on_header_field = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+
+        if (_current_header == null)
+          _current_header = new String(data);
+        else
+          _current_header.concat(new String(data));
+
+        return 0;
+      }
+    };
+    this.settings.on_header_value = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        byte[] data = fetchBytes(buf, pos, len);
+        ThreadContext context = headers.getRuntime().getCurrentContext();
+        IRubyObject key, val;
+
+        if (_current_header != null) {
+          _last_header = _current_header;
+          _current_header = null;
+
+          key = (IRubyObject)runtime.newString(_last_header);
+          val = headers.op_aref(context, key);
+
+          if (!val.isNil())
+            ((RubyString)val).cat(", ".getBytes());
+        } else {
+          key = (IRubyObject)runtime.newString(_last_header);
+          val = headers.op_aref(context, key);
+        }
+
+        if (val.isNil())
+          headers.op_aset(context, key, runtime.newString(new String(data)));
+        else
+          ((RubyString)val).cat(data);
+
+        return 0;
+      }
+    };
+
+    this.settings.on_message_begin = new HTTPCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p) {
+        headers = new RubyHash(runtime);
+
+        requestUrl = runtime.newString("");
+        requestPath = runtime.newString("");
+        queryString = runtime.newString("");
+        fragment = runtime.newString("");
+
+        IRubyObject ret = runtime.getNil();
+
+        if (callback_object != null) {
+          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_message_begin")).toJava(Boolean.class) == Boolean.TRUE) {
+            ThreadContext context = callback_object.getRuntime().getCurrentContext();
+            ret = callback_object.callMethod(context, "on_message_begin");
+          }
+        } else if (on_message_begin != null) {
+          ThreadContext context = on_message_begin.getRuntime().getCurrentContext();
+          ret = on_message_begin.callMethod(context, "call");
+        }
+
+        if (ret == runtime.newSymbol("stop")) {
+          throw new StopException();
+        } else {
+          return 0;
+        }
+      }
+    };
+    this.settings.on_message_complete = new HTTPCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p) {
+        IRubyObject ret = runtime.getNil();
+
+        if (callback_object != null) {
+          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_message_complete")).toJava(Boolean.class) == Boolean.TRUE) {
+            ThreadContext context = callback_object.getRuntime().getCurrentContext();
+            ret = callback_object.callMethod(context, "on_message_complete");
+          }
+        } else if (on_message_complete != null) {
+          ThreadContext context = on_message_complete.getRuntime().getCurrentContext();
+          ret = on_message_complete.callMethod(context, "call");
+        }
+
+        if (ret == runtime.newSymbol("stop")) {
+          throw new StopException();
+        } else {
+          return 0;
+        }
+      }
+    };
+    this.settings.on_headers_complete = new HTTPCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p) {
+        IRubyObject ret = runtime.getNil();
+
+        if (callback_object != null) {
+          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_headers_complete")).toJava(Boolean.class) == Boolean.TRUE) {
+            ThreadContext context = callback_object.getRuntime().getCurrentContext();
+            ret = callback_object.callMethod(context, "on_headers_complete", headers);
+          }
+        } else if (on_headers_complete != null) {
+          ThreadContext context = on_headers_complete.getRuntime().getCurrentContext();
+          ret = on_headers_complete.callMethod(context, "call", headers);
+        }
+
+        if (ret == runtime.newSymbol("stop")) {
+          throw new StopException();
+        } else {
+          return 0;
+        }
+      }
+    };
+    this.settings.on_body = new HTTPDataCallback() {
+      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
+        IRubyObject ret = runtime.getNil();
+        byte[] data = fetchBytes(buf, pos, len);
+
+        if (callback_object != null) {
+          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_body")).toJava(Boolean.class) == Boolean.TRUE) {
+            ThreadContext context = callback_object.getRuntime().getCurrentContext();
+            ret = callback_object.callMethod(context, "on_body", callback_object.getRuntime().newString(new String(data)));
+          }
+        } else if (on_body != null) {
+          ThreadContext context = on_body.getRuntime().getCurrentContext();
+          ret = on_body.callMethod(context, "call", on_body.getRuntime().newString(new String(data)));
+        }
+
+        if (ret == runtime.newSymbol("stop")) {
+          throw new StopException();
+        } else {
+          return 0;
+        }
+      }
+    };
+  }
+
+  private void init() {
+    this.parser = new HTTPParser();
+    this.headers = null;
+
+    this.requestUrl = runtime.getNil();
+    this.requestPath = runtime.getNil();
+    this.queryString = runtime.getNil();
+    this.fragment = runtime.getNil();
+  }
+
+  @JRubyMethod(name = "initialize")
+  public IRubyObject initialize() {
+    return this;
+  }
+
+  @JRubyMethod(name = "initialize")
+  public IRubyObject initialize(IRubyObject arg) {
+    callback_object = arg;
+    return initialize();
+  }
+
+  @JRubyMethod(name = "on_message_begin=")
+  public IRubyObject set_on_message_begin(IRubyObject cb) {
+    on_message_begin = cb;
+    return cb;
+  }
+
+  @JRubyMethod(name = "on_headers_complete=")
+  public IRubyObject set_on_headers_complete(IRubyObject cb) {
+    on_headers_complete = cb;
+    return cb;
+  }
+
+  @JRubyMethod(name = "on_body=")
+  public IRubyObject set_on_body(IRubyObject cb) {
+    on_body = cb;
+    return cb;
+  }
+
+  @JRubyMethod(name = "on_message_complete=")
+  public IRubyObject set_on_message_complete(IRubyObject cb) {
+    on_message_complete = cb;
+    return cb;
+  }
+
+  @JRubyMethod(name = "<<")
+  public IRubyObject execute(IRubyObject data) {
+    RubyString str = (RubyString)data;
+    ByteBuffer buf = ByteBuffer.wrap(str.getBytes());
+    boolean stopped = false;
+
+    try {
+      this.parser.execute(this.settings, buf);
+    } catch (HTTPException e) {
+      throw new RaiseException(runtime, eParserError, e.getMessage(), true);
+    } catch (StopException e) {
+      stopped = true;
+    }
+
+    if (parser.getUpgrade()) {
+      // upgrade request
+    } else if (buf.hasRemaining()) {
+      if (!stopped)
+        throw new RaiseException(runtime, eParserError, "Could not parse data entirely", true);
+    }
+
+    return RubyNumeric.int2fix(runtime, buf.position());
+  }
+
+  @JRubyMethod(name = "keep_alive?")
+  public IRubyObject shouldKeepAlive() {
+    return parser.shouldKeepAlive() ? runtime.getTrue() : runtime.getFalse();
+  }
+
+  @JRubyMethod(name = "upgrade?")
+  public IRubyObject shouldUpgrade() {
+    return parser.getUpgrade() ? runtime.getTrue() : runtime.getFalse();
+  }
+
+  @JRubyMethod(name = "http_major")
+  public IRubyObject httpMajor() {
+    if (parser.getMajor() == 0 && parser.getMinor() == 0)
+      return runtime.getNil();
+    else
+      return RubyNumeric.int2fix(runtime, parser.getMajor());
+  }
+
+  @JRubyMethod(name = "http_minor")
+  public IRubyObject httpMinor() {
+    if (parser.getMajor() == 0 && parser.getMinor() == 0)
+      return runtime.getNil();
+    else
+      return RubyNumeric.int2fix(runtime, parser.getMinor());
+  }
+
+  @JRubyMethod(name = "http_version")
+  public IRubyObject httpVersion() {
+    if (parser.getMajor() == 0 && parser.getMinor() == 0)
+      return runtime.getNil();
+    else
+      return runtime.newArray(httpMajor(), httpMinor());
+  }
+
+  @JRubyMethod(name = "http_method")
+  public IRubyObject httpMethod() {
+    HTTPMethod method = parser.getHTTPMethod();
+    if (method != null)
+      return runtime.newString(new String(method.bytes));
+    else
+      return runtime.getNil();
+  }
+
+  @JRubyMethod(name = "status_code")
+  public IRubyObject statusCode() {
+    int code = parser.getStatusCode();
+    if (code != 0)
+      return RubyNumeric.int2fix(runtime, code);
+    else
+      return runtime.getNil();
+  }
+
+  @JRubyMethod(name = "headers")
+  public IRubyObject getHeaders() {
+    return headers == null ? runtime.getNil() : headers;
+  }
+
+  @JRubyMethod(name = "request_url")
+  public IRubyObject getRequestUrl() {
+    return requestUrl == null ? runtime.getNil() : requestUrl;
+  }
+
+  @JRubyMethod(name = "request_path")
+  public IRubyObject getRequestPath() {
+    return requestPath == null ? runtime.getNil() : requestPath;
+  }
+
+  @JRubyMethod(name = "query_string")
+  public IRubyObject getQueryString() {
+    return queryString == null ? runtime.getNil() : queryString;
+  }
+
+  @JRubyMethod(name = "fragment")
+  public IRubyObject getFragment() {
+    return fragment == null ? runtime.getNil() : fragment;
+  }
+
+  @JRubyMethod(name = "reset!")
+  public IRubyObject reset() {
+    init();
+    return runtime.getTrue();
+  }
+
+}