http_parser.rb 0.5.1 → 0.6.0.beta.1

data/.gitmodules

@@ -1,6 +1,6 @@
-[submodule "ext/ruby_http_parser/vendor/http-parser"]
+[submodule "http-parser"]
 	path = ext/ruby_http_parser/vendor/http-parser
-	url = git://github.com/ry/http-parser.git
-[submodule "ext/ruby_http_parser/vendor/http-parser-java"]
+	url = git://github.com/joyent/http-parser.git
+[submodule "http-parser-java"]
 	path = ext/ruby_http_parser/vendor/http-parser-java
-	url = git://github.com/a2800276/http-parser.java.git
+	url = git://github.com/http-parser/http-parser.java

data/Gemfile

@@ -1,2 +1,2 @@
-source :rubygems
+source 'https://rubygems.org'
 gemspec

data/Gemfile.lock

@@ -1,16 +1,19 @@
 PATH
   remote: .
   specs:
-    http_parser.rb (0.5.1)
+    http_parser.rb (0.5.3)
 
 GEM
   remote: http://rubygems.org/
   specs:
+    benchmark_suite (0.8.0)
     diff-lcs (1.1.2)
+    ffi (1.0.11)
+    ffi (1.0.11-java)
     json (1.5.1)
     json (1.5.1-java)
-    rake (0.8.7)
-    rake-compiler (0.7.5)
+    rake (0.9.2)
+    rake-compiler (0.7.9)
       rake
     rspec (2.4.0)
       rspec-core (~> 2.4.0)
@@ -20,13 +23,17 @@ GEM
     rspec-expectations (2.4.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.4.0)
+    yajl-ruby (1.1.0)
 
 PLATFORMS
   java
   ruby
 
 DEPENDENCIES
+  benchmark_suite
+  ffi
   http_parser.rb!
   json (>= 1.4.6)
-  rake-compiler (>= 0.7.5)
+  rake-compiler (>= 0.7.9)
   rspec (>= 2.0.1)
+  yajl-ruby (>= 0.8.1)

data/LICENSE-MIT

@@ -0,0 +1,20 @@
+Copyright 2009,2010 Marc-André Cournoyer <macournoyer@gmail.com>
+Copyright 2010,2011 Aman Gupta <aman@tmm1.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

data/README.md

@@ -3,7 +3,7 @@
 A simple callback-based HTTP request/response parser for writing http
 servers, clients and proxies.
 
-This gem is built on top of [ry/http-parser](http://github.com/ry/http-parser) and its java port [a2800276/http-parser.java](http://github.com/a2800276/http-parser.java).
+This gem is built on top of [joyent/http-parser](http://github.com/joyent/http-parser) and its java port [a2800276/http-parser.java](http://github.com/a2800276/http-parser.java).
 
 ## Supported Platforms
 
@@ -16,70 +16,75 @@ This gem aims to work on all major Ruby platforms, including:
 
 ## Usage
 
-    require "http/parser"
+```ruby
+require "http/parser"
 
-    parser = Http::Parser.new
+parser = Http::Parser.new
 
-    parser.on_headers_complete = proc do
-      p parser.http_version
+parser.on_headers_complete = proc do
+  p parser.http_version
 
-      p parser.http_method # for requests
-      p parser.request_url
+  p parser.http_method # for requests
+  p parser.request_url
 
-      p parser.status_code # for responses
+  p parser.status_code # for responses
 
-      p parser.headers
-    end
+  p parser.headers
+end
 
-    parser.on_body = proc do |chunk|
-      # One chunk of the body
-      p chunk
-    end
+parser.on_body = proc do |chunk|
+  # One chunk of the body
+  p chunk
+end
 
-    parser.on_message_complete = proc do |env|
-      # Headers and body is all parsed
-      puts "Done!"
-    end
+parser.on_message_complete = proc do |env|
+  # Headers and body is all parsed
+  puts "Done!"
+end
+```
 
-    # Feed raw data from the socket to the parser
-    parser << raw_data
+# Feed raw data from the socket to the parser
+`parser << raw_data`
 
 ## Advanced Usage
 
 ### Accept callbacks on an object
 
-    module MyHttpConnection
-      def connection_completed
-        @parser = Http::Parser.new(self)
-      end
+```ruby
+module MyHttpConnection
+  def connection_completed
+    @parser = Http::Parser.new(self)
+  end
 
-      def receive_data(data)
-        @parser << data
-      end
+  def receive_data(data)
+    @parser << data
+  end
 
-      def on_message_begin
-        @headers = nil
-        @body = ''
-      end
+  def on_message_begin
+    @headers = nil
+    @body = ''
+  end
 
-      def on_headers_complete
-        @headers = @parser.headers
-      end
+  def on_headers_complete(headers)
+    @headers = headers
+  end
 
-      def on_body(chunk)
-        @body << chunk
-      end
+  def on_body(chunk)
+    @body << chunk
+  end
 
-      def on_message_complete
-        p [@headers, @body]
-      end
-    end
+  def on_message_complete
+    p [@headers, @body]
+  end
+end
+```
 
 ### Stop parsing after headers
 
-    parser = Http::Parser.new
-    parser.on_headers_complete = proc{ :stop }
-
-    offset = parser << request_data
-    body = request_data[offset..-1]
+```ruby
+parser = Http::Parser.new
+parser.on_headers_complete = proc{ :stop }
 
+offset = parser << request_data
+body = request_data[offset..-1]
+```

data/bench/standalone.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require "rubygems"
+require "http/parser"
+require "benchmark/ips"
+
+request = <<-REQUEST
+GET / HTTP/1.1
+Host: www.example.com
+Connection: keep-alive
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.78 S
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Accept-Encoding: gzip,deflate,sdch
+Accept-Language: en-US,en;q=0.8
+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
+
+REQUEST
+request.gsub!(/\n/m, "\r\n")
+
+Benchmark.ips do |ips|
+  ips.report("instance") { Http::Parser.new }
+  ips.report("parsing")  { Http::Parser.new << request }
+end

data/bench/thin.rb

@@ -3,6 +3,7 @@ require "rubygems"
 require "thin_parser"
 require "http_parser"
 require "benchmark"
+require "stringio"
 
 data = "POST /postit HTTP/1.1\r\n" +
        "Host: localhost:3000\r\n" +

data/ext/ruby_http_parser/org/ruby_http_parser/RubyHttpParser.java

@@ -1,12 +1,14 @@
 package org.ruby_http_parser;
 
 import org.jruby.Ruby;
+import org.jruby.RubyArray;
 import org.jruby.RubyClass;
 import org.jruby.RubyHash;
 import org.jruby.RubyModule;
 import org.jruby.RubyNumeric;
 import org.jruby.RubyObject;
 import org.jruby.RubyString;
+import org.jruby.RubySymbol;
 
 import org.jruby.runtime.ObjectAllocator;
 import org.jruby.runtime.ThreadContext;
@@ -15,6 +17,10 @@ import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.anno.JRubyMethod;
 import org.jruby.exceptions.RaiseException;
 
+import org.jruby.util.ByteList;
+
+import org.jcodings.specific.UTF8Encoding;
+
 import java.nio.ByteBuffer;
 import http_parser.*;
 import http_parser.lolevel.ParserSettings;
@@ -23,6 +29,11 @@ import http_parser.lolevel.HTTPDataCallback;
 
 public class RubyHttpParser extends RubyObject {
 
+  @JRubyMethod(name = "strict?", module = true)
+  public static IRubyObject strict(IRubyObject recv) {
+    return recv.getRuntime().newBoolean(true);
+  }
+
   public static ObjectAllocator ALLOCATOR = new ObjectAllocator() {
     public IRubyObject allocate(Ruby runtime, RubyClass klass) {
       return new RubyHttpParser(runtime, klass);
@@ -59,10 +70,15 @@ public class RubyHttpParser extends RubyObject {
   private IRubyObject queryString;
   private IRubyObject fragment;
 
+  private IRubyObject header_value_type;
+  private IRubyObject upgradeData;
+
   private IRubyObject callback_object;
 
-  private String _current_header;
-  private String _last_header;
+  private boolean completed;
+
+  private byte[] _current_header;
+  private byte[] _last_header;
 
   public RubyHttpParser(final Ruby runtime, RubyClass clazz) {
     super(runtime,clazz);
@@ -77,6 +93,10 @@ public class RubyHttpParser extends RubyObject {
 
     this.callback_object = null;
 
+    this.completed = false;
+
+    this.header_value_type = runtime.getModule("HTTP").getClass("Parser").getInstanceVariable("@default_header_value_type");
+
     initSettings();
     init();
   }
@@ -87,28 +107,7 @@ public class RubyHttpParser extends RubyObject {
     this.settings.on_url = new HTTPDataCallback() {
       public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
         byte[] data = fetchBytes(buf, pos, len);
-        ((RubyString)requestUrl).concat(runtime.newString(new String(data)));
-        return 0;
-      }
-    };
-    this.settings.on_path = new HTTPDataCallback() {
-      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
-        byte[] data = fetchBytes(buf, pos, len);
-        ((RubyString)requestPath).concat(runtime.newString(new String(data)));
-        return 0;
-      }
-    };
-    this.settings.on_query_string = new HTTPDataCallback() {
-      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
-        byte[] data = fetchBytes(buf, pos, len);
-        ((RubyString)queryString).concat(runtime.newString(new String(data)));
-        return 0;
-      }
-    };
-    this.settings.on_fragment = new HTTPDataCallback() {
-      public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
-        byte[] data = fetchBytes(buf, pos, len);
-        ((RubyString)fragment).concat(runtime.newString(new String(data)));
+        ((RubyString)requestUrl).cat(data);
         return 0;
       }
     };
@@ -118,37 +117,65 @@ public class RubyHttpParser extends RubyObject {
         byte[] data = fetchBytes(buf, pos, len);
 
         if (_current_header == null)
-          _current_header = new String(data);
-        else
-          _current_header = _current_header.concat(new String(data));
+          _current_header = data;
+        else {
+          byte[] tmp = new byte[_current_header.length + data.length];
+          System.arraycopy(_current_header, 0, tmp, 0, _current_header.length);
+          System.arraycopy(data, 0, tmp, _current_header.length, data.length);
+          _current_header = tmp;
+       }
 
         return 0;
       }
     };
+    final RubySymbol arraysSym = runtime.newSymbol("arrays");
+    final RubySymbol mixedSym = runtime.newSymbol("mixed");
+    final RubySymbol stopSym = runtime.newSymbol("stop");
+    final RubySymbol resetSym = runtime.newSymbol("reset");
     this.settings.on_header_value = new HTTPDataCallback() {
       public int cb (http_parser.lolevel.HTTPParser p, ByteBuffer buf, int pos, int len) {
         byte[] data = fetchBytes(buf, pos, len);
         ThreadContext context = headers.getRuntime().getCurrentContext();
         IRubyObject key, val;
+        int new_field = 0;
 
         if (_current_header != null) {
+          new_field = 1;
           _last_header = _current_header;
           _current_header = null;
+        }
 
-          key = (IRubyObject)runtime.newString(_last_header);
+        key = RubyString.newString(runtime, new ByteList(_last_header, UTF8Encoding.INSTANCE, false));
+        val = headers.op_aref(context, key);
+
+        if (new_field == 1) {
+          if (val.isNil()) {
+            if (header_value_type == arraysSym) {
+              headers.op_aset(context, key, RubyArray.newArrayLight(runtime, RubyString.newStringLight(runtime, 10)));
+            } else {
+              headers.op_aset(context, key, RubyString.newStringLight(runtime, 10));
+            }
+          } else {
+            if (header_value_type == mixedSym) {
+              if (val instanceof RubyString) {
+                headers.op_aset(context, key, RubyArray.newArrayLight(runtime, val, RubyString.newStringLight(runtime, 10)));
+              } else {
+                ((RubyArray)val).add(RubyString.newStringLight(runtime, 10));
+              }
+            } else if (header_value_type == arraysSym) {
+              ((RubyArray)val).add(RubyString.newStringLight(runtime, 10));
+            } else {
+              ((RubyString)val).cat(',').cat(' ');
+            }
+          }
           val = headers.op_aref(context, key);
+        }
 
-          if (!val.isNil())
-            ((RubyString)val).cat(", ".getBytes());
-        } else {
-          key = (IRubyObject)runtime.newString(_last_header);
-          val = headers.op_aref(context, key);
+        if (val instanceof RubyArray) {
+          val = ((RubyArray)val).entry(-1);
         }
 
-        if (val.isNil())
-          headers.op_aset(context, key, runtime.newString(new String(data)));
-        else
-          ((RubyString)val).cat(data);
+        ((RubyString)val).cat(data);
 
         return 0;
       }
@@ -158,15 +185,17 @@ public class RubyHttpParser extends RubyObject {
       public int cb (http_parser.lolevel.HTTPParser p) {
         headers = new RubyHash(runtime);
 
-        requestUrl = runtime.newString("");
-        requestPath = runtime.newString("");
-        queryString = runtime.newString("");
-        fragment = runtime.newString("");
+        requestUrl = RubyString.newEmptyString(runtime);
+        requestPath = RubyString.newEmptyString(runtime);
+        queryString = RubyString.newEmptyString(runtime);
+        fragment = RubyString.newEmptyString(runtime);
+
+        upgradeData = RubyString.newEmptyString(runtime);
 
         IRubyObject ret = runtime.getNil();
 
         if (callback_object != null) {
-          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_message_begin")).toJava(Boolean.class) == Boolean.TRUE) {
+          if (((RubyObject)callback_object).respondsTo("on_message_begin")) {
             ThreadContext context = callback_object.getRuntime().getCurrentContext();
             ret = callback_object.callMethod(context, "on_message_begin");
           }
@@ -175,7 +204,7 @@ public class RubyHttpParser extends RubyObject {
           ret = on_message_begin.callMethod(context, "call");
         }
 
-        if (ret == runtime.newSymbol("stop")) {
+        if (ret == stopSym) {
           throw new StopException();
         } else {
           return 0;
@@ -186,8 +215,10 @@ public class RubyHttpParser extends RubyObject {
       public int cb (http_parser.lolevel.HTTPParser p) {
         IRubyObject ret = runtime.getNil();
 
+        completed = true;
+
         if (callback_object != null) {
-          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_message_complete")).toJava(Boolean.class) == Boolean.TRUE) {
+          if (((RubyObject)callback_object).respondsTo("on_message_complete")) {
             ThreadContext context = callback_object.getRuntime().getCurrentContext();
             ret = callback_object.callMethod(context, "on_message_complete");
           }
@@ -196,7 +227,7 @@ public class RubyHttpParser extends RubyObject {
           ret = on_message_complete.callMethod(context, "call");
         }
 
-        if (ret == runtime.newSymbol("stop")) {
+        if (ret == stopSym) {
           throw new StopException();
         } else {
           return 0;
@@ -208,7 +239,7 @@ public class RubyHttpParser extends RubyObject {
         IRubyObject ret = runtime.getNil();
 
         if (callback_object != null) {
-          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_headers_complete")).toJava(Boolean.class) == Boolean.TRUE) {
+          if (((RubyObject)callback_object).respondsTo("on_headers_complete")) {
             ThreadContext context = callback_object.getRuntime().getCurrentContext();
             ret = callback_object.callMethod(context, "on_headers_complete", headers);
           }
@@ -217,8 +248,10 @@ public class RubyHttpParser extends RubyObject {
           ret = on_headers_complete.callMethod(context, "call", headers);
         }
 
-        if (ret == runtime.newSymbol("stop")) {
+        if (ret == stopSym) {
           throw new StopException();
+        } else if (ret == resetSym) {
+          return 1;
         } else {
           return 0;
         }
@@ -230,16 +263,16 @@ public class RubyHttpParser extends RubyObject {
         byte[] data = fetchBytes(buf, pos, len);
 
         if (callback_object != null) {
-          if (((RubyObject)callback_object).respond_to_p(runtime.newSymbol("on_body")).toJava(Boolean.class) == Boolean.TRUE) {
+          if (((RubyObject)callback_object).respondsTo("on_body")) {
             ThreadContext context = callback_object.getRuntime().getCurrentContext();
-            ret = callback_object.callMethod(context, "on_body", callback_object.getRuntime().newString(new String(data)));
+            ret = callback_object.callMethod(context, "on_body", RubyString.newString(runtime, new ByteList(data, UTF8Encoding.INSTANCE, false)));
           }
         } else if (on_body != null) {
           ThreadContext context = on_body.getRuntime().getCurrentContext();
-          ret = on_body.callMethod(context, "call", on_body.getRuntime().newString(new String(data)));
+          ret = on_body.callMethod(context, "call", RubyString.newString(runtime, new ByteList(data, UTF8Encoding.INSTANCE, false)));
         }
 
-        if (ret == runtime.newSymbol("stop")) {
+        if (ret == stopSym) {
           throw new StopException();
         } else {
           return 0;
@@ -250,12 +283,15 @@ public class RubyHttpParser extends RubyObject {
 
   private void init() {
     this.parser = new HTTPParser();
+    this.parser.HTTP_PARSER_STRICT = true;
     this.headers = null;
 
     this.requestUrl = runtime.getNil();
     this.requestPath = runtime.getNil();
     this.queryString = runtime.getNil();
     this.fragment = runtime.getNil();
+
+    this.upgradeData = runtime.getNil();
   }
 
   @JRubyMethod(name = "initialize")
@@ -269,6 +305,12 @@ public class RubyHttpParser extends RubyObject {
     return initialize();
   }
 
+  @JRubyMethod(name = "initialize")
+  public IRubyObject initialize(IRubyObject arg, IRubyObject arg2) {
+    header_value_type = arg2;
+    return initialize(arg);
+  }
+
   @JRubyMethod(name = "on_message_begin=")
   public IRubyObject set_on_message_begin(IRubyObject cb) {
     on_message_begin = cb;
@@ -296,7 +338,8 @@ public class RubyHttpParser extends RubyObject {
   @JRubyMethod(name = "<<")
   public IRubyObject execute(IRubyObject data) {
     RubyString str = (RubyString)data;
-    ByteBuffer buf = ByteBuffer.wrap(str.getBytes());
+    ByteList byteList = str.getByteList();
+    ByteBuffer buf = ByteBuffer.wrap(byteList.getUnsafeBytes(), byteList.getBegin(), byteList.getRealSize());
     boolean stopped = false;
 
     try {
@@ -308,8 +351,10 @@ public class RubyHttpParser extends RubyObject {
     }
 
     if (parser.getUpgrade()) {
-      // upgrade request
-    } else if (buf.hasRemaining()) {
+      byte[] upData = fetchBytes(buf, buf.position(), buf.limit() - buf.position());
+      ((RubyString)upgradeData).cat(upData);
+
+    } else if (buf.hasRemaining() && !completed) {
       if (!stopped)
         throw new RaiseException(runtime, eParserError, "Could not parse data entirely", true);
     }
@@ -319,12 +364,12 @@ public class RubyHttpParser extends RubyObject {
 
   @JRubyMethod(name = "keep_alive?")
   public IRubyObject shouldKeepAlive() {
-    return parser.shouldKeepAlive() ? runtime.getTrue() : runtime.getFalse();
+    return runtime.newBoolean(parser.shouldKeepAlive());
   }
 
   @JRubyMethod(name = "upgrade?")
   public IRubyObject shouldUpgrade() {
-    return parser.getUpgrade() ? runtime.getTrue() : runtime.getFalse();
+    return runtime.newBoolean(parser.getUpgrade());
   }
 
   @JRubyMethod(name = "http_major")
@@ -394,6 +439,26 @@ public class RubyHttpParser extends RubyObject {
     return fragment == null ? runtime.getNil() : fragment;
   }
 
+  @JRubyMethod(name = "header_value_type")
+  public IRubyObject getHeaderValueType() {
+    return header_value_type == null ? runtime.getNil() : header_value_type;
+  }
+
+  @JRubyMethod(name = "header_value_type=")
+  public IRubyObject set_header_value_type(IRubyObject val) {
+    String valString = val.toString();
+    if (valString != "mixed" && valString != "arrays" && valString != "strings") {
+      throw runtime.newArgumentError("Invalid header value type");
+    }
+    header_value_type = val;
+    return val;
+  }
+
+  @JRubyMethod(name = "upgrade_data")
+  public IRubyObject upgradeData() {
+    return upgradeData == null ? runtime.getNil() : upgradeData;
+  }
+
   @JRubyMethod(name = "reset!")
   public IRubyObject reset() {
     init();