http_parser.rb 0.5.3 → 0.6.0.beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. data/.gitmodules +3 -3
  2. data/Gemfile +1 -1
  3. data/Gemfile.lock +9 -2
  4. data/README.md +50 -45
  5. data/bench/standalone.rb +23 -0
  6. data/bench/thin.rb +1 -0
  7. data/ext/ruby_http_parser/org/ruby_http_parser/RubyHttpParser.java +66 -58
  8. data/ext/ruby_http_parser/ruby_http_parser.c +10 -41
  9. data/ext/ruby_http_parser/vendor/http-parser-java/AUTHORS +32 -0
  10. data/ext/ruby_http_parser/vendor/http-parser-java/LICENSE-MIT +5 -1
  11. data/ext/ruby_http_parser/vendor/http-parser-java/README.md +133 -1
  12. data/ext/ruby_http_parser/vendor/http-parser-java/TODO +6 -0
  13. data/ext/ruby_http_parser/vendor/http-parser-java/http_parser.c +1029 -615
  14. data/ext/ruby_http_parser/vendor/http-parser-java/http_parser.gyp +79 -0
  15. data/ext/ruby_http_parser/vendor/http-parser-java/http_parser.h +177 -43
  16. data/ext/ruby_http_parser/vendor/http-parser-java/src/Http-parser.java.iml +22 -0
  17. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/FieldData.java +41 -0
  18. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/HTTPHeadersCompleteCallback.java +13 -0
  19. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/HTTPMethod.java +4 -1
  20. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/HTTPParserUrl.java +76 -0
  21. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/ParserSettings.java +2 -2
  22. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/Util.java +6 -6
  23. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/lolevel/HTTPHeadersCompleteCallback.java +12 -0
  24. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/lolevel/HTTPParser.java +715 -637
  25. data/ext/ruby_http_parser/vendor/http-parser-java/src/impl/http_parser/lolevel/ParserSettings.java +1 -1
  26. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Message.java +71 -21
  27. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/ParseUrl.java +51 -0
  28. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Requests.java +1 -1
  29. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Responses.java +1 -0
  30. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Test.java +2 -1
  31. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/TestHeaderOverflowError.java +1 -0
  32. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/TestLoaderNG.java +6 -17
  33. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/TestNoOverflowLongBody.java +1 -0
  34. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/UnitTest.java +1 -0
  35. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Upgrade.java +1 -0
  36. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Url.java +127 -0
  37. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/Util.java +80 -9
  38. data/ext/ruby_http_parser/vendor/http-parser-java/src/test/http_parser/lolevel/WrongContentLength.java +2 -1
  39. data/ext/ruby_http_parser/vendor/http-parser-java/test.c +1141 -210
  40. data/ext/ruby_http_parser/vendor/http-parser-java/tests.dumped +230 -71
  41. data/ext/ruby_http_parser/vendor/http-parser/AUTHORS +32 -0
  42. data/ext/ruby_http_parser/vendor/http-parser/LICENSE-MIT +5 -1
  43. data/ext/ruby_http_parser/vendor/http-parser/README.md +9 -2
  44. data/ext/ruby_http_parser/vendor/http-parser/http_parser.c +1029 -615
  45. data/ext/ruby_http_parser/vendor/http-parser/http_parser.gyp +79 -0
  46. data/ext/ruby_http_parser/vendor/http-parser/http_parser.h +145 -16
  47. data/ext/ruby_http_parser/vendor/http-parser/test.c +1065 -141
  48. data/http_parser.rb.gemspec +3 -1
  49. data/spec/parser_spec.rb +41 -17
  50. data/spec/support/requests.json +236 -24
  51. data/spec/support/responses.json +182 -36
  52. data/tasks/compile.rake +2 -2
  53. data/tasks/fixtures.rake +7 -1
  54. metadata +57 -19
  55. data/ext/ruby_http_parser/vendor/http-parser-java/compile +0 -1
  56. data/ext/ruby_http_parser/vendor/http-parser-java/test_permutations +0 -1
  57. data/ext/ruby_http_parser/vendor/http-parser-java/test_unit +0 -1
  58. data/ext/ruby_http_parser/vendor/http-parser-java/test_utf8 +0 -1
@@ -10,7 +10,7 @@ public class ParserSettings {
10
10
  public HTTPDataCallback on_fragment;
11
11
  public HTTPDataCallback on_header_field;
12
12
  public HTTPDataCallback on_header_value;
13
- public HTTPCallback on_headers_complete;
13
+ public HTTPHeadersCompleteCallback on_headers_complete;
14
14
  public HTTPDataCallback on_body;
15
15
  public HTTPCallback on_message_complete;
16
16
  public HTTPErrorCallback on_error;
@@ -5,11 +5,13 @@ import java.io.*;
5
5
  import java.util.*;
6
6
 
7
7
  import http_parser.HTTPMethod;
8
+ import http_parser.HTTPParserUrl;
8
9
  import http_parser.ParserType;
9
- import http_parser.lolevel.TestLoaderNG.TestSettings;
10
10
  import http_parser.lolevel.TestLoaderNG.Header;
11
11
  import http_parser.lolevel.TestLoaderNG.LastHeader;
12
12
 
13
+ import primitive.collection.ByteList;
14
+
13
15
  import static http_parser.lolevel.Util.str;
14
16
 
15
17
  public class Message {
@@ -30,7 +32,10 @@ public class Message {
30
32
  List<Header> headers;
31
33
  boolean should_keep_alive;
32
34
 
33
- boolean upgrade;
35
+ byte[] upgrade;
36
+ boolean upgrade() {
37
+ return null != upgrade;
38
+ }
34
39
 
35
40
  int http_major;
36
41
  int http_minor;
@@ -49,6 +54,7 @@ public class Message {
49
54
 
50
55
  public String toString() {
51
56
  StringBuilder b = new StringBuilder();
57
+ b.append("NAME: "); b.append(name);b.append("\n");
52
58
  b.append("type: "); b.append(type);b.append("\n");
53
59
  b.append("method: "); b.append(method);b.append("\n");
54
60
  b.append("status_code: "); b.append(status_code);b.append("\n");
@@ -97,9 +103,12 @@ public class Message {
97
103
  // //throw new RuntimeException(name);
98
104
  // }
99
105
  // }
100
- String str = str(b, pos, len);
101
- String prev_val = settings.map.get(mes);
102
- settings.map.put(mes, prev_val + str);
106
+ //String str = str(b, pos, len);
107
+ ByteList list = settings.map.get(mes);
108
+ for (int i=0; i!=len; ++i) {
109
+ list.add(b.get(pos+i));
110
+ }
111
+ //settings.map.put(mes, prev_val + str);
103
112
  //check(value.equals(str), "incorrect "+mes+": "+str);
104
113
  if (-1 == pos) {
105
114
  throw new RuntimeException("he?");
@@ -118,8 +127,13 @@ public class Message {
118
127
 
119
128
 
120
129
  p.execute(s, buf);
130
+ if (!p.upgrade) {
131
+ // call execute again, else parser can't know message is done
132
+ // if no content length is set.
133
+ p.execute(s, buf);
134
+ }
121
135
  if (!s.success) {
122
- throw new RuntimeException("Test: "+name+"failed");
136
+ throw new RuntimeException("Test: "+name+" failed");
123
137
  }
124
138
  } // execute
125
139
 
@@ -134,7 +148,7 @@ public class Message {
134
148
  */
135
149
  p(name);
136
150
  for (int i = 2; i != raw.length; ++i) {
137
- // p(i);
151
+ // p(i);
138
152
  HTTPParser p = new HTTPParser();
139
153
  TestSettings s = settings();
140
154
  ByteBuffer buf = ByteBuffer.wrap(raw);
@@ -142,13 +156,24 @@ public class Message {
142
156
  buf.limit(i);
143
157
 
144
158
  parse(p,s,buf);
159
+ if (!p.upgrade) {
160
+ buf.position(i);
161
+ buf.limit(olimit);
145
162
 
146
- buf.position(i);
147
- buf.limit(olimit);
148
-
149
- parse(p,s,buf);
150
- parse(p,s,buf);
151
-
163
+ parse(p,s,buf);
164
+ if (!p.upgrade) {
165
+ parse(p,s,buf);
166
+ } else {
167
+ if (!upgrade()) {
168
+ throw new RuntimeException("Test:"+name+"parsed as upgrade, is not");
169
+ }
170
+ }
171
+
172
+ } else {
173
+ if (!upgrade()) {
174
+ throw new RuntimeException("Test:"+name+"parsed as upgrade, is not");
175
+ }
176
+ }
152
177
  if (!s.success) {
153
178
  p(this);
154
179
  throw new RuntimeException("Test: "+name+" failed");
@@ -164,10 +189,7 @@ public class Message {
164
189
 
165
190
  TestSettings settings() {
166
191
  final TestSettings s = new TestSettings();
167
- s.on_path = getCB(request_path, "path", s);
168
- s.on_query_string = getCB(query_string, "query_string", s);
169
192
  s.on_url = getCB(request_url, "url", s);
170
- s.on_fragment = getCB(fragment, "fragment", s);
171
193
  s.on_message_begin = new HTTPCallback() {
172
194
  public int cb (HTTPParser p) {
173
195
  message_begin_called = true;
@@ -206,13 +228,29 @@ public class Message {
206
228
  return 0;
207
229
  }
208
230
  };
209
- s.on_headers_complete = new HTTPCallback() {
231
+ s.on_headers_complete = new HTTPHeadersCompleteCallback() {
210
232
  public int cb (HTTPParser p) {
211
233
  headers_complete_called = true;
212
- String parsed_path = s.map.get("path");
213
- String parsed_query = s.map.get("query_string");
214
- String parsed_url = s.map.get("url");
215
- String parsed_frag = s.map.get("fragment");
234
+ String parsed_path = null;
235
+ String parsed_query = null;
236
+ String parsed_url = null;
237
+ String parsed_frag = null;
238
+
239
+ try {
240
+ parsed_url = new String(s.map.get("url").toArray(), "UTF8");
241
+
242
+ HTTPParserUrl u = new HTTPParserUrl();
243
+ HTTPParser pp = new HTTPParser();
244
+ ByteBuffer data = Util.buffer(parsed_url);
245
+ pp.parse_url(data,false, u);
246
+
247
+ parsed_path = u.getFieldValue(HTTPParser.UrlFields.UF_PATH, data);
248
+ parsed_query = u.getFieldValue(HTTPParser.UrlFields.UF_QUERY, data);
249
+ parsed_frag = u.getFieldValue(HTTPParser.UrlFields.UF_FRAGMENT, data);
250
+
251
+ } catch (java.io.UnsupportedEncodingException uee) {
252
+ throw new RuntimeException(uee);
253
+ }
216
254
 
217
255
  if (!request_path.equals(parsed_path)) {
218
256
  throw new RuntimeException(name+": invalid path: "+parsed_path+" should be: "+request_path);
@@ -321,4 +359,16 @@ public class Message {
321
359
  static void p(Object o) {
322
360
  System.out.println(o);
323
361
  }
362
+
363
+ static class TestSettings extends ParserSettings {
364
+ public boolean success;
365
+ Map<String, ByteList> map;
366
+ TestSettings () {
367
+ map = new HashMap<String, ByteList>();
368
+ map.put("path", new ByteList());
369
+ map.put("query_string", new ByteList());
370
+ map.put("url", new ByteList());
371
+ map.put("fragment", new ByteList());
372
+ }
373
+ }
324
374
  }
@@ -0,0 +1,51 @@
1
+ package http_parser.lolevel;
2
+
3
+ import http_parser.HTTPParserUrl;
4
+ import static http_parser.lolevel.Util.*;
5
+
6
+ public class ParseUrl {
7
+ public static void test(int i) {
8
+ HTTPParserUrl u = new HTTPParserUrl();
9
+ HTTPParser p = new HTTPParser();
10
+ Url test = Url.URL_TESTS[i];
11
+ // System.out.println(":: " + test.name);
12
+ int rv = p.parse_url(Util.buffer(test.url),test.is_connect,u);
13
+ UnitTest.check_equals(rv, test.rv);
14
+ if(test.rv == 0){
15
+ UnitTest.check_equals(u, test.u);
16
+ }
17
+
18
+ }
19
+ public static void test() {
20
+ p(ParseUrl.class);
21
+
22
+ for (int i = 0; i < Url.URL_TESTS.length; i++) {
23
+ test(i);
24
+ }
25
+ }
26
+
27
+ static void usage() {
28
+ p("usage: [jre] http_parser.lolevel.ParseUrl [i]");
29
+ p(" i : optional test case id");
30
+ p("---------------------------------------------");
31
+ p("Test Cases:");
32
+ for (int i =0; i!= Url.URL_TESTS.length; ++i) {
33
+ p(" "+i+": "+Url.URL_TESTS[i].name);
34
+ }
35
+ }
36
+
37
+ public static void main (String [] args) {
38
+ if (0 == args.length) {
39
+ test();
40
+ } else {
41
+ try {
42
+ int i = Integer.parseInt(args[0]);
43
+ test(i);
44
+ } catch (Throwable t) {
45
+ t.printStackTrace();
46
+ usage();
47
+ }
48
+
49
+ }
50
+ }
51
+ }
@@ -35,7 +35,7 @@ public class Requests {
35
35
  }
36
36
 
37
37
  public static void test () {
38
-
38
+ p(Requests.class);
39
39
  simple_tests();
40
40
 
41
41
  List<Message> all = TestLoaderNG.load("tests.dumped");
@@ -13,6 +13,7 @@ public class Responses {
13
13
 
14
14
 
15
15
  public static void test () {
16
+ p(Responses.class);
16
17
  List<Message> all = TestLoaderNG.load("tests.dumped");
17
18
  List<Message> responses = new LinkedList<Message>();
18
19
  for (Message m : all) {
@@ -7,7 +7,8 @@ public class Test {
7
7
  TestHeaderOverflowError.test();
8
8
  TestNoOverflowLongBody.test();
9
9
  Responses.test();
10
- //Requests.test();
10
+ ParseUrl.test();
11
+ Requests.test();
11
12
  Upgrade.test();
12
13
  WrongContentLength.test();
13
14
  }
@@ -39,6 +39,7 @@ public class TestHeaderOverflowError {
39
39
  }
40
40
 
41
41
  public static void test () {
42
+ p(TestHeaderOverflowError.class);
42
43
  test(http_parser.ParserType.HTTP_REQUEST);
43
44
  test(http_parser.ParserType.HTTP_RESPONSE);
44
45
  }
@@ -99,7 +99,7 @@ public class TestLoaderNG {
99
99
  }
100
100
  else if ("should_keep_alive".equals(key))
101
101
  {curr.should_keep_alive = (1 == Integer.parseInt(value));}
102
- else if ("upgrade".equals(key)) {curr.upgrade = (1 == Integer.parseInt(value));}
102
+ else if ("upgrade".equals(key)) { curr.upgrade = toByteArray(value);}
103
103
  else if ("http_major".equals(key)) {curr.http_major = Integer.parseInt(value);}
104
104
  else if ("http_minor".equals(key)) {curr.http_minor = Integer.parseInt(value);}
105
105
  } else {
@@ -189,12 +189,12 @@ public class TestLoaderNG {
189
189
  // for (int i =0; i!= t.raw.length; ++i) {
190
190
  // p(i+":"+t.raw[i]);
191
191
  // }
192
- try {
192
+ // try {
193
193
  t.execute_permutations();
194
- } catch (Throwable th) {
195
- p("failed: "+t.name);
196
- }
197
- // t.execute();
194
+ // } catch (Throwable th) {
195
+ // p("failed: "+t.name);
196
+ // }
197
+ t.execute();
198
198
  // System.exit(0);
199
199
  }
200
200
  }
@@ -209,15 +209,4 @@ public class TestLoaderNG {
209
209
  ,VALUE
210
210
  }
211
211
 
212
- static class TestSettings extends ParserSettings {
213
- public boolean success;
214
- Map<String, String> map;
215
- TestSettings () {
216
- map = new HashMap<String, String>();
217
- map.put("path", "");
218
- map.put("query_string", "");
219
- map.put("url", "");
220
- map.put("fragment", "");
221
- }
222
- }
223
212
  }
@@ -50,6 +50,7 @@ public class TestNoOverflowLongBody {
50
50
  }
51
51
 
52
52
  public static void test () {
53
+ p(TestNoOverflowLongBody.class);
53
54
  test(http_parser.ParserType.HTTP_REQUEST, 1000);
54
55
  test(http_parser.ParserType.HTTP_REQUEST, 100000);
55
56
  test(http_parser.ParserType.HTTP_RESPONSE, 1000);
@@ -110,6 +110,7 @@ public class UnitTest {
110
110
 
111
111
 
112
112
  public static void test () {
113
+ p(UnitTest.class);
113
114
  testErrorFormat();
114
115
  testErrorCallback();
115
116
  }
@@ -13,6 +13,7 @@ public class Upgrade {
13
13
  "Upgrade: WebSocket\r\n\r\n" +
14
14
  "third key data";
15
15
  static void test () {
16
+ p(Upgrade.class);
16
17
  HTTPParser parser = new HTTPParser(ParserType.HTTP_REQUEST);
17
18
  ByteBuffer buf = buffer(upgrade);
18
19
 
@@ -0,0 +1,127 @@
1
+ package http_parser.lolevel;
2
+
3
+ import http_parser.FieldData;
4
+ import http_parser.HTTPParserUrl;
5
+
6
+ import static http_parser.HTTPParserUrl.*;
7
+ import static http_parser.lolevel.HTTPParser.*;
8
+
9
+ /**
10
+ */
11
+ public class Url {
12
+
13
+ public static Url[] URL_TESTS = new Url[]{
14
+ new Url("proxy request", "http://hostname/", false,
15
+ new HTTPParserUrl(
16
+ (1 << UrlFields.UF_SCHEMA.getIndex()) | (1 << UrlFields.UF_HOST.getIndex()) | (1 << UrlFields.UF_PATH.getIndex()),
17
+ 0,
18
+ new FieldData[]{
19
+ new FieldData(0,4),
20
+ new FieldData(7,8),
21
+ new FieldData(0,0),
22
+ new FieldData(15,1),
23
+ new FieldData(0,0),
24
+ new FieldData(0,0)
25
+ }),
26
+ 0),
27
+ new Url("CONNECT request", "hostname:443", true,
28
+ new HTTPParserUrl(
29
+ (1 << UrlFields.UF_HOST.getIndex()) | (1 << UrlFields.UF_PORT.getIndex()),
30
+ 443,
31
+ new FieldData[]{
32
+ new FieldData(0,0),
33
+ new FieldData(0,8),
34
+ new FieldData(9,3),
35
+ new FieldData(0,0),
36
+ new FieldData(0,0),
37
+ new FieldData(0,0)
38
+ }),
39
+ 0),
40
+ new Url("proxy ipv6 request", "http://[1:2::3:4]/", false,
41
+ new HTTPParserUrl(
42
+ (1 << UrlFields.UF_SCHEMA.getIndex()) | (1 << UrlFields.UF_HOST.getIndex()) | (1 << UrlFields.UF_PATH.getIndex()),
43
+ 0,
44
+ new FieldData[]{
45
+ new FieldData(0,4),
46
+ new FieldData(8,8),
47
+ new FieldData(0,0),
48
+ new FieldData(17,1),
49
+ new FieldData(0,0),
50
+ new FieldData(0,0)
51
+ }),
52
+ 0),
53
+ new Url("CONNECT ipv6 address", "[1:2::3:4]:443", true,
54
+ new HTTPParserUrl(
55
+ (1 << UrlFields.UF_HOST.getIndex()) | (1 << UrlFields.UF_PORT.getIndex()),
56
+ 443,
57
+ new FieldData[]{
58
+ new FieldData(0,0),
59
+ new FieldData(1,8),
60
+ new FieldData(11,3),
61
+ new FieldData(0,0),
62
+ new FieldData(0,0),
63
+ new FieldData(0,0)
64
+ }),
65
+ 0),
66
+ new Url("extra ? in query string",
67
+ "http://a.tbcdn.cn/p/fp/2010c/??fp-header-min.css,fp-base-min.css,fp-channel-min.css,fp-product-min.css,fp-mall-min.css,fp-category-min.css,fp-sub-min.css,fp-gdp4p-min.css,fp-css3-min.css,fp-misc-min.css?t=20101022.css",
68
+ false,
69
+ new HTTPParserUrl(
70
+ (1 << UrlFields.UF_SCHEMA.getIndex()) |
71
+ (1 << UrlFields.UF_HOST.getIndex()) |
72
+ (1 << UrlFields.UF_PATH.getIndex()) |
73
+ (1 << UrlFields.UF_QUERY.getIndex()),
74
+ 0,
75
+ new FieldData[]{
76
+ new FieldData(0,4),
77
+ new FieldData(7,10),
78
+ new FieldData(0,0),
79
+ new FieldData(17,12),
80
+ new FieldData(30,187),
81
+ new FieldData(0,0)
82
+ }),
83
+ 0),
84
+ new Url("proxy empty host",
85
+ "http://:443/",
86
+ false,
87
+ null,
88
+ 1),
89
+ new Url("proxy empty port",
90
+ "http://hostname:/",
91
+ false,
92
+ null,
93
+ 1),
94
+ new Url("CONNECT empty host",
95
+ ":443",
96
+ true,
97
+ null,
98
+ 1),
99
+ new Url("CONNECT empty port",
100
+ "hostname:",
101
+ true,
102
+ null,
103
+ 1),
104
+ new Url("CONNECT with extra bits",
105
+ "hostname:443/",
106
+ true,
107
+ null,
108
+ 1),
109
+
110
+ };
111
+
112
+ String name;
113
+ String url;
114
+ boolean is_connect;
115
+ HTTPParserUrl u;
116
+ int rv;
117
+
118
+ public Url(String name, String url, boolean is_connect, HTTPParserUrl u, int rv) {
119
+ this.name = name;
120
+ this.url = url;
121
+ this.is_connect = is_connect;
122
+ this.u = u;
123
+ this.rv = rv;
124
+ }
125
+
126
+
127
+ }