http_tools 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +6 -3
- data/lib/http_tools/builder.rb +2 -3
- data/lib/http_tools/encoding.rb +8 -8
- data/lib/http_tools/parser.rb +40 -8
- data/test/parser/request_test.rb +7 -6
- data/test/parser/response_test.rb +150 -1
- metadata +2 -2
data/README.rdoc
CHANGED
@@ -12,9 +12,12 @@ Written purely in Ruby, with no dependencies outside of the standard library, it
|
|
12
12
|
should run across all Ruby implementations compatible with 1.8 or later, and
|
13
13
|
install in environments without a compiler available.
|
14
14
|
|
15
|
-
Tests are currently run on travis-ci[http://travis-ci.org/]
|
15
|
+
Tests are currently run on travis-ci[http://travis-ci.org/matsadler/http_tools]
|
16
|
+
against Ruby 1.8.6, 1.8.7, 1.9.2, 1.9.3, JRuby, Rubinius, Rubinius 2, and Ruby
|
17
|
+
Enterprise Edition. Additionally tests are run against MacRuby 0.10
|
16
18
|
|
17
|
-
Performance tuning is mainly aimed at Ruby 1.9, with Ruby 1.8 and JRuby taken in
|
19
|
+
Performance tuning is mainly aimed at Ruby 1.9, with Ruby 1.8 and JRuby taken in
|
20
|
+
to consideration. JRuby is generally fastest.
|
18
21
|
|
19
22
|
== HTTPTools::Parser
|
20
23
|
|
@@ -62,7 +65,7 @@ responses. It can be used as a mixin or class methods on HTTPTools::Builder.
|
|
62
65
|
|
63
66
|
=== Example
|
64
67
|
|
65
|
-
Builder.request(:get, "example.com")
|
68
|
+
Builder.request(:get, "example.com")
|
66
69
|
#=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
|
67
70
|
|
68
71
|
== Licence
|
data/lib/http_tools/builder.rb
CHANGED
@@ -31,9 +31,8 @@ module HTTPTools
|
|
31
31
|
# Builder.request(:get, "example.com")
|
32
32
|
# #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
|
33
33
|
#
|
34
|
-
# Builder.request(:post, "example.com", "/
|
35
|
-
|
36
|
-
#=> "POST" /form HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
|
34
|
+
# Builder.request(:post, "example.com", "/s", "Accept" => "text/html")
|
35
|
+
# #=> "POST /s HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
|
37
36
|
#
|
38
37
|
def request(method, host, path="/", headers={})
|
39
38
|
"#{method.to_s.upcase} #{path} HTTP/1.1\r\nHost: #{host}\r\n#{
|
data/lib/http_tools/encoding.rb
CHANGED
@@ -109,27 +109,27 @@ module HTTPTools
|
|
109
109
|
# the decoded response and nil.
|
110
110
|
# Example:
|
111
111
|
# encoded_string = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n"
|
112
|
-
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
113
|
-
#
|
112
|
+
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
113
|
+
# #=> ["foobar", nil]
|
114
114
|
#
|
115
115
|
# Decoding a partial response will return an array of the response decoded
|
116
116
|
# so far, and the remainder of the encoded string.
|
117
117
|
# Example
|
118
118
|
# encoded_string = "3\r\nfoo\r\n3\r\nba"
|
119
|
-
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
120
|
-
#
|
119
|
+
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
120
|
+
# #=> ["foo", "3\r\nba"]
|
121
121
|
#
|
122
122
|
# If the chunks are complete, but there is no empty terminating chunk, the
|
123
123
|
# second element in the array will be an empty string.
|
124
124
|
# encoded_string = "3\r\nfoo\r\n3\r\nbar"
|
125
|
-
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
126
|
-
#
|
125
|
+
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
126
|
+
# #=> ["foobar", ""]
|
127
127
|
#
|
128
128
|
# If nothing can be decoded the first element in the array will be nil and
|
129
129
|
# the second the remainder
|
130
130
|
# encoded_string = "3\r\nfo"
|
131
|
-
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
132
|
-
#
|
131
|
+
# Encoding.transfer_encoding_chunked_decode(encoded_string)
|
132
|
+
# #=> [nil, "3\r\nfo"]
|
133
133
|
#
|
134
134
|
# Example use:
|
135
135
|
# include Encoding
|
data/lib/http_tools/parser.rb
CHANGED
@@ -31,6 +31,7 @@ module HTTPTools
|
|
31
31
|
# :stopdoc:
|
32
32
|
EMPTY = "".freeze
|
33
33
|
COLON = ":".freeze
|
34
|
+
SPACE = " ".freeze
|
34
35
|
KEY_TERMINATOR = ": ".freeze
|
35
36
|
CONTENT_LENGTH = "Content-Length".freeze
|
36
37
|
TRANSFER_ENCODING = "Transfer-Encoding".freeze
|
@@ -124,10 +125,15 @@ module HTTPTools
|
|
124
125
|
#
|
125
126
|
# Returns a Rack compatible environment hash. Will return nil if called
|
126
127
|
# before headers are complete.
|
128
|
+
#
|
129
|
+
# "SERVER_NAME" and "SERVER_PORT" are only supplied if they can be
|
130
|
+
# determined from the request (eg, they are present in the "Host" header).
|
127
131
|
#
|
128
132
|
# "rack.input" is only supplied if #env is called after parsing the request
|
129
|
-
# has finsished, and no listener is set for the `stream` event
|
130
|
-
#
|
133
|
+
# has finsished, and no listener is set for the `stream` event
|
134
|
+
#
|
135
|
+
# If not supplied, you must ensure "SERVER_NAME", "SERVER_PORT", and
|
136
|
+
# "rack.input" are present to make the environment hash fully Rack compliant
|
131
137
|
#
|
132
138
|
def env
|
133
139
|
return unless @header_complete
|
@@ -141,8 +147,8 @@ module HTTPTools
|
|
141
147
|
env[upper_key.freeze] = value
|
142
148
|
end
|
143
149
|
host, port = env[HTTP_HOST].split(COLON) if env.key?(HTTP_HOST)
|
144
|
-
env[SERVER_NAME] = host
|
145
|
-
env[SERVER_PORT] = port
|
150
|
+
env[SERVER_NAME] = host if host
|
151
|
+
env[SERVER_PORT] = port if port
|
146
152
|
@trailer.each {|k, val| env[HTTP_ + k.tr(LOWERCASE, UPPERCASE)] = val}
|
147
153
|
if @body || @stream_callback == method(:setup_stream_callback)
|
148
154
|
env[RACK_INPUT] = StringIO.new(@body || "")
|
@@ -409,7 +415,7 @@ module HTTPTools
|
|
409
415
|
def value
|
410
416
|
value = @buffer.scan(/[^\x00\n\x7f]*\n/i)
|
411
417
|
if value
|
412
|
-
value.
|
418
|
+
value.strip!
|
413
419
|
if @header.key?(@last_key)
|
414
420
|
@header[@last_key] << "\n#{value}"
|
415
421
|
else
|
@@ -420,7 +426,7 @@ module HTTPTools
|
|
420
426
|
elsif TRANSFER_ENCODING.casecmp(@last_key) == 0
|
421
427
|
@chunked = CHUNKED.casecmp(value) == 0
|
422
428
|
end
|
423
|
-
|
429
|
+
value_extention
|
424
430
|
elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
|
425
431
|
:value
|
426
432
|
else
|
@@ -428,6 +434,19 @@ module HTTPTools
|
|
428
434
|
end
|
429
435
|
end
|
430
436
|
|
437
|
+
def value_extention
|
438
|
+
if @buffer.check(/[^ \t]/)
|
439
|
+
key_or_newline
|
440
|
+
elsif value_extra = @buffer.scan(/[ \t]+[^\x00\n\x7f]*\n/)
|
441
|
+
value_extra.sub!(/^[ \t]+/, SPACE)
|
442
|
+
value_extra.chop!
|
443
|
+
(@header[@last_key] << value_extra).strip!
|
444
|
+
value_extention
|
445
|
+
else
|
446
|
+
:value_extention
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
431
450
|
def start_body
|
432
451
|
if @request_method && !(@content_left || @chunked) ||
|
433
452
|
NO_BODY.key?(@status_code) || @force_no_body
|
@@ -532,13 +551,13 @@ module HTTPTools
|
|
532
551
|
def trailer_value
|
533
552
|
value = @buffer.scan(/[^\000\n\177]+\n/i)
|
534
553
|
if value
|
535
|
-
value.
|
554
|
+
value.strip!
|
536
555
|
if @trailer.key?(@last_key)
|
537
556
|
@trailer[@last_key] << "\n#{value}"
|
538
557
|
else
|
539
558
|
@trailer[@last_key.freeze] = value
|
540
559
|
end
|
541
|
-
|
560
|
+
trailer_value_extention
|
542
561
|
elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
|
543
562
|
:trailer_value
|
544
563
|
else
|
@@ -546,6 +565,19 @@ module HTTPTools
|
|
546
565
|
end
|
547
566
|
end
|
548
567
|
|
568
|
+
def trailer_value_extention
|
569
|
+
if @buffer.check(/[^ \t]/)
|
570
|
+
trailer_key_or_newline
|
571
|
+
elsif value_extra = @buffer.scan(/[ \t]+[^\x00\n\x7f]*\n/)
|
572
|
+
value_extra.sub!(/^[ \t]+/, SPACE)
|
573
|
+
value_extra.chop!
|
574
|
+
(@trailer[@last_key] << value_extra).strip!
|
575
|
+
trailer_value_extention
|
576
|
+
else
|
577
|
+
:trailer_value_extention
|
578
|
+
end
|
579
|
+
end
|
580
|
+
|
549
581
|
def end_of_message
|
550
582
|
raise EndOfMessageError.new("Message ended") if @state == :end_of_message
|
551
583
|
@finish_callback.call if @finish_callback
|
data/test/parser/request_test.rb
CHANGED
@@ -589,7 +589,7 @@ class ParserRequestTest < Test::Unit::TestCase
|
|
589
589
|
assert_equal("/test", env["PATH_INFO"])
|
590
590
|
assert_equal("q=foo", env["QUERY_STRING"])
|
591
591
|
assert_equal("www.example.com", env["SERVER_NAME"])
|
592
|
-
|
592
|
+
assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
|
593
593
|
assert_equal("www.example.com", env["HTTP_HOST"])
|
594
594
|
assert_equal("text/html", env["HTTP_ACCEPT"])
|
595
595
|
|
@@ -623,7 +623,7 @@ class ParserRequestTest < Test::Unit::TestCase
|
|
623
623
|
assert_equal("/submit", env["PATH_INFO"])
|
624
624
|
assert_equal("", env["QUERY_STRING"])
|
625
625
|
assert_equal("www.example.com", env["SERVER_NAME"])
|
626
|
-
|
626
|
+
assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
|
627
627
|
assert_equal("www.example.com", env["HTTP_HOST"])
|
628
628
|
assert_equal("chunked", env["HTTP_TRANSFER_ENCODING"])
|
629
629
|
assert_equal("X-Checksum", env["HTTP_TRAILER"])
|
@@ -703,15 +703,16 @@ class ParserRequestTest < Test::Unit::TestCase
|
|
703
703
|
|
704
704
|
def test_env_no_host
|
705
705
|
parser = HTTPTools::Parser.new
|
706
|
+
env = nil
|
706
707
|
|
707
708
|
parser << "GET /test?q=foo HTTP/1.1\r\n\r\n"
|
708
709
|
|
709
710
|
assert_nothing_raised(NoMethodError) do
|
710
|
-
parser.env
|
711
|
+
env = parser.env
|
711
712
|
end
|
712
|
-
|
713
|
-
|
714
|
-
|
713
|
+
assert(!env.key?("HTTP_HOST"), "env must not contain HTTP_HOST")
|
714
|
+
assert(!env.key?("SERVER_NAME"), "env must not contain SERVER_NAME")
|
715
|
+
assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
|
715
716
|
end
|
716
717
|
|
717
718
|
def test_inspect
|
@@ -235,6 +235,73 @@ class ParserResponseTest < Test::Unit::TestCase
|
|
235
235
|
assert_equal({"Set-Cookie" => "foo=bar\nbaz=qux"}, headers)
|
236
236
|
end
|
237
237
|
|
238
|
+
def test_multi_line_header_value
|
239
|
+
parser = HTTPTools::Parser.new
|
240
|
+
headers = nil
|
241
|
+
|
242
|
+
parser.add_listener(:header) {headers = parser.header}
|
243
|
+
|
244
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
245
|
+
parser << "Content-Type: text/html;\r\n"
|
246
|
+
parser << " charset=utf-8\r\n\r\n"
|
247
|
+
|
248
|
+
assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
|
249
|
+
end
|
250
|
+
|
251
|
+
def test_multi_line_header_value_in_one_chunk
|
252
|
+
parser = HTTPTools::Parser.new
|
253
|
+
headers = nil
|
254
|
+
|
255
|
+
parser.add_listener(:header) {headers = parser.header}
|
256
|
+
|
257
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
258
|
+
parser << "Content-Type: text/html;\r\n charset=utf-8\r\n\r\n"
|
259
|
+
|
260
|
+
assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
|
261
|
+
end
|
262
|
+
|
263
|
+
def test_multi_line_header_value_sub_line_chunks
|
264
|
+
parser = HTTPTools::Parser.new
|
265
|
+
headers = nil
|
266
|
+
|
267
|
+
parser.add_listener(:header) {headers = parser.header}
|
268
|
+
|
269
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
270
|
+
parser << "Content-Type:"
|
271
|
+
parser << " text/"
|
272
|
+
parser << "html;\r\n "
|
273
|
+
parser << "charset="
|
274
|
+
parser << "utf-8\r\n\r\n"
|
275
|
+
|
276
|
+
assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
|
277
|
+
end
|
278
|
+
|
279
|
+
def test_header_value_separated_by_newline
|
280
|
+
parser = HTTPTools::Parser.new
|
281
|
+
headers = nil
|
282
|
+
|
283
|
+
parser.add_listener(:header) {headers = parser.header}
|
284
|
+
|
285
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
286
|
+
parser << "Content-Type:\r\n"
|
287
|
+
parser << "\t text/html;\r\n"
|
288
|
+
parser << "\t charset=utf-8\r\n\r\n"
|
289
|
+
|
290
|
+
assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
|
291
|
+
end
|
292
|
+
|
293
|
+
def test_header_value_leading_and_trailing_whitespace_is_stripped
|
294
|
+
parser = HTTPTools::Parser.new
|
295
|
+
headers = nil
|
296
|
+
|
297
|
+
parser.add_listener(:header) {headers = parser.header}
|
298
|
+
|
299
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
300
|
+
parser << "Content-Type:\t text/html \t\r\n\r\n"
|
301
|
+
|
302
|
+
assert_equal({"Content-Type" => "text/html"}, headers)
|
303
|
+
end
|
304
|
+
|
238
305
|
def test_skip_junk_headers_at_end
|
239
306
|
parser = HTTPTools::Parser.new
|
240
307
|
code, message, headers = nil
|
@@ -526,7 +593,7 @@ class ParserResponseTest < Test::Unit::TestCase
|
|
526
593
|
parser << "\r\n"
|
527
594
|
parser << "<h1>Hello world</h1>"
|
528
595
|
|
529
|
-
assert_equal({"Page-Completion-Status" => "Normal
|
596
|
+
assert_equal({"Page-Completion-Status" => "Normal", "Content-Length" => "20"}, headers)
|
530
597
|
assert_equal("<h1>Hello world</h1>", body)
|
531
598
|
assert(parser.finished?, "parser should be finished")
|
532
599
|
end
|
@@ -1021,6 +1088,88 @@ class ParserResponseTest < Test::Unit::TestCase
|
|
1021
1088
|
assert_equal({"X-Test" => "1\n2"}, trailer)
|
1022
1089
|
end
|
1023
1090
|
|
1091
|
+
def test_multi_line_trailer_value
|
1092
|
+
parser = HTTPTools::Parser.new
|
1093
|
+
trailer = nil
|
1094
|
+
|
1095
|
+
parser.add_listener(:trailer) {trailer = parser.trailer}
|
1096
|
+
|
1097
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
1098
|
+
parser << "Transfer-Encoding: chunked\r\n"
|
1099
|
+
parser << "Trailer: X-Test\r\n\r\n"
|
1100
|
+
parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
|
1101
|
+
parser << "X-Test: one\r\n"
|
1102
|
+
parser << " two\r\n\r\n"
|
1103
|
+
|
1104
|
+
assert_equal({"X-Test" => "one two"}, trailer)
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
def test_multi_line_trailer_value_in_one_chunk
|
1108
|
+
parser = HTTPTools::Parser.new
|
1109
|
+
trailer = nil
|
1110
|
+
|
1111
|
+
parser.add_listener(:trailer) {trailer = parser.trailer}
|
1112
|
+
|
1113
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
1114
|
+
parser << "Transfer-Encoding: chunked\r\n"
|
1115
|
+
parser << "Trailer: X-Test\r\n\r\n"
|
1116
|
+
parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
|
1117
|
+
parser << "X-Test: one\r\n two\r\n\r\n"
|
1118
|
+
|
1119
|
+
assert_equal({"X-Test" => "one two"}, trailer)
|
1120
|
+
end
|
1121
|
+
|
1122
|
+
def test_multi_line_trailer_value_sub_line_chunks
|
1123
|
+
parser = HTTPTools::Parser.new
|
1124
|
+
trailer = nil
|
1125
|
+
|
1126
|
+
parser.add_listener(:trailer) {trailer = parser.trailer}
|
1127
|
+
|
1128
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
1129
|
+
parser << "Transfer-Encoding: chunked\r\n"
|
1130
|
+
parser << "Trailer: X-Test\r\n\r\n"
|
1131
|
+
parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
|
1132
|
+
parser << "X-Test:"
|
1133
|
+
parser << " on"
|
1134
|
+
parser << "e\r\n "
|
1135
|
+
parser << "t"
|
1136
|
+
parser << "wo\r\n\r\n"
|
1137
|
+
|
1138
|
+
assert_equal({"X-Test" => "one two"}, trailer)
|
1139
|
+
end
|
1140
|
+
|
1141
|
+
def test_trailer_value_separated_by_newline
|
1142
|
+
parser = HTTPTools::Parser.new
|
1143
|
+
trailer = nil
|
1144
|
+
|
1145
|
+
parser.add_listener(:trailer) {trailer = parser.trailer}
|
1146
|
+
|
1147
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
1148
|
+
parser << "Transfer-Encoding: chunked\r\n"
|
1149
|
+
parser << "Trailer: X-Test\r\n\r\n"
|
1150
|
+
parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
|
1151
|
+
parser << "X-Test:\r\n"
|
1152
|
+
parser << "\t one\r\n"
|
1153
|
+
parser << "\t two\r\n\r\n"
|
1154
|
+
|
1155
|
+
assert_equal({"X-Test" => "one two"}, trailer)
|
1156
|
+
end
|
1157
|
+
|
1158
|
+
def test_trailer_value_leading_and_trailing_whitespace_is_stripped
|
1159
|
+
parser = HTTPTools::Parser.new
|
1160
|
+
trailer = nil
|
1161
|
+
|
1162
|
+
parser.add_listener(:trailer) {trailer = parser.trailer}
|
1163
|
+
|
1164
|
+
parser << "HTTP/1.1 200 OK\r\n"
|
1165
|
+
parser << "Transfer-Encoding: chunked\r\n"
|
1166
|
+
parser << "Trailer: X-Test\r\n\r\n"
|
1167
|
+
parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
|
1168
|
+
parser << "X-Test:\t one \t\r\n\r\n"
|
1169
|
+
|
1170
|
+
assert_equal({"X-Test" => "one"}, trailer)
|
1171
|
+
end
|
1172
|
+
|
1024
1173
|
def test_messed_up_iis_header_style_trailer_1
|
1025
1174
|
parser = HTTPTools::Parser.new
|
1026
1175
|
trailer = nil
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: http_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-09
|
12
|
+
date: 2011-10-09 00:00:00.000000000Z
|
13
13
|
dependencies: []
|
14
14
|
description: A fast-as-possible pure Ruby HTTP parser plus associated lower level
|
15
15
|
utilities to aid working with HTTP and the web.
|