http_tools 0.4.2 → 0.4.3

data/README.rdoc

@@ -12,9 +12,12 @@ Written purely in Ruby, with no dependencies outside of the standard library, it
 should run across all Ruby implementations compatible with 1.8 or later, and
 install in environments without a compiler available.
 
-Tests are currently run on travis-ci[http://travis-ci.org/] against Ruby 1.8.6, 1.8.7, 1.9.2, 1.9.3, JRuby, Rubinius, Rubinius 2, and Ruby Enterprise Edition. Additionally tests are run against MacRuby 0.10
+Tests are currently run on travis-ci[http://travis-ci.org/matsadler/http_tools]
+against Ruby 1.8.6, 1.8.7, 1.9.2, 1.9.3, JRuby, Rubinius, Rubinius 2, and Ruby
+Enterprise Edition. Additionally tests are run against MacRuby 0.10
 
-Performance tuning is mainly aimed at Ruby 1.9, with Ruby 1.8 and JRuby taken in to consideration. JRuby is generally fastest.
+Performance tuning is mainly aimed at Ruby 1.9, with Ruby 1.8 and JRuby taken in
+to consideration. JRuby is generally fastest.
 
 == HTTPTools::Parser
 
@@ -62,7 +65,7 @@ responses. It can be used as a mixin or class methods on HTTPTools::Builder.
 
 === Example
 
-  Builder.request(:get, "example.com")\
+  Builder.request(:get, "example.com")
   #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
 
 == Licence

data/lib/http_tools/builder.rb

@@ -31,9 +31,8 @@ module HTTPTools
     #   Builder.request(:get, "example.com")
     #   #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
     #   
-    #   Builder.request(:post, "example.com", "/form", "Accept" => "text/html")
-    #\
-    #=> "POST" /form HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
+    #   Builder.request(:post, "example.com", "/s", "Accept" => "text/html")
+    #   #=> "POST /s HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
     # 
     def request(method, host, path="/", headers={})
       "#{method.to_s.upcase} #{path} HTTP/1.1\r\nHost: #{host}\r\n#{

data/lib/http_tools/encoding.rb

@@ -109,27 +109,27 @@ module HTTPTools
     # the decoded response and nil.
     # Example:
     #   encoded_string = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n"
-    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
-    # => ["foobar", nil]
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)
+    #   #=> ["foobar", nil]
     # 
     # Decoding a partial response will return an array of the response decoded
     # so far, and the remainder of the encoded string.
     # Example
     #   encoded_string = "3\r\nfoo\r\n3\r\nba"
-    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
-    # => ["foo", "3\r\nba"]
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)
+    #   #=> ["foo", "3\r\nba"]
     # 
     # If the chunks are complete, but there is no empty terminating chunk, the
     # second element in the array will be an empty string.
     #   encoded_string = "3\r\nfoo\r\n3\r\nbar"
-    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
-    # => ["foobar", ""]
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)
+    #   #=> ["foobar", ""]
     # 
     # If nothing can be decoded the first element in the array will be nil and
     # the second the remainder
     #   encoded_string = "3\r\nfo"
-    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
-    # => [nil, "3\r\nfo"]
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)
+    #   #=> [nil, "3\r\nfo"]
     # 
     # Example use:
     #   include Encoding

data/lib/http_tools/parser.rb

@@ -31,6 +31,7 @@ module HTTPTools
     # :stopdoc:
     EMPTY = "".freeze
     COLON = ":".freeze
+    SPACE = " ".freeze
     KEY_TERMINATOR = ": ".freeze
     CONTENT_LENGTH = "Content-Length".freeze
     TRANSFER_ENCODING = "Transfer-Encoding".freeze
@@ -124,10 +125,15 @@ module HTTPTools
     # 
     # Returns a Rack compatible environment hash. Will return nil if called
     # before headers are complete.
+    #
+    # "SERVER_NAME" and "SERVER_PORT" are only supplied if they can be
+    # determined from the request (eg, they are present in the "Host" header).
     # 
     # "rack.input" is only supplied if #env is called after parsing the request
-    # has finsished, and no listener is set for the `stream` event, otherwise
-    # you must add it yourself to make the environment hash fully Rack compliant
+    # has finsished, and no listener is set for the `stream` event
+    # 
+    # If not supplied, you must ensure "SERVER_NAME", "SERVER_PORT", and
+    # "rack.input" are present to make the environment hash fully Rack compliant
     # 
     def env
       return unless @header_complete
@@ -141,8 +147,8 @@ module HTTPTools
         env[upper_key.freeze] = value
       end
       host, port = env[HTTP_HOST].split(COLON) if env.key?(HTTP_HOST)
-      env[SERVER_NAME] = host
-      env[SERVER_PORT] = port || "80"
+      env[SERVER_NAME] = host if host
+      env[SERVER_PORT] = port if port
       @trailer.each {|k, val| env[HTTP_ + k.tr(LOWERCASE, UPPERCASE)] = val}
       if @body || @stream_callback == method(:setup_stream_callback)
         env[RACK_INPUT] = StringIO.new(@body || "")
@@ -409,7 +415,7 @@ module HTTPTools
     def value
       value = @buffer.scan(/[^\x00\n\x7f]*\n/i)
       if value
-        value.chop!
+        value.strip!
         if @header.key?(@last_key)
           @header[@last_key] << "\n#{value}"
         else
@@ -420,7 +426,7 @@ module HTTPTools
         elsif TRANSFER_ENCODING.casecmp(@last_key) == 0
           @chunked = CHUNKED.casecmp(value) == 0
         end
-        key_or_newline
+        value_extention
       elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
         :value
       else
@@ -428,6 +434,19 @@ module HTTPTools
       end
     end
     
+    def value_extention
+      if @buffer.check(/[^ \t]/)
+        key_or_newline
+      elsif value_extra = @buffer.scan(/[ \t]+[^\x00\n\x7f]*\n/)
+        value_extra.sub!(/^[ \t]+/, SPACE)
+        value_extra.chop!
+        (@header[@last_key] << value_extra).strip!
+        value_extention
+      else
+        :value_extention
+      end
+    end
+    
     def start_body
       if @request_method && !(@content_left || @chunked) ||
         NO_BODY.key?(@status_code) || @force_no_body
@@ -532,13 +551,13 @@ module HTTPTools
     def trailer_value
       value = @buffer.scan(/[^\000\n\177]+\n/i)
       if value
-        value.chop!
+        value.strip!
         if @trailer.key?(@last_key)
           @trailer[@last_key] << "\n#{value}"
         else
           @trailer[@last_key.freeze] = value
         end
-        trailer_key_or_newline
+        trailer_value_extention
       elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
         :trailer_value
       else
@@ -546,6 +565,19 @@ module HTTPTools
       end
     end
     
+    def trailer_value_extention
+      if @buffer.check(/[^ \t]/)
+        trailer_key_or_newline
+      elsif value_extra = @buffer.scan(/[ \t]+[^\x00\n\x7f]*\n/)
+        value_extra.sub!(/^[ \t]+/, SPACE)
+        value_extra.chop!
+        (@trailer[@last_key] << value_extra).strip!
+        trailer_value_extention
+      else
+        :trailer_value_extention
+      end
+    end
+    
     def end_of_message
       raise EndOfMessageError.new("Message ended") if @state == :end_of_message
       @finish_callback.call if @finish_callback

data/test/parser/request_test.rb

@@ -589,7 +589,7 @@ class ParserRequestTest < Test::Unit::TestCase
     assert_equal("/test", env["PATH_INFO"])
     assert_equal("q=foo", env["QUERY_STRING"])
     assert_equal("www.example.com", env["SERVER_NAME"])
-    assert_equal("80", env["SERVER_PORT"])
+    assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
     assert_equal("www.example.com", env["HTTP_HOST"])
     assert_equal("text/html", env["HTTP_ACCEPT"])
     
@@ -623,7 +623,7 @@ class ParserRequestTest < Test::Unit::TestCase
     assert_equal("/submit", env["PATH_INFO"])
     assert_equal("", env["QUERY_STRING"])
     assert_equal("www.example.com", env["SERVER_NAME"])
-    assert_equal("80", env["SERVER_PORT"])
+    assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
     assert_equal("www.example.com", env["HTTP_HOST"])
     assert_equal("chunked", env["HTTP_TRANSFER_ENCODING"])
     assert_equal("X-Checksum", env["HTTP_TRAILER"])
@@ -703,15 +703,16 @@ class ParserRequestTest < Test::Unit::TestCase
   
   def test_env_no_host
     parser = HTTPTools::Parser.new
+    env = nil
     
     parser << "GET /test?q=foo HTTP/1.1\r\n\r\n"
     
     assert_nothing_raised(NoMethodError) do
-      parser.env
+      env = parser.env
     end
-    assert_equal(nil, parser.env["HTTP_HOST"])
-    assert_equal(nil, parser.env["SERVER_NAME"])
-    assert_equal("80", parser.env["SERVER_PORT"])
+    assert(!env.key?("HTTP_HOST"), "env must not contain HTTP_HOST")
+    assert(!env.key?("SERVER_NAME"), "env must not contain SERVER_NAME")
+    assert(!env.key?("SERVER_PORT"), "env must not contain SERVER_PORT")
   end
   
   def test_inspect

data/test/parser/response_test.rb

@@ -235,6 +235,73 @@ class ParserResponseTest < Test::Unit::TestCase
     assert_equal({"Set-Cookie" => "foo=bar\nbaz=qux"}, headers)
   end
   
+  def test_multi_line_header_value
+    parser = HTTPTools::Parser.new
+    headers = nil
+    
+    parser.add_listener(:header) {headers = parser.header}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Content-Type: text/html;\r\n"
+    parser << " charset=utf-8\r\n\r\n"
+    
+    assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
+  end
+  
+  def test_multi_line_header_value_in_one_chunk
+    parser = HTTPTools::Parser.new
+    headers = nil
+    
+    parser.add_listener(:header) {headers = parser.header}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Content-Type: text/html;\r\n charset=utf-8\r\n\r\n"
+    
+    assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
+  end
+  
+  def test_multi_line_header_value_sub_line_chunks
+    parser = HTTPTools::Parser.new
+    headers = nil
+    
+    parser.add_listener(:header) {headers = parser.header}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Content-Type:"
+    parser << " text/"
+    parser << "html;\r\n "
+    parser << "charset="
+    parser << "utf-8\r\n\r\n"
+    
+    assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
+  end
+  
+  def test_header_value_separated_by_newline
+    parser = HTTPTools::Parser.new
+    headers = nil
+    
+    parser.add_listener(:header) {headers = parser.header}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Content-Type:\r\n"
+    parser << "\t     text/html;\r\n"
+    parser << "\t     charset=utf-8\r\n\r\n"
+    
+    assert_equal({"Content-Type" => "text/html; charset=utf-8"}, headers)
+  end
+  
+  def test_header_value_leading_and_trailing_whitespace_is_stripped
+    parser = HTTPTools::Parser.new
+    headers = nil
+    
+    parser.add_listener(:header) {headers = parser.header}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Content-Type:\t text/html \t\r\n\r\n"
+    
+    assert_equal({"Content-Type" => "text/html"}, headers)
+  end
+  
   def test_skip_junk_headers_at_end
     parser = HTTPTools::Parser.new
     code, message, headers = nil
@@ -526,7 +593,7 @@ class ParserResponseTest < Test::Unit::TestCase
     parser << "\r\n"
     parser << "<h1>Hello world</h1>"
     
-    assert_equal({"Page-Completion-Status" => "Normal\r", "Content-Length" => "20"}, headers)
+    assert_equal({"Page-Completion-Status" => "Normal", "Content-Length" => "20"}, headers)
     assert_equal("<h1>Hello world</h1>", body)
     assert(parser.finished?, "parser should be finished")
   end
@@ -1021,6 +1088,88 @@ class ParserResponseTest < Test::Unit::TestCase
     assert_equal({"X-Test" => "1\n2"}, trailer)
   end
   
+  def test_multi_line_trailer_value
+    parser = HTTPTools::Parser.new
+    trailer = nil
+    
+    parser.add_listener(:trailer) {trailer = parser.trailer}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Transfer-Encoding: chunked\r\n"
+    parser << "Trailer: X-Test\r\n\r\n"
+    parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
+    parser << "X-Test: one\r\n"
+    parser << " two\r\n\r\n"
+    
+    assert_equal({"X-Test" => "one two"}, trailer)
+  end
+  
+  def test_multi_line_trailer_value_in_one_chunk
+    parser = HTTPTools::Parser.new
+    trailer = nil
+    
+    parser.add_listener(:trailer) {trailer = parser.trailer}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Transfer-Encoding: chunked\r\n"
+    parser << "Trailer: X-Test\r\n\r\n"
+    parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
+    parser << "X-Test: one\r\n two\r\n\r\n"
+    
+    assert_equal({"X-Test" => "one two"}, trailer)
+  end
+  
+  def test_multi_line_trailer_value_sub_line_chunks
+    parser = HTTPTools::Parser.new
+    trailer = nil
+    
+    parser.add_listener(:trailer) {trailer = parser.trailer}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Transfer-Encoding: chunked\r\n"
+    parser << "Trailer: X-Test\r\n\r\n"
+    parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
+    parser << "X-Test:"
+    parser << " on"
+    parser << "e\r\n "
+    parser << "t"
+    parser << "wo\r\n\r\n"
+    
+    assert_equal({"X-Test" => "one two"}, trailer)
+  end
+  
+  def test_trailer_value_separated_by_newline
+    parser = HTTPTools::Parser.new
+    trailer = nil
+    
+    parser.add_listener(:trailer) {trailer = parser.trailer}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Transfer-Encoding: chunked\r\n"
+    parser << "Trailer: X-Test\r\n\r\n"
+    parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
+    parser << "X-Test:\r\n"
+    parser << "\t     one\r\n"
+    parser << "\t     two\r\n\r\n"
+    
+    assert_equal({"X-Test" => "one two"}, trailer)
+  end
+  
+  def test_trailer_value_leading_and_trailing_whitespace_is_stripped
+    parser = HTTPTools::Parser.new
+    trailer = nil
+    
+    parser.add_listener(:trailer) {trailer = parser.trailer}
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    parser << "Transfer-Encoding: chunked\r\n"
+    parser << "Trailer: X-Test\r\n\r\n"
+    parser << "14\r\n<h1>Hello world</h1>\r\n0\r\n"
+    parser << "X-Test:\t one \t\r\n\r\n"
+    
+    assert_equal({"X-Test" => "one"}, trailer)
+  end
+  
   def test_messed_up_iis_header_style_trailer_1
     parser = HTTPTools::Parser.new
     trailer = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http_tools
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-25 00:00:00.000000000 Z
+date: 2011-10-09 00:00:00.000000000Z
 dependencies: []
 description: A fast-as-possible pure Ruby HTTP parser plus associated lower level
   utilities to aid working with HTTP and the web.