http_tools 0.3.0 → 0.4.0

data/lib/http_tools/parser.rb

@@ -1,4 +1,5 @@
 require 'strscan'
+require 'stringio'
 
 module HTTPTools
   
@@ -15,7 +16,7 @@ module HTTPTools
   #     puts parser.status_code + " " + parser.request_method
   #     puts parser.header.inspect
   #   end
-  #   parser.on(:stream) {|chunk| print chunk}
+  #   parser.on(:finish) {print parser.body}
   #   
   #   parser << "HTTP/1.1 200 OK\r\n"
   #   parser << "Content-Length: 20\r\n\r\n"
@@ -27,8 +28,7 @@ module HTTPTools
   #   <h1>Hello world</h1>
   # 
   class Parser
-    include Encoding
-    
+    # :stopdoc:
     COLON = ":".freeze
     KEY_TERMINATOR = ": ".freeze
     CONTENT_LENGTH = "Content-Length".freeze
@@ -37,7 +37,6 @@ module HTTPTools
     CONNECTION = "Connection".freeze
     CLOSE = "close".freeze
     CHUNKED = "chunked".freeze
-    EVENTS = %W{header stream trailer finish error}.map {|e| e.freeze}.freeze
     
     REQUEST_METHOD = "REQUEST_METHOD".freeze
     PATH_INFO = "PATH_INFO".freeze
@@ -45,6 +44,7 @@ module HTTPTools
     SERVER_NAME = "SERVER_NAME".freeze
     SERVER_PORT = "SERVER_PORT".freeze
     HTTP_HOST = "HTTP_HOST".freeze
+    RACK_INPUT = "rack.input".freeze
     
     PROTOTYPE_ENV = {
       "SCRIPT_NAME" => "".freeze,
@@ -59,10 +59,12 @@ module HTTPTools
     LOWERCASE = "a-z-".freeze
     UPPERCASE = "A-Z_".freeze
     NO_HTTP_ = {"CONTENT_LENGTH" => true, "CONTENT_TYPE" => true}
+    # :startdoc:
+    EVENTS = %W{header stream trailer finish error}.map {|e| e.freeze}.freeze
     
     attr_reader :state # :nodoc:
     attr_reader :request_method, :path_info, :query_string, :request_uri,
-      :version, :status_code, :message, :header, :trailer
+      :version, :status_code, :message, :header, :body, :trailer
     
     # Force parser to expect and parse a trailer when Trailer header missing.
     attr_accessor :force_trailer
@@ -79,9 +81,21 @@ module HTTPTools
     # 
     def initialize
       @state = :start
-      @buffer = StringScanner.new("")
+      @buffer = @scanner = StringScanner.new("")
       @header = {}
       @trailer = {}
+      @force_no_body = nil
+      @allow_html_without_header = nil
+      @force_trailer = nil
+      @status_code = nil
+      @content_left = nil
+      @chunked = nil
+      @body = nil
+      @header_callback = nil
+      @stream_callback = method(:setup_stream_callback)
+      @trailer_callback = nil
+      @finish_callback = nil
+      @error_callback = nil
     end
     
     # :call-seq: parser.concat(data) -> parser
@@ -105,8 +119,9 @@ module HTTPTools
     # Returns a Rack compatible environment hash. Will return nil if called
     # before headers are complete.
     # 
-    # "rack.input" is not supplied and must be added to make the environment
-    # hash fully Rack compliant.
+    # "rack.input" is only supplied if #env is called after parsing the request
+    # has finsished, and no listener is set for the `stream` event, otherwise
+    # you must add it yourself to make the environment hash fully Rack compliant
     # 
     def env
       return unless @header_complete
@@ -123,6 +138,9 @@ module HTTPTools
       env[SERVER_NAME] = host
       env[SERVER_PORT] = port || "80"
       @trailer.each {|k, val| env[HTTP_ + k.tr(LOWERCASE, UPPERCASE)] = val}
+      if @body || @stream_callback == method(:setup_stream_callback)
+        env[RACK_INPUT] = StringIO.new(@body || "")
+      end
       env
     end
     
@@ -151,9 +169,10 @@ module HTTPTools
     # 
     def finish
       if @state == :body_on_close
+        @buffer = @scanner
         @state = end_of_message
-      elsif @state == :body_chunked && @header[CONNECTION] == CLOSE &&
-        !@header[TRAILER] && @buffer.eos?
+      elsif @state == :body_chunked && @buffer.eos? && !@trailer_expected &&
+        @header.any? {|k,v| CONNECTION.casecmp(k) == 0 && CLOSE.casecmp(v) == 0}
         @state = end_of_message
       elsif @state == :start && @buffer.string.length < 1
         raise EmptyMessageError.new("Message empty")
@@ -204,6 +223,8 @@ module HTTPTools
       @trailer = {}
       @last_key = nil
       @content_left = nil
+      @chunked = nil
+      @trailer_expected = nil
       self
     end
     
@@ -214,14 +235,15 @@ module HTTPTools
     # 
     # Available events are :header, :stream, :trailer, :finish, and :error.
     # 
-    # Adding a second callback for an event will overwite the existing callback
-    # or delegate.
+    # Adding a second callback for an event will overwite the existing callback.
     # 
     # Events:
     # [header]     Called when headers are complete
     # 
     # [stream]     Supplied with one argument, the last chunk of body data fed
-    #              in to the parser as a String, e.g. "<h1>Hello"
+    #              in to the parser as a String, e.g. "<h1>Hello". If no
+    #              listener is set for this event the body can be retrieved with
+    #              #body
     # 
     # [trailer]    Called on the completion of the trailer, if present
     # 
@@ -240,6 +262,12 @@ module HTTPTools
     end
     alias on add_listener
     
+    def inspect # :nodoc:
+      str = to_s
+      str[-1, 0] = " #{posstr} #{state}"
+      str
+    end
+    
     private
     def start
       @request_method = @buffer.scan(/[a-z]+ /i)
@@ -254,7 +282,7 @@ module HTTPTools
       elsif @allow_html_without_header && @buffer.check(/\s*</i)
         skip_header
       else
-        raise ParseError.new("Protocol or method not recognised")
+        raise ParseError.new("Protocol or method not recognised at " + posstr)
       end
     end
     
@@ -269,7 +297,7 @@ module HTTPTools
       elsif @buffer.check(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]+\Z/i)
         :uri
       else
-        raise ParseError.new("URI or path not recognised")
+        raise ParseError.new("URI or path not recognised at " + posstr)
       end
     end
     
@@ -285,7 +313,7 @@ module HTTPTools
         @buffer.check(/ (H(T(T(P(\/(\d+(\.(\d+\r?)?)?)?)?)?)?)?)?\Z/i)
         :request_http_version
       else
-        raise ParseError.new("Invalid version specifier")
+        raise ParseError.new("Invalid version specifier at " + posstr)
       end
     end
     
@@ -299,7 +327,7 @@ module HTTPTools
         @buffer.check(/H(T(T(P(\/(\d+(\.(\d+\r?)?)?)?)?)?)?)?\Z/i)
         :response_http_version
       else
-        raise ParseError.new("Invalid version specifier")
+        raise ParseError.new("Invalid version specifier at " + posstr)
       end
     end
     
@@ -309,7 +337,7 @@ module HTTPTools
       @message = ""
       @header_complete = true
       @header_callback.call if @header_callback
-      body
+      start_body
     end
     
     def status
@@ -322,7 +350,7 @@ module HTTPTools
         @buffer.check(/\d(\d(\d( ([^\x00-\x1f\x7f]+\r?)?)?)?)?\Z/i)
         :status
       else
-        raise ParseError.new("Invalid status line")
+        raise ParseError.new("Invalid status line at " + posstr)
       end
     end
     
@@ -334,7 +362,7 @@ module HTTPTools
       elsif @buffer.skip(/\r?\n/i)
         @header_complete = true
         @header_callback.call if @header_callback
-        body
+        start_body
       elsif @buffer.eos? || @buffer.check(/([ -9;-~]+:?|\r)\Z/i)
         :key_or_newline
       elsif @last_key = @buffer.scan(/[ -9;-~]+:(?=[^ ])/i)
@@ -351,7 +379,7 @@ module HTTPTools
       elsif @buffer.check(/[^\x00\n\x7f]+\Z/)
         :skip_bad_header
       else
-        raise ParseError.new("Illegal character in field name")
+        raise ParseError.new("Illegal character in field name at " + posstr)
       end
     end
     
@@ -359,51 +387,58 @@ module HTTPTools
       value = @buffer.scan(/[^\x00\n\x7f]*\n/i)
       if value
         value.chop!
-        if ARRAY_VALUE_HEADERS.key?(@last_key)
-          @header.fetch(@last_key) {@header[@last_key] = []}.push(value)
+        if @header.key?(@last_key)
+          @header[@last_key] << "\n#{value}"
         else
           @header[@last_key] = value
         end
+        if CONTENT_LENGTH.casecmp(@last_key) == 0
+          @content_left = value.to_i
+        elsif TRANSFER_ENCODING.casecmp(@last_key) == 0
+          @chunked = CHUNKED.casecmp(value) == 0
+        end
         key_or_newline
       elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
         :value
       else
-        raise ParseError.new("Illegal character in field body")
+        raise ParseError.new("Illegal character in field body at " + posstr)
       end
     end
     
-    def body
-      if @request_method &&
-        !(@header.key?(CONTENT_LENGTH) || @header.key?(TRANSFER_ENCODING)) ||
+    def start_body
+      if @request_method && !(@content_left || @chunked) ||
         NO_BODY.key?(@status_code) || @force_no_body
         end_of_message
+      elsif @content_left
+        @buffer = [@buffer.rest]
+        body_with_length
+      elsif @chunked
+        @trailer_expected = @header.any? {|k,v| TRAILER.casecmp(k) == 0}
+        body_chunked
       else
-        length = @header[CONTENT_LENGTH]
-        if length
-          @content_left = length.to_i
-          body_with_length
-        elsif @header[TRANSFER_ENCODING] == CHUNKED
-          body_chunked
-        else
-          body_on_close
-        end
+        @buffer = [@buffer.rest]
+        body_on_close
       end
     end
     
     def body_with_length
-      if !@buffer.eos?
-        chunk = @buffer.string.slice(@buffer.pos, @content_left)
-        @stream_callback.call(chunk) if @stream_callback
+      chunk = @buffer.shift
+      if !chunk.empty?
         chunk_length = chunk.length
-        @buffer.pos += chunk_length
+        if chunk_length > @content_left
+          @scanner << chunk.slice!(@content_left..-1)
+        end
+        @stream_callback.call(chunk) if @stream_callback
         @content_left -= chunk_length
         if @content_left < 1
+          @buffer = @scanner
           end_of_message
         else
           :body_with_length
         end
       elsif @content_left < 1 # zero length body
         @stream_callback.call("") if @stream_callback
+        @buffer = @scanner
         end_of_message
       else
         :body_with_length
@@ -411,24 +446,38 @@ module HTTPTools
     end
     
     def body_chunked
-      decoded, remainder = transfer_encoding_chunked_decode(nil, @buffer)
-      if decoded
-        @stream_callback.call(decoded) if @stream_callback
-      end
-      if remainder
-        :body_chunked
-      else
-        if @header[TRAILER] || @force_trailer
-          trailer_key_or_newline
-        else
-          end_of_message
+      while true
+        start_pos = @buffer.pos
+        hex_chunk_length = @buffer.scan(/[0-9a-f]+ *\r?\n/i)
+        break :body_chunked unless hex_chunk_length
+        
+        chunk_length = hex_chunk_length.to_i(16)
+        if chunk_length == 0
+          if @trailer_expected || @force_trailer
+            break trailer_key_or_newline
+          else
+            break end_of_message
+          end
+        end
+        
+        begin
+          chunk = @buffer.rest.slice(0, chunk_length)
+          @buffer.pos += chunk_length
+          if chunk && @buffer.skip(/\r?\n/i)
+            @stream_callback.call(chunk) if @stream_callback
+          else
+            @buffer.pos = start_pos
+            break :body_chunked
+          end
+        rescue RangeError
+          @buffer.pos = start_pos
+          break :body_chunked
         end
       end
     end
     
     def body_on_close
-      chunk = @buffer.rest
-      @buffer.terminate
+      chunk = @buffer.shift
       @stream_callback.call(chunk) if @stream_callback
       :body_on_close
     end
@@ -446,7 +495,7 @@ module HTTPTools
         @last_key.chomp!(COLON)
         trailer_value
       else
-        raise ParseError.new("Illegal character in field name")
+        raise ParseError.new("Illegal character in field name at " + posstr)
       end
     end
     
@@ -454,12 +503,16 @@ module HTTPTools
       value = @buffer.scan(/[^\000\n\177]+\n/i)
       if value
         value.chop!
-        @trailer[@last_key] = value
+        if @trailer.key?(@last_key)
+          @trailer[@last_key] << "\n#{value}"
+        else
+          @trailer[@last_key] = value
+        end
         trailer_key_or_newline
       elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\Z/i)
         :trailer_value
       else
-        raise ParseError.new("Illegal character in field body")
+        raise ParseError.new("Illegal character in field body at " + posstr)
       end
     end
     
@@ -477,5 +530,31 @@ module HTTPTools
     end
     alias error raise
     
+    def setup_stream_callback(chunk)
+      @body = ""
+      stream_callback(chunk)
+      @stream_callback = method(:stream_callback)
+    end
+    
+    def stream_callback(chunk)
+      @body << chunk
+    end
+    
+    def line_char(string, position)
+      line_count = 1
+      char_count = 0
+      string.each_line do |line|
+        break if line.length + char_count > position
+        line_count += 1
+        char_count += line.length
+      end
+      [line_count, position + 1 - char_count]
+    end
+    
+    def posstr
+      line, char = line_char(@buffer.string, @buffer.pos)
+      "line #{line}, char #{char}"
+    end
+    
   end
-end
+end

data/profile/parser/large_response_profile.rb

@@ -0,0 +1,16 @@
+base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+require base + '/http_tools'
+require 'rubygems'
+require 'ruby-prof'
+
+body = "x" * 1024 * 1024 * 1
+response = "HTTP/1.1 200 OK\r\nDate: Mon, 06 Jun 2011 14:55:51 GMT\r\nServer: Apache/2.2.17 (Unix) mod_ssl/2.2.17 OpenSSL/0.9.8l DAV/2 mod_fastcgi/2.4.2\r\nLast-Modified: Mon, 06 Jun 2011 14:55:49 GMT\r\nETag: \"3f18045-400-4a50c4c87c740\"\r\nAccept-Ranges: bytes\r\nContent-Length: #{body.length}\r\nContent-Type: text/plain\r\n\r\n"
+chunks = []
+64.times {|i| chunks << body[i * 64, body.length / 64]}
+parser = HTTPTools::Parser.new
+
+result = RubyProf.profile do
+  parser << response
+  chunks.each {|chunk| parser << chunk}
+end
+RubyProf::FlatPrinter.new(result).print(STDOUT, 0)

data/test/builder/request_test.rb

@@ -3,7 +3,7 @@ require base + '/http_tools'
 require 'test/unit'
 require 'uri'
 
-class RequestTest < Test::Unit::TestCase
+class BuilderRequestTest < Test::Unit::TestCase
   
   def test_get
     result = HTTPTools::Builder.request(:get, "www.example.com", "/test")
@@ -18,9 +18,27 @@ class RequestTest < Test::Unit::TestCase
   end
   
   def test_headers
-    result = HTTPTools::Builder.request(:get, "www.foobar.com", "/", "x-test" => "foo")
+    result = HTTPTools::Builder.request(:get, "www.foobar.com", "/", "X-Test" => "foo")
     
-    assert_equal("GET / HTTP/1.1\r\nHost: www.foobar.com\r\nx-test: foo\r\n\r\n", result)
+    assert_equal("GET / HTTP/1.1\r\nHost: www.foobar.com\r\nX-Test: foo\r\n\r\n", result)
+  end
+  
+  def test_newline_separated_multi_value_headers
+    result = HTTPTools::Builder.request(:get, "www.foo.com", "/", "X-Test" => "foo\nbar")
+    
+    assert_equal("GET / HTTP/1.1\r\nHost: www.foo.com\r\nX-Test: foo\r\nX-Test: bar\r\n\r\n", result)
+  end
+  
+  def test_array_multi_value_headers
+    result = HTTPTools::Builder.request(:get, "www.foo.com", "/", "X-Test" => ["foo", "bar"])
+    
+    assert_equal("GET / HTTP/1.1\r\nHost: www.foo.com\r\nX-Test: foo\r\nX-Test: bar\r\n\r\n", result)
+  end
+  
+  def test_non_string_headers
+    result = HTTPTools::Builder.request(:get, "www.foobar.com", "/", "X-Test" => 42)
+    
+    assert_equal("GET / HTTP/1.1\r\nHost: www.foobar.com\r\nX-Test: 42\r\n\r\n", result)
   end
   
 end