http_tools 0.1.0 → 0.2.0

data/lib/http_tools/parser.rb

@@ -11,9 +11,11 @@ module HTTPTools
   # 
   # Example:
   #   parser = HTTPTools::Parser.new
-  #   parser.on(:status) {|status, message| puts "#{status} #{message}"}
-  #   parser.on(:headers) {|headers| puts headers.inspect}
-  #   parser.on(:body) {|body| puts body}
+  #   parser.on(:header) do |header|
+  #     puts parser.status_code + " " + parser.method
+  #     puts parser.header.inspect
+  #   end
+  #   parser.on(:stream) {|chunk| print chunk}
   #   
   #   parser << "HTTP/1.1 200 OK\r\n"
   #   parser << "Content-Length: 20\r\n\r\n"
@@ -27,16 +29,42 @@ module HTTPTools
   class Parser
     include Encoding
     
+    COLON = ":".freeze
     KEY_TERMINATOR = ": ".freeze
     CONTENT_LENGTH = "Content-Length".freeze
     TRANSFER_ENCODING = "Transfer-Encoding".freeze
     TRAILER = "Trailer".freeze
+    CONNECTION = "Connection".freeze
+    CLOSE = "close".freeze
     CHUNKED = "chunked".freeze
-    EVENTS = ["method", "path", "uri", "fragment", "version", "status", "key",
-      "value", "headers", "stream", "body", "trailers", "finished",
-      "error"].map {|event| event.freeze}.freeze
+    EVENTS = %W{header stream trailer finish error}.map do |event|
+      event.freeze
+    end.freeze
+    
+    REQUEST_METHOD = "REQUEST_METHOD".freeze
+    PATH_INFO = "PATH_INFO".freeze
+    QUERY_STRING = "QUERY_STRING".freeze
+    REQUEST_URI = "REQUEST_URI".freeze
+    FRAGMENT = "FRAGMENT".freeze
+    
+    PROTOTYPE_ENV = {
+      "SCRIPT_NAME" => "".freeze,
+      PATH_INFO => "/".freeze,
+      QUERY_STRING => "".freeze,
+      "rack.version" => [1, 1].freeze,
+      "rack.url_scheme" => "http".freeze,
+      "rack.errors" => STDERR,
+      "rack.multithread" => false,
+      "rack.multiprocess" => false,
+      "rack.run_once" => false}.freeze
+    
+    HTTP_ = "HTTP_".freeze
+    LOWERCASE = "a-z-".freeze
+    UPPERCASE = "A-Z_".freeze
     
     attr_reader :state # :nodoc:
+    attr_reader :request_method, :path_info, :query_string, :request_uri,
+      :fragment, :version, :status_code, :message, :header, :trailer
     
     # Force parser to expect and parse a trailer when Trailer header missing.
     attr_accessor :force_trailer
@@ -44,40 +72,19 @@ module HTTPTools
     # Skip parsing the body, e.g. with the response to a HEAD request.
     attr_accessor :force_no_body
     
-    # :call-seq: Parser.new(delegate=nil) -> parser
+    # Allow responses with no status line or headers if it looks like HTML.
+    attr_accessor :allow_html_without_header
+    
+    # :call-seq: Parser.new -> parser
     # 
     # Create a new HTTPTools::Parser.
     # 
-    # delegate is an object that will recieve callbacks for events during
-    # parsing. The delegate's methods should be named on_[event name], e.g.
-    # on_status, on_body, etc. See #add_listener for more.
-    # 
-    # Example:
-    #   class ExampleDelegate
-    #     def on_status(status, message)
-    #       puts "#{status} #{message}"
-    #     end
-    #   end
-    #   parser = HTTPTools::Parser.new(ExampleDelegate.new)
-    # 
-    # If a callback is set for an event, it will take precedence over the
-    # delegate for that event.
-    # 
-    def initialize(delegate=nil)
+    def initialize
       @state = :start
       @buffer = StringScanner.new("")
       @buffer_backup_reference = @buffer
-      @status = nil
-      @headers = {}
-      @last_key = nil
-      @content_left = nil
-      @body = nil
-      if delegate
-        EVENTS.each do |event|
-          id = "on_#{event}"
-          add_listener(event, delegate.method(id)) if delegate.respond_to?(id)
-        end
-      end
+      @header = {}
+      @trailer = {}
     end
     
     # :call-seq: parser.concat(data) -> parser
@@ -96,6 +103,29 @@ module HTTPTools
     end
     alias << concat
     
+    # :call-seq: parser.env -> hash or nil
+    # 
+    # Returns a Rack compatible environment hash. Will return nil if called
+    # before headers are complete.
+    # 
+    # The following are not supplied, and must be added to make the environment
+    # hash fully Rack compliant: SERVER_NAME, SERVER_PORT, rack.input
+    # 
+    def env
+      return unless @header_complete
+      env = PROTOTYPE_ENV.merge(
+        REQUEST_METHOD => @request_method,
+        REQUEST_URI => @request_uri)
+      if @path_info
+        env[PATH_INFO] = @path_info
+        env[QUERY_STRING] = @query_string
+      end
+      env[FRAGMENT] = @fragment if @fragment
+      @header.each {|k, val| env[HTTP_ + k.tr(LOWERCASE, UPPERCASE)] = val}
+      @trailer.each {|k, val| env[HTTP_ + k.tr(LOWERCASE, UPPERCASE)] = val}
+      env
+    end
+    
     # :call-seq: parser.finish -> parser
     # 
     # Used to notify the parser that the request has finished in a case where it
@@ -121,8 +151,12 @@ module HTTPTools
     # 
     def finish
       if @state == :body_on_close
-        @body_callback.call(@body) if @body_callback
         @state = end_of_message
+      elsif @state == :body_chunked && @header[CONNECTION] == CLOSE &&
+        !@header[TRAILER] && @buffer.eos?
+        @state = end_of_message
+      elsif @state == :start && @buffer.string.length < 1
+        raise EmptyMessageError.new("Message empty")
       else
         raise MessageIncompleteError.new("Message ended early")
       end
@@ -153,12 +187,17 @@ module HTTPTools
       @buffer = @buffer_backup_reference
       @buffer.string.replace("")
       @buffer.reset
-      # @status = nil
-      @headers = {}
+      @request_method = nil
+      @path_info = nil
+      @query_string = nil
+      @request_uri = nil
+      @fragment = nil
+      @version = nil
+      @status_code = nil
+      @header = {}
       @trailer = {}
-      # @last_key = nil
-      # @content_left = nil
-      @body = nil
+      @last_key = nil
+      @content_left = nil
       self
     end
     
@@ -167,44 +206,20 @@ module HTTPTools
     # parser.on(event) {|arg1 [, arg2]| block} -> parser
     # parser.on(event, proc) -> parser
     # 
-    # Available events are :method, :path, :version, :status, :headers, :stream,
-    # :body, and :error.
+    # Available events are :header, :stream, :trailer, :finish, and :error.
     # 
     # Adding a second callback for an event will overwite the existing callback
     # or delegate.
     # 
     # Events:
-    # [method]     Supplied with one argument, the HTTP method as a String,
-    #              e.g. "GET"
-    # 
-    # [path]       Supplied with two arguments, the request path as a String,
-    #              e.g. "/example.html", and the query string as a String,
-    #              e.g. "query=foo"
-    #              (this callback is only called if the request uri is a path)
-    # 
-    # [uri]        Supplied with one argument, the request uri as a String,
-    #              e.g. "/example.html?query=foo"
-    # 
-    # [fragment]   Supplied with one argument, the fragment from the request
-    #              uri, if present
-    # 
-    # [version]    Supplied with one argument, the HTTP version as a String,
-    #              e.g. "1.1"
-    # 
-    # [status]     Supplied with two arguments, the HTTP status code as a
-    #              Numeric, e.g. 200, and the HTTP status message as a String,
-    #              e.g. "OK"
-    # 
-    # [headers]    Supplied with one argument, the message headers as a Hash,
-    #              e.g. {"Content-Length" => "20"}
+    # [header]     Called when headers are complete
     # 
     # [stream]     Supplied with one argument, the last chunk of body data fed
     #              in to the parser as a String, e.g. "<h1>Hello"
     # 
-    # [body]       Supplied with one argument, the message body as a String,
-    #              e.g. "<h1>Hello world</h1>"
+    # [trailer]    Called on the completion of the trailer, if present
     # 
-    # [finished]   Supplied with one argument, any data left in the parser's
+    # [finish]     Supplied with one argument, any data left in the parser's
     #              buffer after the end of the HTTP message (likely nil, but
     #              possibly the start of the next message)
     # 
@@ -221,37 +236,31 @@ module HTTPTools
     
     private
     def start
-      method = @buffer.scan(/[a-z]+ /i)
-      if method
-        if @method_callback
-          method.chop!
-          method.upcase!
-          @method_callback.call(method)
-        end
+      @request_method = @buffer.scan(/[a-z]+ /i)
+      if @request_method
+        @request_method.chop!
+        @request_method.upcase!
         uri
       elsif @buffer.skip(/HTTP\//i)
         response_http_version
-      elsif @buffer.check(/[a-z]+\Z/i)
+      elsif @buffer.check(/[a-z]*\Z/i)
         :start
+      elsif @allow_html_without_header && @buffer.check(/\s*</i)
+        skip_header
       else
         raise ParseError.new("Protocol or method not recognised")
       end
     end
     
     def uri
-      uri = @buffer.scan(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]*(?=( |\r\n))/i)
-      if uri
-        fragment = uri.slice!(/#[a-z0-9;\/?:@&=+$,%_.!~*')(-]+\Z/i)
-        if @path_callback && uri =~ /^\//i
-          path = uri.dup
-          query = path.slice!(/\?[a-z0-9;\/?:@&=+$,%_.!~*')(-]*/i)
-          query.slice!(0) if query
-          @path_callback.call(path, query)
-        end
-        @uri_callback.call(uri) if @uri_callback
-        if fragment && @fragment_callback
-          fragment.slice!(0)
-          @fragment_callback.call(fragment)
+      @request_uri= @buffer.scan(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]*(?=( |\r\n))/i)
+      if @request_uri
+        @fragment = @request_uri.slice!(/#[a-z0-9;\/?:@&=+$,%_.!~*')(-]+\Z/i)
+        @fragment.slice!(0) if @fragment
+        if @request_uri =~ /^\//i
+          @path_info = @request_uri.dup
+          @query_string = @path_info.slice!(/\?[a-z0-9;\/?:@&=+$,%_.!~*')(-]*/i)
+          @query_string ? @query_string.slice!(0) : @query_string = ""
         end
         space_before_http
       elsif @buffer.check(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]+\Z/i)
@@ -280,12 +289,9 @@ module HTTPTools
     end
     
     def request_http_version
-      version = @buffer.scan(/[0-9]+\.[0-9]+\r\n/i)
-      if version
-        if @version_callback
-          version.chop!
-          @version_callback.call(version)
-        end
+      @version = @buffer.scan(/[0-9]+\.[0-9x]+\r\n/i)
+      if @version
+        @version.chop!
         key_or_newline
       elsif @buffer.eos? || @buffer.check(/\d+(\.(\d+\r?)?)?\Z/i)
         :request_http_version
@@ -295,12 +301,9 @@ module HTTPTools
     end
     
     def response_http_version
-      version = @buffer.scan(/[0-9]+\.[0-9]+ /i)
-      if version
-        if @version_callback
-          version.chop!
-          @version_callback.call(version)
-        end
+      @version = @buffer.scan(/[0-9]+\.[0-9x]+ /i)
+      if @version
+        version.chop!
         status
       elsif @buffer.eos? || @buffer.check(/\d+(\.(\d+)?)?\Z/i)
         :response_http_version
@@ -309,13 +312,23 @@ module HTTPTools
       end
     end
     
+    def skip_header
+      @version = "0.0"
+      @status_code = 200
+      @message = ""
+      @header_complete = true
+      @header_callback.call if @header_callback
+      body
+    end
+    
     def status
-      status = @buffer.scan(/\d\d\d [a-z -]+\r?\n/i)
+      status = @buffer.scan(/\d\d\d[^\x00-\x1f\x7f]*\r?\n/i)
       if status
-        @status = status.slice!(0, 3).to_i
-        @status_callback.call(@status, status.strip) if @status_callback
+        @status_code = status.slice!(0, 3).to_i
+        @message = status.strip
         key_or_newline
-      elsif @buffer.eos? || @buffer.check(/\d(\d(\d( ([a-z]+\r?)?)?)?)?\Z/i)
+      elsif @buffer.eos? ||
+        @buffer.check(/\d(\d(\d( ([^\x00-\x1f\x7f]+\r?)?)?)?)?\Z/i)
         :status
       else
         raise ParseError.new("Invalid status line")
@@ -323,27 +336,45 @@ module HTTPTools
     end
     
     def key_or_newline
-      @last_key = @buffer.scan(/[!-9;-~]+: /i)
+      @last_key = @buffer.scan(/[ -9;-~]+: /i)
       if @last_key
         @last_key.chomp!(KEY_TERMINATOR)
         value
-      elsif @buffer.skip(/\n|\r\n/i)
-        @headers_callback.call(@headers) if @headers_callback
+      elsif @buffer.skip(/\r?\n/i)
+        @header_complete = true
+        @header_callback.call if @header_callback
         body
-      elsif @buffer.eos? || @buffer.check(/[!-9;-~]+:?\Z/i)
+      elsif @buffer.eos? || @buffer.check(/([ -9;-~]+:?|\r)\Z/i)
         :key_or_newline
+      elsif @last_key = @buffer.scan(/[ -9;-~]+:(?=[^ ])/i)
+        @last_key.chomp!(COLON)
+        value
+      else
+        skip_bad_header
+      end
+    end
+    
+    def skip_bad_header
+      if @buffer.skip(/[^\x00\n\x7f]*\n/)
+        key_or_newline
+      elsif @buffer.check(/[^\x00\n\x7f]+\Z/)
+        :skip_bad_header
       else
         raise ParseError.new("Illegal character in field name")
       end
     end
     
     def value
-      value = @buffer.scan(/[ -~]+\r?\n/i)
+      value = @buffer.scan(/[^\x00\n\x7f]*\r?\n/i)
       if value
         value.chop!
-        @headers[@last_key] = value
+        if ARRAY_VALUE_HEADERS[@last_key]
+          @header.fetch(@last_key) {@header[@last_key] = []}.push(value)
+        else
+          @header[@last_key] = value
+        end
         key_or_newline
-      elsif @buffer.eos? || @buffer.check(/[ -~]+\Z/i)
+      elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\r?\Z/i)
         :value
       else
         raise ParseError.new("Illegal character in field body")
@@ -351,18 +382,14 @@ module HTTPTools
     end
     
     def body
-      if @force_no_body || NO_BODY[@status]
+      if @force_no_body || NO_BODY[@status_code]
         end_of_message
-      elsif @buffer.eos?
-        :body
       else
-        @body = "" if @body_callback
-        @buffer = @buffer.rest # Switch @buffer from StringScanner to String
-        length = @headers[CONTENT_LENGTH]
+        length = @header[CONTENT_LENGTH]
         if length
           @content_left = length.to_i
           body_with_length
-        elsif @headers[TRANSFER_ENCODING] == CHUNKED
+        elsif @header[TRANSFER_ENCODING] == CHUNKED
           body_chunked
         else
           body_on_close
@@ -370,48 +397,35 @@ module HTTPTools
       end
     end
     
-    #--
-    # From this point on @buffer is a String, not a StringScanner.
-    # This is because 1. we don't need a StringScanner anymore, 2. if we
-    # switched to a diffrent instace variable we'd need a condition in #concat
-    # to feed the data in to the new instace variable, which would slow us down.
-    #++
-    
     def body_with_length
-      if @buffer.length > 0
-        chunk = @buffer.slice!(0, @content_left)
+      if !@buffer.eos?
+        chunk = @buffer.string.slice(@buffer.pos, @content_left)
         @stream_callback.call(chunk) if @stream_callback
-        @body << chunk if @body_callback
-        @content_left -= chunk.length
+        chunk_length = chunk.length
+        @buffer.pos += chunk_length
+        @content_left -= chunk_length
         if @content_left < 1
-          @body_callback.call(@body) if @body_callback
           end_of_message
         else
           :body_with_length
         end
+      elsif @content_left < 1 # zero length body
+        @stream_callback.call("") if @stream_callback
+        end_of_message
       else
         :body_with_length
       end
     end
     
     def body_chunked
-      decoded, remainder = transfer_encoding_chunked_decode(@buffer)
+      decoded, remainder = transfer_encoding_chunked_decode(nil, @buffer)
       if decoded
         @stream_callback.call(decoded) if @stream_callback
-        @body << decoded if @body_callback
       end
       if remainder
-        @buffer = remainder
         :body_chunked
       else
-        @buffer.slice!(/.*0\r\n/m)
-        @body_callback.call(@body) if @body_callback
-        if @headers[TRAILER] || @force_trailer
-          @trailer = {}
-          # @buffer switches back to a StringScanner for the trailer.
-          @buffer_backup_reference.string.replace(@buffer)
-          @buffer_backup_reference.reset
-          @buffer = @buffer_backup_reference
+        if @header[TRAILER] || @force_trailer
           trailer_key_or_newline
         else
           end_of_message
@@ -420,37 +434,36 @@ module HTTPTools
     end
     
     def body_on_close
-      @stream_callback.call(@buffer) if @stream_callback
-      @body << @buffer if @body_callback
-      @buffer = ""
+      chunk = @buffer.rest
+      @buffer.terminate
+      @stream_callback.call(chunk) if @stream_callback
       :body_on_close
     end
     
-    #--
-    # @buffer switches back to a StringScanner for the trailer.
-    #++
-    
     def trailer_key_or_newline
-      if @last_key = @buffer.scan(/[!-9;-~]+: /i)
+      if @last_key = @buffer.scan(/[ -9;-~]+: /i)
         @last_key.chomp!(KEY_TERMINATOR)
         trailer_value
-      elsif @buffer.skip(/\n|\r\n/i)
-        @trailer_callback.call(@trailer) if @trailer_callback
+      elsif @buffer.skip(/\r?\n/i)
+        @trailer_callback.call if @trailer_callback
         end_of_message
-      elsif @buffer.eos? || @buffer.check(/[!-9;-~]+:?\Z/i)
+      elsif @buffer.eos? || @buffer.check(/([ -9;-~]+:?|\r)\Z/i)
         :trailer_key_or_newline
+      elsif @last_key = @buffer.scan(/[ -9;-~]+:(?=[^ ])/i)
+        @last_key.chomp!(COLON)
+        trailer_value
       else
         raise ParseError.new("Illegal character in field name")
       end
     end
     
     def trailer_value
-      value = @buffer.scan(/[ -~]+\r?\n/i)
+      value = @buffer.scan(/[^\000\n\177]+\r?\n/i)
       if value
         value.chop!
         @trailer[@last_key] = value
         trailer_key_or_newline
-      elsif @buffer.eos? || @buffer.check(/[ -~]+\Z/i)
+      elsif @buffer.eos? || @buffer.check(/[^\x00\n\x7f]+\r?\Z/i)
         :trailer_value
       else
         raise ParseError.new("Illegal character in field body")
@@ -460,8 +473,8 @@ module HTTPTools
     def end_of_message
       raise EndOfMessageError.new("Message ended") if @state == :end_of_message
       remainder = @buffer.respond_to?(:rest) ? @buffer.rest : @buffer
-      if @finished_callback
-        @finished_callback.call((remainder if remainder.length > 0))
+      if @finish_callback
+        @finish_callback.call((remainder if remainder.length > 0))
       end
       :end_of_message
     end