http_tools 0.1.0

data/lib/http_tools/encoding.rb

@@ -0,0 +1,169 @@
+require 'strscan'
+
+module HTTPTools
+  
+  # HTTPTools::Encoding provides methods to deal with several HTTP related
+  # encodings including url, www-form, and chunked transfer encoding. It can be
+  # used as a mixin or class methods on HTTPTools::Encoding.
+  # 
+  module Encoding
+    HEX_BIG_ENDIAN_2_BYTES = "H2".freeze
+    HEX_BIG_ENDIAN_REPEATING = "H*".freeze
+    PERCENT = "%".freeze
+    PLUS = "+".freeze
+    AMPERSAND = "&".freeze
+    EQUALS = "=".freeze
+    CHUNK_FORMAT = "%x\r\n%s\r\n".freeze
+    
+    module_function
+    
+    # :call-seq: Encoding.url_encode(string) -> encoded_string
+    # 
+    # URL encode a string, eg "le café" becomes "le+caf%c3%a9"
+    # 
+    def url_encode(string)
+      string.gsub(/[^a-zA-Z0-9._~-]+/) do |match|
+        length = match.respond_to?(:bytesize) ? match.bytesize : match.length
+        PERCENT + match.unpack(HEX_BIG_ENDIAN_2_BYTES * length).join(PERCENT)
+      end
+    end
+    
+    # :call-seq: Encoding.url_decode(encoded_string) -> string
+    # 
+    # URL decode a string, eg "le+caf%c3%a9" becomes "le café"
+    # 
+    def url_decode(string)
+      string.tr(PLUS, SPACE).gsub(/(%[0-9a-fA-F]{2})+/) do |match|
+        r = [match.delete(PERCENT)].pack(HEX_BIG_ENDIAN_REPEATING)
+        r.respond_to?(:force_encoding) ? r.force_encoding(string.encoding) : r
+      end
+    end
+    
+    # :call-seq: Encoding.www_form_encode(hash) -> string
+    # 
+    # Takes a Hash and converts it to a String as if it was a HTML form being
+    # submitted, eg
+    # {"query" => "fish", "lang" => "en"} becomes "lang=en&query=fish"
+    # 
+    # To get multiple key value pairs with the same key use an array as the
+    # value, eg
+    # {"lang" => ["en", "fr"]} become "lang=en&lang=fr"
+    # 
+    def www_form_encode(hash)
+      hash.map do |key, value|
+        if value.respond_to?(:map) && !value.is_a?(String)
+          value.map {|val| www_form_encode(key => val.to_s)}.join(AMPERSAND)
+        else
+          url_encode(key.to_s) << EQUALS << url_encode(value.to_s)
+        end
+      end.join(AMPERSAND)
+    end
+    
+    # :call-seq: Encoding.www_form_decode(string) -> hash
+    # 
+    # Takes a String resulting from a HTML form being submitted, and converts it
+    # to a hash,
+    # eg "lang=en&query=fish" becomes {"query" => "fish", "lang" => "en"}
+    # 
+    # Multiple key value pairs with the same key will become a single key with
+    # an array value, eg "lang=en&lang=fr" becomes {"lang" => ["en", "fr"]}
+    #
+    def www_form_decode(string)
+      string.split(AMPERSAND).inject({}) do |memo, key_value|
+        key, value = key_value.split(EQUALS)
+        key, value = url_decode(key), url_decode(value)
+        if memo.key?(key)
+          memo.merge(key => [*memo[key]].push(value))
+        else
+          memo.merge(key => value)
+        end
+      end
+    end
+    
+    # :call-seq:
+    # Encoding.transfer_encoding_chunked_encode(string) -> encoded_string
+    # 
+    # Returns string as a 'chunked' transfer encoding encoded string, suitable
+    # for a streaming response from a HTTP server, eg
+    # "foo" becomes "3\r\nfoo\r\n"
+    #
+    # chunked responses should be terminted with a empty chunk, eg "0\r\n",
+    # passing an empty string or nil will generate the empty chunk.
+    # 
+    def transfer_encoding_chunked_encode(string)
+      if string && string.length > 0
+        sprintf(CHUNK_FORMAT, string.length, string)
+      else
+        "0\r\n"
+      end
+    end
+    
+    # :call-seq:
+    # Encoding.transfer_encoding_chunked_decode(encoded_string) -> array
+    # 
+    # Decoding a complete chunked response will return an array containing
+    # the decoded response and nil.
+    # Example:
+    #   encoded_string = "3\r\nfoo\r\n3\r\nbar\r\n0\r\n"
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
+    # => ["foobar", nil]
+    # 
+    # Decoding a partial response will return an array of the response decoded
+    # so far, and the remainder of the encoded string.
+    # Example
+    #   encoded_string = "3\r\nfoo\r\n3\r\nba"
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
+    # => ["foo", "3\r\nba"]
+    # 
+    # If the chunks are complete, but there is no empty terminating chunk, the
+    # second element in the array will be an empty string.
+    #   encoded_string = "3\r\nfoo\r\n3\r\nbar"
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
+    # => ["foobar", ""]
+    # 
+    # If nothing can be decoded the first element in the array will be an empty
+    # string and the second the remainder
+    #   encoded_string = "3\r\nfo"
+    #   Encoding.transfer_encoding_chunked_decode(encoded_string)\
+    # => ["", "3\r\nfo"]
+    # 
+    # Example use:
+    #   include Encoding
+    #   decoded = ""
+    #   remainder = ""
+    #   while remainder
+    #     remainder << get_data
+    #     chunk, remainder = transfer_encoding_chunked_decode(remainder)
+    #     decoded << chunk
+    #   end
+    # 
+    def transfer_encoding_chunked_decode(scanner)
+      unless scanner.is_a?(StringScanner)
+        scanner = StringScanner.new(scanner.dup)
+      end
+      hex_chunk_length = scanner.scan(/[0-9a-fA-F]+\r?\n/)
+      return [nil, scanner.string] unless hex_chunk_length
+      
+      chunk_length = hex_chunk_length.to_i(16)
+      return [nil, nil] if chunk_length == 0
+      
+      chunk = scanner.rest.slice(0, chunk_length)
+      begin
+        scanner.pos += chunk_length
+        separator = scanner.scan(/\n|\r\n/)
+      rescue RangeError
+      end
+      
+      if separator && chunk.length == chunk_length
+        scanner.string.replace(scanner.rest)
+        scanner.reset
+        rest, remainder = transfer_encoding_chunked_decode(scanner)
+        chunk << rest if rest
+        [chunk, remainder]
+      else
+        [nil, scanner.string]
+      end
+    end
+    
+  end
+end

data/lib/http_tools/errors.rb

@@ -0,0 +1,5 @@
+module HTTPTools
+  class ParseError < StandardError; end
+  class EndOfMessageError < ParseError; end
+  class MessageIncompleteError < EndOfMessageError; end
+end

data/lib/http_tools/parser.rb

@@ -0,0 +1,478 @@
+require 'strscan'
+
+module HTTPTools
+  
+  # HTTPTools::Parser is a pure Ruby HTTP request & response parser with an
+  # evented API.
+  # 
+  # The HTTP message can be fed in to the parser piece by piece as it comes over
+  # the wire, and the parser will call its callbacks as it works it's way
+  # through the message.
+  # 
+  # Example:
+  #   parser = HTTPTools::Parser.new
+  #   parser.on(:status) {|status, message| puts "#{status} #{message}"}
+  #   parser.on(:headers) {|headers| puts headers.inspect}
+  #   parser.on(:body) {|body| puts body}
+  #   
+  #   parser << "HTTP/1.1 200 OK\r\n"
+  #   parser << "Content-Length: 20\r\n\r\n"
+  #   parser << "<h1>Hello world</h1>"
+  # 
+  # Prints:
+  #   200 OK
+  #   {"Content-Length" => "20"}
+  #   <h1>Hello world</h1>
+  # 
+  class Parser
+    include Encoding
+    
+    KEY_TERMINATOR = ": ".freeze
+    CONTENT_LENGTH = "Content-Length".freeze
+    TRANSFER_ENCODING = "Transfer-Encoding".freeze
+    TRAILER = "Trailer".freeze
+    CHUNKED = "chunked".freeze
+    EVENTS = ["method", "path", "uri", "fragment", "version", "status", "key",
+      "value", "headers", "stream", "body", "trailers", "finished",
+      "error"].map {|event| event.freeze}.freeze
+    
+    attr_reader :state # :nodoc:
+    
+    # Force parser to expect and parse a trailer when Trailer header missing.
+    attr_accessor :force_trailer
+    
+    # Skip parsing the body, e.g. with the response to a HEAD request.
+    attr_accessor :force_no_body
+    
+    # :call-seq: Parser.new(delegate=nil) -> parser
+    # 
+    # Create a new HTTPTools::Parser.
+    # 
+    # delegate is an object that will recieve callbacks for events during
+    # parsing. The delegate's methods should be named on_[event name], e.g.
+    # on_status, on_body, etc. See #add_listener for more.
+    # 
+    # Example:
+    #   class ExampleDelegate
+    #     def on_status(status, message)
+    #       puts "#{status} #{message}"
+    #     end
+    #   end
+    #   parser = HTTPTools::Parser.new(ExampleDelegate.new)
+    # 
+    # If a callback is set for an event, it will take precedence over the
+    # delegate for that event.
+    # 
+    def initialize(delegate=nil)
+      @state = :start
+      @buffer = StringScanner.new("")
+      @buffer_backup_reference = @buffer
+      @status = nil
+      @headers = {}
+      @last_key = nil
+      @content_left = nil
+      @body = nil
+      if delegate
+        EVENTS.each do |event|
+          id = "on_#{event}"
+          add_listener(event, delegate.method(id)) if delegate.respond_to?(id)
+        end
+      end
+    end
+    
+    # :call-seq: parser.concat(data) -> parser
+    # parser << data -> parser
+    # 
+    # Feed data in to the parser and trigger callbacks.
+    # 
+    # Will raise HTTPTools::ParseError on error, unless a callback has been set
+    # for the :error event, in which case the callback will recieve the error
+    # insted.
+    # 
+    def concat(data)
+      @buffer << data
+      @state = send(@state)
+      self
+    end
+    alias << concat
+    
+    # :call-seq: parser.finish -> parser
+    # 
+    # Used to notify the parser that the request has finished in a case where it
+    # can not be determined by the request itself.
+    # 
+    # For example, when a server does not set a content length, and instead
+    # relies on closing the connection to signify the body end.
+    #   until parser.finished?
+    #     begin
+    #       parser << socket.sysread(1024 * 16)
+    #     rescue EOFError
+    #       parser.finish
+    #       break
+    #     end
+    #   end
+    # 
+    # This method can not be used to interrupt parsing from within a callback.
+    # 
+    # Will raise HTTPTools::MessageIncompleteError if called too early, or
+    # HTTPTools::EndOfMessageError if the message has already finished, unless
+    # a callback has been set for the :error event, in which case the callback
+    # will recieve the error insted.
+    # 
+    def finish
+      if @state == :body_on_close
+        @body_callback.call(@body) if @body_callback
+        @state = end_of_message
+      else
+        raise MessageIncompleteError.new("Message ended early")
+      end
+      self
+    end
+    
+    # :call-seq: parser.finished? -> bool
+    # 
+    # Returns true when the parser has come to the end of the message, false
+    # otherwise.
+    # 
+    # Some HTTP servers may not supply the necessary information in the response
+    # to determine the end of the message (e.g., no content length) and insted
+    # close the connection to signify the end of the message, see #finish for
+    # how to deal with this.
+    # 
+    def finished?
+      @state == :end_of_message
+    end
+    
+    # :call-seq: parser.reset -> parser
+    # 
+    # Reset the parser so it can be used to process a new request.
+    # Callbacks/delegates will not be removed.
+    # 
+    def reset
+      @state = :start
+      @buffer = @buffer_backup_reference
+      @buffer.string.replace("")
+      @buffer.reset
+      # @status = nil
+      @headers = {}
+      @trailer = {}
+      # @last_key = nil
+      # @content_left = nil
+      @body = nil
+      self
+    end
+    
+    # :call-seq: parser.add_listener(event) {|arg1 [, arg2]| block} -> parser
+    # parser.add_listener(event, proc) -> parser
+    # parser.on(event) {|arg1 [, arg2]| block} -> parser
+    # parser.on(event, proc) -> parser
+    # 
+    # Available events are :method, :path, :version, :status, :headers, :stream,
+    # :body, and :error.
+    # 
+    # Adding a second callback for an event will overwite the existing callback
+    # or delegate.
+    # 
+    # Events:
+    # [method]     Supplied with one argument, the HTTP method as a String,
+    #              e.g. "GET"
+    # 
+    # [path]       Supplied with two arguments, the request path as a String,
+    #              e.g. "/example.html", and the query string as a String,
+    #              e.g. "query=foo"
+    #              (this callback is only called if the request uri is a path)
+    # 
+    # [uri]        Supplied with one argument, the request uri as a String,
+    #              e.g. "/example.html?query=foo"
+    # 
+    # [fragment]   Supplied with one argument, the fragment from the request
+    #              uri, if present
+    # 
+    # [version]    Supplied with one argument, the HTTP version as a String,
+    #              e.g. "1.1"
+    # 
+    # [status]     Supplied with two arguments, the HTTP status code as a
+    #              Numeric, e.g. 200, and the HTTP status message as a String,
+    #              e.g. "OK"
+    # 
+    # [headers]    Supplied with one argument, the message headers as a Hash,
+    #              e.g. {"Content-Length" => "20"}
+    # 
+    # [stream]     Supplied with one argument, the last chunk of body data fed
+    #              in to the parser as a String, e.g. "<h1>Hello"
+    # 
+    # [body]       Supplied with one argument, the message body as a String,
+    #              e.g. "<h1>Hello world</h1>"
+    # 
+    # [finished]   Supplied with one argument, any data left in the parser's
+    #              buffer after the end of the HTTP message (likely nil, but
+    #              possibly the start of the next message)
+    # 
+    # [error]      Supplied with one argument, an error encountered while
+    #              parsing as a HTTPTools::ParseError. If a listener isn't
+    #              registered for this event, an exception will be raised when
+    #              an error is encountered
+    # 
+    def add_listener(event, proc=nil, &block)
+      instance_variable_set(:"@#{event}_callback", proc || block)
+      self
+    end
+    alias on add_listener
+    
+    private
+    def start
+      method = @buffer.scan(/[a-z]+ /i)
+      if method
+        if @method_callback
+          method.chop!
+          method.upcase!
+          @method_callback.call(method)
+        end
+        uri
+      elsif @buffer.skip(/HTTP\//i)
+        response_http_version
+      elsif @buffer.check(/[a-z]+\Z/i)
+        :start
+      else
+        raise ParseError.new("Protocol or method not recognised")
+      end
+    end
+    
+    def uri
+      uri = @buffer.scan(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]*(?=( |\r\n))/i)
+      if uri
+        fragment = uri.slice!(/#[a-z0-9;\/?:@&=+$,%_.!~*')(-]+\Z/i)
+        if @path_callback && uri =~ /^\//i
+          path = uri.dup
+          query = path.slice!(/\?[a-z0-9;\/?:@&=+$,%_.!~*')(-]*/i)
+          query.slice!(0) if query
+          @path_callback.call(path, query)
+        end
+        @uri_callback.call(uri) if @uri_callback
+        if fragment && @fragment_callback
+          fragment.slice!(0)
+          @fragment_callback.call(fragment)
+        end
+        space_before_http
+      elsif @buffer.check(/[a-z0-9;\/?:@&=+$,%_.!~*')(#-]+\Z/i)
+        :uri
+      else
+        raise ParseError.new("URI or path not recognised")
+      end
+    end
+    
+    def space_before_http
+      if @buffer.skip(/ /i)
+        http
+      elsif @buffer.skip(/\r\n/i)
+        key_or_newline
+      end
+    end
+    
+    def http
+      if @buffer.skip(/HTTP\//i)
+        request_http_version
+      elsif @buffer.eos? || @buffer.check(/H(T(T(P\/?)?)?)?\Z/i)
+        :http
+      else
+        raise ParseError.new("Protocol not recognised")
+      end
+    end
+    
+    def request_http_version
+      version = @buffer.scan(/[0-9]+\.[0-9]+\r\n/i)
+      if version
+        if @version_callback
+          version.chop!
+          @version_callback.call(version)
+        end
+        key_or_newline
+      elsif @buffer.eos? || @buffer.check(/\d+(\.(\d+\r?)?)?\Z/i)
+        :request_http_version
+      else
+        raise ParseError.new("Invalid version specifier")
+      end
+    end
+    
+    def response_http_version
+      version = @buffer.scan(/[0-9]+\.[0-9]+ /i)
+      if version
+        if @version_callback
+          version.chop!
+          @version_callback.call(version)
+        end
+        status
+      elsif @buffer.eos? || @buffer.check(/\d+(\.(\d+)?)?\Z/i)
+        :response_http_version
+      else
+        raise ParseError.new("Invalid version specifier")
+      end
+    end
+    
+    def status
+      status = @buffer.scan(/\d\d\d [a-z -]+\r?\n/i)
+      if status
+        @status = status.slice!(0, 3).to_i
+        @status_callback.call(@status, status.strip) if @status_callback
+        key_or_newline
+      elsif @buffer.eos? || @buffer.check(/\d(\d(\d( ([a-z]+\r?)?)?)?)?\Z/i)
+        :status
+      else
+        raise ParseError.new("Invalid status line")
+      end
+    end
+    
+    def key_or_newline
+      @last_key = @buffer.scan(/[!-9;-~]+: /i)
+      if @last_key
+        @last_key.chomp!(KEY_TERMINATOR)
+        value
+      elsif @buffer.skip(/\n|\r\n/i)
+        @headers_callback.call(@headers) if @headers_callback
+        body
+      elsif @buffer.eos? || @buffer.check(/[!-9;-~]+:?\Z/i)
+        :key_or_newline
+      else
+        raise ParseError.new("Illegal character in field name")
+      end
+    end
+    
+    def value
+      value = @buffer.scan(/[ -~]+\r?\n/i)
+      if value
+        value.chop!
+        @headers[@last_key] = value
+        key_or_newline
+      elsif @buffer.eos? || @buffer.check(/[ -~]+\Z/i)
+        :value
+      else
+        raise ParseError.new("Illegal character in field body")
+      end
+    end
+    
+    def body
+      if @force_no_body || NO_BODY[@status]
+        end_of_message
+      elsif @buffer.eos?
+        :body
+      else
+        @body = "" if @body_callback
+        @buffer = @buffer.rest # Switch @buffer from StringScanner to String
+        length = @headers[CONTENT_LENGTH]
+        if length
+          @content_left = length.to_i
+          body_with_length
+        elsif @headers[TRANSFER_ENCODING] == CHUNKED
+          body_chunked
+        else
+          body_on_close
+        end
+      end
+    end
+    
+    #--
+    # From this point on @buffer is a String, not a StringScanner.
+    # This is because 1. we don't need a StringScanner anymore, 2. if we
+    # switched to a diffrent instace variable we'd need a condition in #concat
+    # to feed the data in to the new instace variable, which would slow us down.
+    #++
+    
+    def body_with_length
+      if @buffer.length > 0
+        chunk = @buffer.slice!(0, @content_left)
+        @stream_callback.call(chunk) if @stream_callback
+        @body << chunk if @body_callback
+        @content_left -= chunk.length
+        if @content_left < 1
+          @body_callback.call(@body) if @body_callback
+          end_of_message
+        else
+          :body_with_length
+        end
+      else
+        :body_with_length
+      end
+    end
+    
+    def body_chunked
+      decoded, remainder = transfer_encoding_chunked_decode(@buffer)
+      if decoded
+        @stream_callback.call(decoded) if @stream_callback
+        @body << decoded if @body_callback
+      end
+      if remainder
+        @buffer = remainder
+        :body_chunked
+      else
+        @buffer.slice!(/.*0\r\n/m)
+        @body_callback.call(@body) if @body_callback
+        if @headers[TRAILER] || @force_trailer
+          @trailer = {}
+          # @buffer switches back to a StringScanner for the trailer.
+          @buffer_backup_reference.string.replace(@buffer)
+          @buffer_backup_reference.reset
+          @buffer = @buffer_backup_reference
+          trailer_key_or_newline
+        else
+          end_of_message
+        end
+      end
+    end
+    
+    def body_on_close
+      @stream_callback.call(@buffer) if @stream_callback
+      @body << @buffer if @body_callback
+      @buffer = ""
+      :body_on_close
+    end
+    
+    #--
+    # @buffer switches back to a StringScanner for the trailer.
+    #++
+    
+    def trailer_key_or_newline
+      if @last_key = @buffer.scan(/[!-9;-~]+: /i)
+        @last_key.chomp!(KEY_TERMINATOR)
+        trailer_value
+      elsif @buffer.skip(/\n|\r\n/i)
+        @trailer_callback.call(@trailer) if @trailer_callback
+        end_of_message
+      elsif @buffer.eos? || @buffer.check(/[!-9;-~]+:?\Z/i)
+        :trailer_key_or_newline
+      else
+        raise ParseError.new("Illegal character in field name")
+      end
+    end
+    
+    def trailer_value
+      value = @buffer.scan(/[ -~]+\r?\n/i)
+      if value
+        value.chop!
+        @trailer[@last_key] = value
+        trailer_key_or_newline
+      elsif @buffer.eos? || @buffer.check(/[ -~]+\Z/i)
+        :trailer_value
+      else
+        raise ParseError.new("Illegal character in field body")
+      end
+    end
+    
+    def end_of_message
+      raise EndOfMessageError.new("Message ended") if @state == :end_of_message
+      remainder = @buffer.respond_to?(:rest) ? @buffer.rest : @buffer
+      if @finished_callback
+        @finished_callback.call((remainder if remainder.length > 0))
+      end
+      :end_of_message
+    end
+    
+    def raise(*args)
+      @state = :error
+      super unless @error_callback
+      @error_callback.call(args.first)
+      :error
+    end
+    alias error raise
+    
+  end
+end