http_tools 0.1.0

data/README.rdoc

@@ -0,0 +1,80 @@
+= HTTPTools
+
+HTTPTools is a collection of lower level utilities to aid working with HTTP,
+including a fast-as-possible pure Ruby HTTP parser.
+
+* rdoc[http://sourcetagsandcodes.com/http_tools/doc/]
+* source[https://github.com/matsadler/http_tools]
+
+== HTTPTools::Parser
+
+HTTPTools::Parser is a HTTP request & response parser with an evented API.
+Written purely in Ruby, with no dependencies, it should run across all Ruby
+implementations, and install in environments without a compiler available.
+Despite being just Ruby, every effort has been made to ensure it is as fast as
+possible.
+
+=== Example
+
+  parser = HTTPTools::Parser.new
+  parser.on(:status) {|status, message| puts "#{status} #{message}"}
+  parser.on(:headers) {|headers| puts headers.inspect}
+  parser.on(:body) {|body| puts body}
+  
+  parser << "HTTP/1.1 200 OK\r\n"
+  parser << "Content-Length: 20\r\n\r\n"
+  parser << "<h1>Hello world</h1>"
+
+Prints:
+  200 OK
+  {"Content-Length" => "20"}
+  <h1>Hello world</h1>
+
+== HTTPTools::Encoding
+
+HTTPTools::Encoding provides methods to deal with several HTTP related encodings
+including url, www-form, and chunked transfer encoding. It can be used as a
+mixin or class methods on HTTPTools::Encoding.
+
+=== Example
+
+  HTTPTools::Encoding.www_form_encode({"query" => "fish", "lang" => "en"})
+  #=> "lang=en&query=fish"
+  
+  include HTTPTools::Encoding
+  www_form_decode("lang=en&query=fish")
+  #=> {"query" => "fish", "lang" => "en"}
+
+== HTTPTools::Builder
+
+HTTPTools::Builder is a provides a simple interface to build HTTP requests &
+responses. It can be used as a mixin or class methods on HTTPTools::Builder.
+
+=== Example
+
+  Builder.request(:get, "example.com")\
+  #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+
+== Licence
+
+(The MIT License)
+
+Copyright (c) 2011 Matthew Sadler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/bench/parser/request_bench.rb

@@ -0,0 +1,59 @@
+base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+require base + '/http_tools'
+require 'benchmark'
+
+request = "GET / HTTP/1.1\r\nHost: example.com\r\nUser-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-gb) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16\r\nAccept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5\r\nAccept-Language: en-gb\r\nAccept-Encoding: gzip, deflate\r\nConnection: keep-alive\r\n\r\n"
+
+Benchmark.bm(41) do |x|
+  x.report("HTTPTools::Parser") do
+    10_000.times do
+       HTTPTools::Parser.new << request
+    end
+  end
+  
+  x.report("HTTPTools::Parser (reset)") do
+    parser = HTTPTools::Parser.new
+    10_000.times do
+       parser << request
+       parser.reset
+    end
+  end
+  
+  x.report("HTTPTools::Parser (reset, with callbacks)") do
+    parser = HTTPTools::Parser.new
+    parser.on(:method) {|arg|}
+    parser.on(:path) {|arg, arg2|}
+    parser.on(:headers) {|arg|}
+    10_000.times do
+       parser << request
+       parser.reset
+    end
+  end
+  
+  x.report("HTTPTools::Parser (reset, with delegate)") do
+    class TestDelegate
+      def on_method(arg)
+      end
+      def on_path(arg, arg2)
+      end
+      def on_headers(arg)
+      end
+    end
+    parser = HTTPTools::Parser.new(TestDelegate.new)
+    10_000.times do
+       parser << request
+       parser.reset
+    end
+  end
+  
+  begin
+    require 'rubygems'
+    require 'http11'
+    x.report("Mongrel::HttpParser") do
+      10_000.times do
+        Mongrel::HttpParser.new.execute({}, request.dup, 0)
+      end
+    end
+  rescue LoadError
+  end
+end

data/bench/parser/response_bench.rb

@@ -0,0 +1,21 @@
+base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+require base + '/http_tools'
+require 'benchmark'
+
+response = "HTTP/1.1 200 OK\r\nServer: Apache/2.2.3 (CentOS)\r\nLast-Modified: Thu, 03 Jun 2010 17:40:12 GMT\r\nETag: \"4d2c-23e-48823b2cf3700\"\r\nAccept-Ranges: bytes\r\nContent-Type: text/html; charset=UTF-8\r\nConnection: Keep-Alive\r\nDate: Wed, 21 Jul 2010 16:26:04 GMT\r\nAge: 7985   \r\nContent-Length: 574\r\n\r\n<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">\r\n<HTML>\r\n<HEAD>\r\n  <META http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\r\n  <TITLE>Example Web Page</TITLE>\r\n</HEAD> \r\n<body>  \r\n<p>You have reached this web page by typing &quot;example.com&quot;,\r\n&quot;example.net&quot;,\r\n  or &quot;example.org&quot; into your web browser.</p>\r\n<p>These domain names are reserved for use in documentation and are not available \r\n  for registration. See <a href=\"http://www.rfc-editor.org/rfc/rfc2606.txt\">RFC \r\n  2606</a>, Section 3.</p>\r\n</BODY>\r\n</HTML>\r\n\r\n"
+
+Benchmark.bm(25) do |x|
+  x.report("HTTPTools::Parser") do
+    10_000.times do
+       HTTPTools::Parser.new << response
+    end
+  end
+  
+  x.report("HTTPTools::Parser (reset)") do
+    parser = HTTPTools::Parser.new
+    10_000.times do
+       parser << response
+       parser.reset
+    end
+  end
+end

data/example/http_client.rb

@@ -0,0 +1,132 @@
+require 'uri'
+require 'socket'
+require 'stringio'
+require 'rubygems'
+require 'http_tools'
+
+# Usage:
+#   uri = URI.parse("http://example.com/")
+#   client = HTTP::Client.new(uri.host, uri.port)
+#   response = client.get(uri.path)
+#   
+#   puts "#{response.status} #{response.message}"
+#   puts response.headers.inspect
+#   puts response.body
+# 
+# Streaming response:
+#   client.get(uri.path) do |response|
+#     puts "#{response.status} #{response.message}"
+#     response.stream do |chunk|
+#       print chunk
+#     end
+#   end
+#   
+module HTTP
+  class Client
+    include HTTPTools::Encoding
+    
+    CONTENT_TYPE = "Content-Type".freeze
+    CONTENT_LENGTH = "Content-Length".freeze
+    WWW_FORM = "application/x-www-form-urlencoded".freeze
+    
+    def initialize(host, port=80)
+      @host = host
+      @port = port
+    end
+    
+    def socket
+      @socket ||= TCPSocket.new(@host, @port)
+    end
+    
+    def head(path, headers={})
+      request(:head, path, nil, headers, false)
+    end
+    
+    def get(path, headers={}, &block)
+      request(:get, path, nil, headers, &block)
+    end
+    
+    def post(path, body="", headers={}, &block)
+      headers[CONTENT_TYPE] ||= WWW_FORM
+      unless body.respond_to?(:read)
+        if headers[CONTENT_TYPE] == WWW_FORM && body.respond_to?(:map) &&
+          !body.kind_of?(String)
+          body = www_form_encode(body)
+        end
+        body = StringIO.new(body.to_s)
+      end
+      if headers[CONTENT_LENGTH]
+        # ok
+      elsif body.respond_to?(:length)
+        headers[CONTENT_LENGTH] ||= body.length
+      elsif body.respond_to?(:stat)
+        headers[CONTENT_LENGTH] ||= body.stat.size
+      else
+        raise "Content-Length must be supplied"
+      end
+      
+      request(:post, path, body, headers, &block)
+    end
+    
+    private
+    def request(method, path, request_body=nil, request_headers={}, response_has_body=true, &block)
+      parser = HTTPTools::Parser.new
+      parser.force_no_body = !response_has_body
+      response = nil
+      
+      parser.add_listener(:status) {|s, m| response = Response.new(s, m)}
+      parser.add_listener(:headers) do |headers|
+        response.headers = headers
+        if block
+          response.parser = parser
+          block.call(response)
+          response.parser = nil
+        end
+      end
+      parser.add_listener(:body) {|body| response.body = body} unless block
+      
+      socket << HTTPTools::Builder.request(method, @host, path, request_headers)
+      if request_body
+        socket << request_body.read(1024 * 16) until request_body.eof?
+      end
+      
+      until parser.finished?
+        begin
+          readable, = select([socket], nil, nil)
+          parser << socket.read_nonblock(1024 * 16) if readable.any?
+        rescue EOFError
+          parser.finish
+          break
+        end
+      end
+      response
+    end
+  end
+  
+  class Response
+    attr_reader :status, :message
+    attr_accessor :headers, :body
+    attr_accessor :parser # :nodoc:
+    
+    def initialize(status, message, headers={}, body=nil)
+      @status = status
+      @message = message
+      @headers = headers
+      @body = body
+    end
+    
+    def stream(&block)
+      if parser
+        parser.add_listener(:stream, block)
+      else
+        block.call(body)
+      end
+      nil
+    end
+    
+    def inspect
+      bytesize = body.respond_to?(:bytesize) ? body.bytesize : body.to_s.length
+      "#<Response #{status} #{message}: #{bytesize} bytes>"
+    end
+  end
+end

data/lib/http_tools.rb

@@ -0,0 +1,110 @@
+module HTTPTools
+  STATUS_CODES = {
+    :continue => 100,
+    :switching_protocols => 101,
+    :ok => 200,
+    :created => 201,
+    :accepted => 202,
+    :non_authoritative_information => 203,
+    :no_content => 204,
+    :reset_content => 205,
+    :partial_content => 206,
+    :multiple_choices => 300,
+    :moved_permanently => 301,
+    :found => 302,
+    :see_other => 303,
+    :not_modified => 304,
+    :use_proxy => 305,
+    :temporary_redirect => 307,
+    :bad_request => 400,
+    :unauthorized => 401,
+    :payment_required => 402,
+    :forbidden => 403,
+    :not_found => 404,
+    :method_not_allowed => 405,
+    :not_acceptable => 406,
+    :proxy_authentication_required => 407,
+    :request_timeout => 408,
+    :conflict => 409,
+    :gone => 410,
+    :length_required => 411,
+    :precondition_failed => 412,
+    :request_entity_too_large => 413,
+    :request_uri_too_long => 414,
+    :unsupported_media_type => 415,
+    :requested_range_not_satisfiable => 416,
+    :expectation_failed => 417,
+    :internal_server_error => 500,
+    :not_implemented => 501,
+    :bad_gateway => 502,
+    :service_unavailable => 503,
+    :gateway_timeout => 504,
+    :http_version_not_supported => 505}.freeze
+  
+  STATUS_DESCRIPTIONS = {
+    100 => "Continue",
+    101 => "Switching Protocols",
+    200 => "OK",
+    201 => "Created",
+    202 => "Accepted",
+    203 => "Non-Authoritative Information",
+    204 => "No Content",
+    205 => "Reset Content",
+    206 => "Partial Content",
+    300 => "Multiple Choices",
+    301 => "Moved Permanently",
+    302 => "Found",
+    303 => "See Other",
+    304 => "Not Modified",
+    305 => "Use Proxy",
+    307 => "Temporary Redirect",
+    400 => "Bad Request",
+    401 => "Unauthorized",
+    402 => "Payment Required",
+    403 => "Forbidden",
+    404 => "Not Found",
+    405 => "Method Not Allowed",
+    406 => "Not Acceptable",
+    407 => "Proxy Authentication Required",
+    408 => "Request Timeout",
+    409 => "Conflict",
+    410 => "Gone",
+    411 => "Length Required",
+    412 => "Precondition Failed",
+    413 => "Request Entity Too Large",
+    414 => "Request-URI Too Long",
+    415 => "Unsupported Media Type",
+    416 => "Requested Range Not Satisfiable",
+    417 => "Expectation Failed",
+    500 => "Internal Server Error",
+    501 => "Not Implemented",
+    502 => "Bad Gateway",
+    503 => "Service Unavailable",
+    504 => "Gateway Timeout",
+    505 => "HTTP Version Not Supported"}.freeze
+  STATUS_DESCRIPTIONS.values.each {|val| val.freeze}
+  
+  STATUS_LINES = Hash.new do |hash, key|
+    code = if key.kind_of?(Integer) then key else STATUS_CODES[key] end
+    description = STATUS_DESCRIPTIONS[code]
+    hash[key] = "#{code} #{description}"
+  end
+  
+  METHODS = %W{GET POST HEAD PUT DELETE OPTIONS TRACE CONNECT}.freeze
+  
+  NO_BODY = Hash.new {|hash, key| hash[key] = false}
+  NO_BODY.merge!(204 => true, 304 => true, nil => false)
+  100.upto(199) {|status_code| NO_BODY[status_code] = true}
+  
+  CRLF = "\r\n".freeze
+  SPACE = " ".freeze
+  
+  require_base = File.dirname(__FILE__) + '/http_tools/'
+  autoload :Encoding, require_base + 'encoding'
+  autoload :Parser, require_base + 'parser'
+  autoload :Builder, require_base + 'builder'
+  autoload :ParseError, require_base + 'errors'
+  autoload :EndOfMessageError, require_base + 'errors'
+  autoload :MessageIncompleteError, require_base + 'errors'
+  
+end

data/lib/http_tools/builder.rb

@@ -0,0 +1,49 @@
+module HTTPTools
+  
+  # HTTPTools::Builder is a provides a simple interface to build HTTP requests &
+  # responses. It can be used as a mixin or class methods on HTTPTools::Builder.
+  # 
+  module Builder
+    KEY_VALUE = "%s: %s\r\n".freeze
+    
+    module_function
+    
+    # :call-seq: Builder.response(status, headers={}) -> string
+    # 
+    # Returns a HTTP status line and headers. Status can be a HTTP status code
+    # as an integer, or a HTTP status message as a lowercase, underscored
+    # symbol.
+    #   Builder.response(200, "Content-Type" => "text/html")\
+    #=> "HTTP/1.1 200 ok\r\nContent-Type: text/html\r\n\r\n"
+    #   
+    #   Builder.response(:internal_server_error)\
+    #=> "HTTP/1.1 500 Internal Server Error\r\n\r\n"
+    # 
+    def response(status, headers={})
+      "HTTP/1.1 #{STATUS_LINES[status]}\r\n#{format_headers(headers)}\r\n"
+    end
+    
+    # :call-seq: Builder.request(method, host, path="/", headers={}) -> string
+    # 
+    # Returns a HTTP request line and headers.
+    #   Builder.request(:get, "example.com")\
+    #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    #   
+    #   Builder.request(:post, "example.com", "/form", "Accept" => "text/html")\
+    #=> "POST" /form HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
+    # 
+    def request(method, host, path="/", headers={})
+      "#{method.to_s.upcase} #{path} HTTP/1.1\r\nHost: #{host}\r\n#{
+        format_headers(headers)}\r\n"
+    end
+    
+    def format_headers(headers)
+      headers.inject("") {|buffer, kv| buffer << KEY_VALUE % kv}
+    end
+    private :format_headers
+    class << self
+      private :format_headers
+    end
+    
+  end
+end