http_tools 0.3.0 → 0.4.0

data/README.rdoc

@@ -25,7 +25,7 @@ possible.
     puts parser.status_code + " " + parser.request_method
     puts parser.header.inspect
   end
-  parser.on(:stream) {|chunk| print chunk}
+  parser.on(:finish) {print parser.body}
   
   parser << "HTTP/1.1 200 OK\r\n"
   parser << "Content-Length: 20\r\n\r\n"
@@ -45,11 +45,11 @@ mixin or class methods on HTTPTools::Encoding.
 === Example
 
   HTTPTools::Encoding.www_form_encode({"query" => "fish", "lang" => "en"})
-  #=> "lang=en&query=fish"
+  #=> "query=fish&lang=en"
   
   include HTTPTools::Encoding
   www_form_decode("lang=en&query=fish")
-  #=> {"query" => "fish", "lang" => "en"}
+  #=> {"lang" => "en", "query" => "fish"}
 
 == HTTPTools::Builder
 

data/bench/parser/large_response_bench.rb

@@ -0,0 +1,40 @@
+base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+require base + '/http_tools'
+require 'benchmark'
+
+Benchmark.bm(26) do |x|
+  body = "x" * 1024 * 1024
+  chunks = []
+  64.times {|i| chunks << body[i * 64, body.length / 64]}
+  
+  header = "HTTP/1.1 200 OK\r\nDate: Mon, 06 Jun 2011 14:55:51 GMT\r\nServer: Apache/2.2.17 (Unix) mod_ssl/2.2.17 OpenSSL/0.9.8l DAV/2 mod_fastcgi/2.4.2\r\nLast-Modified: Mon, 06 Jun 2011 14:55:49 GMT\r\nETag: \"3f18045-400-4a50c4c87c740\"\r\nAccept-Ranges: bytes\r\nContent-Length: #{body.length}\r\nContent-Type: text/plain\r\n\r\n"
+  x.report("content_length") do
+    200.times do
+      parser = HTTPTools::Parser.new
+      parser << header
+      chunks.each {|chunk| parser << chunk}
+    end
+  end
+  
+  header = "HTTP/1.1 200 OK\r\nDate: Mon, 06 Jun 2011 14:55:51 GMT\r\nServer: Apache/2.2.17 (Unix) mod_ssl/2.2.17 OpenSSL/0.9.8l DAV/2 mod_fastcgi/2.4.2\r\nLast-Modified: Mon, 06 Jun 2011 14:55:49 GMT\r\nETag: \"3f18045-400-4a50c4c87c740\"\r\nAccept-Ranges: bytes\r\nConnection: close\r\nContent-Type: text/plain\r\n\r\n"
+  x.report("close") do
+    200.times do
+      parser = HTTPTools::Parser.new
+      parser << header
+      chunks.each {|chunk| parser << chunk}
+      parser.finish
+    end
+  end
+  
+  header = "HTTP/1.1 200 OK\r\nDate: Mon, 06 Jun 2011 14:55:51 GMT\r\nServer: Apache/2.2.17 (Unix) mod_ssl/2.2.17 OpenSSL/0.9.8l DAV/2 mod_fastcgi/2.4.2\r\nLast-Modified: Mon, 06 Jun 2011 14:55:49 GMT\r\nETag: \"3f18045-400-4a50c4c87c740\"\r\nAccept-Ranges: bytes\r\nTransfer-Encoding: chunked\r\nContent-Type: text/plain\r\n\r\n"
+  chunks << nil
+  chunks.map!(&HTTPTools::Encoding.method(:transfer_encoding_chunked_encode))
+  x.report("chunked") do
+    200.times do
+      parser = HTTPTools::Parser.new
+      parser << header
+      chunks.each {|chunk| parser << chunk}
+    end
+  end
+  
+end

data/bench/transfer_encoding_chunked_bench.rb

@@ -3,24 +3,24 @@ require base + '/http_tools'
 require 'benchmark'
 
 Benchmark.bm(36) do |x|
+  encoded = "1\r\na\r\n" * 100 + "0\r\n"
   x.report("lots of very short chunks") do
-    encoded = "1\r\na\r\n" * 100 + "0\r\n"
     1_000.times do
        HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
     end
   end
   
+  encoded = "16\r\n<h1>Hello world</h1>\r\n\r\n12\r\n<p>Lorem ipsum</p>\r\n" * 50 + "0\r\n"
   x.report("slightly less slightly longer chunks") do
-    encoded = "16\r\n<h1>Hello world</h1>\r\n\r\n12\r\n<p>Lorem ipsum</p>\r\n" * 50 + "0\r\n"
     1_000.times do
        HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
     end
   end
   
+  encoded = "2710\r\n#{"a" * 10000}\r\n" * 2 + "0\r\n"
   x.report("a couple of big chunks") do
-    encoded = "2710\r\n#{"a" * 10000}\r\n" * 2 + "0\r\n"
     1_000.times do
        HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
     end
   end
-end
+end

data/example/http_client.rb

@@ -1,6 +1,4 @@
-require 'uri'
 require 'socket'
-require 'stringio'
 require 'rubygems'
 require 'http_tools'
 
@@ -12,170 +10,57 @@ require 'http_tools'
 #   puts "#{response.status} #{response.message}"
 #   puts response.headers.inspect
 #   puts response.body
-# 
-# Streaming response:
-#   client.get(uri.path) do |response|
-#     puts "#{response.status} #{response.message}"
-#     response.stream do |chunk|
-#       print chunk
-#     end
-#   end
 #   
 module HTTP
   class Client
-    include HTTPTools::Encoding
-    
-    CONTENT_TYPE = "Content-Type".freeze
-    CONTENT_LENGTH = "Content-Length".freeze
-    WWW_FORM = "application/x-www-form-urlencoded".freeze
-    
-    attr_writer :keepalive
+    Response = Struct.new(:status, :message, :headers, :body)
     
     def initialize(host, port=80)
-      @host = host
-      @port = port
-      @pipeline = []
-    end
-    
-    def socket
-      @socket ||= TCPSocket.new(@host, @port)
+      @host, @port = host, port
     end
     
     def head(path, headers={})
       request(:head, path, nil, headers, false)
     end
     
-    def get(path, headers={}, &block)
-      request(:get, path, nil, headers, &block)
+    def get(path, headers={})
+      request(:get, path, nil, headers)
     end
     
-    def post(path, body="", headers={}, &block)
-      headers[CONTENT_TYPE] ||= WWW_FORM
-      unless body.respond_to?(:read)
-        if headers[CONTENT_TYPE] == WWW_FORM && body.respond_to?(:map) &&
-          !body.kind_of?(String)
-          body = www_form_encode(body)
-        end
-        body = StringIO.new(body.to_s)
-      end
-      if headers[CONTENT_LENGTH]
-        # ok
-      elsif body.respond_to?(:length)
-        headers[CONTENT_LENGTH] = body.length
-      elsif body.respond_to?(:stat)
-        headers[CONTENT_LENGTH] = body.stat.size
-      else
-        raise "Content-Length must be supplied"
-      end
-      
-      request(:post, path, body, headers, &block)
+    def post(path, body=nil, headers={})
+      request(:post, path, body, headers)
     end
     
-    def pipeline
-      @pipelining = true
-      yield self
-      pipeline_requests(@pipeline)
-    ensure
-      @pipelining = false
+    def put(path, body=nil, headers={})
+      request(:put, path, body, headers)
     end
     
-    def keepalive?
-      @keepalive
-    end
-    
-    def keepalive
-      self.keepalive, original = true, keepalive?
-      yield self
-    ensure
-      self.keepalive = original
+    def delete(path, headers={})
+      request(:delete, path, nil, headers)
     end
     
     private
-    def request(method, path, body=nil, headers={}, response_has_body=true, &b)
-      request = {
-        :method => method,
-        :path => path,
-        :body => body,
-        :headers => headers,
-        :response_has_body => response_has_body,
-        :block => b}
-      if @pipelining
-        @pipeline << request
-        nil
-      else
-        pipeline_requests([request]).first
-      end
-    end
-    
-    def pipeline_requests(requests)
+    def request(method, path, body=nil, headers={}, response_has_body=true)
       parser = HTTPTools::Parser.new
-      parser.allow_html_without_header = true
-      responses = []
-      
-      parser.on(:finish) do |remainder|
-        if responses.length < requests.length
-          parser.reset
-          parser << remainder.lstrip if remainder
-          throw :reset
-        end
-      end
-      parser.on(:header) do
-        request = requests[responses.length]
-        parser.force_no_body = !request[:response_has_body]
-        response = Response.new(parser.status_code, parser.message)
-        response.headers = parser.header
-        parser.on(:stream) {|chunk| response.receive_chunk(chunk)}
-        responses.push(response)
-      end
+      parser.force_no_body = !response_has_body
+      response = nil
       
-      requests.each do |r|
-        socket << HTTPTools::Builder.request(r[:method], @host, r[:path], r[:headers])
-        if body = r[:body]
-          socket << body.read(1024 * 16) until body.eof?
-        end
+      parser.on(:finish) do
+        code, message = parser.status_code, parser.message
+        response = Response.new(code, message, parser.header, parser.body)
       end
       
+      socket = TCPSocket.new(@host, @port)
+      socket << HTTPTools::Builder.request(method, @host, path, headers)
+      socket << body if body
       begin
-        catch(:reset) {parser << socket.sysread(1024 * 16)}
+        parser << socket.sysread(1024 * 16)
       rescue EOFError
-        @socket = nil
-        parser.finish
-        break
+        break parser.finish
       end until parser.finished?
+      socket.close
       
-      @socket = nil unless keepalive?
-      responses
-    end
-  end
-  
-  class Response
-    attr_reader :status, :message
-    attr_accessor :headers, :body
-    
-    def initialize(status, message, headers={}, body="")
-      @status = status
-      @message = message
-      @headers = headers
-      @body = body
-    end
-    
-    def stream(&block)
-      @stream_callback = block
-      nil
-    end
-    
-    def receive_chunk(chunk) # :nodoc:
-      body << chunk
-      @stream_callback.call(chunk) if @stream_callback
-    end
-    
-    def inspect
-      bytesize = body.respond_to?(:bytesize) ? body.bytesize : body.to_s.length
-      "#<Response #{status} #{message}: #{bytesize} bytes>"
-    end
-    
-    def to_s
-      body.to_s
+      response
     end
   end
-end
+end

data/example/http_server.rb

@@ -1,11 +1,9 @@
 require 'socket'
-require 'stringio'
 require 'rubygems'
 require 'http_tools'
 
 module HTTP
   class Server
-    RACK_INPUT = "rack.input".freeze
     CONNECTION = "Connection".freeze
     KEEP_ALIVE = "Keep-Alive".freeze
     CLOSE = "close".freeze
@@ -39,16 +37,9 @@ module HTTP
     private
     def on_connection(socket)
       parser = HTTPTools::Parser.new
-      env, input = nil
       
-      parser.on(:header) do
-        input = StringIO.new
-        env = parser.env.merge!(RACK_INPUT => input).merge!(@instance_env)
-      end
-      parser.on(:stream) {|chunk| input << chunk}
       parser.on(:finish) do
-        input.rewind
-        status, header, body = @app.call(env)
+        status, header, body = @app.call(parser.env.merge!(@instance_env))
         keep_alive = keep_alive?(parser.version, parser.header[CONNECTION])
         header[CONNECTION] = keep_alive ? KEEP_ALIVE : CLOSE
         socket << HTTPTools::Builder.response(status, header)
@@ -74,7 +65,3 @@ module HTTP
     
   end
 end
-
-HTTP::Server.run(proc do |env|
-  [200, {"Content-Length" => "5"}, "Hello"]
-end)

data/lib/http_tools.rb

@@ -84,11 +84,13 @@ module HTTPTools
     505 => "HTTP Version Not Supported"}.freeze
   STATUS_DESCRIPTIONS.values.each {|val| val.freeze}
   
+  # :stopdoc: hide from rdoc as it makes a mess
   STATUS_LINES = Hash.new do |hash, key|
     code = if key.kind_of?(Integer) then key else STATUS_CODES[key] end
     description = STATUS_DESCRIPTIONS[code]
     hash[key] = "#{code} #{description}"
   end
+  # :startdoc:
   
   METHODS = %W{GET POST HEAD PUT DELETE OPTIONS TRACE CONNECT}.freeze
   
@@ -96,15 +98,15 @@ module HTTPTools
   100.upto(199) {|status_code| NO_BODY[status_code] = true}
   NO_BODY.freeze
   
-  ARRAY_VALUE_HEADERS = {"Set-Cookie" => true} # presence of key tested, not val
+  Error = Class.new(StandardError)
+  ParseError = Class.new(Error)
+  EndOfMessageError = Class.new(ParseError)
+  MessageIncompleteError = Class.new(EndOfMessageError)
+  EmptyMessageError = Class.new(MessageIncompleteError)
   
   require_base = File.dirname(__FILE__) + '/http_tools/'
   autoload :Encoding, require_base + 'encoding'
   autoload :Parser, require_base + 'parser'
   autoload :Builder, require_base + 'builder'
-  autoload :ParseError, require_base + 'errors'
-  autoload :EndOfMessageError, require_base + 'errors'
-  autoload :MessageIncompleteError, require_base + 'errors'
-  autoload :EmptyMessageError, require_base + 'errors'
   
 end

data/lib/http_tools/builder.rb

@@ -4,8 +4,6 @@ module HTTPTools
   # responses. It can be used as a mixin or class methods on HTTPTools::Builder.
   # 
   module Builder
-    KEY_VALUE = "%s: %s\r\n".freeze
-    
     module_function
     
     # :call-seq: Builder.response(status, headers={}) -> string
@@ -13,11 +11,15 @@ module HTTPTools
     # Returns a HTTP status line and headers. Status can be a HTTP status code
     # as an integer, or a HTTP status message as a lowercase, underscored
     # symbol.
-    #   Builder.response(200, "Content-Type" => "text/html")\
-    #=> "HTTP/1.1 200 ok\r\nContent-Type: text/html\r\n\r\n"
+    #   Builder.response(200, "Content-Type" => "text/html")
+    #   #=> "HTTP/1.1 200 ok\r\nContent-Type: text/html\r\n\r\n"
+    #   
+    #   Builder.response(:internal_server_error)
+    #   #=> "HTTP/1.1 500 Internal Server Error\r\n\r\n"
     #   
-    #   Builder.response(:internal_server_error)\
-    #=> "HTTP/1.1 500 Internal Server Error\r\n\r\n"
+    # To send multiple headers with the same name:
+    #   Builder.response(:ok, "Set-Cookie" => ["a=b", "c=d"])
+    #   Builder.response(:ok, "Set-Cookie" => "a=b\nc=d")
     # 
     def response(status, headers={})
       "HTTP/1.1 #{STATUS_LINES[status]}\r\n#{format_headers(headers)}\r\n"
@@ -26,10 +28,11 @@ module HTTPTools
     # :call-seq: Builder.request(method, host, path="/", headers={}) -> string
     # 
     # Returns a HTTP request line and headers.
-    #   Builder.request(:get, "example.com")\
-    #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    #   Builder.request(:get, "example.com")
+    #   #=> "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
     #   
-    #   Builder.request(:post, "example.com", "/form", "Accept" => "text/html")\
+    #   Builder.request(:post, "example.com", "/form", "Accept" => "text/html")
+    #\
     #=> "POST" /form HTTP/1.1\r\nHost: example.com\r\nAccept: text/html\r\n\r\n"
     # 
     def request(method, host, path="/", headers={})
@@ -37,8 +40,17 @@ module HTTPTools
         format_headers(headers)}\r\n"
     end
     
-    def format_headers(headers)
-      headers.inject("") {|buffer, kv| buffer << KEY_VALUE % kv}
+    def format_headers(headers, buffer="")
+      headers.each do |key, value|
+        if value.respond_to?(:each_line)
+          value.each_line {|val| buffer << "#{key}: #{val.chomp}\r\n"}
+        elsif value.respond_to?(:each)
+          value.each {|val| buffer << "#{key}: #{val}\r\n"}
+        else
+          buffer << "#{key}: #{value}\r\n"
+        end
+      end
+      buffer
     end
     private :format_headers
     class << self

data/lib/http_tools/encoding.rb

@@ -7,6 +7,7 @@ module HTTPTools
   # used as a mixin or class methods on HTTPTools::Encoding.
   # 
   module Encoding
+    # :stopdoc:
     HEX_BIG_ENDIAN_2_BYTES = "H2".freeze
     HEX_BIG_ENDIAN_REPEATING = "H*".freeze
     PERCENT = "%".freeze
@@ -15,6 +16,7 @@ module HTTPTools
     AMPERSAND = "&".freeze
     EQUALS = "=".freeze
     CHUNK_FORMAT = "%x\r\n%s\r\n".freeze
+    # :startdoc:
     
     module_function
     
@@ -44,7 +46,7 @@ module HTTPTools
     # 
     # Takes a Hash and converts it to a String as if it was a HTML form being
     # submitted, eg
-    # {"query" => "fish", "lang" => "en"} becomes "lang=en&query=fish"
+    # {"query" => "fish", "lang" => "en"} becomes "query=fish&lang=en"
     # 
     # To get multiple key value pairs with the same key use an array as the
     # value, eg
@@ -64,7 +66,7 @@ module HTTPTools
     # 
     # Takes a String resulting from a HTML form being submitted, and converts it
     # to a hash,
-    # eg "lang=en&query=fish" becomes {"query" => "fish", "lang" => "en"}
+    # eg "lang=en&query=fish" becomes {"lang" => "en", "query" => "fish"}
     # 
     # Multiple key value pairs with the same key will become a single key with
     # an array value, eg "lang=en&lang=fr" becomes {"lang" => ["en", "fr"]}