httpclient 2.1.2 → 2.1.3

data/lib/httpclient/connection.rb

@@ -0,0 +1,84 @@
+# HTTPClient - HTTP client library.
+# Copyright (C) 2000-2008  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+#
+# This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
+# redistribute it and/or modify it under the same terms of Ruby's license;
+# either the dual license version in 2003, or any later version.
+
+
+class HTTPClient
+
+
+  # Represents a HTTP response to an asynchronous request.  Async methods of
+  # HTTPClient such as get_async, post_async, etc. returns an instance of
+  # Connection.
+  #
+  # == How to use
+  #
+  # 1. Invoke HTTP method asynchronously and check if it's been finished
+  #    periodically.
+  #
+  #     connection = clnt.post_async(url, body)
+  #     print 'posting.'
+  #     while true
+  #       break if connection.finished?
+  #       print '.'
+  #       sleep 1
+  #     end
+  #     puts '.'
+  #     res = connection.pop
+  #     p res.status
+  #
+  # 2. Read the response as an IO.
+  #
+  #     connection = clnt.get_async('http://dev.ctor.org/')
+  #     io = connection.pop.content
+  #     while str = io.read(40)
+  #       p str
+  #     end
+  class Connection
+    attr_accessor :async_thread
+
+    def initialize(header_queue = [], body_queue = []) # :nodoc:
+      @headers = header_queue
+      @body = body_queue
+      @async_thread = nil
+      @queue = Queue.new
+    end
+
+    # Checks if the asynchronous invocation has been finished or not.
+    def finished?
+      if !@async_thread
+        # Not in async mode.
+        true
+      elsif @async_thread.alive?
+        # Working...
+        false
+      else
+        # Async thread have been finished.
+        join
+        true
+      end
+    end
+
+    # Retrieves a HTTP::Message instance of HTTP response.  Do not invoke this
+    # method twice for now.  The second invocation will be blocked.
+    def pop
+      @queue.pop
+    end
+
+    def push(result) # :nodoc:
+      @queue.push(result)
+    end
+
+    # Waits the completion of the asynchronous invocation.
+    def join
+      if @async_thread
+        @async_thread.join
+      end
+      nil
+    end
+  end
+
+
+end

data/lib/httpclient/cookie.rb

@@ -12,6 +12,8 @@
 #    w3m homepage:  http://ei5nazha.yz.yamagata-u.ac.jp/~aito/w3m/eng/
 
 require 'uri'
+require 'time'
+require 'monitor'
 
 class WebAgent
 
@@ -30,18 +32,18 @@ class WebAgent
     end
 
     def domain_match(host, domain)
-      domain = domain.downcase
-      host = host.downcase
+      domainname = domain.sub(/\.\z/, '').downcase
+      hostname = host.sub(/\.\z/, '').downcase
       case domain
       when /\d+\.\d+\.\d+\.\d+/
-	return (host == domain)
+	return (hostname == domainname)
       when '.' 
 	return true
       when /^\./
         # allows; host == rubyforge.org, domain == .rubyforge.org
-	return tail_match?(domain, host) || (domain == '.' + host)
+	return tail_match?(domainname, '.' + hostname)
       else
-	return (host == domain)
+	return (hostname == domainname)
       end
     end
 
@@ -54,9 +56,6 @@ class WebAgent
   class Cookie
     include CookieUtils
 
-    require 'parsedate'
-    include ParseDate
-
     attr_accessor :name, :value
     attr_accessor :domain, :path
     attr_accessor :expires      ## for Netscape Cookie
@@ -73,6 +72,7 @@ class WebAgent
 
     def initialize()
       @discard = @use = @secure = @domain_orig = @path_orig = @override = nil
+      @path = nil
     end
 
     def discard?
@@ -185,7 +185,7 @@ class WebAgent
 	  @domain = value
 	when 'expires'
 	  begin
-	    @expires = Time.gm(*parsedate(value)[0,6])
+	    @expires = Time.parse(value)
 	  rescue ArgumentError
 	    @expires = nil
 	  end
@@ -208,43 +208,52 @@ class WebAgent
     class Error < StandardError; end
     class ErrorOverrideOK < Error; end
     class SpecialError < Error; end
-    class NoDotError < ErrorOverrideOK; end
-
-    SPECIAL_DOMAIN = [".com",".edu",".gov",".mil",".net",".org",".int"]
 
-    attr_accessor :cookies
+    attr_reader :cookies
     attr_accessor :cookies_file
     attr_accessor :accept_domains, :reject_domains
+
+    # for conformance to http://wp.netscape.com/newsref/std/cookie_spec.html
     attr_accessor :netscape_rule
+    SPECIAL_DOMAIN = [".com",".edu",".gov",".mil",".net",".org",".int"]
 
     def initialize(file=nil)
       @cookies = Array.new()
+      @cookies.extend(MonitorMixin)
       @cookies_file = file
       @is_saved = true
       @reject_domains = Array.new()
       @accept_domains = Array.new()
-      # for conformance to http://wp.netscape.com/newsref/std/cookie_spec.html
       @netscape_rule = false
     end
 
+    def cookies=(cookies)
+      @cookies = cookies
+      @cookies.extend(MonitorMixin)
+    end
+
     def save_all_cookies(force = nil, save_unused = true, save_discarded = true)
-      if @is_saved and !force
-	return
-      end
-      File.open(@cookies_file, 'w') do |f|
-	@cookies.each do |cookie|
-          if (cookie.use? or save_unused) and
-              (!cookie.discard? or save_discarded)
-	    f.print(cookie.url.to_s,"\t",
-		    cookie.name,"\t",
-		    cookie.value,"\t",
-		    cookie.expires.to_i,"\t",
-		    cookie.domain,"\t",
-		    cookie.path,"\t",
-		    cookie.flag,"\n")
-	  end
+      @cookies.synchronize do
+        check_expired_cookies()
+        if @is_saved and !force
+          return
+        end
+        File.open(@cookies_file, 'w') do |f|
+          @cookies.each do |cookie|
+            if (cookie.use? or save_unused) and
+                (!cookie.discard? or save_discarded)
+              f.print(cookie.url.to_s,"\t",
+                      cookie.name,"\t",
+                      cookie.value,"\t",
+                      cookie.expires.to_i,"\t",
+                      cookie.domain,"\t",
+                      cookie.path,"\t",
+                      cookie.flag,"\n")
+            end
+          end
         end
       end
+      @is_saved = true
     end
 
     def save_cookies(force = nil)
@@ -253,11 +262,11 @@ class WebAgent
 
     def check_expired_cookies()
       @cookies.reject!{|cookie|
-	is_expired = (cookie.expires && (cookie.expires < Time.now.gmtime))
-	if is_expired && !cookie.discard?
-	  @is_saved = false
-	end
-	is_expired
+        is_expired = (cookie.expires && (cookie.expires < Time.now.gmtime))
+        if is_expired && !cookie.discard?
+          @is_saved = false
+        end
+        is_expired
       }
     end
 
@@ -284,17 +293,16 @@ class WebAgent
 
 
     def find(url)
-
-      check_expired_cookies()
+      return nil if @cookies.empty?
 
       cookie_list = Array.new()
-
       @cookies.each{|cookie|
-	if cookie.use? && cookie.match?(url)
-	  if cookie_list.select{|c1| c1.name == cookie.name}.empty?
-	    cookie_list << cookie
-	  end
-	end
+        is_expired = (cookie.expires && (cookie.expires < Time.now.gmtime))
+        if cookie.use? && !is_expired && cookie.match?(url)
+          if cookie_list.select{|c1| c1.name == cookie.name}.empty?
+            cookie_list << cookie
+          end
+        end
       }
       return make_cookie_str(cookie_list)
     end
@@ -306,8 +314,9 @@ class WebAgent
     end
     private :find_cookie_info
 
+    # not tested well; used only netscape_rule = true.
     def cookie_error(err, override)
-      if err.kind_of?(ErrorOverrideOK) || !override
+      if !err.kind_of?(ErrorOverrideOK) || !override
 	raise err
       end
     end
@@ -327,10 +336,6 @@ class WebAgent
       domain_orig, path_orig = domain, path
       use_security = override
 
-      if !domainname
-	cookie_error(NodotError.new(), override)
-      end
-
       if domain
 
 	# [DRAFT 12] s. 4.2.2 (does not apply in the case that
@@ -342,6 +347,7 @@ class WebAgent
 	  domain = '.'+domain
 	end
 
+        # [NETSCAPE] rule
         if @netscape_rule
           n = total_dot_num(domain)
           if n < 2
@@ -357,16 +363,22 @@ class WebAgent
           end
         end
 
+        # this implementation does not check RFC2109 4.3.2 case 2;
+        # the portion of host not in domain does not contain a dot.
+        # according to nsCookieService.cpp in Firefox 3.0.4, Firefox 3.0.4
+        # and IE does not check, too.
       end
 
-      path ||= url.path.sub!(%r|/[^/]*|, '')
+      path ||= url.path.sub(%r|/[^/]*|, '')
       domain ||= domainname
-      cookie = find_cookie_info(domain, path, name)
-
-      if !cookie
-	cookie = WebAgent::Cookie.new()
-	cookie.use = true
-	@cookies << cookie
+      @cookies.synchronize do
+        cookie = find_cookie_info(domain, path, name)
+        if !cookie
+          cookie = WebAgent::Cookie.new()
+          cookie.use = true
+          @cookies << cookie
+        end
+        check_expired_cookies()
       end
 
       cookie.url = url
@@ -386,27 +398,26 @@ class WebAgent
 	cookie.discard = false
 	@is_saved = false
       end
-
-      check_expired_cookies()
-      return false
     end
 
     def load_cookies()
       return if !File.readable?(@cookies_file)
-      File.open(@cookies_file,'r'){|f|
-	while line = f.gets
-	  cookie = WebAgent::Cookie.new()
-	  @cookies << cookie
-	  col = line.chomp.split(/\t/)
-	  cookie.url = URI.parse(col[0])
-	  cookie.name = col[1]
-	  cookie.value = col[2]
-	  cookie.expires = Time.at(col[3].to_i)
-	  cookie.domain = col[4]
-	  cookie.path = col[5]
-	  cookie.set_flag(col[6])
-	end
-      }
+      @cookies.synchronize do
+        File.open(@cookies_file,'r'){|f|
+          while line = f.gets
+            cookie = WebAgent::Cookie.new()
+            @cookies << cookie
+            col = line.chomp.split(/\t/)
+            cookie.url = URI.parse(col[0])
+            cookie.name = col[1]
+            cookie.value = col[2]
+            cookie.expires = Time.at(col[3].to_i)
+            cookie.domain = col[4]
+            cookie.path = col[5]
+            cookie.set_flag(col[6])
+          end
+        }
+      end
     end
 
     def check_cookie_accept_domain(domain)

data/lib/httpclient/http.rb

@@ -1,604 +1,846 @@
-# HTTP - HTTP container.
-# Copyright (C) 2001-2007 NAKAMURA, Hiroshi.
+# HTTPClient - HTTP client library.
+# Copyright (C) 2000-2008  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
-# This module is copyrighted free software by NAKAMURA, Hiroshi.
-# You can redistribute it and/or modify it under the same term as Ruby.
+# This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
+# redistribute it and/or modify it under the same terms of Ruby's license;
+# either the dual license version in 2003, or any later version.
+
 
-require 'uri'
 require 'time'
 
+
+# A namespace module for HTTP Message definitions used by HTTPClient.
 module HTTP
 
 
-module Status
-  OK = 200
-  CREATED = 201
-  ACCEPTED = 202
-  NON_AUTHORITATIVE_INFORMATION = 203
-  NO_CONTENT = 204
-  RESET_CONTENT = 205
-  PARTIAL_CONTENT = 206
-  MOVED_PERMANENTLY = 301
-  FOUND = 302
-  SEE_OTHER = 303
-  TEMPORARY_REDIRECT = MOVED_TEMPORARILY = 307
-  BAD_REQUEST = 400
-  UNAUTHORIZED = 401
-  PROXY_AUTHENTICATE_REQUIRED = 407
-  INTERNAL = 500
-
-  def self.successful?(status)
-    [
+  # Represents HTTP response status code.  Defines constants for HTTP response
+  # and some conditional methods.
+  module Status
+    OK = 200
+    CREATED = 201
+    ACCEPTED = 202
+    NON_AUTHORITATIVE_INFORMATION = 203
+    NO_CONTENT = 204
+    RESET_CONTENT = 205
+    PARTIAL_CONTENT = 206
+    MOVED_PERMANENTLY = 301
+    FOUND = 302
+    SEE_OTHER = 303
+    TEMPORARY_REDIRECT = MOVED_TEMPORARILY = 307
+    BAD_REQUEST = 400
+    UNAUTHORIZED = 401
+    PROXY_AUTHENTICATE_REQUIRED = 407
+    INTERNAL = 500
+
+    # Status codes for successful HTTP response.
+    SUCCESSFUL_STATUS = [
       OK, CREATED, ACCEPTED,
       NON_AUTHORITATIVE_INFORMATION, NO_CONTENT,
       RESET_CONTENT, PARTIAL_CONTENT
-    ].include?(status)
-  end
+    ]
 
-  def self.redirect?(status)
-    [
+    # Status codes which is a redirect.
+    REDIRECT_STATUS = [
       MOVED_PERMANENTLY, FOUND, SEE_OTHER,
       TEMPORARY_REDIRECT, MOVED_TEMPORARILY
-    ].include?(status)
-  end
-end
-
-
-class Error < StandardError; end
-class BadResponseError < Error; end
+    ]
 
-  class << self
-    def http_date(a_time)
-      a_time.gmtime.strftime("%a, %d %b %Y %H:%M:%S GMT")
+    # Returns true if the given status represents successful HTTP response.
+    # See also SUCCESSFUL_STATUS.
+    def self.successful?(status)
+      SUCCESSFUL_STATUS.include?(status)
     end
 
-    ProtocolVersionRegexp = Regexp.new('^(?:HTTP/|)(\d+)\.(\d+)$')
-    def keep_alive_enabled?(version)
-      ProtocolVersionRegexp =~ version
-      if !($1 and $2)
-        false
-      elsif $1.to_i > 1
-	true
-      elsif $1.to_i == 1 and $2.to_i >= 1
-	true
-      else
-	false
-      end
+    # Returns true if the given status is thought to be redirect.
+    # See also REDIRECT_STATUS.
+    def self.redirect?(status)
+      REDIRECT_STATUS.include?(status)
     end
   end
 
 
-# HTTP::Message -- HTTP message.
-# 
-# DESCRIPTION
-#   A class that describes 1 HTTP request / response message.
-#
-class Message
-  CRLF = "\r\n"
-
-  # HTTP::Message::Headers -- HTTP message header.
-  # 
-  # DESCRIPTION
-  #   A class that describes header part of HTTP message.
+  # Represents a HTTP message.  A message is for a request or a response.
   #
-  class Headers
-    # HTTP version string in a HTTP header.
-    attr_accessor :http_version
-    # Content-type.
-    attr_accessor :body_type
-    # Charset.
-    attr_accessor :body_charset
-    # Size of body.
-    attr_reader :body_size
-    # A milestone of body.
-    attr_accessor :body_date
-    # Chunked or not.
-    attr_reader :chunked
-    # Request method.
-    attr_reader :request_method
-    # Requested URI.
-    attr_reader :request_uri
-    # HTTP status reason phrase.
-    attr_accessor :reason_phrase
-
-    StatusCodeMap = {
-      Status::OK => 'OK',
-      Status::CREATED => "Created",
-      Status::NON_AUTHORITATIVE_INFORMATION => "Non-Authoritative Information",
-      Status::NO_CONTENT => "No Content",
-      Status::RESET_CONTENT => "Reset Content",
-      Status::PARTIAL_CONTENT => "Partial Content",
-      Status::MOVED_PERMANENTLY => 'Moved Permanently',
-      Status::FOUND => 'Found',
-      Status::SEE_OTHER => 'See Other',
-      Status::TEMPORARY_REDIRECT => 'Temporary Redirect',
-      Status::MOVED_TEMPORARILY => 'Temporary Redirect',
-      Status::BAD_REQUEST => 'Bad Request',
-      Status::INTERNAL => 'Internal Server Error',
-    }
-
-    CharsetMap = {
-      'NONE' => 'us-ascii',
-      'EUC'  => 'euc-jp',
-      'SJIS' => 'shift_jis',
-      'UTF8' => 'utf-8',
-    }
-
-    # SYNOPSIS
-    #   HTTP::Message.new
-    #
-    # ARGS
-    #   N/A
-    #
-    # DESCRIPTION
-    #   Create a instance of HTTP request or HTTP response.  Specify
-    #   status_code for HTTP response.
-    #
-    def initialize
-      @is_request = nil	# true, false and nil
-      @http_version = 'HTTP/1.1'
-      @body_type = nil
-      @body_charset = nil
-      @body_size = nil
-      @body_date = nil
-      @header_item = []
-      @chunked = false
-      @response_status_code = nil
-      @reason_phrase = nil
-      @request_method = nil
-      @request_uri = nil
-      @request_query = nil
-      @request_via_proxy = nil
-    end
+  # Request message is generated from given parameters internally so users
+  # don't need to care about it.  Response message is the instance that
+  # methods of HTTPClient returns so users need to know how to extract
+  # HTTP response data from Message.
+  #
+  # Some attributes are only for a request or a response, not both.
+  #
+  # == How to use HTTP response message
+  #
+  # 1. Gets response message body.
+  #
+  #     res = clnt.get(url)
+  #     p res.content #=> String
+  #
+  # 2. Gets response status code.
+  #
+  #     res = clnt.get(url)
+  #     p res.status #=> 200, 501, etc. (Integer)
+  #
+  # 3. Gets response header.
+  #
+  #     res = clnt.get(url)
+  #     res.header['set-cookie'].each do |value|
+  #       p value
+  #     end
+  #     assert_equal(1, res.header['last-modified'].size)
+  #     p res.header['last-modified'].first
+  #
+  class Message
+
+    CRLF = "\r\n"
+
+    # Represents HTTP message header.
+    class Headers
+      # HTTP version in a HTTP header.  Float.
+      attr_accessor :http_version
+      # Size of body.  nil when size is unknown (e.g. chunked response).
+      attr_reader :body_size
+      # Request/Response is chunked or not.
+      attr_accessor :chunked
+
+      # Request only.  Requested method.
+      attr_reader :request_method
+      # Request only.  Requested URI.
+      attr_accessor :request_uri
+      # Request only.  Requested query.
+      attr_accessor :request_query
+      # Request only.  Requested via proxy or not.
+      attr_accessor :request_via_proxy
+
+      # Response only.  HTTP status
+      attr_reader :status_code
+      # Response only.  HTTP status reason phrase.
+      attr_accessor :reason_phrase
+
+      # Used for dumping response.
+      attr_accessor :body_type # :nodoc:
+      # Used for dumping response.
+      attr_accessor :body_charset # :nodoc:
+      # Used for dumping response.
+      attr_accessor :body_date # :nodoc:
+
+      # HTTP response status code to reason phrase mapping definition.
+      STATUS_CODE_MAP = {
+        Status::OK => 'OK',
+        Status::CREATED => "Created",
+        Status::NON_AUTHORITATIVE_INFORMATION => "Non-Authoritative Information",
+        Status::NO_CONTENT => "No Content",
+        Status::RESET_CONTENT => "Reset Content",
+        Status::PARTIAL_CONTENT => "Partial Content",
+        Status::MOVED_PERMANENTLY => 'Moved Permanently',
+        Status::FOUND => 'Found',
+        Status::SEE_OTHER => 'See Other',
+        Status::TEMPORARY_REDIRECT => 'Temporary Redirect',
+        Status::MOVED_TEMPORARILY => 'Temporary Redirect',
+        Status::BAD_REQUEST => 'Bad Request',
+        Status::INTERNAL => 'Internal Server Error',
+      }
+
+      # $KCODE to charset mapping definition.
+      CHARSET_MAP = {
+        'NONE' => 'us-ascii',
+        'EUC'  => 'euc-jp',
+        'SJIS' => 'shift_jis',
+        'UTF8' => 'utf-8',
+      }
+
+      # Creates a Message::Headers.  Use init_request, init_response, or
+      # init_connect_request for acutual initialize.
+      def initialize
+        @http_version = 1.1
+        @body_size = nil
+        @chunked = false
+
+        @request_method = nil
+        @request_uri = nil
+        @request_query = nil
+        @request_via_proxy = nil
+
+        @status_code = nil
+        @reason_phrase = nil
+
+        @body_type = nil
+        @body_charset = nil
+        @body_date = nil
+
+        @is_request = nil
+        @header_item = []
+        @dumped = false
+      end
 
-    def init_request(method, uri, query = nil, via_proxy = nil)
-      @is_request = true
-      @request_method = method
-      @request_uri = if uri.is_a?(URI)
-	  uri
-	else
-	  URI.parse(uri.to_s)
-	end
-      @request_query = create_query_uri(@request_uri, query)
-      @request_via_proxy = via_proxy
-    end
+      # Initialize this instance as a CONNECT request.
+      def init_connect_request(uri)
+        @is_request = true
+        @request_method = 'CONNECT'
+        @request_uri = uri
+        @request_query = nil
+        @http_version = 1.0
+      end
 
-    def init_response(status_code)
-      @is_request = false
-      self.response_status_code = status_code
-    end
+      # Placeholder URI object for nil uri.
+      NIL_URI = URI.parse('http://nil-uri-given/')
+      # Initialize this instance as a general request.
+      def init_request(method, uri, query = nil)
+        @is_request = true
+        @request_method = method
+        @request_uri = uri || NIL_URI
+        @request_query = query
+        @request_via_proxy = false
+      end
 
-    attr_accessor :request_via_proxy
+      # Initialize this instance as a response.
+      def init_response(status_code)
+        @is_request = false
+        self.status_code = status_code
+      end
 
-    attr_reader :response_status_code
-    def response_status_code=(status_code)
-      @response_status_code = status_code
-      @reason_phrase = StatusCodeMap[@response_status_code]
-    end
+      # Sets status code and reason phrase.
+      def status_code=(status_code)
+        @status_code = status_code
+        @reason_phrase = STATUS_CODE_MAP[@status_code]
+      end
 
-    def contenttype
-      self['content-type'][0]
-    end
+      # Returns 'Content-Type' header value.
+      def contenttype
+        self['Content-Type'][0]
+      end
 
-    def contenttype=(contenttype)
-      self['content-type'] = contenttype
-    end
+      # Sets 'Content-Type' header value.  Overrides if already exists.
+      def contenttype=(contenttype)
+        delete('Content-Type')
+        self['Content-Type'] = contenttype
+      end
 
-    # body_size == nil means that the body is_a? IO
-    def body_size=(body_size)
-      @body_size = body_size
-      if @body_size
-	@chunked = false
-      else
-	@chunked = true
+      # Sets byte size of message body.
+      # body_size == nil means that the body is_a? IO
+      def body_size=(body_size)
+        @body_size = body_size
       end
-    end
 
-    def dump(dev = '')
-      set_header
-      str = nil
-      if @is_request
-	str = request_line
-      else
-	str = response_status_line
+      # Dumps message header part and returns a dumped String.
+      def dump
+        set_header
+        str = nil
+        if @is_request
+          str = request_line
+        else
+          str = response_status_line
+        end
+        str + @header_item.collect { |key, value|
+          "#{ key }: #{ value }#{ CRLF }"
+        }.join
       end
-      str += @header_item.collect { |key, value|
-	  dump_line("#{ key }: #{ value }")
-	}.join
-      dev << str
-      dev
-    end
 
-    def set(key, value)
-      @header_item.push([key, value])
-    end
+      # Adds a header.  Addition order is preserved.
+      def set(key, value)
+        @header_item.push([key, value])
+      end
 
-    def get(key = nil)
-      if !key
-	@header_item
-      else
-	@header_item.find_all { |pair| pair[0].upcase == key.upcase }
+      # Returns an Array of headers for the given key.  Each element is a pair
+      # of key and value.  It returns an single element Array even if the only
+      # one header exists.  If nil key given, it returns all headers.
+      def get(key = nil)
+        if key.nil?
+          all
+        else
+          key = key.upcase
+          @header_item.find_all { |k, v| k.upcase == key }
+        end
       end
-    end
 
-    def delete(key)
-      key = key.upcase
-      @header_item.delete_if { |k, v| k.upcase == key }
-    end
+      # Returns an Array of all headers.
+      def all
+        @header_item
+      end
 
-    def []=(key, value)
-      set(key, value)
-    end
+      # Deletes headers of the given key.
+      def delete(key)
+        key = key.upcase
+        @header_item.delete_if { |k, v| k.upcase == key }
+      end
 
-    def [](key)
-      get(key).collect { |item| item[1] }
-    end
+      # Adds a header.  See set.
+      def []=(key, value)
+        set(key, value)
+      end
 
-  private
+      # Returns an Array of header values for the given key.
+      def [](key)
+        get(key).collect { |item| item[1] }
+      end
 
-    def request_line
-      path = if @request_via_proxy
-	if @request_uri.port
-	  "#{ @request_uri.scheme }://#{ @request_uri.host }:#{ @request_uri.port }#{ @request_query }"
-	else
-	  "#{ @request_uri.scheme }://#{ @request_uri.host }#{ @request_query }"
-	end
-      else
-	@request_query
+    private
+
+      def request_line
+        path = create_query_uri(@request_uri, @request_query)
+        if @request_via_proxy
+          path = "#{ @request_uri.scheme }://#{ @request_uri.host }:#{ @request_uri.port }#{ path }"
+        end
+        "#{ @request_method } #{ path } HTTP/#{ @http_version }#{ CRLF }"
       end
-      dump_line("#{ @request_method } #{ path } #{ @http_version }")
-    end
 
-    def response_status_line
-      if defined?(Apache)
-	dump_line("#{ @http_version } #{ response_status_code } #{ @reason_phrase }")
-      else
-	dump_line("Status: #{ response_status_code } #{ @reason_phrase }")
+      def response_status_line
+        if defined?(Apache)
+          "HTTP/#{ @http_version } #{ @status_code } #{ @reason_phrase }#{ CRLF }"
+        else
+          "Status: #{ @status_code } #{ @reason_phrase }#{ CRLF }"
+        end
       end
-    end
 
-    def set_header
-      if defined?(Apache) && !self['Date']
-	set('Date', HTTP.http_date(Time.now))
+      def set_header
+        if @is_request
+          set_request_header
+        else
+          set_response_header
+        end
       end
 
-      keep_alive = HTTP.keep_alive_enabled?(@http_version)
-      set('Connection', 'close') unless keep_alive
+      def set_request_header
+        return if @dumped
+        @dumped = true
+        keep_alive = Message.keep_alive_enabled?(@http_version)
+        if !keep_alive and @request_method != 'CONNECT'
+          set('Connection', 'close')
+        end
+        if @chunked
+          set('Transfer-Encoding', 'chunked')
+        elsif keep_alive or @body_size != 0
+          set('Content-Length', @body_size.to_s)
+        end
+        if @http_version >= 1.1
+	  set('Host', "#{@request_uri.host}:#{@request_uri.port}")
+        end
+      end
 
-      if @chunked
-	set('Transfer-Encoding', 'chunked')
-      else
-	if keep_alive or @body_size != 0
-	  set('Content-Length', @body_size.to_s)
-	end
+      def set_response_header
+        return if @dumped
+        @dumped = true
+        if defined?(Apache) && self['Date'].empty?
+          set('Date', Time.now.httpdate)
+        end
+        keep_alive = Message.keep_alive_enabled?(@http_version)
+        if @chunked
+          set('Transfer-Encoding', 'chunked')
+        else
+          if keep_alive or @body_size != 0
+            set('Content-Length', @body_size.to_s)
+          end
+        end
+        if @body_date
+          set('Last-Modified', @body_date.httpdate)
+        end
+        if self['Content-Type'].empty?
+          set('Content-Type', "#{ @body_type || 'text/html' }; charset=#{ charset_label(@body_charset || $KCODE) }")
+        end
       end
 
-      if @body_date
-	set('Last-Modified', HTTP.http_date(@body_date))
+      def charset_label(charset)
+        CHARSET_MAP[charset] || 'us-ascii'
       end
 
-      if @is_request == true
-	if @http_version >= 'HTTP/1.1'
-          if @request_uri.port == @request_uri.default_port
-            set('Host', "#{@request_uri.host}")
+      def create_query_uri(uri, query)
+        if @request_method == 'CONNECT'
+          return "#{uri.host}:#{uri.port}"
+        end
+        path = uri.path
+        path = '/' if path.nil? or path.empty?
+        query_str = nil
+        if uri.query
+          query_str = uri.query
+        end
+        if query
+          if query_str
+            query_str += "&#{Message.create_query_part_str(query)}"
           else
-            set('Host', "#{@request_uri.host}:#{@request_uri.port}")
+            query_str = Message.create_query_part_str(query)
           end
-	end
-      elsif @is_request == false
-	set('Content-Type', "#{ @body_type || 'text/html' }; charset=#{ CharsetMap[@body_charset || $KCODE] }")
+        end
+        if query_str
+          path += "?#{query_str}"
+        end
+        path
       end
     end
 
-    def dump_line(str)
-      str + CRLF
-    end
 
-    def create_query_uri(uri, query)
-      path = uri.path
-      path = '/' if path.nil? or path.empty?
-      query_str = nil
-      if uri.query
-	query_str = uri.query
+    # Represents HTTP message body.
+    class Body
+      # Size of body.  nil when size is unknown (e.g. chunked response).
+      attr_reader :size
+      # maxbytes of IO#read for streaming request.  See DEFAULT_CHUNK_SIZE.
+      attr_accessor :chunk_size
+
+      # Default value for chunk_size
+      DEFAULT_CHUNK_SIZE = 1024 * 16
+
+      # Creates a Message::Body.  Use init_request or init_response
+      # for acutual initialize.
+      def initialize
+        @body = nil
+        @size = nil
+        @positions = nil
+        @chunk_size = nil
       end
-      if query
-	if query_str
-	  query_str << '&' << Message.create_query_part_str(query)
-	else
-	  query_str = Message.create_query_part_str(query)
-	end
+
+      # Initialize this instance as a request.
+      def init_request(body = nil, boundary = nil)
+        @boundary = boundary
+        @positions = {}
+        set_content(body, boundary)
+        @chunk_size = DEFAULT_CHUNK_SIZE
       end
-      if query_str
-	path += "?#{query_str}"
+
+      # Initialize this instance as a response.
+      def init_response(body = nil)
+        @body = body
+        if @body.respond_to?(:size)
+          @size = @body.size
+        else
+          @size = nil
+        end
       end
-      path
-    end
-  end
 
-  class Body
-    attr_accessor :type, :charset, :date, :chunk_size
-
-    def initialize(body = nil, date = nil, type = nil, charset = nil,
-        boundary = nil)
-      @body = nil
-      @boundary = boundary
-      set_content(body || '', boundary)
-      @type = type
-      @charset = charset
-      @date = date
-      @chunk_size = 4096
-    end
+      # Dumps message body to given dev.
+      # dev needs to respond to <<.
+      #
+      # Message header must be given as the first argument for performance
+      # reason. (header is dumped to dev, too)
+      # If no dev (the second argument) given, this method returns a dumped
+      # String.
+      def dump(header = '', dev = '')
+        if @body.is_a?(Parts)
+          dev << header
+          buf = ''
+          @body.parts.each do |part|
+            if Message.file?(part)
+              reset_pos(part)
+              while !part.read(@chunk_size, buf).nil?
+                dev << buf
+              end
+            else
+              dev << part
+            end
+          end
+        elsif @body
+          dev << header + @body
+        else
+          dev << header
+        end
+        dev
+      end
 
-    def size
-      if @body.respond_to?(:read)
-	nil
-      else
-	@body.size
+      # Dumps message body with chunked encoding to given dev.
+      # dev needs to respond to <<.
+      #
+      # Message header must be given as the first argument for performance
+      # reason. (header is dumped to dev, too)
+      # If no dev (the second argument) given, this method returns a dumped
+      # String.
+      def dump_chunked(header = '', dev = '')
+        dev << header
+        if @body.is_a?(Parts)
+          @body.parts.each do |part|
+            if Message.file?(part)
+              reset_pos(part)
+              dump_chunks(part, dev)
+            else
+              dev << dump_chunk(part)
+            end
+          end
+          dev << (dump_last_chunk + CRLF)
+        elsif @body
+          reset_pos(@body)
+          dump_chunks(@body, dev)
+          dev << (dump_last_chunk + CRLF)
+        end
+        dev
       end
-    end
 
-    def dump(dev = '')
-      if @body.respond_to?(:read)
-	begin
-	  while true
-	    chunk = @body.read(@chunk_size)
-	    break if chunk.nil?
-	    dev << dump_chunk(chunk)
-	  end
-	rescue EOFError
-	end
-	dev << (dump_last_chunk + CRLF)
-      else
-	dev << @body
+      # Returns a message body itself.
+      def content
+        @body
       end
-      dev
-    end
 
-    def content
-      @body
-    end
+    private
+
+      def set_content(body, boundary = nil)
+        if body.respond_to?(:read)
+          # uses Transfer-Encoding: chunked.  bear in mind that server may not
+          # support it.  at least ruby's CGI doesn't.
+          @body = body
+          remember_pos(@body)
+          @size = nil
+        elsif boundary and Message.multiparam_query?(body)
+          @body = build_query_multipart_str(body, boundary)
+          @size = @body.size
+        else
+          @body = Message.create_query_part_str(body)
+          @size = @body.size
+        end
+      end
 
-    def set_content(body, boundary = nil)
-      if body.respond_to?(:read)
-	@body = body
-      elsif boundary
-	@body = Message.create_query_multipart_str(body, boundary)
-      else
-	@body = Message.create_query_part_str(body)
+      def remember_pos(io)
+        # IO may not support it (ex. IO.pipe)
+        @positions[io] = io.pos rescue nil
       end
-    end
 
-  private
+      def reset_pos(io)
+        io.pos = @positions[io] if @positions.key?(io)
+      end
 
-    def dump_chunk(str)
-      dump_chunk_size(str.size) << (str + CRLF)
-    end
+      def dump_chunks(io, dev)
+        buf = ''
+        while !io.read(@chunk_size, buf).nil?
+          dev << dump_chunk(buf)
+        end
+      end
 
-    def dump_last_chunk
-      dump_chunk_size(0)
-    end
+      def dump_chunk(str)
+        dump_chunk_size(str.size) + (str + CRLF)
+      end
 
-    def dump_chunk_size(size)
-      sprintf("%x", size) << CRLF
-    end
-  end
+      def dump_last_chunk
+        dump_chunk_size(0)
+      end
 
-  attr_reader :header
-  attr_reader :body
-  attr_accessor :peer_cert
+      def dump_chunk_size(size)
+        sprintf("%x", size) + CRLF
+      end
 
-  def initialize
-    @body = @header = @peer_cert = nil
-  end
+      class Parts
+        attr_reader :size
 
-  class << self
-    alias __new new
-    undef new
-  end
+        def initialize
+          @body = []
+          @size = 0
+          @as_stream = false
+        end
 
-  def self.new_request(method, uri, query = nil, body = nil, proxy = nil,
-      boundary = nil)
-    m = self.__new
-    m.header = Headers.new
-    m.header.init_request(method, uri, query, proxy)
-    m.body = Body.new(body, nil, nil, nil, boundary)
-    m
-  end
+        def add(part)
+          if Message.file?(part)
+            @as_stream = true
+            @body << part
+            if part.respond_to?(:size)
+              if sz = part.size
+                @size += sz
+              else
+                @size = nil
+              end
+            elsif part.respond_to?(:lstat)
+              @size += part.lstat.size
+            else
+              # use chunked upload
+              @size = nil
+            end
+          elsif @body[-1].is_a?(String)
+            @body[-1] += part.to_s
+            @size += part.to_s.size if @size
+          else
+            @body << part.to_s
+            @size += part.to_s.size if @size
+          end
+        end
 
-  def self.new_response(body = '')
-    m = self.__new
-    m.header = Headers.new
-    m.header.init_response(Status::OK)
-    m.body = Body.new(body)
-    m
-  end
+        def parts
+          if @as_stream
+            @body
+          else
+            [@body.join]
+          end
+        end
+      end
 
-  def dump(dev = '')
-    sync_header
-    dev = header.dump(dev)
-    dev << CRLF
-    dev = body.dump(dev) if body
-    dev
-  end
+      def build_query_multipart_str(query, boundary)
+        parts = Parts.new
+        query.each do |attr, value|
+          value ||= ''
+          headers = ["--#{boundary}"]
+          if Message.file?(value)
+            remember_pos(value)
+            param_str = params_from_file(value).collect { |k, v|
+              "#{k}=\"#{v}\""
+            }.join("; ")
+            if value.respond_to?(:mime_type)
+              content_type = value.mime_type
+            else
+              content_type = Message.mime_type(value.path)
+            end
+            headers << %{Content-Disposition: form-data; name="#{attr}"; #{param_str}}
+            headers << %{Content-Type: #{content_type}}
+          else
+            headers << %{Content-Disposition: form-data; name="#{attr}"}
+          end
+          parts.add(headers.join(CRLF) + CRLF + CRLF)
+          parts.add(value)
+          parts.add(CRLF)
+        end
+        parts.add("--#{boundary}--" + CRLF + CRLF) # empty epilogue
+        parts
+      end
 
-  def load(str)
-    buf = str.dup
-    unless self.header.load(buf)
-      self.body.load(buf)
+      def params_from_file(value)
+        params = {}
+        params['filename'] = File.basename(value.path || '')
+        # Creation time is not available from File::Stat
+        if value.respond_to?(:mtime)
+          params['modification-date'] = value.mtime.rfc822
+        end
+        if value.respond_to?(:atime)
+          params['read-date'] = value.atime.rfc822
+        end
+        params
+      end
     end
-  end
 
-  def header=(header)
-    @header = header
-    sync_body
-  end
 
-  def content
-    @body.content
-  end
+    class << self
+      private :new
 
-  def body=(body)
-    @body = body
-    sync_header
-  end
+      # Creates a Message instance of 'CONNECT' request.
+      # 'CONNECT' request does not have Body.
+      # uri:: an URI that need to connect.  Only uri.host and uri.port are used.
+      def new_connect_request(uri)
+        m = new
+        m.header.init_connect_request(uri)
+        m.header.body_size = 0
+        m
+      end
 
-  def status
-    @header.response_status_code
-  end
+      # Creates a Message instance of general request.
+      # method:: HTTP method String.
+      # uri:: an URI object which represents an URL of web resource.
+      # query:: a Hash or an Array of query part of URL.
+      #         e.g. { "a" => "b" } => 'http://host/part?a=b'
+      #         Give an array to pass multiple value like
+      #         [["a", "b"], ["a", "c"]] => 'http://host/part?a=b&a=c'
+      # body:: a Hash or an Array of body part.
+      #        e.g. { "a" => "b" } => 'a=b'.
+      #        Give an array to pass multiple value like
+      #        [["a", "b"], ["a", "c"]] => 'a=b&a=c'.
+      # boundary:: When the boundary given, it is sent as
+      #            a multipart/form-data using this boundary String.
+      def new_request(method, uri, query = nil, body = nil, boundary = nil)
+        m = new
+        m.header.init_request(method, uri, query)
+        m.body = Body.new
+        m.body.init_request(body || '', boundary)
+        m.header.body_size = m.body.size
+        m.header.chunked = true if m.body.size.nil?
+        m
+      end
 
-  def status=(status)
-    @header.response_status_code = status
-  end
+      # Creates a Message instance of response.
+      # body:: a String or an IO of response message body.
+      def new_response(body)
+        m = new
+        m.header.init_response(Status::OK)
+        m.body = Body.new
+        m.body.init_response(body)
+        m.header.body_size = m.body.size || 0
+        m
+      end
 
-  def version
-    @header.http_version
-  end
+      @@mime_type_handler = nil
+
+      # Sets MIME type handler.
+      #
+      # handler must respond to :call with a single argument :path and returns
+      # a MIME type String e.g. 'text/html'.
+      # When the handler returns nil or an empty String,
+      # 'application/octet-stream' is used.
+      #
+      # When you set nil to the handler, internal_mime_type is used instead.
+      # The handler is nil by default.
+      def mime_type_handler=(handler)
+        @@mime_type_handler = handler
+      end
 
-  def version=(version)
-    @header.http_version = version
-  end
+      # Returns MIME type handler.
+      def mime_type_handler
+        @@mime_type_handler
+      end
 
-  def reason
-    @header.reason_phrase
-  end
+      # For backward compatibility.
+      alias set_mime_type_func mime_type_handler=
+      alias get_mime_type_func mime_type_handler
 
-  def reason=(reason)
-    @header.reason_phrase = reason
-  end
+      def mime_type(path) # :nodoc:
+        if @@mime_type_handler
+          res = @@mime_type_handler.call(path)
+          if !res || res.to_s == ''
+            return 'application/octet-stream'
+          else
+            return res
+          end
+        else
+          internal_mime_type(path)
+        end
+      end
 
-  def contenttype
-    @header.contenttype
-  end
+      # Default MIME type handler.
+      # See mime_type_handler=.
+      def internal_mime_type(path)
+        case path
+        when /\.txt$/i
+          'text/plain'
+        when /\.(htm|html)$/i
+          'text/html'
+        when /\.doc$/i
+          'application/msword'
+        when /\.png$/i
+          'image/png'
+        when /\.gif$/i
+          'image/gif'
+        when /\.(jpg|jpeg)$/i
+          'image/jpeg'
+        else
+          'application/octet-stream'
+        end
+      end
 
-  def contenttype=(contenttype)
-    @header.contenttype = contenttype
-  end
+      # Returns true if the given HTTP version allows keep alive connection.
+      # version:: Float
+      def keep_alive_enabled?(version)
+        version >= 1.1
+      end
+
+      # Returns true if the given query (or body) has a multiple parameter.
+      def multiparam_query?(query)
+        query.is_a?(Array) or query.is_a?(Hash)
+      end
+
+      # Returns true if the given object is a File.  In HTTPClient, a file is;
+      # * must respond to :read for retrieving String chunks.
+      # * must respond to :path and returns a path for Content-Disposition.
+      # * must respond to :pos and :pos= to rewind for reading.
+      #   Rewinding is only needed for following HTTP redirect.  Some IO impl
+      #   defines :pos= but raises an Exception for pos= such as StringIO
+      #   but there's no problem as far as using it for non-following methods
+      #   (get/post/etc.)
+      def file?(obj)
+        obj.respond_to?(:read) and obj.respond_to?(:path) and
+          obj.respond_to?(:pos) and obj.respond_to?(:pos=)
+      end
 
-  class << self
-    @@mime_type_func = nil
+      def create_query_part_str(query) # :nodoc:
+        if multiparam_query?(query)
+          escape_query(query)
+        elsif query.respond_to?(:read)
+          query = query.read
+        else
+          query.to_s
+        end
+      end
 
-    def set_mime_type_func(val)
-      @@mime_type_func = val
+      def escape_query(query) # :nodoc:
+        query.collect { |attr, value|
+          if value.respond_to?(:read)
+            value = value.read
+          end
+          escape(attr.to_s) << '=' << escape(value.to_s)
+        }.join('&')
+      end
+
+      # from CGI.escape
+      def escape(str) # :nodoc:
+        str.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+          '%' + $1.unpack('H2' * $1.size).join('%').upcase
+        }.tr(' ', '+')
+      end
     end
 
-    def get_mime_type_func
-      @@mime_type_func
+
+    # HTTP::Message::Headers:: message header.
+    attr_accessor :header
+
+    # HTTP::Message::Body:: message body.
+    attr_reader :body
+
+    # OpenSSL::X509::Certificate:: response only.  server certificate which is
+    #                              used for retrieving the response.
+    attr_accessor :peer_cert
+
+    # Creates a Message.  This method should be used internally.
+    # Use Message.new_connect_request, Message.new_request or
+    # Message.new_response instead.
+    def initialize # :nodoc:
+      @header = Headers.new
+      @body = @peer_cert = nil
     end
 
-    def create_query_part_str(query)
-      if multiparam_query?(query)
-	escape_query(query)
+    # Dumps message (header and body) to given dev.
+    # dev needs to respond to <<.
+    def dump(dev = '')
+      str = header.dump + CRLF
+      if header.chunked
+        dev = body.dump_chunked(str, dev)
+      elsif body
+        dev = body.dump(str, dev)
       else
-	query.to_s
+        dev << str
       end
+      dev
     end
 
-    def create_query_multipart_str(query, boundary)
-      if multiparam_query?(query)
-        query.collect { |attr, value|
-          value ||= ''
-          extra_content_disposition = content_type = content = nil
-          if value.is_a? File
-            params = {
-              'filename' => File.basename(value.path),
-              # Creation time is not available from File::Stat
-              # 'creation-date' => value.ctime.rfc822,
-              'modification-date' => value.mtime.rfc822,
-              'read-date' => value.atime.rfc822,
-            }
-            param_str = params.to_a.collect { |k, v|
-              "#{k}=\"#{v}\""
-            }.join("; ")
-            extra_content_disposition = " #{param_str}"
-            content_type = mime_type(value.path)
-            content = value.read
-          else
-            extra_content_disposition = ''
-            content_type = mime_type(nil)
-            content = value.to_s
-          end
-          "--#{boundary}" + CRLF +
-            %{Content-Disposition: form-data; name="#{attr.to_s}";} +
-              extra_content_disposition + CRLF +
-            "Content-Type: " + content_type + CRLF +
-            CRLF +
-            content + CRLF
-        }.join('') + "--#{boundary}--" + CRLF + CRLF # empty epilogue
-      else
-        query.to_s
-      end
+    # Sets a new body.  header.body_size is updated with new body.size.
+    def body=(body)
+      @body = body
+      @header.body_size = @body.size if @header
     end
 
-    def multiparam_query?(query)
-      query.is_a?(Array) or query.is_a?(Hash)
+    # Returns HTTP version in a HTTP header.  Float.
+    def version
+      @header.http_version
     end
 
-    def escape_query(query)
-      query.collect { |attr, value|
-	escape(attr.to_s) << '=' << escape(value.to_s)
-      }.join('&')
+    # Sets HTTP version in a HTTP header.  Float.
+    def version=(version)
+      @header.http_version = version
     end
 
-    # from CGI.escape
-    def escape(str)
-      str.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
-  	'%' + $1.unpack('H2' * $1.size).join('%').upcase
-      }.tr(' ', '+')
+    # Returns HTTP status code in response.  Integer.
+    def status
+      @header.status_code
     end
 
-    def mime_type(path)
-      if @@mime_type_func
-        res = @@mime_type_func.call(path)
-        if !res || res.to_s == ''
-          return 'application/octet-stream'
-        else
-          return res
-        end
-      else
-        internal_mime_type(path)
-      end
+    alias code status
+    alias status_code status
+
+    # Sets HTTP status code of response.  Integer.
+    # Reason phrase is updated, too.
+    def status=(status)
+      @header.status_code = status
     end
 
-    def internal_mime_type(path)
-      case path
-      when /\.txt$/i
-        'text/plain'
-      when /\.(htm|html)$/i
-        'text/html'
-      when /\.doc$/i
-        'application/msword'
-      when /\.png$/i
-        'image/png'
-      when /\.gif$/i
-        'image/gif'
-      when /\.(jpg|jpeg)$/i
-        'image/jpeg'
-      else
-        'application/octet-stream'
-      end
+    # Returns  HTTP status reason phrase in response.  String.
+    def reason
+      @header.reason_phrase
     end
-  end
 
-private
+    # Sets  HTTP status reason phrase of response.  String.
+    def reason=(reason)
+      @header.reason_phrase = reason
+    end
 
-  def sync_header
-    if @header and @body
-      @header.body_type = @body.type
-      @header.body_charset = @body.charset
-      @header.body_size = @body.size
-      @header.body_date = @body.date
+    # Sets 'Content-Type' header value.  Overrides if already exists.
+    def contenttype
+      @header.contenttype
+    end
+
+    # Returns 'Content-Type' header value.
+    def contenttype=(contenttype)
+      @header.contenttype = contenttype
     end
-  end
 
-  def sync_body
-    if @header and @body
-      @body.type = @header.body_type
-      @body.charset = @header.body_charset
-      @body.size = @header.body_size
-      @body.date = @header.body_date
+    # Returns a content of message body.  A String or an IO.
+    def content
+      @body.content
     end
   end
-end
 
 
 end