httpclient 2.1.5 → 2.8.3

data/lib/httpclient/session.rb

@@ -1,22 +1,31 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;
 # either the dual license version in 2003, or any later version.
-
-# httpclient/session.rb is based on http-access.rb in http-access/0.0.4.
-# Some part of code in http-access.rb was recycled in httpclient.rb.
-# Those part is copyrighted by Maehashi-san.
+#
+# httpclient/session.rb is based on http-access.rb in http-access/0.0.4.  Some
+# part of it is copyrighted by Maebashi-san who made and published
+# http-access/0.0.4.  http-access/0.0.4 did not include license notice but when
+# I asked Maebashi-san he agreed that I can redistribute it under the same terms
+# of Ruby.  Many thanks to Maebashi-san.
 
 
 require 'socket'
 require 'thread'
+require 'timeout'
 require 'stringio'
+require 'zlib'
 
-require 'httpclient/timeout'
+require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
 require 'httpclient/ssl_config'
 require 'httpclient/http'
+if RUBY_ENGINE == 'jruby'
+  require 'httpclient/jruby_ssl_socket'
+else
+  require 'httpclient/ssl_socket'
+end
 
 
 class HTTPClient
@@ -27,15 +36,16 @@ class HTTPClient
     # Protocol scheme.
     attr_accessor :scheme
     # Host String.
-    attr_reader :host
+    attr_accessor :host
+    alias hostname host
     # Port number.
-    attr_reader :port
+    attr_accessor :port
 
     # Creates a new Site based on the given URI.
     def initialize(uri = nil)
       if uri
-        @scheme = uri.scheme
-        @host = uri.host
+        @scheme = uri.scheme || 'tcp'
+        @host = uri.hostname || '0.0.0.0'
         @port = uri.port.to_i
       else
         @scheme = 'tcp'
@@ -66,7 +76,7 @@ class HTTPClient
     def to_s # :nodoc:
       addr
     end
-    
+
     # Returns true if scheme, host and port of the given URI matches with this.
     def match(uri)
       (@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
@@ -75,6 +85,8 @@ class HTTPClient
     def inspect # :nodoc:
       sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
     end
+
+    EMPTY = Site.new.freeze
   end
 
 
@@ -93,19 +105,30 @@ class HTTPClient
     attr_accessor :debug_dev
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
 
     attr_accessor :connect_timeout
     # Maximum retry count.  0 for infinite.
     attr_accessor :connect_retry
     attr_accessor :send_timeout
     attr_accessor :receive_timeout
+    attr_accessor :keep_alive_timeout
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    # Raise BadResponseError if response size does not match with Content-Length header in response.
+    attr_accessor :strict_response_size_check
+
+    # Local address to bind local side of the socket to
+    attr_accessor :socket_local
+
     attr_accessor :ssl_config
 
     attr_reader :test_loopback_http_response
 
+    attr_accessor :transparent_gzip_decompression
+
     def initialize(client)
       @client = client
       @proxy = client.proxy
@@ -116,20 +139,27 @@ class HTTPClient
       @protocol_version = nil
       @debug_dev = client.debug_dev
       @socket_sync = true
-      @chunk_size = 4096
+      @tcp_keepalive = false
+      @chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
 
       @connect_timeout = 60
       @connect_retry = 1
       @send_timeout = 120
-      @receive_timeout = 60       # For each read_block_size bytes
+      @receive_timeout = 60        # For each read_block_size bytes
+      @keep_alive_timeout = 15     # '15' is from Apache 2 default
       @read_block_size = 1024 * 16 # follows net/http change in 1.8.7
       @protocol_retry_count = 5
 
       @ssl_config = nil
       @test_loopback_http_response = []
 
-      @sess_pool = []
+      @transparent_gzip_decompression = false
+      @strict_response_size_check = false
+      @socket_local = Site.new
+
+      @sess_pool = {}
       @sess_pool_mutex = Mutex.new
+      @sess_pool_last_checked = Time.now
     end
 
     def proxy=(proxy)
@@ -141,8 +171,8 @@ class HTTPClient
     end
 
     def query(req, via_proxy)
-      req.body.chunk_size = @chunk_size
-      sess = open(req.header.request_uri, via_proxy)
+      req.http_body.chunk_size = @chunk_size if req.http_body
+      sess = get_session(req, via_proxy)
       begin
         sess.query(req)
       rescue
@@ -161,45 +191,65 @@ class HTTPClient
       close_all
     end
 
+    # assert: sess.last_used must not be nil
     def keep(sess)
       add_cached_session(sess)
     end
 
   private
 
-    def open(uri, via_proxy = false)
-      sess = nil
-      if cached = get_cached_session(uri)
-        sess = cached
+    # TODO: create PR for webmock's httpclient adapter to use get_session
+    # instead of open so that we can remove duplicated Site creation for
+    # each session.
+    def get_session(req, via_proxy = false)
+      uri = req.header.request_uri
+      if uri.scheme.nil?
+        raise ArgumentError.new("Request URI must have schema. Possibly add 'http://' to the request URI?")
+      end
+      site = Site.new(uri)
+      if cached = get_cached_session(site)
+        cached
       else
-        sess = Session.new(@client, Site.new(uri), @agent_name, @from)
-        sess.proxy = via_proxy ? @proxy : nil
-        sess.socket_sync = @socket_sync
-        sess.requested_version = @protocol_version if @protocol_version
-        sess.connect_timeout = @connect_timeout
-        sess.connect_retry = @connect_retry
-        sess.send_timeout = @send_timeout
-        sess.receive_timeout = @receive_timeout
-        sess.read_block_size = @read_block_size
-        sess.protocol_retry_count = @protocol_retry_count
-        sess.ssl_config = @ssl_config
-        sess.debug_dev = @debug_dev
-        sess.test_loopback_http_response = @test_loopback_http_response
+        open(uri, via_proxy)
       end
+    end
+
+    def open(uri, via_proxy = false)
+      site = Site.new(uri)
+      sess = Session.new(@client, site, @agent_name, @from)
+      sess.proxy = via_proxy ? @proxy : nil
+      sess.socket_sync = @socket_sync
+      sess.tcp_keepalive = @tcp_keepalive
+      sess.requested_version = @protocol_version if @protocol_version
+      sess.connect_timeout = @connect_timeout
+      sess.connect_retry = @connect_retry
+      sess.send_timeout = @send_timeout
+      sess.receive_timeout = @receive_timeout
+      sess.read_block_size = @read_block_size
+      sess.protocol_retry_count = @protocol_retry_count
+      sess.ssl_config = @ssl_config
+      sess.debug_dev = @debug_dev
+      sess.strict_response_size_check = @strict_response_size_check
+      sess.socket_local = @socket_local
+      sess.test_loopback_http_response = @test_loopback_http_response
+      sess.transparent_gzip_decompression = @transparent_gzip_decompression
       sess
     end
 
     def close_all
       @sess_pool_mutex.synchronize do
-        @sess_pool.each do |sess|
-          sess.close
+        @sess_pool.each do |site, pool|
+          pool.each do |sess|
+            sess.close
+          end
         end
       end
       @sess_pool.clear
     end
 
+    # This method might not work as you expected...
     def close(dest)
-      if cached = get_cached_session(dest)
+      if cached = get_cached_session(Site.new(dest))
         cached.close
         true
       else
@@ -207,136 +257,48 @@ class HTTPClient
       end
     end
 
-    def get_cached_session(uri)
-      cached = nil
+    def get_cached_session(site)
+      if Thread.current[:HTTPClient_AcquireNewConnection]
+        return nil
+      end
       @sess_pool_mutex.synchronize do
-        new_pool = []
-        @sess_pool.each do |s|
-          if s.dest.match(uri)
-            cached = s
-          else
-            new_pool << s
+        now = Time.now
+        if now > @sess_pool_last_checked + @keep_alive_timeout
+          scrub_cached_session(now)
+          @sess_pool_last_checked = now
+        end
+        if pool = @sess_pool[site]
+          pool.each_with_index do |sess, idx|
+            if valid_session?(sess, now)
+              return pool.slice!(idx)
+            end
           end
         end
-        @sess_pool = new_pool
       end
-      cached
-    end
-
-    def add_cached_session(sess)
-      @sess_pool_mutex.synchronize do
-        @sess_pool << sess
-      end
-    end
-  end
-
-
-  # Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
-  class SSLSocketWrap
-    def initialize(socket, context, debug_dev = nil)
-      unless SSLEnabled
-        raise ConfigurationError.new('Ruby/OpenSSL module is required')
-      end
-      @context = context
-      @socket = socket
-      @ssl_socket = create_openssl_socket(@socket)
-      @debug_dev = debug_dev
-    end
-
-    def ssl_connect
-      @ssl_socket.connect
+      nil
     end
 
-    def post_connection_check(host)
-      verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
-      if verify_mode == OpenSSL::SSL::VERIFY_NONE
-        return
-      elsif @ssl_socket.peer_cert.nil? and
-        check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
-        raise OpenSSL::SSL::SSLError.new('no peer cert')
-      end
-      hostname = host.host
-      if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
-        @ssl_socket.post_connection_check(hostname)
-      else
-        @context.post_connection_check(@ssl_socket.peer_cert, hostname)
+    def scrub_cached_session(now)
+      @sess_pool.each do |site, pool|
+        pool.replace(pool.select { |sess|
+          if valid_session?(sess, now)
+            true
+          else
+            sess.close # close & remove from the pool
+            false
+          end
+        })
       end
     end
 
-    def peer_cert
-      @ssl_socket.peer_cert
-    end
-
-    def close
-      @ssl_socket.close
-      @socket.close
-    end
-
-    def closed?
-      @socket.closed?
-    end
-
-    def eof?
-      @ssl_socket.eof?
-    end
-
-    def gets(*args)
-      str = @ssl_socket.gets(*args)
-      debug(str)
-      str
-    end
-
-    def read(*args)
-      str = @ssl_socket.read(*args)
-      debug(str)
-      str
-    end
-
-    def readpartial(*args)
-      str = @ssl_socket.readpartial(*args)
-      debug(str)
-      str
-    end
-
-    def <<(str)
-      rv = @ssl_socket.write(str)
-      debug(str)
-      rv
-    end
-
-    def flush
-      @ssl_socket.flush
-    end
-
-    def sync
-      @ssl_socket.sync
-    end
-
-    def sync=(sync)
-      @ssl_socket.sync = sync
+    def valid_session?(sess, now)
+      (now <= sess.last_used + @keep_alive_timeout)
     end
 
-  private
-
-    def check_mask(value, mask)
-      value & mask == mask
-    end
-
-    def create_openssl_socket(socket)
-      ssl_socket = nil
-      if OpenSSL::SSL.const_defined?("SSLContext")
-        ctx = OpenSSL::SSL::SSLContext.new
-        @context.set_context(ctx)
-        ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
-      else
-        ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
-        @context.set_context(ssl_socket)
+    def add_cached_session(sess)
+      @sess_pool_mutex.synchronize do
+        (@sess_pool[sess.dest] ||= []).unshift(sess)
       end
-      ssl_socket
-    end
-
-    def debug(str)
-      @debug_dev << str if @debug_dev && str
     end
   end
 
@@ -360,20 +322,20 @@ class HTTPClient
       @socket.eof?
     end
 
-    def gets(*args)
-      @socket.gets(*args)
+    def gets(rs)
+      @socket.gets(rs)
     end
 
-    def read(*args)
-      @socket.read(*args)
+    def read(size, buf = nil)
+      @socket.read(size, buf)
     end
 
-    def readpartial(*args)
+    def readpartial(size, buf = nil)
       # StringIO doesn't support :readpartial
       if @socket.respond_to?(:readpartial)
-        @socket.readpartial(*args)
+        @socket.readpartial(size, buf)
       else
-        @socket.read(*args)
+        @socket.read(size, buf)
       end
     end
 
@@ -409,19 +371,19 @@ class HTTPClient
       debug("! CONNECTION CLOSED\n")
     end
 
-    def gets(*args)
+    def gets(rs)
       str = super
       debug(str)
       str
     end
 
-    def read(*args)
+    def read(size, buf = nil)
       str = super
       debug(str)
       str
     end
 
-    def readpartial(*args)
+    def readpartial(size, buf = nil)
       str = super
       debug(str)
       str
@@ -435,7 +397,15 @@ class HTTPClient
   private
 
     def debug(str)
-      @debug_dev << str if str && @debug_dev
+      if str && @debug_dev
+        if str.index("\0")
+          require 'hexdump'
+          str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+          @debug_dev << HexDump.encode(str).join("\n")
+        else
+          @debug_dev << str
+        end
+      end
     end
   end
 
@@ -453,12 +423,17 @@ class HTTPClient
     def <<(str)
       # ignored
     end
+
+    def peer_cert
+      nil
+    end
   end
 
 
   # Manages a HTTP session with a Site.
   class Session
     include HTTPClient::Timeout
+    include Util
 
     # Destination site
     attr_reader :dest
@@ -466,6 +441,8 @@ class HTTPClient
     attr_accessor :proxy
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
     # Requested protocol version
     attr_accessor :requested_version
     # Device for dumping log for debugging
@@ -478,15 +455,22 @@ class HTTPClient
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    attr_accessor :strict_response_size_check
+    attr_accessor :socket_local
+
     attr_accessor :ssl_config
     attr_reader :ssl_peer_cert
     attr_accessor :test_loopback_http_response
 
+    attr_accessor :transparent_gzip_decompression
+    attr_reader :last_used
+
     def initialize(client, dest, agent_name, from)
       @client = client
       @dest = dest
       @proxy = nil
       @socket_sync = true
+      @tcp_keepalive = false
       @requested_version = nil
 
       @debug_dev = nil
@@ -502,6 +486,8 @@ class HTTPClient
       @ssl_peer_cert = nil
 
       @test_loopback_http_response = nil
+      @strict_response_size_check = false
+      @socket_local = Site::EMPTY
 
       @agent_name = agent_name
       @from = from
@@ -515,28 +501,34 @@ class HTTPClient
 
       @socket = nil
       @readbuf = nil
+
+      @transparent_gzip_decompression = false
+      @last_used = nil
     end
 
     # Send a request to the server
     def query(req)
       connect if @state == :INIT
-      req.header.request_via_proxy = !@proxy.nil?
+      # Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
+      req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
       begin
-        timeout(@send_timeout, SendTimeoutError) do
+        ::Timeout.timeout(@send_timeout, SendTimeoutError) do
           set_header(req)
           req.dump(@socket)
           # flush the IO stream as IO::sync mode is false
           @socket.flush unless @socket_sync
         end
-      rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
+      rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
+        # JRuby can raise IOError instead of ECONNRESET for now
         close
-        raise KeepAliveDisconnected.new
+        raise KeepAliveDisconnected.new(self, $!)
       rescue HTTPClient::TimeoutError
         close
         raise
-      rescue
-        if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
-          raise KeepAliveDisconnected.new
+      rescue => e
+        close
+        if SSLEnabled and e.is_a?(OpenSSL::SSL::SSLError)
+          raise KeepAliveDisconnected.new(self, e)
         else
           raise
         end
@@ -545,6 +537,7 @@ class HTTPClient
       @state = :META if @state == :WAIT
       @next_connection = nil
       @requests.push(req)
+      @last_used = Time.now
     end
 
     def close
@@ -585,6 +578,9 @@ class HTTPClient
       begin
         read_header if @state == :META
         return nil if @state != :DATA
+        if @transparent_gzip_decompression
+          block = content_inflater_block(@content_encoding, block)
+        end
         if @chunked
           read_body_chunked(&block)
         elsif @content_length
@@ -606,21 +602,142 @@ class HTTPClient
       nil
     end
 
+    def create_socket(host, port)
+      socket = nil
+      begin
+        @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
+        clean_host = host.delete("[]")
+        if @socket_local == Site::EMPTY
+          socket = TCPSocket.new(clean_host, port)
+        else
+          clean_local = @socket_local.host.delete("[]")
+          socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
+        end
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
+        if @debug_dev
+          @debug_dev << "! CONNECTION ESTABLISHED\n"
+          socket.extend(DebugSocket)
+          socket.debug_dev = @debug_dev
+        end
+      rescue SystemCallError => e
+        raise e.class, e.message + " (#{host}:#{port})"
+      rescue SocketError => e
+        raise e.class, e.message + " (#{host}:#{port})"
+      end
+      socket
+    end
+
+    def create_loopback_socket(host, port, str)
+      @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
+      socket = LoopBackSocket.new(host, port, str)
+      if @debug_dev
+        @debug_dev << "! CONNECTION ESTABLISHED\n"
+        socket.extend(DebugSocket)
+        socket.debug_dev = @debug_dev
+      end
+      if https?(@dest) && @proxy
+        connect_ssl_proxy(socket, Util.urify(@dest.to_s))
+      end
+      socket
+    end
+
+    def connect_ssl_proxy(socket, uri)
+      req = HTTP::Message.new_connect_request(uri)
+      @client.request_filter.each do |filter|
+        filter.filter_request(req)
+      end
+      set_header(req)
+      req.dump(socket)
+      socket.flush unless @socket_sync
+      res = HTTP::Message.new_response('')
+      parse_header(socket)
+      res.http_version, res.status, res.reason = @version, @status, @reason
+      @headers.each do |key, value|
+        res.header.set(key.to_s, value)
+      end
+      commands = @client.request_filter.collect { |filter|
+        filter.filter_response(req, res)
+      }
+      if commands.find { |command| command == :retry }
+        raise RetryableResponse.new(res)
+      end
+      unless @status == 200
+        raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
+      end
+    end
+
   private
 
+    # This inflater allows deflate compression with/without zlib header
+    class LenientInflater
+      def initialize
+        @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
+        @first = true
+      end
+
+      def inflate(body)
+        if @first
+          first_inflate(body)
+        else
+          @inflater.inflate(body)
+        end
+      end
+
+    private
+
+      def first_inflate(body)
+        @first = false
+        begin
+          @inflater.inflate(body)
+        rescue Zlib::DataError
+          # fallback to deflate without zlib header
+          @inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+          @inflater.inflate(body)
+        end
+      end
+    end
+
+    def content_inflater_block(content_encoding, block)
+      case content_encoding
+      when 'gzip', 'x-gzip'
+        # zlib itself has a functionality to decompress gzip stream.
+        # - zlib 1.2.5 Manual
+        #   http://www.zlib.net/manual.html#Advanced
+        # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
+        # > windowBits to enable zlib and gzip decoding with automatic header detection,
+        # > or add 16 to decode only the gzip format
+        inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+      when 'deflate'
+        inflate_stream = LenientInflater.new
+      else
+        return block
+      end
+      Proc.new { |buf|
+        block.call(inflate_stream.inflate(buf))
+      }
+    end
+
     def set_header(req)
       if @requested_version
         if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
-          req.version = $1.to_f
+          req.http_version = $1
         end
       end
-      if @agent_name
+      if @agent_name && req.header.get('User-Agent').empty?
         req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
       end
-      if @from
+      if @from && req.header.get('From').empty?
         req.header.set('From', @from)
       end
-      req.header.set('Date', Time.now.httpdate)
+      if req.header.get('Accept').empty?
+        req.header.set('Accept', '*/*')
+      end
+      if @transparent_gzip_decompression
+        req.header.set('Accept-Encoding', 'gzip,deflate')
+      end
+      if req.header.get('Date').empty?
+        req.header.set_date_header
+      end
     end
 
     # Connect to the server
@@ -628,22 +745,15 @@ class HTTPClient
       site = @proxy || @dest
       retry_number = 0
       begin
-        timeout(@connect_timeout, ConnectTimeoutError) do
-          @socket = create_socket(site)
-          if @dest.scheme == 'https'
-            if @socket.is_a?(LoopBackSocket)
-              connect_ssl_proxy(@socket, URI.parse(@dest.to_s)) if @proxy
-            else
-              @socket = create_ssl_socket(@socket)
-              connect_ssl_proxy(@socket, URI.parse(@dest.to_s)) if @proxy
-              @socket.ssl_connect
-              @socket.post_connection_check(@dest)
-              @ssl_peer_cert = @socket.peer_cert
-            end
+        ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
+          if str = @test_loopback_http_response.shift
+            @socket = create_loopback_socket(site.host, site.port, str)
+          elsif https?(@dest)
+            @socket = SSLSocket.create_socket(self)
+            @ssl_peer_cert = @socket.peer_cert
+          else
+            @socket = create_socket(site.host, site.port)
           end
-          # Use Ruby internal buffering instead of passing data immediately
-          # to the underlying layer
-          # => we need to to call explicitly flush on the socket
           @socket.sync = @socket_sync
         end
       rescue RetryableResponse
@@ -665,72 +775,17 @@ class HTTPClient
       @state = :WAIT
     end
 
-    def create_socket(site)
-      socket = nil
-      begin
-        @debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
-        if str = @test_loopback_http_response.shift
-          socket = LoopBackSocket.new(site.host, site.port, str)
-        else
-          socket = TCPSocket.new(site.host, site.port)
-        end
-        if @debug_dev
-          @debug_dev << "! CONNECTION ESTABLISHED\n"
-          socket.extend(DebugSocket)
-          socket.debug_dev = @debug_dev
-        end
-      rescue SystemCallError => e
-        e.message << " (#{site})"
-        raise
-      rescue SocketError => e
-        e.message << " (#{site})"
-        raise
-      end
-      socket
-    end
-
-    # wrap socket with OpenSSL.
-    def create_ssl_socket(raw_socket)
-      SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
-    end
-
-    def connect_ssl_proxy(socket, uri)
-      req = HTTP::Message.new_connect_request(uri)
-      @client.request_filter.each do |filter|
-        filter.filter_request(req)
-      end
-      set_header(req)
-      req.dump(@socket)
-      @socket.flush unless @socket_sync
-      res = HTTP::Message.new_response('')
-      parse_header
-      res.version, res.status, res.reason = @version, @status, @reason
-      @headers.each do |key, value|
-        res.header.set(key, value)
-      end
-      commands = @client.request_filter.collect { |filter|
-        filter.filter_response(req, res)
-      }
-      if commands.find { |command| command == :retry }
-        raise RetryableResponse.new
-      end
-      unless @status == 200
-        raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
-      end
-    end
-
     # Read status block.
     def read_header
       @content_length = nil
       @chunked = false
+      @content_encoding = nil
       @chunk_length = 0
-      parse_header
-
-      # Head of the request has been parsed.
+      parse_header(@socket)
+      # Header of the request has been parsed.
       @state = :DATA
       req = @requests.shift
-
-      if req.header.request_method == 'HEAD'
+      if req.header.request_method == 'HEAD' or no_message_body?(@status)
         @content_length = 0
         if @next_connection
           @state = :WAIT
@@ -738,16 +793,24 @@ class HTTPClient
           close
         end
       end
-      @next_connection = false unless @content_length
+      @next_connection = false if !@content_length and !@chunked
     end
 
     StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
-    def parse_header
-      timeout(@receive_timeout, ReceiveTimeoutError) do
+    def parse_header(socket)
+      ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
+        initial_line = nil
         begin
-          initial_line = @socket.gets("\n")
-          if initial_line.nil?
-            raise KeepAliveDisconnected.new
+          begin
+            initial_line = socket.gets("\n")
+            if initial_line.nil?
+              close
+              raise KeepAliveDisconnected.new(self)
+            end
+          rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
+            # JRuby can raise IOError instead of ECONNRESET for now
+            close
+            raise KeepAliveDisconnected.new(self, $!)
           end
           if StatusParseRegexp !~ initial_line
             @version = '0.9'
@@ -759,32 +822,48 @@ class HTTPClient
             break
           end
           @version, @status, @reason = $1, $2.to_i, $3
-          @next_connection = HTTP::Message.keep_alive_enabled?(@version.to_f)
+          @next_connection = HTTP::Message.keep_alive_enabled?(@version)
           @headers = []
           while true
-            line = @socket.gets("\n")
+            line = socket.gets("\n")
             unless line
               raise BadResponseError.new('unexpected EOF')
             end
             line.chomp!
             break if line.empty?
-            key, value = line.split(/\s*:\s*/, 2)
-            parse_keepalive_header(key, value)
-            @headers << [key, value]
+            if line[0] == ?\  or line[0] == ?\t
+              last = @headers.last[1]
+              last << ' ' unless last.empty?
+              last << line.strip
+            else
+              key, value = line.strip.split(/\s*:\s*/, 2)
+              parse_content_header(key, value)
+              @headers << [key, value]
+            end
           end
         end while (@version == '1.1' && @status == 100)
       end
     end
 
-    def parse_keepalive_header(key, value)
+    def no_message_body?(status)
+      !status.nil? && # HTTP/0.9
+        ((status >= 100 && status < 200) || status == 204 || status == 304)
+    end
+
+    def parse_content_header(key, value)
       key = key.downcase
-      if key == 'content-length'
+      case key
+      when 'content-length'
         @content_length = value.to_i
-      elsif key == 'transfer-encoding' and value.downcase == 'chunked'
-        @chunked = true
-        @chunk_length = 0
-        @content_length = nil
-      elsif key == 'connection' or key == 'proxy-connection'
+      when 'content-encoding'
+        @content_encoding = value.downcase
+      when 'transfer-encoding'
+        if value.downcase == 'chunked'
+          @chunked = true
+          @chunk_length = 0
+          @content_length = nil
+        end
+      when 'connection', 'proxy-connection'
         if value.downcase == 'keep-alive'
           @next_connection = true
         else
@@ -795,19 +874,23 @@ class HTTPClient
 
     def read_body_length(&block)
       return nil if @content_length == 0
-      buf = ''
       while true
+        buf = empty_bin_str
         maxbytes = @read_block_size
-        maxbytes = @content_length if maxbytes > @content_length
-        timeout(@receive_timeout, ReceiveTimeoutError) do
+        maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
+        ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(maxbytes, buf)
           rescue EOFError
+            close
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
+            end
           end
         end
-        if buf && buf.length > 0
-          @content_length -= buf.length
+        if buf && buf.bytesize > 0
+          @content_length -= buf.bytesize
           yield buf
         else
           @content_length = 0
@@ -818,45 +901,59 @@ class HTTPClient
 
     RS = "\r\n"
     def read_body_chunked(&block)
-      buf = ''
+      buf = empty_bin_str
       while true
-        len = @socket.gets(RS)
-        @chunk_length = len.hex
-        if @chunk_length == 0
-          @content_length = 0
-          @socket.gets(RS)
-          return
-        end
-        timeout(@receive_timeout, ReceiveTimeoutError) do
-          @socket.read(@chunk_length + 2, buf)
+       ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
+          len = @socket.gets(RS)
+          if len.nil? # EOF
+            close
+            return
+          end
+          @chunk_length = len.hex
+          if @chunk_length == 0
+            @content_length = 0
+            @socket.gets(RS)
+            return
+          end
+          @socket.read(@chunk_length, buf)
+          @socket.read(2)
         end
         unless buf.empty?
-          yield buf.slice(0, @chunk_length)
+          yield buf
         end
       end
     end
 
     def read_body_rest
-      if @readbuf and @readbuf.length > 0
+      if @readbuf and @readbuf.bytesize > 0
         yield @readbuf
         @readbuf = nil
       end
-      buf = ''
       while true
-        timeout(@receive_timeout, ReceiveTimeoutError) do
+        buf = empty_bin_str
+        ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(@read_block_size, buf)
           rescue EOFError
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading chunked response")
+            end
           end
         end
-        if buf && buf.length > 0
+        if buf && buf.bytesize > 0
           yield buf
         else
           return
         end
       end
     end
+
+    def empty_bin_str
+      str = ''
+      str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+      str
+    end
   end