httpclient 2.3.0.1 → 2.8.3

data/test/test_http-access2.rb

@@ -1,5 +1,7 @@
+# -*- encoding: utf-8 -*-
 require 'http-access2'
 require File.expand_path('helper', File.dirname(__FILE__))
+require 'tempfile'
 
 
 module HTTPAccess2
@@ -47,7 +49,7 @@ class TestClient < Test::Unit::TestCase
     @client.get(serverurl)
     lines = str.split(/(?:\r?\n)+/)
     assert_equal("= Request", lines[0])
-    assert_match(/^From: from_bar/, lines[4])
+    assert_match(/^From: from_bar/, lines[5])
   end
 
   def test_debug_dev
@@ -81,8 +83,8 @@ class TestClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET /hello HTTP/1.0", lines[3])
-    assert_equal("Connection: close", lines[6])
-    assert_equal("= Response", lines[7])
+    assert_equal("Connection: close", lines[7])
+    assert_equal("= Response", lines[8])
   end
 
   def test_protocol_version_http11
@@ -93,7 +95,7 @@ class TestClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET / HTTP/1.1", lines[3])
-    assert_equal("Host: localhost:#{serverport}", lines[6])
+    assert_equal("Host: localhost:#{serverport}", lines[7])
     @client.protocol_version = 'HTTP/1.1'
     str = ""
     @client.debug_dev = str
@@ -116,9 +118,9 @@ class TestClient < Test::Unit::TestCase
     setup_proxyserver
     escape_noproxy do
       begin
-       	@client.proxy = "http://"
-      rescue
-        assert_match(/InvalidURIError/, $!.class.to_s)
+       	@client.proxy = "http://あ"
+      rescue => e
+        assert_match(/InvalidURIError/, e.class.to_s)
       end
       @client.proxy = ""
       assert_nil(@client.proxy)
@@ -243,7 +245,7 @@ class TestClient < Test::Unit::TestCase
   end
 
   def test_post
-    assert_equal("post", @client.post(serverurl + 'servlet').content)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content)
     param = {'1'=>'2', '3'=>'4'}
     res = @client.get(serverurl + 'servlet', param)
     assert_equal(param, params(res.header["x-query"][0]))
@@ -345,27 +347,26 @@ class TestClient < Test::Unit::TestCase
   def test_receive_timeout
     # this test takes 2 sec
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
+    @client.reset_all
     @client.receive_timeout = 1
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=0'))
     assert_raise(HTTPClient::ReceiveTimeoutError) do
       @client.get_content(serverurl + 'sleep?sec=2')
     end
+    @client.reset_all
     @client.receive_timeout = 3
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
   end
 
   def test_cookies
-    cookiefile = File.join(File.dirname(File.expand_path(__FILE__)),
-      'test_cookies_file')
+    cookiefile = Tempfile.new('test_cookies_file')
     # from [ruby-talk:164079]
-    File.open(cookiefile, "wb") do |f|
-      f << "http://rubyforge.org//account/login.php	session_ser	LjEwMy45Ni40Ni0q%2A-fa0537de8cc31	1131676286	.rubyforge.org	/	13\n"
+    File.open(cookiefile.path, "wb") do |f|
+      f << "http://rubyforge.org//account/login.php	session_ser	LjEwMy45Ni40Ni0q%2A-fa0537de8cc31	2131676286	.rubyforge.org	/	13\n"
     end
-    cm = WebAgent::CookieManager::new(cookiefile)
+    cm = WebAgent::CookieManager::new(cookiefile.path)
     cm.load_cookies
-    cookie = cm.cookies.first
-    url = cookie.url
-    assert(cookie.domain_match(url.host, cookie.domain))
+    assert_equal(1, cm.cookies.size)
   end
 
 private

data/test/test_httpclient.rb

@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 require File.expand_path('helper', File.dirname(__FILE__))
+require 'tempfile'
 
 
 class TestHTTPClient < Test::Unit::TestCase
@@ -44,7 +45,7 @@ class TestHTTPClient < Test::Unit::TestCase
     @client.get(serverurl)
     lines = str.split(/(?:\r?\n)+/)
     assert_equal("= Request", lines[0])
-    assert_match(/^From: from_bar/, lines[4])
+    assert_match(/^From: from_bar/, lines[5])
   end
 
   def test_debug_dev
@@ -77,10 +78,10 @@ class TestHTTPClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET /hello HTTP/0.9", lines[3])
-    assert_equal("Connection: close", lines[6])
-    assert_equal("= Response", lines[7])
-    assert_match(/^hello$/, lines[8])
-    assert_match(/^world$/, lines[9])
+    assert_equal("Connection: close", lines[7])
+    assert_equal("= Response", lines[8])
+    assert_match(/^hello$/, lines[9])
+    assert_match(/^world$/, lines[10])
   end
 
   def test_protocol_version_http10
@@ -94,8 +95,8 @@ class TestHTTPClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET /hello HTTP/1.0", lines[3])
-    assert_equal("Connection: close", lines[6])
-    assert_equal("= Response", lines[7])
+    assert_equal("Connection: close", lines[7])
+    assert_equal("= Response", lines[8])
   end
 
   def test_header_accept_by_default
@@ -103,7 +104,7 @@ class TestHTTPClient < Test::Unit::TestCase
     @client.debug_dev = str
     @client.get(serverurl)
     lines = str.split(/(?:\r?\n)+/)
-    assert_equal("Accept: */*", lines[4])
+    assert_equal("Accept: */*", lines[5])
   end
 
   def test_header_accept
@@ -114,6 +115,14 @@ class TestHTTPClient < Test::Unit::TestCase
     assert_equal("Accept: text/html", lines[4])
   end
 
+  def test_header_symbol
+    str = ""
+    @client.debug_dev = str
+    @client.post(serverurl + 'servlet', :header => {:'Content-Type' => 'application/json'}, :body => 'hello')
+    lines = str.split(/(?:\r?\n)+/).grep(/^Content-Type/)
+    assert_equal(2, lines.size) # 1 for both request and response
+  end
+
   def test_host_given
     str = ""
     @client.debug_dev = str
@@ -122,7 +131,7 @@ class TestHTTPClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET / HTTP/1.1", lines[3])
-    assert_equal("Host: localhost:#{serverport}", lines[6])
+    assert_equal("Host: localhost:#{serverport}", lines[7])
     #
     @client.reset_all
     str = ""
@@ -137,12 +146,24 @@ class TestHTTPClient < Test::Unit::TestCase
 
   def test_redirect_returns_not_modified
     assert_nothing_raised do
-      timeout(2) do
+      ::Timeout.timeout(2) do
         @client.get(serverurl + 'status', {:status => 306}, {:follow_redirect => true})
       end
     end
   end
 
+  class LocationRemoveFilter
+    def filter_request(req); end
+    def filter_response(req, res); res.header.delete('Location'); end
+  end
+
+  def test_redirect_without_location_should_gracefully_fail
+    @client.request_filter << LocationRemoveFilter.new
+    assert_raises(HTTPClient::BadResponseError) do
+      @client.get(serverurl + 'redirect1', :follow_redirect => true)
+    end
+  end
+
   def test_protocol_version_http11
     assert_equal(nil, @client.protocol_version)
     str = ""
@@ -152,7 +173,7 @@ class TestHTTPClient < Test::Unit::TestCase
     assert_equal("= Request", lines[0])
     assert_equal("! CONNECTION ESTABLISHED", lines[2])
     assert_equal("GET / HTTP/1.1", lines[3])
-    assert_equal("Host: localhost:#{serverport}", lines[6])
+    assert_equal("Host: localhost:#{serverport}", lines[7])
     @client.protocol_version = 'HTTP/1.1'
     assert_equal('HTTP/1.1', @client.protocol_version)
     str = ""
@@ -176,9 +197,9 @@ class TestHTTPClient < Test::Unit::TestCase
     setup_proxyserver
     escape_noproxy do
       begin
-        @client.proxy = "http://"
-      rescue
-        assert_match(/InvalidURIError/, $!.class.to_s)
+        @client.proxy = "http://あ"
+      rescue => e
+        assert_match(/InvalidURIError/, e.class.to_s)
       end
       @client.proxy = ""
       assert_nil(@client.proxy)
@@ -518,7 +539,7 @@ EOS
 
   def test_no_content
     assert_nothing_raised do
-      timeout(2) do
+      ::Timeout.timeout(2) do
         @client.get(serverurl + 'status', :status => 101)
         @client.get(serverurl + 'status', :status => 204)
         @client.get(serverurl + 'status', :status => 304)
@@ -554,19 +575,46 @@ EOS
     assert(called)
   end
 
+  def test_get_content_with_base_url
+    @client = HTTPClient.new(:base_url => serverurl)
+    assert_equal('hello', @client.get_content('/hello'))
+    assert_equal('hello', @client.get_content('/redirect1'))
+    assert_equal('hello', @client.get_content('/redirect2'))
+    @client.reset('/')
+    assert_raises(HTTPClient::BadResponseError) do
+      @client.get_content('/notfound')
+    end
+    assert_raises(HTTPClient::BadResponseError) do
+      @client.get_content('/redirect_self')
+    end
+    called = false
+    @client.redirect_uri_callback = lambda { |uri, res|
+      newuri = res.header['location'][0]
+      called = true
+      newuri
+    }
+    assert_equal('hello', @client.get_content('/relative_redirect'))
+    assert(called)
+  end
+
   GZIP_CONTENT = "\x1f\x8b\x08\x00\x1a\x96\xe0\x4c\x00\x03\xcb\x48\xcd\xc9\xc9\x07\x00\x86\xa6\x10\x36\x05\x00\x00\x00"
   DEFLATE_CONTENT = "\x78\x9c\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c\x02\x15"
-  GZIP_CONTENT.force_encoding('BINARY') if GZIP_CONTENT.respond_to?(:force_encoding)
-  DEFLATE_CONTENT.force_encoding('BINARY') if DEFLATE_CONTENT.respond_to?(:force_encoding)
+  DEFLATE_NOHEADER_CONTENT = "x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15"
+  [GZIP_CONTENT, DEFLATE_CONTENT, DEFLATE_NOHEADER_CONTENT].each do |content|
+    content.force_encoding('BINARY') if content.respond_to?(:force_encoding)
+  end
   def test_get_gzipped_content
     @client.transparent_gzip_decompression = false
     content = @client.get_content(serverurl + 'compressed?enc=gzip')
     assert_not_equal('hello', content)
     assert_equal(GZIP_CONTENT, content)
     @client.transparent_gzip_decompression = true
+    @client.reset_all
     assert_equal('hello', @client.get_content(serverurl + 'compressed?enc=gzip'))
     assert_equal('hello', @client.get_content(serverurl + 'compressed?enc=deflate'))
+    assert_equal('hello', @client.get_content(serverurl + 'compressed?enc=deflate_noheader'))
     @client.transparent_gzip_decompression = false
+    @client.reset_all
   end
 
   def test_get_content_with_block
@@ -640,6 +688,32 @@ EOS
     assert_nil(res.contenttype)
   end
 
+  def test_get_with_base_url
+    @client = HTTPClient.new(:base_url => serverurl)
+    assert_equal("get", @client.get('/servlet').content)
+    param = {'1'=>'2', '3'=>'4'}
+    res = @client.get('/servlet', param)
+    assert_equal(param, params(res.header["x-query"][0]))
+    assert_nil(res.contenttype)
+    #
+    @client.base_url = serverurl[0...-1] + '/servlet'
+    url = '?5=6&7=8'
+    res = @client.get(url, param)
+    assert_equal(param.merge("5"=>"6", "7"=>"8"), params(res.header["x-query"][0]))
+    assert_nil(res.contenttype)
+  end
+
+  def test_get_with_default_header
+    @client = HTTPClient.new(:base_url => serverurl, :default_header => {'x-header' => 'custom'})
+    assert_equal('custom', @client.get('/servlet').headers['X-Header'])
+    @client.default_header = {'x-header' => 'custom2'}
+    assert_equal('custom2', @client.get('/servlet').headers['X-Header'])
+    # passing Hash overrides
+    assert_equal('custom3', @client.get('/servlet', :header => {'x-header' => 'custom3'}).headers['X-Header'])
+    # passing Array does not override
+    assert_equal('custom2, custom4', @client.get('/servlet', :header => [['x-header', 'custom4']]).headers['X-Header'])
+  end
+
   def test_head_follow_redirect
     expected = urify(serverurl + 'hello')
     assert_equal(expected, @client.head(serverurl + 'hello', :follow_redirect => true).header.request_uri)
@@ -650,7 +724,17 @@ EOS
   def test_get_follow_redirect
     assert_equal('hello', @client.get(serverurl + 'hello', :follow_redirect => true).body)
     assert_equal('hello', @client.get(serverurl + 'redirect1', :follow_redirect => true).body)
-    assert_equal('hello', @client.get(serverurl + 'redirect2', :follow_redirect => true).body)
+
+    res = @client.get(serverurl + 'redirect2', :follow_redirect => true)
+    assert_equal('hello', res.body)
+    assert_equal("http://localhost:#{@serverport}/hello", res.header.request_uri.to_s)
+    assert_equal("http://localhost:#{@serverport}/redirect3", res.previous.header.request_uri.to_s)
+    assert_equal("http://localhost:#{@serverport}/redirect2", res.previous.previous.header.request_uri.to_s)
+    assert_equal(nil, res.previous.previous.previous)
+  end
+
+  def test_get_follow_redirect_with_query
+    assert_equal('hello?1=2&3=4', @client.get(serverurl + 'redirect1', :query => {1 => 2, 3 => 4}, :follow_redirect => true).body)
   end
 
   def test_get_async
@@ -661,12 +745,44 @@ EOS
     assert_equal(param, params(res.header["x-query"][0]))
   end
 
+  def test_get_async_with_base_url
+    param = {'1'=>'2', '3'=>'4'}
+    @client = HTTPClient.new(:base_url => serverurl)
+
+    # Use preconfigured :base_url
+    conn = @client.get_async('servlet', param)
+    Thread.pass while !conn.finished?
+    res = conn.pop
+    assert_equal(param, params(res.header["x-query"][0]))
+    # full URL still works
+    conn = @client.get_async(serverurl + 'servlet', param)
+    Thread.pass while !conn.finished?
+    res = conn.pop
+    assert_equal(param, params(res.header["x-query"][0]))
+  end
+
   def test_get_async_for_largebody
     conn = @client.get_async(serverurl + 'largebody')
     res = conn.pop
     assert_equal(1000*1000, res.content.read.length)
   end
 
+  if RUBY_VERSION > "1.9"
+    def test_post_async_with_default_internal
+      original_encoding = Encoding.default_internal
+      Encoding.default_internal = Encoding::UTF_8
+      begin
+        post_body = StringIO.new("こんにちは")
+        conn = @client.post_async(serverurl + 'servlet', post_body)
+        Thread.pass while !conn.finished?
+        res = conn.pop
+        assert_equal 'post,こんにちは', res.content.read
+      ensure
+        Encoding.default_internal = original_encoding
+      end
+    end
+  end
+
   def test_get_with_block
     called = false
     res = @client.get(serverurl + 'servlet') { |str|
@@ -678,23 +794,101 @@ EOS
     assert_nil(res.content)
   end
 
-  def test_get_with_block_chunk_string_recycle
+  def test_get_with_block_arity_2
+    called = false
+    res = @client.get(serverurl + 'servlet') { |blk_res, str|
+      assert_equal(200, blk_res.status)
+      assert_equal('get', str)
+      called = true
+    }
+    assert(called)
+    # res does not have a content
+    assert_nil(res.content)
+  end
+
+  def test_get_with_block_and_redirects
+    called = false
+    res = @client.get(serverurl + 'servlet', :follow_redirect => true) { |str|
+      assert_equal('get', str)
+      called = true
+    }
+    assert(called)
+    # res does not have a content
+    assert_nil(res.content)
+  end
+
+  def test_get_with_block_arity_2_and_redirects
+    called = false
+    res = @client.get(serverurl + 'servlet', :follow_redirect => true) { |blk_res, str|
+      assert_equal(200, blk_res.status)
+      assert_equal('get', str)
+      called = true
+    }
+    assert(called)
+    # res does not have a content
+    assert_nil(res.content)
+  end
+
+  def test_get_with_block_string_recycle
     @client.read_block_size = 2
     body = []
-    res = @client.get(serverurl + 'servlet') { |str|
+    _res = @client.get(serverurl + 'servlet') { |str|
       body << str
     }
     assert_equal(2, body.size)
     assert_equal("get", body.join) # Was "tt" by String object recycle...
   end
 
+  def test_get_with_block_chunked_string_recycle
+    server = TCPServer.open('localhost', 0)
+    server_thread = Thread.new {
+      Thread.abort_on_exception = true
+      sock = server.accept
+      create_keepalive_thread(1, sock)
+    }
+    url = "http://localhost:#{server.addr[1]}/"
+    body = []
+    begin
+      _res = @client.get(url + 'chunked') { |str|
+        body << str
+      }
+    ensure
+      server.close
+      server_thread.join
+    end
+    assert_equal('abcdefghijklmnopqrstuvwxyz1234567890abcdef', body.join)
+  end
+
   def test_post
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4])
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4])
     param = {'1'=>'2', '3'=>'4'}
     res = @client.post(serverurl + 'servlet', param)
     assert_equal(param, params(res.header["x-query"][0]))
   end
 
+  def test_post_empty
+    @client.debug_dev = str = ''
+    # nil body means 'no content' that is allowed but WEBrick cannot handle it.
+    @client.post(serverurl + 'servlet', :body => nil)
+    # request does not have 'Content-Type'
+    assert_equal(1, str.scan(/content-type/i).size)
+  end
+
+  def test_post_with_query
+    # this {:query => 'query'} recognized as body
+    res = @client.post(serverurl + 'servlet', :query => 'query')
+    assert_equal("post", res.content[0, 4])
+    assert_equal("query=query", res.headers["X-Query"])
+    assert_equal("", res.headers["X-Request-Query"])
+  end
+
+  def test_post_with_query_and_body
+    res = @client.post(serverurl + 'servlet', :query => {:query => 'query'}, :body => {:body => 'body'})
+    assert_equal("post", res.content[0, 4])
+    assert_equal("body=body", res.headers["X-Query"])
+    assert_equal("query=query", res.headers["X-Request-Query"])
+  end
+
   def test_post_follow_redirect
     assert_equal('hello', @client.post(serverurl + 'hello', :follow_redirect => true).body)
     assert_equal('hello', @client.post(serverurl + 'redirect1', :follow_redirect => true).body)
@@ -704,18 +898,18 @@ EOS
   def test_post_with_content_type
     param = [['1', '2'], ['3', '4']]
     ext = {'content-type' => 'application/x-www-form-urlencoded', 'hello' => 'world'}
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     res = @client.post(serverurl + 'servlet', param, ext)
     assert_equal(Hash[param], params(res.header["x-query"][0]))
     #
     ext = [['content-type', 'multipart/form-data'], ['hello', 'world']]
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     res = @client.post(serverurl + 'servlet', param, ext)
     assert_match(/Content-Disposition: form-data; name="1"/, res.content)
     assert_match(/Content-Disposition: form-data; name="3"/, res.content)
     #
     ext = {'content-type' => 'multipart/form-data; boundary=hello'}
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     res = @client.post(serverurl + 'servlet', param, ext)
     assert_match(/Content-Disposition: form-data; name="1"/, res.content)
     assert_match(/Content-Disposition: form-data; name="3"/, res.content)
@@ -725,7 +919,7 @@ EOS
   def test_post_with_custom_multipart_and_boolean_params
     param = [['boolean_true', true]]
     ext = { 'content-type' => 'multipart/form-data' }
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     res = @client.post(serverurl + 'servlet', param, ext)
     assert_match(/Content-Disposition: form-data; name="boolean_true"\r\n\r\ntrue\r\n/, res.content)
     #
@@ -777,7 +971,8 @@ EOS
     assert_match(/\r\n2\r\n/m, res.content)
     assert_match(/\r\nContent-Disposition: form-data; name="3"; filename=""\r\n/m, res.content)
     assert_match(/\r\nContent-Length:/m, str.string)
-    assert_equal(3, myio.called)
+    # HTTPClient reads from head to 'size'; CHUNK_SIZE bytes then 1 byte, that's all.
+    assert_equal(2, myio.called)
   end
 
   def test_post_with_io_nosize # streaming + chunked post
@@ -794,6 +989,42 @@ EOS
     assert_match(/\r\nTransfer-Encoding: chunked\r\n/m, str.string)
   end
 
+  def test_post_with_sized_io
+    myio = StringIO.new("45")
+    def myio.size
+      1
+    end
+    res = @client.post(serverurl + 'servlet', myio)
+    assert_equal('post,4', res.content)
+  end
+
+  def test_post_with_sized_io_part
+    myio = StringIO.new("45")
+    def myio.size
+      1
+    end
+    @client.debug_dev = str = StringIO.new
+    _res = @client.post(serverurl + 'servlet', { :file => myio })
+    assert_match(/\r\n4\r\n/, str.string, 'should send "4" not "45"')
+  end
+
+  def test_post_with_unknown_sized_io_part
+    myio1 = StringIO.new("123")
+    myio2 = StringIO.new("45")
+    class << myio1
+      undef :size
+    end
+    class << myio2
+      # This does not work because other file is 'unknown sized'
+      def size
+        1
+      end
+    end
+    @client.debug_dev = str = StringIO.new
+    _res = @client.post(serverurl + 'servlet', { :file1 => myio1, :file2 => myio2 })
+    assert_match(/\r\n45\r\n/, str.string)
+  end
+
   def test_post_async
     param = {'1'=>'2', '3'=>'4'}
     conn = @client.post_async(serverurl + 'servlet', param)
@@ -804,7 +1035,7 @@ EOS
 
   def test_post_with_block
     called = false
-    res = @client.post(serverurl + 'servlet') { |str|
+    res = @client.post(serverurl + 'servlet', '') { |str|
       assert_equal('post,', str)
       called = true
     }
@@ -824,7 +1055,7 @@ EOS
 
   def test_post_with_custom_multipart
     ext = {'content-type' => 'multipart/form-data'}
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     body = [{ 'Content-Disposition' => 'form-data; name="1"', :content => "2"},
             { 'Content-Disposition' => 'form-data; name="3"', :content => "4"}]
     res = @client.post(serverurl + 'servlet', body, ext)
@@ -832,7 +1063,7 @@ EOS
     assert_match(/Content-Disposition: form-data; name="3"/, res.content)
     #
     ext = {'content-type' => 'multipart/form-data; boundary=hello'}
-    assert_equal("post", @client.post(serverurl + 'servlet').content[0, 4], ext)
+    assert_equal("post", @client.post(serverurl + 'servlet', '').content[0, 4], ext)
     res = @client.post(serverurl + 'servlet', body, ext)
     assert_match(/Content-Disposition: form-data; name="1"/, res.content)
     assert_match(/Content-Disposition: form-data; name="3"/, res.content)
@@ -842,6 +1073,10 @@ EOS
   def test_post_with_custom_multipart_and_file
     STDOUT.sync = true
     File.open(__FILE__) do |file|
+      def file.original_filename
+        'file.txt'
+      end
+
       ext = { 'Content-Type' => 'multipart/alternative' }
       body = [{ 'Content-Type' => 'text/plain', :content => "this is only a test" },
               { 'Content-Type' => 'application/x-ruby', :content => file }]
@@ -849,12 +1084,43 @@ EOS
       assert_match(/^Content-Type: text\/plain\r\n/m, res.content)
       assert_match(/^this is only a test\r\n/m, res.content)
       assert_match(/^Content-Type: application\/x-ruby\r\n/m, res.content)
+      assert_match(/Content-Disposition: form-data; name="3"; filename="file.txt"/, res.content)
       assert_match(/FIND_TAG_IN_THIS_FILE/, res.content)
     end
   end
 
+  def test_patch
+    assert_equal("patch", @client.patch(serverurl + 'servlet', '').content)
+    param = {'1'=>'2', '3'=>'4'}
+    @client.debug_dev = str = ''
+    res = @client.patch(serverurl + 'servlet', param)
+    assert_equal(param, params(res.header["x-query"][0]))
+    assert_equal('Content-Type: application/x-www-form-urlencoded', str.split(/\r?\n/)[5])
+  end
+
+  def test_patch_with_query_and_body
+    res = @client.patch(serverurl + 'servlet', :query => {:query => 'query'}, :body => {:body => 'body'})
+    assert_equal("patch", res.content)
+    assert_equal("body=body", res.headers["X-Query"])
+    assert_equal("query=query", res.headers["X-Request-Query"])
+  end
+
+  def test_patch_bytesize
+    res = @client.patch(serverurl + 'servlet', 'txt' => 'あいうえお')
+    assert_equal('txt=%E3%81%82%E3%81%84%E3%81%86%E3%81%88%E3%81%8A', res.header["x-query"][0])
+    assert_equal('15', res.header["x-size"][0])
+  end
+
+  def test_patch_async
+    param = {'1'=>'2', '3'=>'4'}
+    conn = @client.patch_async(serverurl + 'servlet', param)
+    Thread.pass while !conn.finished?
+    res = conn.pop
+    assert_equal(param, params(res.header["x-query"][0]))
+  end
+
   def test_put
-    assert_equal("put", @client.put(serverurl + 'servlet').content)
+    assert_equal("put", @client.put(serverurl + 'servlet', '').content)
     param = {'1'=>'2', '3'=>'4'}
     @client.debug_dev = str = ''
     res = @client.put(serverurl + 'servlet', param)
@@ -862,6 +1128,13 @@ EOS
     assert_equal('Content-Type: application/x-www-form-urlencoded', str.split(/\r?\n/)[5])
   end
 
+  def test_put_with_query_and_body
+    res = @client.put(serverurl + 'servlet', :query => {:query => 'query'}, :body => {:body => 'body'})
+    assert_equal("put", res.content)
+    assert_equal("body=body", res.headers["X-Query"])
+    assert_equal("query=query", res.headers["X-Request-Query"])
+  end
+
   def test_put_bytesize
     res = @client.put(serverurl + 'servlet', 'txt' => 'あいうえお')
     assert_equal('txt=%E3%81%82%E3%81%84%E3%81%86%E3%81%88%E3%81%8A', res.header["x-query"][0])
@@ -880,6 +1153,19 @@ EOS
     assert_equal("delete", @client.delete(serverurl + 'servlet').content)
   end
 
+  def test_delete_with_query
+    res = @client.delete(serverurl + 'servlet', :query => {:query => 'query'})
+    assert_equal("delete", res.content)
+    assert_equal('query=query', res.headers['X-Request-Query'])
+  end
+
+  def test_delete_with_query_and_body
+    res = @client.delete(serverurl + 'servlet', :query => {:query => 'query'}, :body => {:body => 'body'})
+    assert_equal("delete", res.content)
+    assert_equal('query=query', res.headers['X-Request-Query'])
+    assert_equal('body=body', res.headers['X-Query'])
+  end
+
   # Not prohibited by spec, but normally it's ignored
   def test_delete_with_body
     param = {'1'=>'2', '3'=>'4'}
@@ -896,7 +1182,23 @@ EOS
   end
 
   def test_options
-    assert_equal("options", @client.options(serverurl + 'servlet').content)
+    assert_equal('options', @client.options(serverurl + 'servlet').content)
+  end
+
+  def test_options_with_header
+    res = @client.options(serverurl + 'servlet', {'x-header' => 'header'})
+    assert_equal('header', res.headers['X-Header'])
+  end
+
+  def test_options_with_body
+    res = @client.options(serverurl + 'servlet', :body => 'body')
+    assert_equal('body', res.headers['X-Body'])
+  end
+
+  def test_options_with_body_and_header
+    res = @client.options(serverurl + 'servlet', :body => 'body', :header => {'x-header' => 'header'})
+    assert_equal('header', res.headers['X-Header'])
+    assert_equal('body', res.headers['X-Body'])
   end
 
   def test_options_async
@@ -951,6 +1253,7 @@ EOS
 
   def test_chunked
     assert_equal('chunked', @client.get_content(serverurl + 'chunked', { 'msg' => 'chunked' }))
+    assert_equal('あいうえお', @client.get_content(serverurl + 'chunked', { 'msg' => 'あいうえお' }))
   end
 
   def test_chunked_empty
@@ -1024,7 +1327,7 @@ EOS
 
   def test_http_custom_date_header
     @client.debug_dev = (str = "")
-    res = @client.get(serverurl + 'hello', :header => {'Date' => 'foo'})
+    _res = @client.get(serverurl + 'hello', :header => {'Date' => 'foo'})
     lines = str.split(/(?:\r?\n)+/)
     assert_equal('Date: foo', lines[4])
   end
@@ -1052,35 +1355,32 @@ EOS
 
   def test_receive_timeout
     # this test takes 2 sec
-    assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
+    assert_equal('hello?sec=2', @client.get_content(serverurl + 'sleep?sec=2'))
     @client.receive_timeout = 1
-    assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=0'))
+    @client.reset_all
+    assert_equal('hello?sec=0', @client.get_content(serverurl + 'sleep?sec=0'))
     assert_raise(HTTPClient::ReceiveTimeoutError) do
       @client.get_content(serverurl + 'sleep?sec=2')
     end
     @client.receive_timeout = 3
-    assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
+    @client.reset_all
+    assert_equal('hello?sec=2', @client.get_content(serverurl + 'sleep?sec=2'))
   end
 
   def test_receive_timeout_post
     # this test takes 2 sec
     assert_equal('hello', @client.post(serverurl + 'sleep', :sec => 2).content)
     @client.receive_timeout = 1
+    @client.reset_all
     assert_equal('hello', @client.post(serverurl + 'sleep', :sec => 0).content)
     assert_raise(HTTPClient::ReceiveTimeoutError) do
       @client.post(serverurl + 'sleep', :sec => 2)
     end
     @client.receive_timeout = 3
+    @client.reset_all
     assert_equal('hello', @client.post(serverurl + 'sleep', :sec => 2).content)
   end
 
-  def test_async_error
-    assert_raise( SocketError ) do
-      conn = @client.get_async("http://non-existing-host/")
-      conn.pop
-    end
-  end
-
   def test_reset
     url = serverurl + 'servlet'
     assert_nothing_raised do
@@ -1101,22 +1401,18 @@ EOS
   end
 
   def test_cookies
-    cookiefile = File.join(File.dirname(File.expand_path(__FILE__)), 'test_cookies_file')
-    File.open(cookiefile, "wb") do |f|
-      f << "http://rubyforge.org/account/login.php\tsession_ser\tLjEwMy45Ni40Ni0q%2A-fa0537de8cc31\t2000000000\t.rubyforge.org\t/\t13\n"
-    end
-    @client.set_cookie_store(cookiefile)
-    cookie = @client.cookie_manager.cookies.first
-    url = cookie.url
-    assert(cookie.domain_match(url.host, cookie.domain))
+    cookiefile = Tempfile.new('test_cookies_file')
+    File.open(cookiefile.path, "wb") do |f|
+      f << "http://rubyforge.org/account/login.php\tsession_ser\tLjEwMy45Ni40Ni0q%2A-fa0537de8cc31\t2000000000\trubyforge.org\t/account/\t9\n"
+    end
+    @client.set_cookie_store(cookiefile.path)
     #
     @client.reset_all
-    @client.test_loopback_http_response << "HTTP/1.0 200 OK\nSet-Cookie: foo=bar; expires=#{Time.at(1924873200).gmtime.httpdate}\n\nOK"
+    @client.test_loopback_http_response << "HTTP/1.0 200 OK\nSet-Cookie: session_ser=bar; expires=#{Time.at(1924873200).gmtime.httpdate}\n\nOK"
     @client.get_content('http://rubyforge.org/account/login.php')
     @client.save_cookie_store
-    str = File.read(cookiefile)
-    assert_match(%r(http://rubyforge.org/account/login.php\tfoo\tbar\t1924873200\trubyforge.org\t/account\t1), str)
-    File.unlink(cookiefile)
+    str = File.read(cookiefile.path)
+    assert_match(%r(http://rubyforge.org/account/login.php\tsession_ser\tbar\t1924873200\trubyforge.org\t/account/\t9), str)
   end
 
   def test_eof_error_length
@@ -1240,6 +1536,7 @@ EOS
     assert_equal('text/plain', HTTP::Message.mime_type('foo.txt'))
     assert_equal('text/html', HTTP::Message.mime_type('foo.html'))
     assert_equal('text/html', HTTP::Message.mime_type('foo.htm'))
+    assert_equal('text/xml', HTTP::Message.mime_type('foo.xml'))
     assert_equal('application/msword', HTTP::Message.mime_type('foo.doc'))
     assert_equal('image/png', HTTP::Message.mime_type('foo.png'))
     assert_equal('image/gif', HTTP::Message.mime_type('foo.gif'))
@@ -1342,6 +1639,7 @@ EOS
     res = HTTP::Message.new_response('response')
     res.contenttype = 'text/plain'
     res.header.body_date = Time.at(946652400)
+    res.header.request_uri = 'http://www.example.com/'
     assert_nil(res.cookies)
     #
     res.header['Set-Cookie'] = [
@@ -1418,8 +1716,11 @@ EOS
     server = TCPServer.open('127.0.0.1', 0)
     server.listen(30) # set enough backlogs
     endpoint = "http://127.0.0.1:#{server.addr[1]}/"
-    Thread.new {
+    queue = Queue.new
+    Thread.new(queue) { |qs|
       Thread.abort_on_exception = true
+      # want 5 requests issued
+      5.times { qs.pop }
       # emulate 10 keep-alive connections
       10.times do |idx|
         sock = server.accept
@@ -1448,22 +1749,29 @@ EOS
         sock.close
       end
     }
-    # allocate 10 keep-alive connections
+    # try to allocate 10 keep-alive connections; it's a race so some
+    # threads can reuse the connection so actual number of keep-alive
+    # connections should be smaller than 10.
     (0...10).to_a.map {
-      Thread.new {
-        assert_equal("12345", client.get(endpoint).content)
+      Thread.new(queue) { |qc|
+        Thread.abort_on_exception = true
+        conn = client.get_async(endpoint)
+        qc.push(true)
+        assert_equal("12345", conn.pop.content.read)
       }
     }.each { |th| th.join }
-    # send 5 requests, which should get KeepAliveDesconnected.
-    # doing these requests, rest keep-alive connections are invalidated.
+    # send 5 requests, some of these should get KeepAliveDesconnected
+    # but should retry with new connection.
     (0...5).to_a.map {
       Thread.new {
+        Thread.abort_on_exception = true
         assert_equal("23456", client.get(endpoint).content)
       }
     }.each { |th| th.join }
-    # rest requests won't get KeepAliveDisconnected; how can I check this?
+    # rest requests won't get KeepAliveDisconnected
     (0...10).to_a.map {
       Thread.new {
+        Thread.abort_on_exception = true
         assert_equal("34567", client.get(endpoint).content)
       }
     }.each { |th| th.join }
@@ -1522,6 +1830,19 @@ EOS
     end
   end
 
+  def test_strict_response_size_check
+    @client.strict_response_size_check = false
+    @client.test_loopback_http_response << "HTTP/1.0 200 OK\r\nContent-Length: 12345\r\n\r\nhello world"
+    assert_equal('hello world', @client.get_content('http://dummy'))
+
+    @client.reset_all
+    @client.strict_response_size_check = true
+    @client.test_loopback_http_response << "HTTP/1.0 200 OK\r\nContent-Length: 12345\r\n\r\nhello world"
+    assert_raise(HTTPClient::BadResponseError) do
+      @client.get_content('http://dummy')
+    end
+  end
+
   def test_socket_local
     @client.socket_local.host = '127.0.0.1'
     assert_equal('hello', @client.get_content(serverurl + 'hello'))
@@ -1557,6 +1878,48 @@ EOS
     end
   end
 
+  def test_ipv6literaladdress_in_uri
+    server = TCPServer.open('::1', 0) rescue return # Skip if IPv6 is unavailable.
+    server_thread = Thread.new {
+      Thread.abort_on_exception = true
+      sock = server.accept
+      while line = sock.gets
+        break if line.chomp.empty?
+      end
+      sock.write("HTTP/1.1 200 OK\r\n")
+      sock.write("Content-Length: 5\r\n")
+      sock.write("\r\n")
+      sock.write("12345")
+      sock.close
+    }
+    uri = "http://[::1]:#{server.addr[1]}/"
+    begin
+      assert_equal('12345', @client.get(uri).body)
+    ensure
+      server.close
+      server_thread.kill
+      server_thread.join
+    end
+  end
+
+  def test_uri_no_schema
+    assert_raise(ArgumentError) do
+      @client.get_content("www.example.com")
+    end
+  end
+
+  def test_tcp_keepalive
+    @client.tcp_keepalive = true
+    @client.get(serverurl)
+
+    # expecting HTTP keepalive caches the socket
+    session = @client.instance_variable_get(:@session_manager).send(:get_cached_session, HTTPClient::Site.new(URI.parse(serverurl)))
+    socket = session.instance_variable_get(:@socket)
+
+    assert_true(session.tcp_keepalive)
+    assert_equal(Socket::SO_KEEPALIVE, socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE).optname)
+  end
+
 private
 
   def check_query_get(query)
@@ -1594,53 +1957,53 @@ private
     @server_thread = start_server_thread(@server)
   end
 
-  def escape_noproxy
-    backup = HTTPClient::NO_PROXY_HOSTS.dup
-    HTTPClient::NO_PROXY_HOSTS.clear
-    yield
-  ensure
-    HTTPClient::NO_PROXY_HOSTS.replace(backup)
+  def add_query_string(req)
+    if req.query_string
+      '?' + req.query_string
+    else
+      ''
+    end
   end
 
   def do_hello(req, res)
     res['content-type'] = 'text/html'
-    res.body = "hello"
+    res.body = "hello" + add_query_string(req)
   end
 
   def do_sleep(req, res)
     sec = req.query['sec'].to_i
     sleep sec
     res['content-type'] = 'text/html'
-    res.body = "hello"
+    res.body = "hello" + add_query_string(req)
   end
 
   def do_servlet_redirect(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "servlet")
+    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "servlet" + add_query_string(req))
   end
 
   def do_redirect1(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::MovedPermanently, serverurl + "hello")
+    res.set_redirect(WEBrick::HTTPStatus::MovedPermanently, serverurl + "hello" + add_query_string(req))
   end
 
   def do_redirect2(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, serverurl + "redirect3")
+    res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, serverurl + "redirect3" + add_query_string(req))
   end
 
   def do_redirect3(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "hello")
+    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "hello" + add_query_string(req))
   end
 
   def do_redirect_self(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "redirect_self")
+    res.set_redirect(WEBrick::HTTPStatus::Found, serverurl + "redirect_self" + add_query_string(req))
   end
 
   def do_relative_redirect(req, res)
-    res.set_redirect(WEBrick::HTTPStatus::Found, "hello")
+    res.set_redirect(WEBrick::HTTPStatus::Found, "hello" + add_query_string(req))
   end
 
   def do_redirect_see_other(req, res)
     if req.request_method == 'POST'
-      res.set_redirect(WEBrick::HTTPStatus::SeeOther, serverurl + "redirect_see_other") # self
+      res.set_redirect(WEBrick::HTTPStatus::SeeOther, serverurl + "redirect_see_other" + add_query_string(req)) # self
     else
       res.body = 'hello'
     end
@@ -1648,6 +2011,7 @@ private
 
   def do_chunked(req, res)
     res.chunked = true
+    res['content-type'] = 'text/plain; charset=UTF-8'
     piper, pipew = IO.pipe
     res.body = piper
     pipew << req.query['msg']
@@ -1667,6 +2031,9 @@ private
     elsif req.query['enc'] == 'deflate'
       res['content-encoding'] = 'deflate'
       res.body = DEFLATE_CONTENT
+    elsif req.query['enc'] == 'deflate_noheader'
+      res['content-encoding'] = 'deflate'
+      res.body = DEFLATE_NOHEADER_CONTENT
     end
   end
 
@@ -1700,6 +2067,7 @@ private
 
     def do_GET(req, res)
       res.body = 'get'
+      res['x-header'] = req['X-Header']
       res["x-query"] = query_response(req)
     end
 
@@ -1707,6 +2075,15 @@ private
       res["content-type"] = "text/plain" # iso-8859-1, not US-ASCII
       res.body = 'post,' + req.body.to_s
       res["x-query"] = body_response(req)
+      res["x-request-query"] = req.query_string
+    end
+
+    def do_PATCH(req, res)
+      res["x-query"] = body_response(req)
+      param = WEBrick::HTTPUtils.parse_query(req.body) || {}
+      res["x-size"] = (param['txt'] || '').size
+      res.body = param['txt'] || 'patch'
+      res["x-request-query"] = req.query_string
     end
 
     def do_PUT(req, res)
@@ -1714,15 +2091,19 @@ private
       param = WEBrick::HTTPUtils.parse_query(req.body) || {}
       res["x-size"] = (param['txt'] || '').size
       res.body = param['txt'] || 'put'
+      res["x-request-query"] = req.query_string
     end
 
     def do_DELETE(req, res)
       res.body = 'delete'
+      res["x-query"] = body_response(req)
+      res["x-request-query"] = req.query_string
     end
 
     def do_OPTIONS(req, res)
-      # check RFC for legal response.
       res.body = 'options'
+      res['x-header'] = req['X-Header']
+      res['x-body'] = req.body
     end
 
     def do_PROPFIND(req, res)