httpclient 2.3.0.1 → 2.8.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 45217dcc777d36d71246dd468e40b79caad351d6
+  data.tar.gz: afcf1a175414e0a1dde95eb5823b0fa339655d9c
+SHA512:
+  metadata.gz: f5ae105eb3b269d67521a35446e3518b359e7a359c0c8a14500ef9e9ff8c4681c20b103bb4d5a2f96cdd9caa337fc149f39df3754bb9f3185b6914f284adfdc0
+  data.tar.gz: 005d1769b6906e0c107ba63e7a178c4d73aae59198ed3efe866e8722fdadcf4c4142c3724c167b6db5f8a45cb03f517083164c18fdd1aa65074dc4ab362bc9de

data/README.md

@@ -0,0 +1,85 @@
+httpclient - HTTP accessing library.  [![Gem Version](https://badge.fury.io/rb/httpclient.svg)](http://badge.fury.io/rb/httpclient)
+
+Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+
+'httpclient' gives something like the functionality of libwww-perl (LWP) in
+Ruby.  'httpclient' formerly known as 'http-access2'.
+
+See [HTTPClient](http://www.rubydoc.info/gems/httpclient/frames) for documentation.
+
+
+## Features
+
+* methods like GET/HEAD/POST/* via HTTP/1.1.
+* HTTPS(SSL), Cookies, proxy, authentication(Digest, NTLM, Basic), etc.
+* asynchronous HTTP request, streaming HTTP request.
+* debug mode CLI.
+* by contrast with net/http in standard distribution;
+  * Cookies support
+  * MT-safe
+  * streaming POST (POST with File/IO)
+  * Digest auth
+  * Negotiate/NTLM auth for WWW-Authenticate (requires net/ntlm module; rubyntlm gem)
+  * NTLM auth for Proxy-Authenticate (requires 'win32/sspi' module; rubysspi gem)
+  * extensible with filter interface
+  * you don't have to care HTTP/1.1 persistent connection
+    (httpclient cares instead of you)
+* Not supported now
+  * Cache
+  * Rather advanced HTTP/1.1 usage such as Range, deflate, etc.
+    (of course you can set it in header by yourself)
+
+## httpclient command
+
+Usage: 1) `httpclient get https://www.google.co.jp/?q=ruby`  
+Usage: 2) `httpclient`
+
+For 1) it issues a GET request to the given URI and shows the wiredump and
+the parsed result.  For 2) it invokes irb shell with the binding that has a
+HTTPClient as 'self'.  You can call HTTPClient instance methods like;
+
+```ruby
+get "https://www.google.co.jp/", :q => :ruby
+```
+
+## Author
+
+ * Name:: Hiroshi Nakamura
+ * E-mail:: nahi@ruby-lang.org
+ * Project web site:: http://github.com/nahi/httpclient
+
+
+## License
+
+This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
+redistribute it and/or modify it under the same terms of Ruby's license;
+either the dual license version in 2003, or any later version.
+
+httpclient/session.rb is based on http-access.rb in http-access/0.0.4.  Some
+part of it is copyrighted by Maebashi-san who made and published
+http-access/0.0.4.  http-access/0.0.4 did not include license notice but when
+I asked Maebashi-san he agreed that I can redistribute it under the same terms
+of Ruby.  Many thanks to Maebashi-san.
+
+
+## Install
+
+You can install httpclient via rubygems: `gem install httpclient`
+
+
+## Usage
+
+See [HTTPClient](http://www.rubydoc.info/gems/httpclient/frames) for documentation.
+You can also check sample/howto.rb how to use APIs.
+
+## Bug report or Feature request
+
+Please file a ticket at the project web site.
+
+1. find a similar ticket from https://github.com/nahi/httpclient/issues
+2. create a new ticket by clicking 'Create Issue' button.
+3. you can use github features such as pull-request if you like.
+
+## Changes
+
+See [ChangeLog](https://github.com/nahi/httpclient/blob/master/CHANGELOG.md)

data/bin/httpclient

@@ -11,13 +11,24 @@
 # > get "https://www.google.co.jp/", :q => :ruby
 require 'httpclient'
 
-METHODS = ['head', 'get', 'post', 'put', 'delete', 'options', 'propfind', 'proppatch', 'trace']
-if ARGV.size >= 2 && METHODS.include?(ARGV[0])
+method = ARGV.shift
+if method == 'version'
+  puts HTTPClient::VERSION
+  exit
+end
+
+url = ARGV.shift
+if method && url
   client = HTTPClient.new
-  client.debug_dev = STDERR
-  $DEBUG = true
-  require 'pp'
-  pp client.send(*ARGV)
+  client.strict_response_size_check = true
+  if method == 'download'
+    print client.get_content(url)
+  else
+    client.debug_dev = STDERR
+    $DEBUG = true
+    require 'pp'
+    pp client.send(method, url, *ARGV)
+  end
   exit
 end
 
@@ -27,6 +38,7 @@ require 'irb/completion'
 class Runner
   def initialize
     @httpclient = HTTPClient.new
+    @httpclient.strict_response_size_check = true
   end
 
   def method_missing(msg, *a, &b)

data/bin/jsonclient

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+
+# jsonclient shell command.
+#
+# Usage: 1) % jsonclient post https://www.example.com/ content.json
+# Usage: 2) % jsonclient
+#
+# For 1) it issues a GET request to the given URI and shows the wiredump and
+# the parsed result.  For 2) it invokes irb shell with the binding that has a
+# JSONClient as 'self'.  You can call JSONClient instance methods like;
+# > post "https://www.example.com/resource", {'hello' => 'world'}
+require 'jsonclient'
+ 
+method = ARGV.shift
+url = ARGV.shift
+body = []
+if ['post', 'put'].include?(method)
+  if ARGV.size == 1 && File.exist?(ARGV[0])
+    body << File.read(ARGV[0])
+  else
+    body << ARGF.read
+  end
+end
+if method && url
+  require 'pp'
+  client = JSONClient.new
+  client.debug_dev = STDERR if $DEBUG
+  res = client.send(method, url, *body)
+  STDERR.puts('RESPONSE HEADER: ')
+  PP.pp(res.headers, STDERR)
+  if res.ok?
+    begin
+      puts JSON.pretty_generate(res.content)
+    rescue JSON::GeneratorError
+      puts res.content
+    end
+    exit 0
+  else
+    STDERR.puts res.content
+    exit 1
+  end
+end
+
+require 'irb'
+require 'irb/completion'
+
+class Runner
+  def initialize
+    @httpclient = JSONClient.new
+  end
+
+  def method_missing(msg, *a, &b)
+    debug, $DEBUG = $DEBUG, true
+    begin
+      @httpclient.send(msg, *a, &b)
+    ensure
+      $DEBUG = debug
+    end
+  end
+
+  def run
+    IRB.setup(nil)
+    ws = IRB::WorkSpace.new(binding)
+    irb = IRB::Irb.new(ws)
+    IRB.conf[:MAIN_CONTEXT] = irb.context
+
+    trap("SIGINT") do
+      irb.signal_handle
+    end
+
+    begin
+      catch(:IRB_EXIT) do
+        irb.eval_input
+      end
+    ensure
+      IRB.irb_at_exit
+    end
+  end
+
+  def to_s
+    'JSONClient'
+  end
+end
+
+Runner.new.run

data/lib/http-access2.rb

@@ -39,7 +39,7 @@ module HTTPAccess2
   Site = ::HTTPClient::Site
   Connection = ::HTTPClient::Connection
   SessionManager = ::HTTPClient::SessionManager
-  SSLSocketWrap = ::HTTPClient::SSLSocketWrap
+  SSLSocketWrap = ::HTTPClient::SSLSocket
   DebugSocket = ::HTTPClient::DebugSocket
 
   class Session < ::HTTPClient::Session

data/lib/httpclient.rb

@@ -1,5 +1,5 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;
@@ -116,7 +116,7 @@ require 'httpclient/cookie'
 #       res = clnt.post(uri, body)
 #     end
 #
-# 3. Do multipart wth custom body.
+# 3. Do multipart with custom body.
 #
 #     File.open('/tmp/post_data') do |file|
 #       body = [{ 'Content-Type' => 'application/atom+xml; charset=UTF-8',
@@ -149,6 +149,16 @@ require 'httpclient/cookie'
 #     clnt.ssl_config.set_client_cert_file(user_cert_file, user_key_file)
 #     clnt.get(https_url)
 #
+# 4. Revocation check. On JRuby you can set following options to let
+#    HTTPClient to perform revocation check with CRL and OCSP:
+#
+#     -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true
+#     ex. jruby -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true app.rb
+#     Revoked cert example: https://test-sspev.verisign.com:2443/test-SSPEV-revoked-verisign.html
+#
+#    On other platform you can download CRL by yourself and set it via
+#    SSLConfig#add_crl.
+#
 # === Handling Cookies
 #
 # 1. Using volatile Cookies.  Nothing to do.  HTTPClient handles Cookies.
@@ -229,7 +239,7 @@ require 'httpclient/cookie'
 #   ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
 #
 class HTTPClient
-  RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE})"
   LIB_NAME = "(#{VERSION}, #{RUBY_VERSION_STRING})"
 
   include Util
@@ -299,7 +309,6 @@ class HTTPClient
       if assignable
         aname = name + '='
         define_method(aname) { |rhs|
-          reset_all
           @session_manager.__send__(aname, rhs)
         }
       end
@@ -308,7 +317,7 @@ class HTTPClient
 
   # HTTPClient::SSLConfig:: SSL configurator.
   attr_reader :ssl_config
-  # WebAgent::CookieManager:: Cookies configurator.
+  # HTTPClient::CookieManager:: Cookies configurator.
   attr_accessor :cookie_manager
   # An array of response HTTP message body String which is used for loop-back
   # test.  See test/* to see how to use it.  If you want to do loop-back test
@@ -324,6 +333,10 @@ class HTTPClient
   # How many times get_content and post_content follows HTTP redirect.
   # 10 by default.
   attr_accessor :follow_redirect_count
+  # Base url of resources.
+  attr_accessor :base_url
+  # Default request header.
+  attr_accessor :default_header
 
   # Set HTTP version as a String:: 'HTTP/1.0' or 'HTTP/1.1'
   attr_proxy(:protocol_version, true)
@@ -342,6 +355,8 @@ class HTTPClient
   # if your ruby is older than 2005-09-06, do not set socket_sync = false to
   # avoid an SSL socket blocking bug in openssl/buffering.rb.
   attr_proxy(:socket_sync, true)
+  # Enables TCP keepalive; no timing settings exist at present
+  attr_proxy(:tcp_keepalive, true)
   # User-Agent header in HTTP request.
   attr_proxy(:agent_name, true)
   # From header in HTTP request.
@@ -351,48 +366,90 @@ class HTTPClient
   attr_proxy(:test_loopback_http_response)
   # Decompress a compressed (with gzip or deflate) content body transparently. false by default.
   attr_proxy(:transparent_gzip_decompression, true)
+  # Raise BadResponseError if response size does not match with Content-Length header in response. false by default.
+  # TODO: enable by default
+  attr_proxy(:strict_response_size_check, true)
   # Local socket address. Set HTTPClient#socket_local.host and HTTPClient#socket_local.port to specify local binding hostname and port of TCP socket.
   attr_proxy(:socket_local, true)
 
   # Default header for PROPFIND request.
   PROPFIND_DEFAULT_EXTHEADER = { 'Depth' => '0' }
 
+  # Default User-Agent header
+  DEFAULT_AGENT_NAME = 'HTTPClient/1.0'
+
   # Creates a HTTPClient instance which manages sessions, cookies, etc.
   #
-  # HTTPClient.new takes 3 optional arguments for proxy url string,
-  # User-Agent String and From header String.  User-Agent and From are embedded
-  # in HTTP request Header if given.  No User-Agent and From header added
-  # without setting it explicitly.
+  # HTTPClient.new takes optional arguments as a Hash.
+  #  * :proxy - proxy url string
+  #  * :agent_name - User-Agent String
+  #  * :from - from header String
+  #  * :base_url - base URL of resources
+  #  * :default_header - header Hash all HTTP requests should have
+  #  * :force_basic_auth - flag for sending Authorization header w/o gettin 401 first
+  # User-Agent and From are embedded in HTTP request Header if given.
+  # From header is not set without setting it explicitly.
   #
   #   proxy = 'http://myproxy:8080'
   #   agent_name = 'MyAgent/0.1'
   #   from = 'from@example.com'
   #   HTTPClient.new(proxy, agent_name, from)
   #
-  # You can use a keyword argument style Hash.  Keys are :proxy, :agent_name
-  # and :from.
+  # After you set :base_url, all resources you pass to get, post and other
+  # methods are recognized to be prefixed with base_url. Say base_url is
+  # 'https://api.example.com/v1/, get('users') is the same as
+  # get('https://api.example.com/v1/users') internally. You can also pass
+  # full URL from 'http://' even after setting base_url.
+  #
+  # The expected base_url and path behavior is the following. Please take
+  # care of '/' in base_url and path.
   #
-  #   HTTPClient.new(:agent_name => 'MyAgent/0.1')
-  def initialize(*args)
-    proxy, agent_name, from = keyword_argument(args, :proxy, :agent_name, :from)
+  # The last '/' is important for base_url:
+  #   1. http://foo/bar/baz/ + path -> http://foo/bar/baz/path
+  #   2. http://foo/bar/baz + path -> http://foo/bar/path
+  # Relative path handling:
+  #   3. http://foo/bar/baz/ + ../path -> http://foo/bar/path
+  #   4. http://foo/bar/baz + ../path -> http://foo/path
+  #   5. http://foo/bar/baz/ + ./path -> http://foo/bar/baz/path
+  #   6. http://foo/bar/baz + ./path -> http://foo/bar/path
+  # The leading '/' of path means absolute path:
+  #   7. http://foo/bar/baz/ + /path -> http://foo/path
+  #   8. http://foo/bar/baz + /path -> http://foo/path
+  #
+  # :default_header is for providing default headers Hash that all HTTP
+  # requests should have, such as custom 'Authorization' header in API.
+  # You can override :default_header with :header Hash parameter in HTTP
+  # request methods.
+  #
+  # :force_basic_auth turns on/off the BasicAuth force flag. Generally
+  # HTTP client must send Authorization header after it gets 401 error
+  # from server from security reason. But in some situation (e.g. API
+  # client) you might want to send Authorization from the beginning.
+  def initialize(*args, &block)
+    proxy, agent_name, from, base_url, default_header, force_basic_auth =
+      keyword_argument(args, :proxy, :agent_name, :from, :base_url, :default_header, :force_basic_auth)
     @proxy = nil        # assigned later.
     @no_proxy = nil
     @no_proxy_regexps = []
+    @base_url = base_url
+    @default_header = default_header || {}
     @www_auth = WWWAuth.new
     @proxy_auth = ProxyAuth.new
+    @www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
     @request_filter = [@proxy_auth, @www_auth]
     @debug_dev = nil
     @redirect_uri_callback = method(:default_redirect_uri_callback)
     @test_loopback_response = []
     @session_manager = SessionManager.new(self)
-    @session_manager.agent_name = agent_name
+    @session_manager.agent_name = agent_name || DEFAULT_AGENT_NAME
     @session_manager.from = from
     @session_manager.ssl_config = @ssl_config = SSLConfig.new(self)
-    @cookie_manager = WebAgent::CookieManager.new
+    @cookie_manager = CookieManager.new
     @follow_redirect_count = 10
     load_environment
     self.proxy = proxy if proxy
     keep_webmock_compat
+    instance_eval(&block) if block
   end
 
   # webmock 1.6.2 depends on HTTP::Message#body.content to work.
@@ -480,7 +537,7 @@ class HTTPClient
     @no_proxy = no_proxy
     @no_proxy_regexps.clear
     if @no_proxy
-      @no_proxy.scan(/([^:,]+)(?::(\d+))?/) do |host, port|
+      @no_proxy.scan(/([^\s:,]+)(?::(\d+))?/) do |host, port|
         if host[0] == ?.
           regexp = /#{Regexp.quote(host)}\z/i
         else
@@ -506,14 +563,14 @@ class HTTPClient
   #
   # Calling this method resets all existing sessions.
   def set_auth(domain, user, passwd)
-    uri = urify(domain)
+    uri = to_resource_url(domain)
     @www_auth.set_auth(uri, user, passwd)
     reset_all
   end
 
   # Deprecated.  Use set_auth instead.
   def set_basic_auth(domain, user, passwd)
-    uri = urify(domain)
+    uri = to_resource_url(domain)
     @www_auth.basic_auth.set(uri, user, passwd)
     reset_all
   end
@@ -528,6 +585,14 @@ class HTTPClient
     reset_all
   end
 
+  # Turn on/off the BasicAuth force flag. Generally HTTP client must
+  # send Authorization header after it gets 401 error from server from
+  # security reason. But in some situation (e.g. API client) you might
+  # want to send Authorization from the beginning.
+  def force_basic_auth=(force_basic_auth)
+    @www_auth.basic_auth.force_auth = @proxy_auth.basic_auth.force_auth = force_basic_auth
+  end
+
   # Sets the filename where non-volatile Cookies be saved by calling
   # save_cookie_store.
   # This method tries to load and managing Cookies from the specified file.
@@ -541,7 +606,6 @@ class HTTPClient
 
   # Try to save Cookies to the file specified in set_cookie_store.  Unexpected
   # error will be raised if you don't call set_cookie_store first.
-  # (interface mismatch between WebAgent::CookieManager implementation)
   def save_cookie_store
     @cookie_manager.save_cookies
   end
@@ -624,8 +688,13 @@ class HTTPClient
   # If you need to get full HTTP response including HTTP status and headers,
   # use post method.
   def post_content(uri, *args, &block)
-    body, header = keyword_argument(args, :body, :header)
-    success_content(follow_redirect(:post, uri, nil, body, header || {}, &block))
+    if hashy_argument_has_keys(args, :query, :body)
+      query, body, header = keyword_argument(args, :query, :body, :header)
+    else
+      query = nil
+      body, header = keyword_argument(args, :body, :header)
+    end
+    success_content(follow_redirect(:post, uri, query, body, header || {}, &block))
   end
 
   # A method for redirect uri callback.  How to use:
@@ -653,9 +722,9 @@ class HTTPClient
   def default_redirect_uri_callback(uri, res)
     newuri = urify(res.header['location'][0])
     if !http?(newuri) && !https?(newuri)
-      newuri = uri + newuri
-      warn("could be a relative URI in location header which is not recommended")
+      warn("#{newuri}: a relative URI in location header which is not recommended")
       warn("'The field value consists of a single absolute URI' in HTTP spec")
+      newuri = uri + newuri
     end
     if https?(uri) && !https?(newuri)
       raise BadResponseError.new("redirecting to non-https resource")
@@ -674,26 +743,47 @@ class HTTPClient
     request(:get, uri, argument_to_hash(args, :query, :header, :follow_redirect), &block)
   end
 
+  # Sends PATCH request to the specified URL.  See request for arguments.
+  def patch(uri, *args, &block)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request(:patch, uri, new_args, &block)
+  end
+
   # Sends POST request to the specified URL.  See request for arguments.
   # You should not depend on :follow_redirect => true for POST method.  It
   # sends the same POST method to the new location which is prohibited in HTTP spec.
   def post(uri, *args, &block)
-    request(:post, uri, argument_to_hash(args, :body, :header, :follow_redirect), &block)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header, :follow_redirect)
+    end
+    request(:post, uri, new_args, &block)
   end
 
   # Sends PUT request to the specified URL.  See request for arguments.
   def put(uri, *args, &block)
-    request(:put, uri, argument_to_hash(args, :body, :header), &block)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request(:put, uri, new_args, &block)
   end
 
   # Sends DELETE request to the specified URL.  See request for arguments.
   def delete(uri, *args, &block)
-    request(:delete, uri, argument_to_hash(args, :body, :header), &block)
+    request(:delete, uri, argument_to_hash(args, :body, :header, :query), &block)
   end
 
   # Sends OPTIONS request to the specified URL.  See request for arguments.
   def options(uri, *args, &block)
-    request(:options, uri, argument_to_hash(args, :header), &block)
+    new_args = argument_to_hash(args, :header, :query, :body)
+    request(:options, uri, new_args, &block)
   end
 
   # Sends PROPFIND request to the specified URL.  See request for arguments.
@@ -747,23 +837,18 @@ class HTTPClient
   #
   # When you pass an IO as a body, HTTPClient sends it as a HTTP request with
   # chunked encoding (Transfer-Encoding: chunked in HTTP header) if IO does not
-  # respond to :read. Bear in mind that some server application does not support
+  # respond to :size. Bear in mind that some server application does not support
   # chunked request.  At least cgi.rb does not support it.
   def request(method, uri, *args, &block)
     query, body, header, follow_redirect = keyword_argument(args, :query, :body, :header, :follow_redirect)
-    if [:post, :put].include?(method)
-      body ||= ''
-    end
     if method == :propfind
       header ||= PROPFIND_DEFAULT_EXTHEADER
     else
       header ||= {}
     end
-    uri = urify(uri)
+    uri = to_resource_url(uri)
     if block
-      filtered_block = proc { |res, str|
-        block.call(str)
-      }
+      filtered_block = adapt_block(&block)
     end
     if follow_redirect
       follow_redirect(method, uri, query, body, header, &block)
@@ -775,64 +860,76 @@ class HTTPClient
   # Sends HEAD request in async style.  See request_async for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def head_async(uri, *args)
-    query, header = keyword_argument(args, :query, :header)
-    request_async(:head, uri, query, nil, header || {})
+    request_async2(:head, uri, argument_to_hash(args, :query, :header))
   end
 
   # Sends GET request in async style.  See request_async for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def get_async(uri, *args)
-    query, header = keyword_argument(args, :query, :header)
-    request_async(:get, uri, query, nil, header || {})
+    request_async2(:get, uri, argument_to_hash(args, :query, :header))
+  end
+
+  # Sends PATCH request in async style.  See request_async2 for arguments.
+  # It immediately returns a HTTPClient::Connection instance as a result.
+  def patch_async(uri, *args)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request_async2(:patch, uri, new_args)
   end
 
   # Sends POST request in async style.  See request_async for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def post_async(uri, *args)
-    body, header = keyword_argument(args, :body, :header)
-    request_async(:post, uri, nil, body || '', header || {})
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request_async2(:post, uri, new_args)
   end
 
-  # Sends PUT request in async style.  See request_async for arguments.
+  # Sends PUT request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def put_async(uri, *args)
-    body, header = keyword_argument(args, :body, :header)
-    request_async(:put, uri, nil, body || '', header || {})
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request_async2(:put, uri, new_args)
   end
 
-  # Sends DELETE request in async style.  See request_async for arguments.
+  # Sends DELETE request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def delete_async(uri, *args)
-    header = keyword_argument(args, :header)
-    request_async(:delete, uri, nil, nil, header || {})
+    request_async2(:delete, uri, argument_to_hash(args, :body, :header, :query))
   end
 
-  # Sends OPTIONS request in async style.  See request_async for arguments.
+  # Sends OPTIONS request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def options_async(uri, *args)
-    header = keyword_argument(args, :header)
-    request_async(:options, uri, nil, nil, header || {})
+    request_async2(:options, uri, argument_to_hash(args, :header, :query, :body))
   end
 
-  # Sends PROPFIND request in async style.  See request_async for arguments.
+  # Sends PROPFIND request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def propfind_async(uri, *args)
-    header = keyword_argument(args, :header)
-    request_async(:propfind, uri, nil, nil, header || PROPFIND_DEFAULT_EXTHEADER)
+    request_async2(:propfind, uri, argument_to_hash(args, :body, :header))
   end
   
-  # Sends PROPPATCH request in async style.  See request_async for arguments.
+  # Sends PROPPATCH request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def proppatch_async(uri, *args)
-    body, header = keyword_argument(args, :body, :header)
-    request_async(:proppatch, uri, nil, body, header || {})
+    request_async2(:proppatch, uri, argument_to_hash(args, :body, :header))
   end
   
-  # Sends TRACE request in async style.  See request_async for arguments.
+  # Sends TRACE request in async style.  See request_async2 for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def trace_async(uri, *args)
-    query, body, header = keyword_argument(args, :query, :body, :header)
-    request_async(:trace, uri, query, body, header || {})
+    request_async2(:trace, uri, argument_to_hash(args, :query, :header))
   end
 
   # Sends a request in async style.  request method creates new Thread for
@@ -840,14 +937,29 @@ class HTTPClient
   #
   # Arguments definition is the same as request.
   def request_async(method, uri, query = nil, body = nil, header = {})
-    uri = urify(uri)
+    uri = to_resource_url(uri)
+    do_request_async(method, uri, query, body, header)
+  end
+
+  # new method that has same signature as 'request'
+  def request_async2(method, uri, *args)
+    query, body, header = keyword_argument(args, :query, :body, :header)
+    if [:post, :put].include?(method)
+      body ||= ''
+    end
+    if method == :propfind
+      header ||= PROPFIND_DEFAULT_EXTHEADER
+    else
+      header ||= {}
+    end
+    uri = to_resource_url(uri)
     do_request_async(method, uri, query, body, header)
   end
 
   # Resets internal session for the given URL.  Keep-alive connection for the
   # site (host-port pair) is disconnected if exists.
   def reset(uri)
-    uri = urify(uri)
+    uri = to_resource_url(uri)
     @session_manager.reset(uri)
   end
 
@@ -859,34 +971,62 @@ class HTTPClient
 private
 
   class RetryableResponse < StandardError # :nodoc:
+    attr_reader :res
+
+    def initialize(res = nil)
+      @res = res
+    end
   end
 
   class KeepAliveDisconnected < StandardError # :nodoc:
     attr_reader :sess
-    def initialize(sess = nil)
+    attr_reader :cause
+    def initialize(sess = nil, cause = nil)
+      super("#{self.class.name}: #{cause ? cause.message : nil}")
       @sess = sess
+      @cause = cause
     end
   end
 
+  def hashy_argument_has_keys(args, *key)
+    # if the given arg is a single Hash and...
+    args.size == 1 and Hash === args[0] and
+      # it has any one of the key
+      key.all? { |e| args[0].key?(e) }
+  end
+
   def do_request(method, uri, query, body, header, &block)
-    conn = Connection.new
     res = nil
     if HTTP::Message.file?(body)
       pos = body.pos rescue nil
     end
     retry_count = @session_manager.protocol_retry_count
     proxy = no_proxy?(uri) ? nil : @proxy
+    previous_request = previous_response = nil
     while retry_count > 0
       body.pos = pos if pos
       req = create_request(method, uri, query, body, header)
+      if previous_request
+        # to remember IO positions to read
+        req.http_body.positions = previous_request.http_body.positions
+      end
       begin
         protect_keep_alive_disconnected do
-          do_get_block(req, proxy, conn, &block)
+          # TODO: remove Connection.new
+          # We want to delete Connection usage in do_get_block but Newrelic gem depends on it.
+          # https://github.com/newrelic/rpm/blob/master/lib/new_relic/agent/instrumentation/httpclient.rb#L34-L36
+          conn = Connection.new
+          res = do_get_block(req, proxy, conn, &block)
+          # Webmock's do_get_block returns ConditionVariable
+          if !res.respond_to?(:previous)
+            res = conn.pop
+          end
         end
-        res = conn.pop
+        res.previous = previous_response
         break
-      rescue RetryableResponse
-        res = conn.pop
+      rescue RetryableResponse => e
+        previous_request = req
+        previous_response = res = e.res
         retry_count -= 1
       end
     end
@@ -914,8 +1054,8 @@ private
             retry_count -= 1
           end
         end
-      rescue Exception
-        conn.push $!
+      rescue Exception => e
+        conn.push e
       end
     }
     conn.async_thread = t
@@ -940,23 +1080,38 @@ private
     ENV[name.downcase] || ENV[name.upcase]
   end
 
+  def adapt_block(&block)
+    return block if block.arity == 2
+    proc { |r, str| block.call(str) }
+  end
+
   def follow_redirect(method, uri, query, body, header, &block)
-    uri = urify(uri)
+    uri = to_resource_url(uri)
     if block
+      b = adapt_block(&block)
       filtered_block = proc { |r, str|
-        block.call(str) if r.ok?
+        b.call(r, str) if r.ok?
       }
     end
     if HTTP::Message.file?(body)
       pos = body.pos rescue nil
     end
     retry_number = 0
+    previous = nil
+    request_query = query
     while retry_number < @follow_redirect_count
       body.pos = pos if pos
-      res = do_request(method, uri, query, body, header, &filtered_block)
+      res = do_request(method, uri, request_query, body, header, &filtered_block)
+      res.previous = previous
       if res.redirect?
+        if res.header['location'].empty?
+          raise BadResponseError.new("Missing Location header for redirect", res)
+        end
         method = :get if res.see_other? # See RFC2616 10.3.4
         uri = urify(@redirect_uri_callback.call(uri, res))
+        # To avoid duped query parameter. 'location' must include query part.
+        request_query = nil
+        previous = res
         retry_number += 1
       else
         return res
@@ -976,25 +1131,28 @@ private
   def protect_keep_alive_disconnected
     begin
       yield
-    rescue KeepAliveDisconnected => e
-      if e.sess
-        @session_manager.invalidate(e.sess.dest)
+    rescue KeepAliveDisconnected
+      # Force to create new connection
+      Thread.current[:HTTPClient_AcquireNewConnection] = true
+      begin
+        yield
+      ensure
+        Thread.current[:HTTPClient_AcquireNewConnection] = false
       end
-      yield
     end
   end
 
   def create_request(method, uri, query, body, header)
     method = method.to_s.upcase
     if header.is_a?(Hash)
-      header = header.to_a
+      header = @default_header.merge(header).to_a
     else
-      header = header.dup
+      header = @default_header.to_a + header.dup
     end
     boundary = nil
     if body
       _, content_type = header.find { |key, value|
-        key.downcase == 'content-type'
+        key.to_s.downcase == 'content-type'
       }
       if content_type
         if /\Amultipart/ =~ content_type
@@ -1003,7 +1161,7 @@ private
           else
             boundary = create_boundary
             content_type = "#{content_type}; boundary=#{boundary}"
-            header = override_header(header, 'Content-Type', content_type)
+            header = override_header(header, 'content-type', content_type)
           end
         end
       else
@@ -1020,8 +1178,11 @@ private
     header.each do |key, value|
       req.header.add(key.to_s, value)
     end
-    if @cookie_manager && cookie = @cookie_manager.find(uri)
-      req.header.add('Cookie', cookie)
+    if @cookie_manager
+      cookie_value = @cookie_manager.cookie_value(uri)
+      if cookie_value
+        req.header.add('Cookie', cookie_value)
+      end
     end
     req
   end
@@ -1038,7 +1199,7 @@ private
   def override_header(header, key, value)
     result = []
     header.each do |k, v|
-      if k.downcase == key.downcase
+      if k.to_s.downcase == key
         result << [key, value]
       else
         result << [k, v]
@@ -1071,8 +1232,9 @@ private
     end
     if str = @test_loopback_response.shift
       dump_dummy_request_response(req.http_body.dump, str) if @debug_dev
-      conn.push(HTTP::Message.new_response(str, req.header))
-      return
+      res = HTTP::Message.new_response(str, req.header)
+      conn.push(res)
+      return res
     end
     content = block ? nil : ''
     res = HTTP::Message.new_response(content, req.header)
@@ -1085,7 +1247,7 @@ private
     sess.get_body do |part|
       set_encoding(part, res.body_encoding)
       if block
-        block.call(res, part)
+        block.call(res, part.dup)
       else
         content << part
       end
@@ -1097,8 +1259,9 @@ private
       filter.filter_response(req, res)
     }
     if commands.find { |command| command == :retry }
-      raise RetryableResponse.new
+      raise RetryableResponse.new(res)
     end
+    res
   end
 
   def do_get_stream(req, proxy, conn)
@@ -1111,6 +1274,7 @@ private
       return
     end
     piper, pipew = IO.pipe
+    pipew.binmode
     res = HTTP::Message.new_response(piper, req.header)
     @debug_dev << "= Request\n\n" if @debug_dev
     sess = @session_manager.query(req, proxy)
@@ -1128,6 +1292,7 @@ private
       filter.filter_response(req, res)
     }
     # ignore commands (not retryable in async mode)
+    res
   end
 
   def do_get_header(req, res, sess)
@@ -1150,4 +1315,13 @@ private
   def set_encoding(str, encoding)
     str.force_encoding(encoding) if encoding
   end
+
+  def to_resource_url(uri)
+    u = urify(uri)
+    if @base_url && u.scheme.nil? && u.host.nil?
+      urify(@base_url) + uri
+    else
+      u
+    end
+  end
 end