httpclient 2.6.0.1 → 2.8.3

data/lib/httpclient.rb

@@ -1,5 +1,5 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;
@@ -149,6 +149,16 @@ require 'httpclient/cookie'
 #     clnt.ssl_config.set_client_cert_file(user_cert_file, user_key_file)
 #     clnt.get(https_url)
 #
+# 4. Revocation check. On JRuby you can set following options to let
+#    HTTPClient to perform revocation check with CRL and OCSP:
+#
+#     -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true
+#     ex. jruby -J-Dcom.sun.security.enableCRLDP=true -J-Dcom.sun.net.ssl.checkRevocation=true app.rb
+#     Revoked cert example: https://test-sspev.verisign.com:2443/test-SSPEV-revoked-verisign.html
+#
+#    On other platform you can download CRL by yourself and set it via
+#    SSLConfig#add_crl.
+#
 # === Handling Cookies
 #
 # 1. Using volatile Cookies.  Nothing to do.  HTTPClient handles Cookies.
@@ -299,7 +309,6 @@ class HTTPClient
       if assignable
         aname = name + '='
         define_method(aname) { |rhs|
-          reset_all
           @session_manager.__send__(aname, rhs)
         }
       end
@@ -326,7 +335,7 @@ class HTTPClient
   attr_accessor :follow_redirect_count
   # Base url of resources.
   attr_accessor :base_url
-  # Defalut request header.
+  # Default request header.
   attr_accessor :default_header
 
   # Set HTTP version as a String:: 'HTTP/1.0' or 'HTTP/1.1'
@@ -346,6 +355,8 @@ class HTTPClient
   # if your ruby is older than 2005-09-06, do not set socket_sync = false to
   # avoid an SSL socket blocking bug in openssl/buffering.rb.
   attr_proxy(:socket_sync, true)
+  # Enables TCP keepalive; no timing settings exist at present
+  attr_proxy(:tcp_keepalive, true)
   # User-Agent header in HTTP request.
   attr_proxy(:agent_name, true)
   # From header in HTTP request.
@@ -355,6 +366,9 @@ class HTTPClient
   attr_proxy(:test_loopback_http_response)
   # Decompress a compressed (with gzip or deflate) content body transparently. false by default.
   attr_proxy(:transparent_gzip_decompression, true)
+  # Raise BadResponseError if response size does not match with Content-Length header in response. false by default.
+  # TODO: enable by default
+  attr_proxy(:strict_response_size_check, true)
   # Local socket address. Set HTTPClient#socket_local.host and HTTPClient#socket_local.port to specify local binding hostname and port of TCP socket.
   attr_proxy(:socket_local, true)
 
@@ -383,10 +397,25 @@ class HTTPClient
   #
   # After you set :base_url, all resources you pass to get, post and other
   # methods are recognized to be prefixed with base_url. Say base_url is
-  # 'https://api.example.com/v1, get('/users') is the same as
+  # 'https://api.example.com/v1/, get('users') is the same as
   # get('https://api.example.com/v1/users') internally. You can also pass
   # full URL from 'http://' even after setting base_url.
   #
+  # The expected base_url and path behavior is the following. Please take
+  # care of '/' in base_url and path.
+  #
+  # The last '/' is important for base_url:
+  #   1. http://foo/bar/baz/ + path -> http://foo/bar/baz/path
+  #   2. http://foo/bar/baz + path -> http://foo/bar/path
+  # Relative path handling:
+  #   3. http://foo/bar/baz/ + ../path -> http://foo/bar/path
+  #   4. http://foo/bar/baz + ../path -> http://foo/path
+  #   5. http://foo/bar/baz/ + ./path -> http://foo/bar/baz/path
+  #   6. http://foo/bar/baz + ./path -> http://foo/bar/path
+  # The leading '/' of path means absolute path:
+  #   7. http://foo/bar/baz/ + /path -> http://foo/path
+  #   8. http://foo/bar/baz + /path -> http://foo/path
+  #
   # :default_header is for providing default headers Hash that all HTTP
   # requests should have, such as custom 'Authorization' header in API.
   # You can override :default_header with :header Hash parameter in HTTP
@@ -396,7 +425,7 @@ class HTTPClient
   # HTTP client must send Authorization header after it gets 401 error
   # from server from security reason. But in some situation (e.g. API
   # client) you might want to send Authorization from the beginning.
-  def initialize(*args)
+  def initialize(*args, &block)
     proxy, agent_name, from, base_url, default_header, force_basic_auth =
       keyword_argument(args, :proxy, :agent_name, :from, :base_url, :default_header, :force_basic_auth)
     @proxy = nil        # assigned later.
@@ -420,6 +449,7 @@ class HTTPClient
     load_environment
     self.proxy = proxy if proxy
     keep_webmock_compat
+    instance_eval(&block) if block
   end
 
   # webmock 1.6.2 depends on HTTP::Message#body.content to work.
@@ -692,9 +722,9 @@ class HTTPClient
   def default_redirect_uri_callback(uri, res)
     newuri = urify(res.header['location'][0])
     if !http?(newuri) && !https?(newuri)
-      newuri = uri + newuri
-      warn("could be a relative URI in location header which is not recommended")
+      warn("#{newuri}: a relative URI in location header which is not recommended")
       warn("'The field value consists of a single absolute URI' in HTTP spec")
+      newuri = uri + newuri
     end
     if https?(uri) && !https?(newuri)
       raise BadResponseError.new("redirecting to non-https resource")
@@ -713,6 +743,16 @@ class HTTPClient
     request(:get, uri, argument_to_hash(args, :query, :header, :follow_redirect), &block)
   end
 
+  # Sends PATCH request to the specified URL.  See request for arguments.
+  def patch(uri, *args, &block)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request(:patch, uri, new_args, &block)
+  end
+
   # Sends POST request to the specified URL.  See request for arguments.
   # You should not depend on :follow_redirect => true for POST method.  It
   # sends the same POST method to the new location which is prohibited in HTTP spec.
@@ -808,13 +848,7 @@ class HTTPClient
     end
     uri = to_resource_url(uri)
     if block
-      if block.arity == 1
-        filtered_block = proc { |res, str|
-          block.call(str)
-        }
-      else
-        filtered_block = block
-      end
+      filtered_block = adapt_block(&block)
     end
     if follow_redirect
       follow_redirect(method, uri, query, body, header, &block)
@@ -835,6 +869,17 @@ class HTTPClient
     request_async2(:get, uri, argument_to_hash(args, :query, :header))
   end
 
+  # Sends PATCH request in async style.  See request_async2 for arguments.
+  # It immediately returns a HTTPClient::Connection instance as a result.
+  def patch_async(uri, *args)
+    if hashy_argument_has_keys(args, :query, :body)
+      new_args = args[0]
+    else
+      new_args = argument_to_hash(args, :body, :header)
+    end
+    request_async2(:patch, uri, new_args)
+  end
+
   # Sends POST request in async style.  See request_async for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def post_async(uri, *args)
@@ -1035,11 +1080,17 @@ private
     ENV[name.downcase] || ENV[name.upcase]
   end
 
+  def adapt_block(&block)
+    return block if block.arity == 2
+    proc { |r, str| block.call(str) }
+  end
+
   def follow_redirect(method, uri, query, body, header, &block)
     uri = to_resource_url(uri)
     if block
+      b = adapt_block(&block)
       filtered_block = proc { |r, str|
-        block.call(str) if r.ok?
+        b.call(r, str) if r.ok?
       }
     end
     if HTTP::Message.file?(body)
@@ -1080,7 +1131,7 @@ private
   def protect_keep_alive_disconnected
     begin
       yield
-    rescue KeepAliveDisconnected => e
+    rescue KeepAliveDisconnected
       # Force to create new connection
       Thread.current[:HTTPClient_AcquireNewConnection] = true
       begin
@@ -1101,7 +1152,7 @@ private
     boundary = nil
     if body
       _, content_type = header.find { |key, value|
-        key.downcase == 'content-type'
+        key.to_s.downcase == 'content-type'
       }
       if content_type
         if /\Amultipart/ =~ content_type
@@ -1110,7 +1161,7 @@ private
           else
             boundary = create_boundary
             content_type = "#{content_type}; boundary=#{boundary}"
-            header = override_header(header, 'Content-Type', content_type)
+            header = override_header(header, 'content-type', content_type)
           end
         end
       else
@@ -1148,7 +1199,7 @@ private
   def override_header(header, key, value)
     result = []
     header.each do |k, v|
-      if k.downcase == key.downcase
+      if k.to_s.downcase == key
         result << [key, value]
       else
         result << [k, v]
@@ -1223,6 +1274,7 @@ private
       return
     end
     piper, pipew = IO.pipe
+    pipew.binmode
     res = HTTP::Message.new_response(piper, req.header)
     @debug_dev << "= Request\n\n" if @debug_dev
     sess = @session_manager.query(req, proxy)
@@ -1267,7 +1319,7 @@ private
   def to_resource_url(uri)
     u = urify(uri)
     if @base_url && u.scheme.nil? && u.host.nil?
-      urify(@base_url + uri)
+      urify(@base_url) + uri
     else
       u
     end

data/lib/oauthclient.rb

@@ -1,5 +1,5 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;
@@ -33,6 +33,7 @@ class OAuthClient < HTTPClient
     @oauth_config = HTTPClient::OAuth::Config.new
     self.www_auth.oauth.set_config(nil, @oauth_config)
     self.www_auth.oauth.challenge(nil)
+    self.strict_response_size_check = true
   end
 
   # Get request token.

data/test/helper.rb

@@ -59,11 +59,13 @@ module Helper
     @client = HTTPClient.new
   end
 
-  #def setup_server
-    # override it
-    # @server = WEBrick::HTTPServer.new(...)
-    # @server_thread = start_server_thread(@server)
-  #end
+  def escape_noproxy
+    backup = HTTPClient::NO_PROXY_HOSTS.dup
+    HTTPClient::NO_PROXY_HOSTS.clear
+    yield
+  ensure
+    HTTPClient::NO_PROXY_HOSTS.replace(backup)
+  end
 
   def setup_proxyserver
     @proxyserver = WEBrick::HTTPProxyServer.new(

data/test/jruby_ssl_socket/test_pemutils.rb

@@ -0,0 +1,32 @@
+require File.expand_path('helper', File.join(File.dirname(__FILE__),  ".."))
+
+
+class PEMUtilsTest < Test::Unit::TestCase
+  include Helper
+
+  def setup
+    @raw_cert = "-----BEGIN CERTIFICATE-----\nMIIDOTCCAiGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBCMRMwEQYKCZImiZPyLGQB\nGRYDb3JnMRkwFwYKCZImiZPyLGQBGRYJcnVieS1sYW5nMRAwDgYDVQQDDAdSdWJ5\nIENBMB4XDTE2MDgxMDE3MjEzNFoXDTE3MDgxMDE3MjEzNFowSzETMBEGCgmSJomT\n8ixkARkWA29yZzEZMBcGCgmSJomT8ixkARkWCXJ1YnktbGFuZzEZMBcGA1UEAwwQ\nUnVieSBjZXJ0aWZpY2F0ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nAJCfsSXpSMpmZCVa+ZCM+QDgomnhDlvnrGDq6pasTaIspGTXgws+7r8Dt/cNe6EH\nHJpRH2cGRiO4yPcfcT9eS4X7k8OC4f33wHfACOmLu6LeoNE8ujmSk6L6WzLUI+sE\nnLZbFrXxoAo4XHsm8vEG9C+jEoXZ1p+47wrAGaDwDQTnzlMy4dT9pRQEJP2G/Rry\nUkuZn8SUWmh3/YS78iaSzsNF1cgE1ealHOrPPFDjiCGDaH/LHyUPYlbFSLZ/B7Qx\nLxi5sePLcywWq/EJrmWpgeVTDjtNijsdKv/A3qkY+fm/oD0pzt7XsfJaP9YKNyJO\nQFdxWZeiPcDF+Hwf+IwSr+kCAwEAAaMxMC8wDgYDVR0PAQH/BAQDAgeAMB0GA1Ud\nDgQWBBQNvzYzJyXemGhxbA8NMXLolDnPyjANBgkqhkiG9w0BAQsFAAOCAQEARIJV\noKejGlOTn71QutnNnu07UtTu0IHs6YqjYzzND+m4JXLN+wvYm72AFUG0b1L7dRg0\niK8XjQrlNQNVqP1Mc6tffchy20neOPOHeiO6qTdRU8P2S8D3Uwe+1qhgxjfE+cWc\nwZmWxYK4HA8c58PxWMqrkr2QqXDplG9KWLvOgrtPGiLLZcQSKhvvB63QzItHBDU6\nRayiJY3oPkK/HrIvFlySqFqzWmuyknkciOFywEHQMz/tcSFJ2QFpPj/tBz9VXohH\nZ8KscmfhZrTPBjo+ky1lz/WraWoz4LMiLnkC2ABczWLRSawu+v3Irx1NFJngt05e\npqwtqIUeg7j+JLiTaA==\n-----END CERTIFICATE-----"
+  end
+
+  def test_read_certificate
+    assert_nothing_raised do
+      binary = HTTPClient::JRubySSLSocket::PEMUtils.read_certificate(@raw_cert)
+    end
+  end
+
+  def test_read_certificate_works_with_random_ascii_text_outside_begin_end
+    raw_cert_with_ascii = "some text before begin\n" + @raw_cert + "\nsome text after end"
+    assert_nothing_raised do
+      binary = HTTPClient::JRubySSLSocket::PEMUtils.read_certificate(raw_cert_with_ascii)
+    end
+  end
+
+  def test_read_certificate_uses_all_content_if_missing_begin_end
+    cert = @raw_cert.sub(/-----BEGIN CERTIFICATE-----/, '').sub(/-----END CERTIFICATE-----/, '')
+    assert_nothing_raised do
+      binary = HTTPClient::JRubySSLSocket::PEMUtils.read_certificate(cert)
+    end
+  end
+
+
+end

data/test/test_auth.rb

@@ -2,7 +2,6 @@ require File.expand_path('helper', File.dirname(__FILE__))
 require 'digest/md5'
 require 'rack'
 require 'rack/lint'
-require 'rack/showexceptions'
 require 'rack-ntlm'
 
 class TestAuth < Test::Unit::TestCase
@@ -124,6 +123,7 @@ class TestAuth < Test::Unit::TestCase
       end
       # Make it work if @value == nil
       class SecurityBuffer < FieldSet
+        remove_method(:data_size) if method_defined?(:data_size)
         def data_size
           @active && @value ? @value.size : 0
         end
@@ -230,7 +230,7 @@ class TestAuth < Test::Unit::TestCase
       c.www_auth.basic_auth.instance_eval { @scheme = "BASIC" }
       c.set_auth("http://localhost:#{serverport}/", 'admin', 'admin')
 
-      threads = 100.times.map { |idx|
+      100.times.map { |idx|
         Thread.new(idx) { |idx2|
           Thread.abort_on_exception = true
           Thread.pass
@@ -251,7 +251,7 @@ class TestAuth < Test::Unit::TestCase
     c.test_loopback_http_response << "HTTP/1.0 200 OK\nContent-Length: 2\n\nOK"
     c.debug_dev = str = ''
     c.get_content("http://localhost:#{serverport}/basic_auth/sub/dir/")
-    assert_match /Authorization: Basic YWRtaW46YWRtaW4=/, str
+    assert_match(/Authorization: Basic YWRtaW46YWRtaW4=/, str)
   end
 
   def test_digest_auth
@@ -267,7 +267,7 @@ class TestAuth < Test::Unit::TestCase
     c.test_loopback_http_response << "HTTP/1.0 200 OK\nContent-Length: 2\n\nOK"
     c.debug_dev = str = ''
     c.get_content("http://localhost:#{serverport}/digest_auth/sub/dir/")
-    assert_match /Authorization: Digest/, str
+    assert_match(/Authorization: Digest/, str)
   end
 
   def test_digest_auth_with_block
@@ -332,6 +332,16 @@ class TestAuth < Test::Unit::TestCase
     assert_match(/Proxy-Authorization: Basic YWRtaW46YWRtaW4=/, str)
   end
 
+  def test_proxy_auth_force
+    c = HTTPClient.new
+    c.set_proxy_auth('admin', 'admin')
+    c.force_basic_auth = true
+    c.test_loopback_http_response << "HTTP/1.0 200 OK\nContent-Length: 2\n\nOK"
+    c.debug_dev = str = ''
+    c.get_content('http://example.com/')
+    assert_match(/Proxy-Authorization: Basic YWRtaW46YWRtaW4=/, str)
+  end
+
   def test_proxy_auth_reuses_credentials
     c = HTTPClient.new
     c.set_proxy_auth('admin', 'admin')
@@ -442,11 +452,20 @@ class TestAuth < Test::Unit::TestCase
   end
 
   def test_basic_auth_post_with_multipart
-    c = HTTPClient.new
-    c.set_auth("http://localhost:#{serverport}/", 'admin', 'admin')
-    File.open(__FILE__) do |f|
-      # read 'f' twice for authorization negotiation
-      assert_equal('basic_auth OK', c.post("http://localhost:#{serverport}/basic_auth", :file => f).content)
+    retry_times = 0
+    begin
+      c = HTTPClient.new
+      c.set_auth("http://localhost:#{serverport}/", 'admin', 'admin')
+      File.open(__FILE__) do |f|
+        # read 'f' twice for authorization negotiation
+        assert_equal('basic_auth OK', c.post("http://localhost:#{serverport}/basic_auth", :file => f).content)
+      end
+    rescue Errno::ECONNRESET, HTTPClient::KeepAliveDisconnected
+      # TODO: WEBrick server returns ECONNRESET/EPIPE before sending Unauthorized response to client?
+      raise if retry_times > 5
+      retry_times += 1
+      sleep 1
+      retry 
     end
   end
 

data/test/test_cookie.rb

@@ -290,8 +290,8 @@ EOF
   def test_load_cookies_escaped
     uri = urify('http://example.org/')
     f = Tempfile.new('test_cookie')
-    File.open(f.path, 'w') do |f|
-      f.write <<EOF
+    File.open(f.path, 'w') do |out|
+      out.write <<EOF
 http://example.org/	key1	"value"	0	.example.org	/	13	0			
 http://example.org/	key2	""	0	.example.org	/	13	0			
 http://example.org/	key3		0	.example.org	/	13	0			

data/test/test_http-access2.rb

@@ -347,11 +347,13 @@ class TestClient < Test::Unit::TestCase
   def test_receive_timeout
     # this test takes 2 sec
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
+    @client.reset_all
     @client.receive_timeout = 1
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=0'))
     assert_raise(HTTPClient::ReceiveTimeoutError) do
       @client.get_content(serverurl + 'sleep?sec=2')
     end
+    @client.reset_all
     @client.receive_timeout = 3
     assert_equal('hello', @client.get_content(serverurl + 'sleep?sec=2'))
   end