httpclient 2.7.2 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3efa89267c49ad8538e958b70fa1d5ad60a51f14
-  data.tar.gz: fddc29793643f39d3d29de2bda66a096353686e0
+SHA256:
+  metadata.gz: e1df0b78f53d502000683d048be1316f156247e5e17995f5fb04a125302bdb4e
+  data.tar.gz: d533eeaf2333f2f35820441a74cdf84075e5227d4fb0648f4e2ea4d2099bd511
 SHA512:
-  metadata.gz: 7f31fd63927c5e74d67fd57710e9a8e8528ef4eeee5ee03accac31629af74ccc19dca082df5d36fb2827e3713d9e36f1e6e444472f56111c9c32805df9a757f1
-  data.tar.gz: a74ae21b44d061adc75c34488de4cde93c55f3e5cb98f22871e7f31cd5fec66f954339a664cd6576b8f796dbec28a9e6f48815e8b34331cafdb5f5fb74b3abe8
+  metadata.gz: 94024e0c5b5bd08800d9b3989151ab919869fabcaed65e4a375490fde0f6ea3b5bfc3ad4a5a9803bb057c5b8d36efc10c6ae5ade8a974c665d443730c63bc7ba
+  data.tar.gz: 768ae9ad96a73740675aca861a81075af3164a4b8875f81368a3228ecfdf523fe6c13cd9355bbe8ddd9155031ff1f908d8ec85e1bd112e0cd31d8b26d809dd6a

data/bin/httpclient

@@ -20,6 +20,7 @@ end
 url = ARGV.shift
 if method && url
   client = HTTPClient.new
+  client.strict_response_size_check = true
   if method == 'download'
     print client.get_content(url)
   else
@@ -37,6 +38,7 @@ require 'irb/completion'
 class Runner
   def initialize
     @httpclient = HTTPClient.new
+    @httpclient.strict_response_size_check = true
   end
 
   def method_missing(msg, *a, &b)

data/lib/hexdump.rb

@@ -11,13 +11,13 @@ module HexDump
     result = []
     while raw = str.slice(offset, 16) and raw.length > 0
       # data field
-      data = ''
+      data = ''.dup
       for v in raw.unpack('N* a*')
-	if v.kind_of? Integer
-	  data << sprintf("%08x ", v)
-	else
-	  v.each_byte {|c| data << sprintf("%02x", c) }
-	end
+        if v.kind_of? Integer
+          data << sprintf("%08x ", v)
+        else
+          v.each_byte {|c| data << sprintf("%02x", c) }
+        end
       end
       # text field
       text = raw.tr("\000-\037\177-\377", ".")
@@ -25,12 +25,12 @@ module HexDump
       offset += 16
       # omit duplicate line
       if /^(#{regex_quote_n(raw)})+/n =~ str[offset .. -1]
-	result << sprintf("%08x  ...", offset)
-	offset += $&.length
-	# should print at the end
-	if offset == str.length
-	  result << sprintf("%08x  %-36s  %s", offset-16, data, text)
-	end
+        result << sprintf("%08x  ...", offset)
+        offset += $&.length
+        # should print at the end
+        if offset == str.length
+          result << sprintf("%08x  %-36s  %s", offset-16, data, text)
+        end
       end
     end
     result

data/lib/httpclient/http.rb

@@ -238,7 +238,7 @@ module HTTP
       if defined?(Encoding::ASCII_8BIT)
         def set_body_encoding
           if type = self.content_type
-            OpenURI::Meta.init(o = '')
+            OpenURI::Meta.init(o = ''.dup)
             o.meta_add_field('content-type', type)
             @body_encoding = o.encoding
           end
@@ -491,7 +491,7 @@ module HTTP
       # String.
       #
       # assert: @size is not nil
-      def dump(header = '', dev = '')
+      def dump(header = '', dev = ''.dup)
         if @body.is_a?(Parts)
           dev << header
           @body.parts.each do |part|
@@ -521,7 +521,7 @@ module HTTP
       # reason. (header is dumped to dev, too)
       # If no dev (the second argument) given, this method returns a dumped
       # String.
-      def dump_chunked(header = '', dev = '')
+      def dump_chunked(header = '', dev = ''.dup)
         dev << header
         if @body.is_a?(Parts)
           @body.parts.each do |part|
@@ -574,7 +574,7 @@ module HTTP
       end
 
       def dump_file(io, dev, sz)
-        buf = ''
+        buf = ''.dup
         rest = sz
         while rest > 0
           n = io.read([rest, @chunk_size].min, buf)
@@ -585,7 +585,7 @@ module HTTP
       end
 
       def dump_chunks(io, dev)
-        buf = ''
+        buf = ''.dup
         while !io.read(@chunk_size, buf).nil?
           dev << dump_chunk(buf)
         end
@@ -618,8 +618,8 @@ module HTTP
           if Message.file?(part)
             @as_stream = true
             @body << part
-            if part.respond_to?(:lstat)
-              sz = part.lstat.size
+            if part.respond_to?(:stat)
+              sz = part.stat.size
               add_size(part, sz)
             elsif part.respond_to?(:size)
               if sz = part.size
@@ -700,8 +700,9 @@ module HTTP
 
       def params_from_file(value)
         params = {}
+        original_filename = value.respond_to?(:original_filename) ? value.original_filename : nil
         path = value.respond_to?(:path) ? value.path : nil
-        params['filename'] = File.basename(path || '')
+        params['filename'] = original_filename || File.basename(path || '')
         # Creation time is not available from File::Stat
         if value.respond_to?(:mtime)
           params['modification-date'] = value.mtime.rfc822
@@ -808,6 +809,8 @@ module HTTP
         case path
         when /\.txt$/i
           'text/plain'
+        when /\.xml$/i
+          'text/xml'
         when /\.(htm|html)$/i
           'text/html'
         when /\.doc$/i
@@ -951,7 +954,7 @@ module HTTP
 
     # Dumps message (header and body) to given dev.
     # dev needs to respond to <<.
-    def dump(dev = '')
+    def dump(dev = ''.dup)
       str = @http_header.dump + CRLF
       if @http_header.chunked
         dev = @http_body.dump_chunked(str, dev)

data/lib/httpclient/jruby_ssl_socket.rb

@@ -15,9 +15,27 @@ class HTTPClient
 unless defined?(SSLSocket)
 
   class JavaSocketWrap
+    java_import 'java.net.InetSocketAddress'
     java_import 'java.io.BufferedInputStream'
+
     BUF_SIZE = 1024 * 16
 
+    def self.normalize_timeout(timeout)
+      [Java::JavaLang::Integer::MAX_VALUE, timeout].min
+    end
+
+    def self.connect(socket, site, opts = {})
+      socket_addr = InetSocketAddress.new(site.host, site.port)
+      if opts[:connect_timeout]
+        socket.connect(socket_addr, normalize_timeout(opts[:connect_timeout]))
+      else
+        socket.connect(socket_addr)
+      end
+      socket.setSoTimeout(normalize_timeout(opts[:so_timeout])) if opts[:so_timeout]
+      socket.setKeepAlive(true) if opts[:tcp_keepalive]
+      socket
+    end
+
     def initialize(socket, debug_dev = nil)
       @socket = socket
       @debug_dev = debug_dev
@@ -39,7 +57,6 @@ unless defined?(SSLSocket)
       @socket.isClosed
     end
 
-
     def gets(rs)
       while (size = @bufstr.index(rs)).nil?
         if fill() == -1
@@ -105,11 +122,15 @@ unless defined?(SSLSocket)
   private
 
     def fill
-      size = @instr.read(@buf)
-      if size > 0
-        @bufstr << String.from_java_bytes(@buf, Encoding::BINARY)[0, size]
+      begin
+        size = @instr.read(@buf)
+        if size > 0
+          @bufstr << String.from_java_bytes(@buf, Encoding::BINARY)[0, size]
+        end
+        size
+      rescue java.io.IOException => e
+        raise OpenSSL::SSL::SSLError.new("#{e.class}: #{e.getMessage}")
       end
-      size
     end
 
     def debug(str)
@@ -267,8 +288,8 @@ unless defined?(SSLSocket)
 
     module PEMUtils
       def self.read_certificate(pem)
-        pem = pem.sub(/-----BEGIN CERTIFICATE-----/, '').sub(/-----END CERTIFICATE-----/, '')
-        der = pem.unpack('m*').first
+        cert = pem.sub(/.*?-----BEGIN CERTIFICATE-----/m, '').sub(/-----END CERTIFICATE-----.*?/m, '')
+        der = cert.unpack('m*').first
         cf = CertificateFactory.getInstance('X.509')
         cf.generateCertificate(ByteArrayInputStream.new(der.to_java_bytes))
       end
@@ -289,11 +310,11 @@ unless defined?(SSLSocket)
         @keystore.load(nil)
       end
 
-      def add(cert_file, key_file, password)
-        cert_str = cert_file.respond_to?(:to_pem) ? cert_file.to_pem : File.read(cert_file.to_s)
+      def add(cert_source, key_source, password)
+        cert_str = cert_source.respond_to?(:to_pem) ? cert_source.to_pem : File.read(cert_source.to_s)
         cert = PEMUtils.read_certificate(cert_str)
         @keystore.setCertificateEntry('client_cert', cert)
-        key_str = key_file.respond_to?(:to_pem) ? key_file.to_pem : File.read(key_file.to_s)
+        key_str = key_source.respond_to?(:to_pem) ? key_source.to_pem : File.read(key_source.to_s)
         key_pair = PEMUtils.read_private_key(key_str, password)
         @keystore.setKeyEntry('client_key', key_pair.getPrivate, PASSWORD, [cert].to_java(Certificate))
       end
@@ -312,20 +333,23 @@ unless defined?(SSLSocket)
         @size = 0
       end
 
-      def add(file_or_dir)
-        return if file_or_dir == :default
-        if File.directory?(file_or_dir)
-          warn("#{file_or_dir}: directory not yet supported")
+      def add(cert_source)
+        return if cert_source == :default
+        if cert_source.respond_to?(:to_pem)
+          pem = cert_source.to_pem
+          load_pem(pem)
+        elsif File.directory?(cert_source)
+          warn("#{cert_source}: directory not yet supported")
+          return
         else
           pem = nil
-          File.read(file_or_dir).each_line do |line|
+          File.read(cert_source).each_line do |line|
             case line
             when /-----BEGIN CERTIFICATE-----/
               pem = ''
             when /-----END CERTIFICATE-----/
-              cert = PEMUtils.read_certificate(pem)
-              @size += 1
-              @trust_store.setCertificateEntry("cert_#{@size}", cert)
+              load_pem(pem)
+              # keep parsing in case where multiple certificates in a file
             else
               if pem
                 pem << line
@@ -342,6 +366,14 @@ unless defined?(SSLSocket)
           @trust_store
         end
       end
+
+    private
+
+      def load_pem(pem)
+        cert = PEMUtils.read_certificate(pem)
+        @size += 1
+        @trust_store.setCertificateEntry("cert_#{@size}", cert)
+      end
     end
 
     # Ported from commons-httpclient 'BrowserCompatHostnameVerifier'
@@ -429,26 +461,58 @@ unless defined?(SSLSocket)
     end
 
     def self.create_socket(session)
-      site = session.proxy || session.dest
-      socket = Socket.new(site.host, site.port)
+      opts = {
+        :connect_timeout => session.connect_timeout * 1000,
+        # send_timeout is ignored in JRuby
+        :so_timeout => session.receive_timeout * 1000,
+        :tcp_keepalive => session.tcp_keepalive,
+        :debug_dev => session.debug_dev
+      }
+      socket = nil
       begin
         if session.proxy
+          site = session.proxy || session.dest
+          socket = JavaSocketWrap.connect(Socket.new, site, opts)
           session.connect_ssl_proxy(JavaSocketWrap.new(socket), Util.urify(session.dest.to_s))
         end
+        new(socket, session.dest, session.ssl_config, opts)
       rescue
-        socket.close
+        socket.close if socket
         raise
       end
-      new(socket, session.dest, session.ssl_config, session.debug_dev)
     end
 
-    DEFAULT_SSL_PROTOCOL = 'TLS'
-    def initialize(socket, dest, config, debug_dev = nil)
+    DEFAULT_SSL_PROTOCOL = (java.lang.System.getProperty('java.specification.version') == '1.7') ? 'TLSv1.2' : 'TLS'
+    def initialize(socket, dest, config, opts = {})
+      @config = config
+      begin
+        @ssl_socket = create_ssl_socket(socket, dest, config, opts)
+        ssl_version = java_ssl_version(config)
+        @ssl_socket.setEnabledProtocols([ssl_version].to_java(java.lang.String)) if ssl_version != DEFAULT_SSL_PROTOCOL
+        if config.ciphers != SSLConfig::CIPHERS_DEFAULT
+          @ssl_socket.setEnabledCipherSuites(config.ciphers.to_java(java.lang.String))
+        end
+        ssl_connect(dest.host)
+      rescue java.security.GeneralSecurityException => e
+        raise OpenSSL::SSL::SSLError.new(e.getMessage)
+      rescue java.net.SocketTimeoutException => e
+        raise HTTPClient::ConnectTimeoutError.new(e.getMessage)
+      rescue java.io.IOException => e
+        raise OpenSSL::SSL::SSLError.new("#{e.class}: #{e.getMessage}")
+      end
+
+      super(@ssl_socket, opts[:debug_dev])
+    end
+
+    def java_ssl_version(config)
       if config.ssl_version == :auto
-        ssl_version = DEFAULT_SSL_PROTOCOL
+        DEFAULT_SSL_PROTOCOL
       else
-        ssl_version = config.ssl_version.to_s.gsub(/_/, '.')
+        config.ssl_version.to_s.tr('_', '.')
       end
+    end
+
+    def create_ssl_context(config)
       unless config.cert_store_crl_items.empty?
         raise NotImplementedError.new('Manual CRL configuration is not yet supported')
       end
@@ -464,7 +528,7 @@ unless defined?(SSLSocket)
 
       trust_store = nil
       verify_callback = config.verify_callback || config.method(:default_verify_callback)
-      if config.verify_mode == nil
+      if !config.verify?
         tmf = VerifyNoneTrustManagerFactory.new(verify_callback)
       else
         tmf = SystemTrustManagerFactory.new(verify_callback)
@@ -477,36 +541,24 @@ unless defined?(SSLSocket)
       tmf.init(trust_store)
       tm = tmf.getTrustManagers
 
-      ctx = SSLContext.getInstance(ssl_version)
+      ctx = SSLContext.getInstance(java_ssl_version(config))
       ctx.init(km, tm, nil)
       if config.timeout
         ctx.getClientSessionContext.setSessionTimeout(config.timeout)
       end
+      ctx
+    end
 
+    def create_ssl_socket(socket, dest, config, opts)
+      ctx = create_ssl_context(config)
       factory = ctx.getSocketFactory
-      begin
-        ssl_socket = factory.createSocket(socket, dest.host, dest.port, true)
-        ssl_socket.setEnabledProtocols([ssl_version].to_java(java.lang.String)) if ssl_version != DEFAULT_SSL_PROTOCOL
-        if config.ciphers != SSLConfig::CIPHERS_DEFAULT
-          ssl_socket.setEnabledCipherSuites(config.ciphers.to_java(java.lang.String))
-        end
-        ssl_socket.startHandshake
-        ssl_session = ssl_socket.getSession
-        @peer_cert = JavaCertificate.new(ssl_session.getPeerCertificates.first)
-        if $DEBUG
-          warn("Protocol version: #{ssl_session.getProtocol}")
-          warn("Cipher: #{ssl_socket.getSession.getCipherSuite}")
-        end
-        post_connection_check(dest.host, @peer_cert)
-      rescue java.security.GeneralSecurityException => e
-        raise OpenSSL::SSL::SSLError.new(e.getMessage)
-      rescue javax.net.ssl.SSLException => e
-        raise OpenSSL::SSL::SSLError.new(e.getMessage)
-      rescue java.net.SocketException => e
-        raise OpenSSL::SSL::SSLError.new(e.getMessage)
+      unless socket
+        # Create a plain socket first to set connection timeouts on,
+        # then wrap it in a SSL socket so that SNI gets setup on it.
+        socket = javax.net.SocketFactory.getDefault.createSocket
+        JavaSocketWrap.connect(socket, dest, opts)
       end
-
-      super(ssl_socket, debug_dev)
+      factory.createSocket(socket, dest.host, dest.port, true)
     end
 
     def peer_cert
@@ -515,8 +567,23 @@ unless defined?(SSLSocket)
 
   private
 
-    def post_connection_check(hostname, wrap_cert)
-      BrowserCompatHostnameVerifier.new.verify(hostname, wrap_cert.cert)
+    def ssl_connect(hostname)
+      @ssl_socket.startHandshake
+      ssl_session = @ssl_socket.getSession
+      @peer_cert = JavaCertificate.new(ssl_session.getPeerCertificates.first)
+      if $DEBUG
+        warn("Protocol version: #{ssl_session.getProtocol}")
+        warn("Cipher: #{@ssl_socket.getSession.getCipherSuite}")
+      end
+      post_connection_check(hostname)
+    end
+
+    def post_connection_check(hostname)
+      if !@config.verify?
+        return
+      else
+        BrowserCompatHostnameVerifier.new.verify(hostname, @peer_cert.cert)
+      end
     end
   end
 

data/lib/httpclient/session.rb

@@ -21,7 +21,7 @@ require 'zlib'
 require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
 require 'httpclient/ssl_config'
 require 'httpclient/http'
-if defined?(JRuby)
+if defined? JRUBY_VERSION
   require 'httpclient/jruby_ssl_socket'
 else
   require 'httpclient/ssl_socket'
@@ -76,7 +76,7 @@ class HTTPClient
     def to_s # :nodoc:
       addr
     end
-    
+
     # Returns true if scheme, host and port of the given URI matches with this.
     def match(uri)
       (@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
@@ -105,6 +105,8 @@ class HTTPClient
     attr_accessor :debug_dev
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
 
     attr_accessor :connect_timeout
     # Maximum retry count.  0 for infinite.
@@ -115,6 +117,9 @@ class HTTPClient
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    # Raise BadResponseError if response size does not match with Content-Length header in response.
+    attr_accessor :strict_response_size_check
+
     # Local address to bind local side of the socket to
     attr_accessor :socket_local
 
@@ -134,6 +139,7 @@ class HTTPClient
       @protocol_version = nil
       @debug_dev = client.debug_dev
       @socket_sync = true
+      @tcp_keepalive = false
       @chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
 
       @connect_timeout = 60
@@ -148,6 +154,7 @@ class HTTPClient
       @test_loopback_http_response = []
 
       @transparent_gzip_decompression = false
+      @strict_response_size_check = false
       @socket_local = Site.new
 
       @sess_pool = {}
@@ -212,6 +219,7 @@ class HTTPClient
       sess = Session.new(@client, site, @agent_name, @from)
       sess.proxy = via_proxy ? @proxy : nil
       sess.socket_sync = @socket_sync
+      sess.tcp_keepalive = @tcp_keepalive
       sess.requested_version = @protocol_version if @protocol_version
       sess.connect_timeout = @connect_timeout
       sess.connect_retry = @connect_retry
@@ -221,6 +229,7 @@ class HTTPClient
       sess.protocol_retry_count = @protocol_retry_count
       sess.ssl_config = @ssl_config
       sess.debug_dev = @debug_dev
+      sess.strict_response_size_check = @strict_response_size_check
       sess.socket_local = @socket_local
       sess.test_loopback_http_response = @test_loopback_http_response
       sess.transparent_gzip_decompression = @transparent_gzip_decompression
@@ -432,6 +441,8 @@ class HTTPClient
     attr_accessor :proxy
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
     # Requested protocol version
     attr_accessor :requested_version
     # Device for dumping log for debugging
@@ -444,6 +455,7 @@ class HTTPClient
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    attr_accessor :strict_response_size_check
     attr_accessor :socket_local
 
     attr_accessor :ssl_config
@@ -458,6 +470,7 @@ class HTTPClient
       @dest = dest
       @proxy = nil
       @socket_sync = true
+      @tcp_keepalive = false
       @requested_version = nil
 
       @debug_dev = nil
@@ -473,6 +486,7 @@ class HTTPClient
       @ssl_peer_cert = nil
 
       @test_loopback_http_response = nil
+      @strict_response_size_check = false
       @socket_local = Site::EMPTY
 
       @agent_name = agent_name
@@ -599,17 +613,16 @@ class HTTPClient
           clean_local = @socket_local.host.delete("[]")
           socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
         end
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
         if @debug_dev
           @debug_dev << "! CONNECTION ESTABLISHED\n"
           socket.extend(DebugSocket)
           socket.debug_dev = @debug_dev
         end
       rescue SystemCallError => e
-        e.message << " (#{host}:#{port})"
-        raise
+        raise e.class, e.message + " (#{host}:#{port})"
       rescue SocketError => e
-        e.message << " (#{host}:#{port})"
-        raise
+        raise e.class, e.message + " (#{host}:#{port})"
       end
       socket
     end
@@ -871,6 +884,9 @@ class HTTPClient
           rescue EOFError
             close
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
+            end
           end
         end
         if buf && buf.bytesize > 0
@@ -887,18 +903,18 @@ class HTTPClient
     def read_body_chunked(&block)
       buf = empty_bin_str
       while true
-        len = @socket.gets(RS)
-        if len.nil? # EOF
-          close
-          return
-        end
-        @chunk_length = len.hex
-        if @chunk_length == 0
-          @content_length = 0
-          @socket.gets(RS)
-          return
-        end
        ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
+          len = @socket.gets(RS)
+          if len.nil? # EOF
+            close
+            return
+          end
+          @chunk_length = len.hex
+          if @chunk_length == 0
+            @content_length = 0
+            @socket.gets(RS)
+            return
+          end
           @socket.read(@chunk_length, buf)
           @socket.read(2)
         end
@@ -920,6 +936,9 @@ class HTTPClient
             @socket.readpartial(@read_block_size, buf)
           rescue EOFError
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading chunked response")
+            end
           end
         end
         if buf && buf.bytesize > 0
@@ -931,7 +950,7 @@ class HTTPClient
     end
 
     def empty_bin_str
-      str = ''
+      str = ''.dup
       str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
       str
     end