httpclient 2.6.0.1 → 2.8.3

data/lib/httpclient/session.rb

@@ -1,5 +1,5 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;
@@ -14,12 +14,18 @@
 
 require 'socket'
 require 'thread'
+require 'timeout'
 require 'stringio'
 require 'zlib'
 
 require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
 require 'httpclient/ssl_config'
 require 'httpclient/http'
+if RUBY_ENGINE == 'jruby'
+  require 'httpclient/jruby_ssl_socket'
+else
+  require 'httpclient/ssl_socket'
+end
 
 
 class HTTPClient
@@ -70,7 +76,7 @@ class HTTPClient
     def to_s # :nodoc:
       addr
     end
-    
+
     # Returns true if scheme, host and port of the given URI matches with this.
     def match(uri)
       (@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
@@ -99,6 +105,8 @@ class HTTPClient
     attr_accessor :debug_dev
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
 
     attr_accessor :connect_timeout
     # Maximum retry count.  0 for infinite.
@@ -109,6 +117,9 @@ class HTTPClient
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    # Raise BadResponseError if response size does not match with Content-Length header in response.
+    attr_accessor :strict_response_size_check
+
     # Local address to bind local side of the socket to
     attr_accessor :socket_local
 
@@ -128,6 +139,7 @@ class HTTPClient
       @protocol_version = nil
       @debug_dev = client.debug_dev
       @socket_sync = true
+      @tcp_keepalive = false
       @chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
 
       @connect_timeout = 60
@@ -142,6 +154,7 @@ class HTTPClient
       @test_loopback_http_response = []
 
       @transparent_gzip_decompression = false
+      @strict_response_size_check = false
       @socket_local = Site.new
 
       @sess_pool = {}
@@ -206,6 +219,7 @@ class HTTPClient
       sess = Session.new(@client, site, @agent_name, @from)
       sess.proxy = via_proxy ? @proxy : nil
       sess.socket_sync = @socket_sync
+      sess.tcp_keepalive = @tcp_keepalive
       sess.requested_version = @protocol_version if @protocol_version
       sess.connect_timeout = @connect_timeout
       sess.connect_retry = @connect_retry
@@ -215,6 +229,7 @@ class HTTPClient
       sess.protocol_retry_count = @protocol_retry_count
       sess.ssl_config = @ssl_config
       sess.debug_dev = @debug_dev
+      sess.strict_response_size_check = @strict_response_size_check
       sess.socket_local = @socket_local
       sess.test_loopback_http_response = @test_loopback_http_response
       sess.transparent_gzip_decompression = @transparent_gzip_decompression
@@ -288,131 +303,6 @@ class HTTPClient
   end
 
 
-  # Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
-  class SSLSocketWrap
-    def initialize(socket, context, debug_dev = nil)
-      unless SSLEnabled
-        raise ConfigurationError.new('Ruby/OpenSSL module is required')
-      end
-      @context = context
-      @socket = socket
-      @ssl_socket = create_openssl_socket(@socket)
-      @debug_dev = debug_dev
-    end
-
-    def ssl_connect(hostname = nil)
-      if hostname && @ssl_socket.respond_to?(:hostname=)
-        @ssl_socket.hostname = hostname
-      end
-      @ssl_socket.connect
-    end
-
-    def post_connection_check(host)
-      verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
-      if verify_mode == OpenSSL::SSL::VERIFY_NONE
-        return
-      elsif @ssl_socket.peer_cert.nil? and
-        check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
-        raise OpenSSL::SSL::SSLError.new('no peer cert')
-      end
-      hostname = host.host
-      if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
-        @ssl_socket.post_connection_check(hostname)
-      else
-        @context.post_connection_check(@ssl_socket.peer_cert, hostname)
-      end
-    end
-
-    def ssl_version
-      @ssl_socket.ssl_version if @ssl_socket.respond_to?(:ssl_version)
-    end
-
-    def ssl_cipher
-      @ssl_socket.cipher
-    end
-
-    def ssl_state
-      @ssl_socket.state
-    end
-
-    def peer_cert
-      @ssl_socket.peer_cert
-    end
-
-    def close
-      @ssl_socket.close
-      @socket.close
-    end
-
-    def closed?
-      @socket.closed?
-    end
-
-    def eof?
-      @ssl_socket.eof?
-    end
-
-    def gets(*args)
-      str = @ssl_socket.gets(*args)
-      debug(str)
-      str
-    end
-
-    def read(*args)
-      str = @ssl_socket.read(*args)
-      debug(str)
-      str
-    end
-
-    def readpartial(*args)
-      str = @ssl_socket.readpartial(*args)
-      debug(str)
-      str
-    end
-
-    def <<(str)
-      rv = @ssl_socket.write(str)
-      debug(str)
-      rv
-    end
-
-    def flush
-      @ssl_socket.flush
-    end
-
-    def sync
-      @ssl_socket.sync
-    end
-
-    def sync=(sync)
-      @ssl_socket.sync = sync
-    end
-
-  private
-
-    def check_mask(value, mask)
-      value & mask == mask
-    end
-
-    def create_openssl_socket(socket)
-      ssl_socket = nil
-      if OpenSSL::SSL.const_defined?("SSLContext")
-        ctx = OpenSSL::SSL::SSLContext.new
-        @context.set_context(ctx)
-        ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
-      else
-        ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
-        @context.set_context(ssl_socket)
-      end
-      ssl_socket
-    end
-
-    def debug(str)
-      @debug_dev << str if @debug_dev && str
-    end
-  end
-
-
   # Wraps up a Socket for method interception.
   module SocketWrap
     def initialize(socket, *args)
@@ -432,20 +322,20 @@ class HTTPClient
       @socket.eof?
     end
 
-    def gets(*args)
-      @socket.gets(*args)
+    def gets(rs)
+      @socket.gets(rs)
     end
 
-    def read(*args)
-      @socket.read(*args)
+    def read(size, buf = nil)
+      @socket.read(size, buf)
     end
 
-    def readpartial(*args)
+    def readpartial(size, buf = nil)
       # StringIO doesn't support :readpartial
       if @socket.respond_to?(:readpartial)
-        @socket.readpartial(*args)
+        @socket.readpartial(size, buf)
       else
-        @socket.read(*args)
+        @socket.read(size, buf)
       end
     end
 
@@ -481,19 +371,19 @@ class HTTPClient
       debug("! CONNECTION CLOSED\n")
     end
 
-    def gets(*args)
+    def gets(rs)
       str = super
       debug(str)
       str
     end
 
-    def read(*args)
+    def read(size, buf = nil)
       str = super
       debug(str)
       str
     end
 
-    def readpartial(*args)
+    def readpartial(size, buf = nil)
       str = super
       debug(str)
       str
@@ -533,6 +423,10 @@ class HTTPClient
     def <<(str)
       # ignored
     end
+
+    def peer_cert
+      nil
+    end
   end
 
 
@@ -547,6 +441,8 @@ class HTTPClient
     attr_accessor :proxy
     # Boolean value for Socket#sync
     attr_accessor :socket_sync
+    # Boolean value to send TCP keepalive packets; no timing settings exist at present
+    attr_accessor :tcp_keepalive
     # Requested protocol version
     attr_accessor :requested_version
     # Device for dumping log for debugging
@@ -559,6 +455,7 @@ class HTTPClient
     attr_accessor :read_block_size
     attr_accessor :protocol_retry_count
 
+    attr_accessor :strict_response_size_check
     attr_accessor :socket_local
 
     attr_accessor :ssl_config
@@ -573,6 +470,7 @@ class HTTPClient
       @dest = dest
       @proxy = nil
       @socket_sync = true
+      @tcp_keepalive = false
       @requested_version = nil
 
       @debug_dev = nil
@@ -588,6 +486,7 @@ class HTTPClient
       @ssl_peer_cert = nil
 
       @test_loopback_http_response = nil
+      @strict_response_size_check = false
       @socket_local = Site::EMPTY
 
       @agent_name = agent_name
@@ -613,7 +512,7 @@ class HTTPClient
       # Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
       req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
       begin
-        timeout(@send_timeout, SendTimeoutError) do
+        ::Timeout.timeout(@send_timeout, SendTimeoutError) do
           set_header(req)
           req.dump(@socket)
           # flush the IO stream as IO::sync mode is false
@@ -679,18 +578,8 @@ class HTTPClient
       begin
         read_header if @state == :META
         return nil if @state != :DATA
-        if @gzipped and @transparent_gzip_decompression
-          # zlib itself has a functionality to decompress gzip stream.
-          # - zlib 1.2.5 Manual
-          #   http://www.zlib.net/manual.html#Advanced
-          # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
-          # > windowBits to enable zlib and gzip decoding with automatic header detection,
-          # > or add 16 to decode only the gzip format
-          inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
-          original_block = block
-          block = Proc.new { |buf|
-            original_block.call(inflate_stream.inflate(buf))
-          }
+        if @transparent_gzip_decompression
+          block = content_inflater_block(@content_encoding, block)
         end
         if @chunked
           read_body_chunked(&block)
@@ -713,8 +602,121 @@ class HTTPClient
       nil
     end
 
+    def create_socket(host, port)
+      socket = nil
+      begin
+        @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
+        clean_host = host.delete("[]")
+        if @socket_local == Site::EMPTY
+          socket = TCPSocket.new(clean_host, port)
+        else
+          clean_local = @socket_local.host.delete("[]")
+          socket = TCPSocket.new(clean_host, port, clean_local, @socket_local.port)
+        end
+        socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if @tcp_keepalive
+        if @debug_dev
+          @debug_dev << "! CONNECTION ESTABLISHED\n"
+          socket.extend(DebugSocket)
+          socket.debug_dev = @debug_dev
+        end
+      rescue SystemCallError => e
+        raise e.class, e.message + " (#{host}:#{port})"
+      rescue SocketError => e
+        raise e.class, e.message + " (#{host}:#{port})"
+      end
+      socket
+    end
+
+    def create_loopback_socket(host, port, str)
+      @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
+      socket = LoopBackSocket.new(host, port, str)
+      if @debug_dev
+        @debug_dev << "! CONNECTION ESTABLISHED\n"
+        socket.extend(DebugSocket)
+        socket.debug_dev = @debug_dev
+      end
+      if https?(@dest) && @proxy
+        connect_ssl_proxy(socket, Util.urify(@dest.to_s))
+      end
+      socket
+    end
+
+    def connect_ssl_proxy(socket, uri)
+      req = HTTP::Message.new_connect_request(uri)
+      @client.request_filter.each do |filter|
+        filter.filter_request(req)
+      end
+      set_header(req)
+      req.dump(socket)
+      socket.flush unless @socket_sync
+      res = HTTP::Message.new_response('')
+      parse_header(socket)
+      res.http_version, res.status, res.reason = @version, @status, @reason
+      @headers.each do |key, value|
+        res.header.set(key.to_s, value)
+      end
+      commands = @client.request_filter.collect { |filter|
+        filter.filter_response(req, res)
+      }
+      if commands.find { |command| command == :retry }
+        raise RetryableResponse.new(res)
+      end
+      unless @status == 200
+        raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
+      end
+    end
+
   private
 
+    # This inflater allows deflate compression with/without zlib header
+    class LenientInflater
+      def initialize
+        @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
+        @first = true
+      end
+
+      def inflate(body)
+        if @first
+          first_inflate(body)
+        else
+          @inflater.inflate(body)
+        end
+      end
+
+    private
+
+      def first_inflate(body)
+        @first = false
+        begin
+          @inflater.inflate(body)
+        rescue Zlib::DataError
+          # fallback to deflate without zlib header
+          @inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+          @inflater.inflate(body)
+        end
+      end
+    end
+
+    def content_inflater_block(content_encoding, block)
+      case content_encoding
+      when 'gzip', 'x-gzip'
+        # zlib itself has a functionality to decompress gzip stream.
+        # - zlib 1.2.5 Manual
+        #   http://www.zlib.net/manual.html#Advanced
+        # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
+        # > windowBits to enable zlib and gzip decoding with automatic header detection,
+        # > or add 16 to decode only the gzip format
+        inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+      when 'deflate'
+        inflate_stream = LenientInflater.new
+      else
+        return block
+      end
+      Proc.new { |buf|
+        block.call(inflate_stream.inflate(buf))
+      }
+    end
+
     def set_header(req)
       if @requested_version
         if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
@@ -743,30 +745,15 @@ class HTTPClient
       site = @proxy || @dest
       retry_number = 0
       begin
-        timeout(@connect_timeout, ConnectTimeoutError) do
-          @socket = create_socket(site)
-          if https?(@dest)
-            if @socket.is_a?(LoopBackSocket)
-              connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
-            else
-              @socket = create_ssl_socket(@socket)
-              connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
-              begin
-                @socket.ssl_connect(@dest.host)
-              ensure
-                if $DEBUG
-                  warn("Protocol version: #{@socket.ssl_version}")
-                  warn("Cipher: #{@socket.ssl_cipher.inspect}")
-                  warn("State: #{@socket.ssl_state}")
-                end
-              end
-              @socket.post_connection_check(@dest)
-              @ssl_peer_cert = @socket.peer_cert
-            end
+        ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do
+          if str = @test_loopback_http_response.shift
+            @socket = create_loopback_socket(site.host, site.port, str)
+          elsif https?(@dest)
+            @socket = SSLSocket.create_socket(self)
+            @ssl_peer_cert = @socket.peer_cert
+          else
+            @socket = create_socket(site.host, site.port)
           end
-          # Use Ruby internal buffering instead of passing data immediately
-          # to the underlying layer
-          # => we need to to call explicitly flush on the socket
           @socket.sync = @socket_sync
         end
       rescue RetryableResponse
@@ -788,71 +775,13 @@ class HTTPClient
       @state = :WAIT
     end
 
-    def create_socket(site)
-      socket = nil
-      begin
-        @debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
-        clean_host = site.host.delete("[]")
-        if str = @test_loopback_http_response.shift
-          socket = LoopBackSocket.new(clean_host, site.port, str)
-        elsif @socket_local == Site::EMPTY
-          socket = TCPSocket.new(clean_host, site.port)
-        else
-          clean_local = @socket_local.host.delete("[]")
-          socket = TCPSocket.new(clean_host, site.port, clean_local, @socket_local.port)
-        end
-        if @debug_dev
-          @debug_dev << "! CONNECTION ESTABLISHED\n"
-          socket.extend(DebugSocket)
-          socket.debug_dev = @debug_dev
-        end
-      rescue SystemCallError => e
-        e.message << " (#{site})"
-        raise
-      rescue SocketError => e
-        e.message << " (#{site})"
-        raise
-      end
-      socket
-    end
-
-    # wrap socket with OpenSSL.
-    def create_ssl_socket(raw_socket)
-      SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
-    end
-
-    def connect_ssl_proxy(socket, uri)
-      req = HTTP::Message.new_connect_request(uri)
-      @client.request_filter.each do |filter|
-        filter.filter_request(req)
-      end
-      set_header(req)
-      req.dump(@socket)
-      @socket.flush unless @socket_sync
-      res = HTTP::Message.new_response('')
-      parse_header
-      res.http_version, res.status, res.reason = @version, @status, @reason
-      @headers.each do |key, value|
-        res.header.set(key.to_s, value)
-      end
-      commands = @client.request_filter.collect { |filter|
-        filter.filter_response(req, res)
-      }
-      if commands.find { |command| command == :retry }
-        raise RetryableResponse.new(res)
-      end
-      unless @status == 200
-        raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
-      end
-    end
-
     # Read status block.
     def read_header
       @content_length = nil
       @chunked = false
-      @gzipped = false
+      @content_encoding = nil
       @chunk_length = 0
-      parse_header
+      parse_header(@socket)
       # Header of the request has been parsed.
       @state = :DATA
       req = @requests.shift
@@ -868,12 +797,12 @@ class HTTPClient
     end
 
     StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
-    def parse_header
-      timeout(@receive_timeout, ReceiveTimeoutError) do
+    def parse_header(socket)
+      ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
         initial_line = nil
         begin
           begin
-            initial_line = @socket.gets("\n")
+            initial_line = socket.gets("\n")
             if initial_line.nil?
               close
               raise KeepAliveDisconnected.new(self)
@@ -896,7 +825,7 @@ class HTTPClient
           @next_connection = HTTP::Message.keep_alive_enabled?(@version)
           @headers = []
           while true
-            line = @socket.gets("\n")
+            line = socket.gets("\n")
             unless line
               raise BadResponseError.new('unexpected EOF')
             end
@@ -908,7 +837,7 @@ class HTTPClient
               last << line.strip
             else
               key, value = line.strip.split(/\s*:\s*/, 2)
-              parse_keepalive_header(key, value)
+              parse_content_header(key, value)
               @headers << [key, value]
             end
           end
@@ -921,18 +850,20 @@ class HTTPClient
         ((status >= 100 && status < 200) || status == 204 || status == 304)
     end
 
-    def parse_keepalive_header(key, value)
+    def parse_content_header(key, value)
       key = key.downcase
-      if key == 'content-length'
+      case key
+      when 'content-length'
         @content_length = value.to_i
-      elsif key == 'content-encoding' and ( value.downcase == 'gzip' or
-                                            value.downcase == 'x-gzip' or value.downcase == 'deflate' )
-        @gzipped = true
-      elsif key == 'transfer-encoding' and value.downcase == 'chunked'
-        @chunked = true
-        @chunk_length = 0
-        @content_length = nil
-      elsif key == 'connection' or key == 'proxy-connection'
+      when 'content-encoding'
+        @content_encoding = value.downcase
+      when 'transfer-encoding'
+        if value.downcase == 'chunked'
+          @chunked = true
+          @chunk_length = 0
+          @content_length = nil
+        end
+      when 'connection', 'proxy-connection'
         if value.downcase == 'keep-alive'
           @next_connection = true
         else
@@ -947,12 +878,15 @@ class HTTPClient
         buf = empty_bin_str
         maxbytes = @read_block_size
         maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
-        timeout(@receive_timeout, ReceiveTimeoutError) do
+        ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(maxbytes, buf)
           rescue EOFError
             close
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading rest #{@content_length} bytes")
+            end
           end
         end
         if buf && buf.bytesize > 0
@@ -969,18 +903,18 @@ class HTTPClient
     def read_body_chunked(&block)
       buf = empty_bin_str
       while true
-        len = @socket.gets(RS)
-        if len.nil? # EOF
-          close
-          return
-        end
-        @chunk_length = len.hex
-        if @chunk_length == 0
-          @content_length = 0
-          @socket.gets(RS)
-          return
-        end
-        timeout(@receive_timeout, ReceiveTimeoutError) do
+       ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
+          len = @socket.gets(RS)
+          if len.nil? # EOF
+            close
+            return
+          end
+          @chunk_length = len.hex
+          if @chunk_length == 0
+            @content_length = 0
+            @socket.gets(RS)
+            return
+          end
           @socket.read(@chunk_length, buf)
           @socket.read(2)
         end
@@ -997,11 +931,14 @@ class HTTPClient
       end
       while true
         buf = empty_bin_str
-        timeout(@receive_timeout, ReceiveTimeoutError) do
+        ::Timeout.timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(@read_block_size, buf)
           rescue EOFError
             buf = nil
+            if @strict_response_size_check
+              raise BadResponseError.new("EOF while reading chunked response")
+            end
           end
         end
         if buf && buf.bytesize > 0