httpclient 2.1.5.2 → 2.1.6

data/lib/httpclient/connection.rb

@@ -64,7 +64,11 @@ class HTTPClient
     # Retrieves a HTTP::Message instance of HTTP response.  Do not invoke this
     # method twice for now.  The second invocation will be blocked.
     def pop
-      @queue.pop
+      response_or_exception = @queue.pop
+      if response_or_exception.is_a? Exception
+        raise response_or_exception
+      end
+      response_or_exception
     end
 
     def push(result) # :nodoc:

data/lib/httpclient/cookie.rb

@@ -337,7 +337,6 @@ class WebAgent
 
       domainname = url.host
       domain_orig, path_orig = domain, path
-      use_security = override
 
       if domain
 
@@ -372,7 +371,7 @@ class WebAgent
         # and IE does not check, too.
       end
 
-      path ||= url.path.sub(%r|/[^/]*|, '')
+      path ||= url.path.sub(%r|/[^/]*\z|, '')
       domain ||= domainname
       @cookies.synchronize do
         cookie = find_cookie_info(domain, path, name)

data/lib/httpclient/http.rb

@@ -95,7 +95,7 @@ module HTTP
 
     # Represents HTTP message header.
     class Headers
-      # HTTP version in a HTTP header.  Float.
+      # HTTP version in a HTTP header.  String.
       attr_accessor :http_version
       # Size of body.  nil when size is unknown (e.g. chunked response).
       attr_reader :body_size
@@ -109,7 +109,7 @@ module HTTP
       # Request only.  Requested query.
       attr_accessor :request_query
       # Request only.  Requested via proxy or not.
-      attr_accessor :request_via_proxy
+      attr_accessor :request_absolute_uri
 
       # Response only.  HTTP status
       attr_reader :status_code
@@ -151,14 +151,14 @@ module HTTP
       # Creates a Message::Headers.  Use init_request, init_response, or
       # init_connect_request for acutual initialize.
       def initialize
-        @http_version = 1.1
+        @http_version = '1.1'
         @body_size = nil
         @chunked = false
 
         @request_method = nil
         @request_uri = nil
         @request_query = nil
-        @request_via_proxy = nil
+        @request_absolute_uri = nil
 
         @status_code = nil
         @reason_phrase = nil
@@ -178,7 +178,7 @@ module HTTP
         @request_method = 'CONNECT'
         @request_uri = uri
         @request_query = nil
-        @http_version = 1.0
+        @http_version = '1.0'
       end
 
       # Placeholder URI object for nil uri.
@@ -189,7 +189,7 @@ module HTTP
         @request_method = method
         @request_uri = uri || NIL_URI
         @request_query = query
-        @request_via_proxy = false
+        @request_absolute_uri = false
       end
 
       # Initialize this instance as a response.
@@ -285,11 +285,38 @@ module HTTP
         get(key).collect { |item| item[1] }
       end
 
+      def create_query_uri()
+        if @request_method == 'CONNECT'
+          return "#{@request_uri.host}:#{@request_uri.port}"
+        end
+        path = @request_uri.path
+        path = '/' if path.nil? or path.empty?
+        if query_str = create_query_part()
+          path += "?#{query_str}"
+        end
+        path
+      end
+
+      def create_query_part()
+        query_str = nil
+        if @request_uri.query
+          query_str = @request_uri.query
+        end
+        if @request_query
+          if query_str
+            query_str += "&#{Message.create_query_part_str(@request_query)}"
+          else
+            query_str = Message.create_query_part_str(@request_query)
+          end
+        end
+        query_str
+      end
+
     private
 
       def request_line
-        path = create_query_uri(@request_uri, @request_query)
-        if @request_via_proxy
+        path = create_query_uri()
+        if @request_absolute_uri
           path = "#{ @request_uri.scheme }://#{ @request_uri.host }:#{ @request_uri.port }#{ path }"
         end
         "#{ @request_method } #{ path } HTTP/#{ @http_version }#{ CRLF }"
@@ -323,7 +350,7 @@ module HTTP
         elsif @body_size and (keep_alive or @body_size != 0)
           set('Content-Length', @body_size.to_s)
         end
-        if @http_version >= 1.1
+        if @http_version >= '1.1' and get('Host').empty?
           if @request_uri.port == @request_uri.default_port
             # GFE/1.3 dislikes default port number (returns 404)
             set('Host', "#{@request_uri.host}")
@@ -358,29 +385,6 @@ module HTTP
       def charset_label(charset)
         CHARSET_MAP[charset] || 'us-ascii'
       end
-
-      def create_query_uri(uri, query)
-        if @request_method == 'CONNECT'
-          return "#{uri.host}:#{uri.port}"
-        end
-        path = uri.path
-        path = '/' if path.nil? or path.empty?
-        query_str = nil
-        if uri.query
-          query_str = uri.query
-        end
-        if query
-          if query_str
-            query_str += "&#{Message.create_query_part_str(query)}"
-          else
-            query_str = Message.create_query_part_str(query)
-          end
-        end
-        if query_str
-          path += "?#{query_str}"
-        end
-        path
-      end
     end
 
 
@@ -414,7 +418,9 @@ module HTTP
       # Initialize this instance as a response.
       def init_response(body = nil)
         @body = body
-        if @body.respond_to?(:size)
+        if @body.respond_to?(:bytesize)
+          @size = @body.bytesize
+        elsif @body.respond_to?(:size)
           @size = @body.size
         else
           @size = nil
@@ -438,6 +444,7 @@ module HTTP
               while !part.read(@chunk_size, buf).nil?
                 dev << buf
               end
+              part.rewind
             else
               dev << part
             end
@@ -496,7 +503,7 @@ module HTTP
           @size = @body.size
         else
           @body = Message.create_query_part_str(body)
-          @size = @body.size
+          @size = @body.bytesize
         end
       end
 
@@ -517,7 +524,7 @@ module HTTP
       end
 
       def dump_chunk(str)
-        dump_chunk_size(str.size) + (str + CRLF)
+        dump_chunk_size(str.bytesize) + (str + CRLF)
       end
 
       def dump_last_chunk
@@ -555,10 +562,10 @@ module HTTP
             end
           elsif @body[-1].is_a?(String)
             @body[-1] += part.to_s
-            @size += part.to_s.size if @size
+            @size += part.to_s.bytesize if @size
           else
             @body << part.to_s
-            @size += part.to_s.size if @size
+            @size += part.to_s.bytesize if @size
           end
         end
 
@@ -583,11 +590,21 @@ module HTTP
             }.join("; ")
             if value.respond_to?(:mime_type)
               content_type = value.mime_type
+            elsif value.respond_to?(:content_type)
+              content_type = value.content_type
             else
-              content_type = Message.mime_type(value.path)
+              path = value.respond_to?(:path) ? value.path : nil
+              content_type = Message.mime_type(path)
             end
             headers << %{Content-Disposition: form-data; name="#{attr}"; #{param_str}}
             headers << %{Content-Type: #{content_type}}
+          elsif attr.is_a?(Hash)
+            h = attr
+            value = h[:content]
+            h.each do |h_key, h_val|
+              headers << %{#{h_key}: #{h_val}} if h_key != :content
+            end
+            remember_pos(value) if Message.file?(value)
           else
             headers << %{Content-Disposition: form-data; name="#{attr}"}
           end
@@ -601,7 +618,8 @@ module HTTP
 
       def params_from_file(value)
         params = {}
-        params['filename'] = File.basename(value.path || '')
+        path = value.respond_to?(:path) ? value.path : nil
+        params['filename'] = File.basename(path || '')
         # Creation time is not available from File::Stat
         if value.respond_to?(:mtime)
           params['modification-date'] = value.mtime.rfc822
@@ -724,9 +742,9 @@ module HTTP
       end
 
       # Returns true if the given HTTP version allows keep alive connection.
-      # version:: Float
+      # version:: String
       def keep_alive_enabled?(version)
-        version >= 1.1
+        version >= '1.1'
       end
 
       # Returns true if the given query (or body) has a multiple parameter.
@@ -736,15 +754,14 @@ module HTTP
 
       # Returns true if the given object is a File.  In HTTPClient, a file is;
       # * must respond to :read for retrieving String chunks.
-      # * must respond to :path and returns a path for Content-Disposition.
       # * must respond to :pos and :pos= to rewind for reading.
       #   Rewinding is only needed for following HTTP redirect.  Some IO impl
       #   defines :pos= but raises an Exception for pos= such as StringIO
       #   but there's no problem as far as using it for non-following methods
       #   (get/post/etc.)
       def file?(obj)
-        obj.respond_to?(:read) and obj.respond_to?(:path) and
-          obj.respond_to?(:pos) and obj.respond_to?(:pos=)
+        obj.respond_to?(:read) and obj.respond_to?(:pos) and
+          obj.respond_to?(:pos=)
       end
 
       def create_query_part_str(query) # :nodoc:
@@ -758,7 +775,7 @@ module HTTP
       end
 
       def escape_query(query) # :nodoc:
-        query.collect { |attr, value|
+        query.sort_by { |attr, value| attr.to_s }.collect { |attr, value|
           if value.respond_to?(:read)
             value = value.read
           end
@@ -768,9 +785,36 @@ module HTTP
 
       # from CGI.escape
       def escape(str) # :nodoc:
-        str.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
-          '%' + $1.unpack('H2' * $1.size).join('%').upcase
-        }.tr(' ', '+')
+        if defined?(Encoding::ASCII_8BIT)
+          str.dup.force_encoding(Encoding::ASCII_8BIT).gsub(/([^ a-zA-Z0-9_.-]+)/) {
+            '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
+          }.tr(' ', '+')
+        else
+          str.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+            '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
+          }.tr(' ', '+')
+        end
+      end
+
+      # from CGI.parse
+      def parse(query)
+        params = Hash.new([].freeze)
+        query.split(/[&;]/n).each do |pairs|
+          key, value = pairs.split('=',2).collect{|v| unescape(v) }
+          if params.has_key?(key)
+            params[key].push(value)
+          else
+            params[key] = [value]
+          end
+        end
+        params
+      end
+
+      # from CGI.unescape
+      def unescape(string)
+        string.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n) do
+          [$1.delete('%')].pack('H*')
+        end
       end
     end
 
@@ -813,13 +857,24 @@ module HTTP
       @header.body_size = @body.size if @header
     end
 
-    # Returns HTTP version in a HTTP header.  Float.
-    def version
+    # Returns HTTP version in a HTTP header.  String.
+    def http_version
       @header.http_version
     end
 
-    # Sets HTTP version in a HTTP header.  Float.
+    # Sets HTTP version in a HTTP header.  String.
+    def http_version=(http_version)
+      @header.http_version = http_version
+    end
+
+    VERSION_WARNING = 'Message#version (Float) is deprecated. Use Message#http_version (String) instead.'
+    def version
+      warn(VERSION_WARNING)
+      @header.http_version.to_f
+    end
+
     def version=(version)
+      warn(VERSION_WARNING)
       @header.http_version = version
     end
 

data/lib/httpclient/session.rb

@@ -13,6 +13,7 @@
 require 'socket'
 require 'thread'
 require 'stringio'
+require 'zlib'
 
 require 'httpclient/timeout'
 require 'httpclient/ssl_config'
@@ -106,6 +107,8 @@ class HTTPClient
 
     attr_reader :test_loopback_http_response
 
+    attr_accessor :transparent_gzip_decompression
+
     def initialize(client)
       @client = client
       @proxy = client.proxy
@@ -128,6 +131,7 @@ class HTTPClient
       @ssl_config = nil
       @test_loopback_http_response = []
 
+      @transparent_gzip_decompression = false
       @sess_pool = []
       @sess_pool_mutex = Mutex.new
     end
@@ -165,6 +169,16 @@ class HTTPClient
       add_cached_session(sess)
     end
 
+    def invalidate(site)
+      @sess_pool_mutex.synchronize do
+        @sess_pool.each do |sess|
+          if sess.dest == site
+            sess.invalidate
+          end
+        end
+      end
+    end
+
   private
 
     def open(uri, via_proxy = false)
@@ -185,6 +199,7 @@ class HTTPClient
         sess.ssl_config = @ssl_config
         sess.debug_dev = @debug_dev
         sess.test_loopback_http_response = @test_loopback_http_response
+        sess.transparent_gzip_decompression = @transparent_gzip_decompression
       end
       sess
     end
@@ -198,6 +213,7 @@ class HTTPClient
       @sess_pool.clear
     end
 
+    # This method might not work as you expected...
     def close(dest)
       if cached = get_cached_session(dest)
         cached.close
@@ -212,7 +228,9 @@ class HTTPClient
       @sess_pool_mutex.synchronize do
         new_pool = []
         @sess_pool.each do |s|
-          if s.dest.match(uri)
+          if s.invalidated?
+            s.close # close & remove from the pool
+          elsif !cached && s.dest.match(uri)
             cached = s
           else
             new_pool << s
@@ -435,7 +453,15 @@ class HTTPClient
   private
 
     def debug(str)
-      @debug_dev << str if str && @debug_dev
+      if str && @debug_dev
+        if str.index("\0")
+          require 'hexdump'
+          str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+          @debug_dev << HexDump.encode(str).join("\n")
+        else
+          @debug_dev << str
+        end
+      end
     end
   end
 
@@ -459,6 +485,7 @@ class HTTPClient
   # Manages a HTTP session with a Site.
   class Session
     include HTTPClient::Timeout
+    include Util
 
     # Destination site
     attr_reader :dest
@@ -482,9 +509,12 @@ class HTTPClient
     attr_reader :ssl_peer_cert
     attr_accessor :test_loopback_http_response
 
+    attr_accessor :transparent_gzip_decompression
+
     def initialize(client, dest, agent_name, from)
       @client = client
       @dest = dest
+      @invalidated = false
       @proxy = nil
       @socket_sync = true
       @requested_version = nil
@@ -515,12 +545,15 @@ class HTTPClient
 
       @socket = nil
       @readbuf = nil
+
+      @transparent_gzip_decompression = false
     end
 
     # Send a request to the server
     def query(req)
       connect if @state == :INIT
-      req.header.request_via_proxy = !@proxy.nil?
+      # Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
+      req.header.request_absolute_uri = !@proxy.nil? and !https?(@dest)
       begin
         timeout(@send_timeout, SendTimeoutError) do
           set_header(req)
@@ -529,14 +562,16 @@ class HTTPClient
           @socket.flush unless @socket_sync
         end
       rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
+        # JRuby can raise IOError instead of ECONNRESET for now
         close
-        raise KeepAliveDisconnected.new
+        raise KeepAliveDisconnected.new(self)
       rescue HTTPClient::TimeoutError
         close
         raise
       rescue
+        close
         if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
-          raise KeepAliveDisconnected.new
+          raise KeepAliveDisconnected.new(self)
         else
           raise
         end
@@ -560,6 +595,14 @@ class HTTPClient
       @state == :INIT
     end
 
+    def invalidate
+      @invalidated = true
+    end
+
+    def invalidated?
+      @invalidated
+    end
+
     def get_header
       begin
         if @state != :META
@@ -585,6 +628,19 @@ class HTTPClient
       begin
         read_header if @state == :META
         return nil if @state != :DATA
+        if @gzipped and @transparent_gzip_decompression
+          # zlib itself has a functionality to decompress gzip stream.
+          # - zlib 1.2.5 Manual
+          #   http://www.zlib.net/manual.html#Advanced
+          # > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
+          # > windowBits to enable zlib and gzip decoding with automatic header detection,
+          # > or add 16 to decode only the gzip format
+          inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+          original_block = block
+          block = Proc.new { |buf|
+            original_block.call(inflate_stream.inflate(buf))
+          }
+        end
         if @chunked
           read_body_chunked(&block)
         elsif @content_length
@@ -611,7 +667,7 @@ class HTTPClient
     def set_header(req)
       if @requested_version
         if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
-          req.version = $1.to_f
+          req.http_version = $1
         end
       end
       if @agent_name
@@ -620,6 +676,9 @@ class HTTPClient
       if @from
         req.header.set('From', @from)
       end
+      if @transparent_gzip_decompression
+        req.header.set('Accept-Encoding', 'gzip,deflate')
+      end
       req.header.set('Date', Time.now.httpdate)
     end
 
@@ -630,7 +689,7 @@ class HTTPClient
       begin
         timeout(@connect_timeout, ConnectTimeoutError) do
           @socket = create_socket(site)
-          if @dest.scheme == 'https'
+          if https?(@dest)
             if @socket.is_a?(LoopBackSocket)
               connect_ssl_proxy(@socket, URI.parse(@dest.to_s)) if @proxy
             else
@@ -704,7 +763,7 @@ class HTTPClient
       @socket.flush unless @socket_sync
       res = HTTP::Message.new_response('')
       parse_header
-      res.version, res.status, res.reason = @version, @status, @reason
+      res.http_version, res.status, res.reason = @version, @status, @reason
       @headers.each do |key, value|
         res.header.set(key, value)
       end
@@ -723,14 +782,13 @@ class HTTPClient
     def read_header
       @content_length = nil
       @chunked = false
+      @gzipped = false
       @chunk_length = 0
       parse_header
-
-      # Head of the request has been parsed.
+      # Header of the request has been parsed.
       @state = :DATA
       req = @requests.shift
-
-      if req.header.request_method == 'HEAD'
+      if req.header.request_method == 'HEAD' or no_message_body?(@status)
         @content_length = 0
         if @next_connection
           @state = :WAIT
@@ -744,11 +802,19 @@ class HTTPClient
     StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
     def parse_header
       timeout(@receive_timeout, ReceiveTimeoutError) do
+        initial_line = nil
         begin
           initial_line = @socket.gets("\n")
           if initial_line.nil?
-            raise KeepAliveDisconnected.new
+            close
+            raise KeepAliveDisconnected.new(self)
           end
+        rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
+          # JRuby can raise IOError instead of ECONNRESET for now
+          close
+          raise KeepAliveDisconnected.new(self)
+        end
+        begin
           if StatusParseRegexp !~ initial_line
             @version = '0.9'
             @status = nil
@@ -759,7 +825,7 @@ class HTTPClient
             break
           end
           @version, @status, @reason = $1, $2.to_i, $3
-          @next_connection = HTTP::Message.keep_alive_enabled?(@version.to_f)
+          @next_connection = HTTP::Message.keep_alive_enabled?(@version)
           @headers = []
           while true
             line = @socket.gets("\n")
@@ -768,18 +834,32 @@ class HTTPClient
             end
             line.chomp!
             break if line.empty?
-            key, value = line.split(/\s*:\s*/, 2)
-            parse_keepalive_header(key, value)
-            @headers << [key, value]
+            if line[0] == ?\  or line[0] == ?\t
+              last = @headers.last[1]
+              last << ' ' unless last.empty?
+              last << line.strip
+            else
+              key, value = line.strip.split(/\s*:\s*/, 2)
+              parse_keepalive_header(key, value)
+              @headers << [key, value]
+            end
           end
         end while (@version == '1.1' && @status == 100)
       end
     end
 
+    def no_message_body?(status)
+      !status.nil? && # HTTP/0.9
+        ((status >= 100 && status < 200) || status == 204 || status == 304)
+    end
+
     def parse_keepalive_header(key, value)
       key = key.downcase
       if key == 'content-length'
         @content_length = value.to_i
+      elsif key == 'content-encoding' and ( value.downcase == 'gzip' or
+                                            value.downcase == 'x-gzip' or value.downcase == 'deflate' )
+        @gzipped = true
       elsif key == 'transfer-encoding' and value.downcase == 'chunked'
         @chunked = true
         @chunk_length = 0
@@ -806,8 +886,8 @@ class HTTPClient
             buf = nil
           end
         end
-        if buf && buf.length > 0
-          @content_length -= buf.length
+        if buf && buf.bytesize > 0
+          @content_length -= buf.bytesize
           yield buf
         else
           @content_length = 0
@@ -837,7 +917,7 @@ class HTTPClient
     end
 
     def read_body_rest
-      if @readbuf and @readbuf.length > 0
+      if @readbuf and @readbuf.bytesize > 0
         yield @readbuf
         @readbuf = nil
       end
@@ -850,7 +930,7 @@ class HTTPClient
             buf = nil
           end
         end
-        if buf && buf.length > 0
+        if buf && buf.bytesize > 0
           yield buf
         else
           return