excon 0.29.0 → 0.30.0

data/lib/excon/middlewares/decompress.rb

@@ -1,14 +1,31 @@
 module Excon
   module Middleware
     class Decompress < Excon::Middleware::Base
+      def request_call(datum)
+        unless datum.has_key?(:response_block)
+          key = datum[:headers].keys.detect {|k| k.to_s.casecmp('Accept-Encoding') == 0 } || 'Accept-Encoding'
+          if datum[:headers][key].to_s.empty?
+            datum[:headers][key] = 'deflate, gzip'
+          end
+        end
+        @stack.request_call(datum)
+      end
+
       def response_call(datum)
         unless datum.has_key?(:response_block)
-          case datum[:response][:headers]['Content-Encoding']
-          when 'deflate'
-            # assume inflate omits header
-            datum[:response][:body] = Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(datum[:response][:body])
-          when 'gzip'
-            datum[:response][:body] = Zlib::GzipReader.new(StringIO.new(datum[:response][:body])).read
+          if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Encoding') == 0 }
+            encodings = Utils.split_header_value(datum[:response][:headers][key])
+            if encoding = encodings.last
+              if encoding.casecmp('deflate') == 0
+                # assume inflate omits header
+                datum[:response][:body] = Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(datum[:response][:body])
+                encodings.pop
+              elsif encoding.casecmp('gzip') == 0 || encoding.casecmp('x-gzip') == 0
+                datum[:response][:body] = Zlib::GzipReader.new(StringIO.new(datum[:response][:body])).read
+                encodings.pop
+              end
+              datum[:response][:headers][key] = encodings.join(', ')
+            end
           end
         end
         @stack.response_call(datum)

data/lib/excon/middlewares/redirect_follower.rb

@@ -23,13 +23,13 @@ module Excon
             params[:headers].delete('Proxy-Authorization')
             params[:headers].delete('Host')
             params.merge!(
-              :scheme     => uri.scheme,
-              :host       => uri.host,
-              :port       => uri.port,
+              :scheme     => uri.scheme || datum[:scheme],
+              :host       => uri.host   || datum[:host],
+              :port       => uri.port   || datum[:port],
               :path       => uri.path,
               :query      => uri.query,
-              :user       => (URI.decode(uri.user) if uri.user),
-              :password   => (URI.decode(uri.password) if uri.password)
+              :user       => (Utils.unescape_uri(uri.user) if uri.user),
+              :password   => (Utils.unescape_uri(uri.password) if uri.password)
             )
 
             response = Excon::Connection.new(params).request

data/lib/excon/middlewares/response_parser.rb

@@ -4,6 +4,24 @@ module Excon
       def response_call(datum)
         unless datum.has_key?(:response)
           datum = Excon::Response.parse(datum[:connection].send(:socket), datum)
+
+          # only requests without a :response_block add 'deflate, gzip' to the TE header.
+          unless datum[:response_block]
+            if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 }
+              encodings = Utils.split_header_value(datum[:response][:headers][key])
+              if encoding = encodings.last
+                if encoding.casecmp('deflate') == 0
+                  # assume inflate omits header
+                  datum[:response][:body] = Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(datum[:response][:body])
+                  encodings.pop
+                elsif encoding.casecmp('gzip') == 0 || encoding.casecmp('x-gzip') == 0
+                  datum[:response][:body] = Zlib::GzipReader.new(StringIO.new(datum[:response][:body])).read
+                  encodings.pop
+                end
+                datum[:response][:headers][key] = encodings.join(', ')
+              end
+            end
+          end
         end
         @stack.response_call(datum)
       end

data/lib/excon/response.rb

@@ -41,51 +41,61 @@ module Excon
         :remote_ip  => socket.respond_to?(:remote_ip) && socket.remote_ip
       }
 
-      until ((data = socket.readline).chop!).empty?
-        key, value = data.split(/:\s*/, 2)
-        datum[:response][:headers][key] = ([*datum[:response][:headers][key]] << value).compact.join(', ')
-        if key.casecmp('Content-Length') == 0
-          content_length = value.to_i
-        elsif (key.casecmp('Transfer-Encoding') == 0) && (value.casecmp('chunked') == 0)
-          transfer_encoding_chunked = true
-        end
-      end
+      parse_headers(socket, datum)
 
       unless (['HEAD', 'CONNECT'].include?(datum[:method].to_s.upcase)) || NO_ENTITY.include?(datum[:response][:status])
 
-        # check to see if expects was set and matched
-        expected_status = !datum.has_key?(:expects) || [*datum[:expects]].include?(datum[:response][:status])
+        if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 }
+          encodings = Utils.split_header_value(datum[:response][:headers][key])
+          if (encoding = encodings.last) && encoding.casecmp('chunked') == 0
+            transfer_encoding_chunked = true
+            encodings.pop
+            datum[:response][:headers][key] = encodings.join(', ')
+          end
+        end
+        unless transfer_encoding_chunked
+          if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Length') == 0 }
+            content_length = datum[:response][:headers][key].to_i
+          end
+        end
 
-        # if expects matched and there is a block, use it
-        if expected_status && datum.has_key?(:response_block)
-          if transfer_encoding_chunked
-            # 2 == "\r\n".length
+        # use :response_block unless :expects would fail
+        if response_block = datum[:response_block]
+          if datum[:middlewares].include?(Excon::Middleware::Expects) && datum[:expects] &&
+                                !Array(datum[:expects]).include?(datum[:response][:status])
+            response_block = nil
+          end
+        end
+
+        if transfer_encoding_chunked
+          # 2 == "\r\n".length
+          if response_block
             while (chunk_size = socket.readline.chop!.to_i(16)) > 0
-              datum[:response_block].call(socket.read(chunk_size + 2).chop!, nil, nil)
-            end
-            socket.read(2) # empty chunk-body
-          elsif remaining = content_length
-            while remaining > 0
-              datum[:response_block].call(socket.read([datum[:chunk_size], remaining].min), [remaining - datum[:chunk_size], 0].max, content_length)
-              remaining -= datum[:chunk_size]
+              response_block.call(socket.read(chunk_size + 2).chop!, nil, nil)
             end
           else
-            while remaining = socket.read(datum[:chunk_size])
-              datum[:response_block].call(remaining, remaining.length, content_length)
-            end
-          end
-        else # no block or unexpected status
-          if transfer_encoding_chunked
-            # 2 == "\r\n".length
             while (chunk_size = socket.readline.chop!.to_i(16)) > 0
               datum[:response][:body] << socket.read(chunk_size + 2).chop!
             end
-            socket.read(2) # empty chunk-body
-          elsif remaining = content_length
+          end
+          parse_headers(socket, datum) # merge trailers into headers
+        elsif remaining = content_length
+          if response_block
+            while remaining > 0
+              response_block.call(socket.read([datum[:chunk_size], remaining].min), [remaining - datum[:chunk_size], 0].max, content_length)
+              remaining -= datum[:chunk_size]
+            end
+          else
             while remaining > 0
               datum[:response][:body] << socket.read([datum[:chunk_size], remaining].min)
               remaining -= datum[:chunk_size]
             end
+          end
+        else
+          if response_block
+            while chunk = socket.read(datum[:chunk_size])
+              response_block.call(chunk, nil, nil)
+            end
           else
             datum[:response][:body] << socket.read
           end
@@ -94,6 +104,23 @@ module Excon
       datum
     end
 
+    def self.parse_headers(socket, datum)
+      last_key = nil
+      until (data = socket.readline.chop!).empty?
+        if !data.lstrip!.nil?
+          raise Excon::Errors::ResponseParseError, 'malformed header' unless last_key
+          # append to last_key's last value
+          datum[:response][:headers][last_key] << ' ' << data.rstrip
+        else
+          key, value = data.split(':', 2)
+          raise Excon::Errors::ResponseParseError, 'malformed header' unless value
+          # add key/value or append value to existing values
+          datum[:response][:headers][key] = ([datum[:response][:headers][key]] << value.strip).compact.join(', ')
+          last_key = key
+        end
+      end
+    end
+
     def initialize(params={})
       @data = {
         :body     => '',

data/lib/excon/socket.rb

@@ -200,11 +200,11 @@ module Excon
             @socket = socket
             break
           rescue SystemCallError => exception
-            socket.close
+            socket.close rescue nil
             next
           end
         rescue SystemCallError => exception
-          socket.close if socket
+          socket.close rescue nil if socket
           next
         end
       end

data/lib/excon/ssl_socket.rb

@@ -1,6 +1,10 @@
 module Excon
   class SSLSocket < Socket
 
+    HAVE_NONBLOCK = [:connect_nonblock, :read_nonblock, :write_nonblock].all? {|m|
+      OpenSSL::SSL::SSLSocket.public_method_defined?(m)
+    }
+
     def initialize(data = {})
       super
 
@@ -87,15 +91,9 @@ module Excon
 
     private
 
-    def check_nonblock_support
-      # backwards compatability for things lacking nonblock
-      if !DEFAULT_NONBLOCK && @nonblock
-        @nonblock = false
-      end
-    end
-
     def connect
-      check_nonblock_support
+      # backwards compatability for things lacking nonblock
+      @nonblock = HAVE_NONBLOCK && @nonblock
       super
     end
 

data/lib/excon/unix_socket.rb

@@ -4,18 +4,34 @@ module Excon
     private
 
     def connect
-      begin
-        @socket = ::UNIXSocket.new(@data[:socket])
-      rescue Errno::ECONNREFUSED
-        @socket.close if @socket
-        raise
-      end
+      @socket  = ::Socket.new(::Socket::AF_UNIX, ::Socket::SOCK_STREAM, 0)
+      sockaddr = ::Socket.sockaddr_un(@data[:socket])
 
-      if @data[:tcp_nodelay]
-        @socket.setsockopt(::Socket::IPPROTO_TCP,
-                           ::Socket::TCP_NODELAY,
-                           true)
+      if @nonblock
+        begin
+          @socket.connect_nonblock(sockaddr)
+        rescue Errno::EINPROGRESS
+          unless IO.select(nil, [@socket], nil, @data[:connect_timeout])
+            raise(Excon::Errors::Timeout.new("connect timeout reached"))
+          end
+          begin
+            @socket.connect_nonblock(sockaddr)
+          rescue Errno::EISCONN
+          end
+        end
+      else
+        begin
+          Timeout.timeout(@data[:connect_timeout]) do
+            @socket.connect(sockaddr)
+          end
+        rescue Timeout::Error
+          raise Excon::Errors::Timeout.new('connect timeout reached')
+        end
       end
+
+    rescue => error
+      @socket.close rescue nil if @socket
+      raise error
     end
 
   end

data/lib/excon/utils.rb

@@ -53,5 +53,21 @@ module Excon
       end
       str
     end
+
+    # Splits a header value +str+ according to HTTP specification.
+    def split_header_value(str)
+      return [] if str.nil?
+      WEBrick::HTTPUtils.split_header_value(str.strip)
+    end
+
+    # Unescapes HTTP reserved and unwise characters in +str+
+    def unescape_uri(str)
+      WEBrick::HTTPUtils.unescape(str)
+    end
+
+    # Unescape form encoded values in +str+
+    def unescape_form(str)
+      WEBrick::HTTPUtils.unescape_form(str)
+    end
   end
 end

data/tests/middlewares/canned_response_tests.rb

@@ -0,0 +1,27 @@
+Shindo.tests("Excon support for middlewares that return canned responses") do
+  the_body = "canned"
+
+  canned_response_middleware = Class.new(Excon::Middleware::Base) do
+    define_method :request_call do |params|
+      params[:response] = {
+        :body     => the_body,
+        :headers  => {},
+        :status   => 200
+      }
+      super(params)
+    end
+  end
+
+  connection = Excon.new(
+    'http://some-host.com/some-path',
+    :method         => :get,
+    :middlewares    => [canned_response_middleware] + Excon.defaults[:middlewares],
+    :response_block => Proc.new { } # to force streaming
+  )
+
+  tests('does not mutate the canned response body').returns("canned") do
+    connection.request
+    the_body
+  end
+end
+

data/tests/middlewares/decompress_tests.rb

@@ -1,23 +1,144 @@
-Shindo.tests('Excon decompression support') do
+Shindo.tests('Excon Decompress Middleware') do
   env_init
 
-  with_rackup('deflater.ru') do
+  with_server('good') do
+
     connection = Excon.new(
-      'http://127.0.0.1:9292/echo',
-      :body         => 'x' * 100,
-      :method       => :post,
-      :middlewares  => Excon.defaults[:middlewares] + [Excon::Middleware::Decompress]
+      'http://127.0.0.1:9292/echo/content-encoded',
+      :method => :post,
+      :body => 'hello world',
+      :middlewares => Excon.defaults[:middlewares] + [Excon::Middleware::Decompress]
     )
 
-    tests('deflate').returns('x' * 100) do
-      response = connection.request(:headers => { 'Accept-Encoding' => 'deflate' })
-      response.body
+    tests('gzip') do
+      resp = connection.request(
+        :headers => { 'Accept-Encoding' => 'gzip, deflate;q=0' }
+      )
+
+      tests('response body decompressed').returns('hello world') do
+        resp[:body]
+      end
+
+      tests('server sent content-encoding').returns('gzip') do
+        resp[:headers]['Content-Encoding-Sent']
+      end
+
+      tests('removes processed encoding from header').returns('') do
+        resp[:headers]['Content-Encoding']
+      end
+    end
+
+    tests('deflate') do
+      resp = connection.request(
+        :headers => { 'Accept-Encoding' => 'gzip;q=0, deflate' }
+      )
+
+      tests('response body decompressed').returns('hello world') do
+        resp[:body]
+      end
+
+      tests('server sent content-encoding').returns('deflate') do
+        resp[:headers]['Content-Encoding-Sent']
+      end
+
+      tests('removes processed encoding from header').returns('') do
+        resp[:headers]['Content-Encoding']
+      end
     end
 
-    tests('gzip').returns('x' * 100) do
-      response = connection.request(:headers => { 'Accept-Encoding' => 'gzip' })
-      response.body
+    tests('with pre-encoding') do
+      resp = connection.request(
+        :headers => { 'Accept-Encoding' => 'gzip, deflate;q=0',
+                      'Content-Encoding-Pre' => 'other' }
+      )
+
+      tests('server sent content-encoding').returns('other, gzip') do
+        resp[:headers]['Content-Encoding-Sent']
+      end
+
+      tests('processed encoding removed from header').returns('other') do
+        resp[:headers]['Content-Encoding']
+      end
+
+      tests('response body decompressed').returns('hello world') do
+        resp[:body]
+      end
+
     end
+
+    tests('with post-encoding') do
+      resp = connection.request(
+        :headers => { 'Accept-Encoding' => 'gzip, deflate;q=0',
+                      'Content-Encoding-Post' => 'other' }
+      )
+
+      tests('server sent content-encoding').returns('gzip, other') do
+        resp[:headers]['Content-Encoding-Sent']
+      end
+
+      tests('unprocessed since last applied is unknown').returns('gzip, other') do
+        resp[:headers]['Content-Encoding']
+      end
+
+      tests('response body still compressed').returns('hello world') do
+        Zlib::GzipReader.new(StringIO.new(resp[:body])).read
+      end
+
+    end
+
+    tests('with a :response_block') do
+      resp = nil
+      captures = capture_response_block do |block|
+        resp = connection.request(
+          :headers => { 'Accept-Encoding' => 'gzip'},
+          :response_block => block
+        )
+      end
+
+      tests('server sent content-encoding').returns('gzip') do
+        resp[:headers]['Content-Encoding-Sent']
+      end
+
+      tests('unprocessed since :response_block was used').returns('gzip') do
+        resp[:headers]['Content-Encoding']
+      end
+
+      tests(':response_block passed unprocessed data').returns('hello world') do
+        body = captures.map {|capture| capture[0] }.join
+        Zlib::GzipReader.new(StringIO.new(body)).read
+      end
+
+    end
+
+    tests('adds Accept-Encoding if needed') do
+
+      tests('without a :response_block').returns('deflate, gzip') do
+        resp = Excon.post(
+          'http://127.0.0.1:9292/echo/request',
+          :body => 'hello world',
+          :middlewares => Excon.defaults[:middlewares] +
+                          [Excon::Middleware::Decompress]
+        )
+        request = Marshal.load(resp.body)
+        request[:headers]['Accept-Encoding']
+      end
+
+      tests('with a :response_block').returns(nil) do
+        captures = capture_response_block do |block|
+          resp = Excon.post(
+            'http://127.0.0.1:9292/echo/request',
+            :body => 'hello world',
+            :response_block => block,
+            :middlewares => Excon.defaults[:middlewares] +
+                            [Excon::Middleware::Decompress]
+          )
+        end
+        request = Marshal.load(captures.map {|capture| capture[0] }.join)
+        request[:headers]['Accept-Encoding']
+      end
+
+    end
+
   end
 
   env_restore