excon 0.62.0 → 0.85.0

data/lib/excon/middlewares/response_parser.rb

@@ -6,6 +6,9 @@ module Excon
         unless datum.has_key?(:response)
           datum = Excon::Response.parse(datum[:connection].send(:socket), datum)
         end
+        if datum.has_key?(:logger)
+          datum[:response][:logger] = datum[:logger]
+        end
         @stack.response_call(datum)
       end
     end

data/lib/excon/pretty_printer.rb

@@ -9,14 +9,7 @@ module Excon
         datum.delete(:connection)
         datum.delete(:stack)
 
-        if datum.has_key?(:headers) && datum[:headers].has_key?('Authorization')
-          datum[:headers] = datum[:headers].dup
-          datum[:headers]['Authorization'] = REDACTED
-        end
-
-        if datum.has_key?(:password)
-          datum[:password] = REDACTED
-        end
+        datum = Utils.redact(datum)
       end
 
       indent += 2

data/lib/excon/response.rb

@@ -59,10 +59,13 @@ module Excon
 
     def self.parse(socket, datum)
       # this will discard any trailing lines from the previous response if any.
-      begin
+      line = nil
+      loop do
         line = socket.readline
-      end until status = line[9, 3].to_i
+        break if line[9,3].to_i != 0
+      end
 
+      status = line[9, 3].to_i
       reason_phrase = line[13..-3] # -3 strips the trailing "\r\n"
 
       datum[:response] = {
@@ -90,7 +93,7 @@ module Excon
 
       unless (['HEAD', 'CONNECT'].include?(datum[:method].to_s.upcase)) || NO_ENTITY.include?(datum[:response][:status])
 
-        if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 }
+        if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 })
           encodings = Utils.split_header_value(datum[:response][:headers][key])
           if (encoding = encodings.last) && encoding.casecmp('chunked') == 0
             transfer_encoding_chunked = true
@@ -103,7 +106,7 @@ module Excon
         end
 
         # use :response_block unless :expects would fail
-        if response_block = datum[:response_block]
+        if (response_block = datum[:response_block])
           if datum[:middlewares].include?(Excon::Middleware::Expects) && datum[:expects] &&
                                 !Array(datum[:expects]).include?(datum[:response][:status])
             response_block = nil
@@ -140,11 +143,11 @@ module Excon
           end
           parse_headers(socket, datum) # merge trailers into headers
         else
-          if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Length') == 0 }
+          if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Length') == 0 })
             content_length = datum[:response][:headers][key].to_i
           end
 
-          if remaining = content_length
+          if (remaining = content_length)
             if response_block
               while remaining > 0
                 chunk = socket.read([datum[:chunk_size], remaining].min) || raise(EOFError)
@@ -160,11 +163,11 @@ module Excon
             end
           else
             if response_block
-              while chunk = socket.read(datum[:chunk_size])
+              while (chunk = socket.read(datum[:chunk_size]))
                 response_block.call(chunk, nil, nil)
               end
             else
-              while chunk = socket.read(datum[:chunk_size])
+              while (chunk = socket.read(datum[:chunk_size]))
                 datum[:response][:body] << chunk
               end
             end
@@ -222,7 +225,7 @@ module Excon
     end
 
     # Retrieve a specific header value. Header names are treated case-insensitively.
-    #   @param [String] name Header name
+    # @param [String] name Header name
     def get_header(name)
       headers[name]
     end

data/lib/excon/socket.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'resolv'
+
 module Excon
   class Socket
     include Utils
@@ -7,6 +9,23 @@ module Excon
 
     attr_accessor :data
 
+    # read/write drawn from https://github.com/ruby-amqp/bunny/commit/75d9dd79551b31a5dd3d1254c537bad471f108cf
+    CONNECT_RETRY_EXCEPTION_CLASSES = if defined?(IO::EINPROGRESSWaitWritable) # Ruby >= 2.1
+      [Errno::EINPROGRESS, IO::EINPROGRESSWaitWritable]
+    else # Ruby <= 2.0
+      [Errno::EINPROGRESS]
+    end
+    READ_RETRY_EXCEPTION_CLASSES = if defined?(IO::EAGAINWaitReadable) # Ruby >= 2.1
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable, IO::EAGAINWaitReadable, IO::EWOULDBLOCKWaitReadable]
+    else # Ruby <= 2.0
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable]
+    end
+    WRITE_RETRY_EXCEPTION_CLASSES = if defined?(IO::EAGAINWaitWritable) # Ruby >= 2.1
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable, IO::EAGAINWaitWritable, IO::EWOULDBLOCKWaitWritable]
+    else # Ruby <= 2.0
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable]
+    end
+
     def params
       Excon.display_warning('Excon::Socket#params is deprecated use Excon::Socket#data instead.')
       @data
@@ -43,18 +62,8 @@ module Excon
     def readline
       return legacy_readline if RUBY_VERSION.to_f <= 1.8_7
       buffer = String.new
-      begin
-        buffer << @socket.read_nonblock(1) while buffer[-1] != "\n"
-        buffer
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
-        select_with_timeout(@socket, :read) && retry
-      rescue OpenSSL::SSL::SSLError => error
-        if error.message == 'read would block'
-          select_with_timeout(@socket, :read) && retry
-        else
-          raise(error)
-        end
-      end
+      buffer << (read_nonblock(1) || raise(EOFError)) while buffer[-1] != "\n"
+      buffer
     end
 
     def legacy_readline
@@ -88,32 +97,30 @@ module Excon
     def connect
       @socket = nil
       exception = nil
+      hostname = @data[:hostname]
+      port = @port
+      family = @data[:family]
 
       if @data[:proxy]
-        family = @data[:proxy][:family] || ::Socket::Constants::AF_UNSPEC
-        args = [@data[:proxy][:hostname], @data[:proxy][:port], family, ::Socket::Constants::SOCK_STREAM]
-      else
-        family = @data[:family] || ::Socket::Constants::AF_UNSPEC
-        args = [@data[:hostname], @port, family, ::Socket::Constants::SOCK_STREAM]
+        hostname = @data[:proxy][:hostname]
+        port = @data[:proxy][:port]
+        family = @data[:proxy][:family]
       end
-      if RUBY_VERSION >= '1.9.2' && defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
-        args << nil << nil << false # no reverse lookup
-      end
-      addrinfo = ::Socket.getaddrinfo(*args)
 
-      addrinfo.each do |_, port, _, ip, a_family, s_type|
+      Resolv.each_address(hostname) do |ip|
         # already succeeded on previous addrinfo
         if @socket
           break
         end
-        
+
         @remote_ip = ip
+        @data[:remote_ip] = ip
 
         # nonblocking connect
         begin
           sockaddr = ::Socket.sockaddr_in(port, ip)
-
-          socket = ::Socket.new(a_family, s_type, 0)
+          addrinfo = Addrinfo.getaddrinfo(ip, port, family, :STREAM).first
+          socket = ::Socket.new(addrinfo.pfamily, addrinfo.socktype, addrinfo.protocol)
 
           if @data[:reuseaddr]
             socket.setsockopt(::Socket::Constants::SOL_SOCKET, ::Socket::Constants::SO_REUSEADDR, true)
@@ -128,7 +135,7 @@ module Excon
             socket.connect(sockaddr)
           end
           @socket = socket
-        rescue Errno::EINPROGRESS
+        rescue *CONNECT_RETRY_EXCEPTION_CLASSES
           select_with_timeout(socket, :connect_write)
           begin
             socket.connect_nonblock(sockaddr)
@@ -143,6 +150,8 @@ module Excon
         end
       end
 
+      exception ||= Resolv::ResolvError.new("no address for #{hostname}")
+
       # this will be our last encountered exception
       fail exception unless @socket
 
@@ -151,6 +160,17 @@ module Excon
                            ::Socket::TCP_NODELAY,
                            true)
       end
+
+      if @data[:keepalive]
+        if [:SOL_SOCKET, :SO_KEEPALIVE, :SOL_TCP, :TCP_KEEPIDLE, :TCP_KEEPINTVL, :TCP_KEEPCNT].all?{|c| ::Socket.const_defined? c}
+          @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true)
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPIDLE, @data[:keepalive][:time])
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPINTVL, @data[:keepalive][:intvl])
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPCNT, @data[:keepalive][:probes])
+        else
+          Excon.display_warning('Excon::Socket keepalive was set, but is not supported by Ruby version.')
+        end
+      end
     end
 
     def read_nonblock(max_length)
@@ -170,7 +190,7 @@ module Excon
         else
           raise(error)
         end
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+      rescue *READ_RETRY_EXCEPTION_CLASSES
         if @read_buffer.empty?
           # if we didn't read anything, try again...
           select_with_timeout(@socket, :read) && retry
@@ -199,7 +219,7 @@ module Excon
       else
         raise(error)
       end
-    rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+    rescue *READ_RETRY_EXCEPTION_CLASSES
       if @read_buffer.empty?
         select_with_timeout(@socket, :read) && retry
       end
@@ -208,9 +228,7 @@ module Excon
     end
 
     def write_nonblock(data)
-      if FORCE_ENC
-        data.force_encoding('BINARY')
-      end
+      data = binary_encode(data)
       loop do
         written = nil
         begin
@@ -225,7 +243,7 @@ module Excon
           else
             raise error
           end
-        rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => error
+        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
           if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
             raise error
           else
@@ -246,7 +264,7 @@ module Excon
 
     def write_block(data)
       @socket.write(data)
-    rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => error
+    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
       if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
         raise error
       else

data/lib/excon/ssl_socket.rb

@@ -12,6 +12,11 @@ module Excon
       # create ssl context
       ssl_context = OpenSSL::SSL::SSLContext.new
 
+      # set the security level before setting other parameters affected by it
+      if @data[:ssl_security_level]
+        ssl_context.security_level = @data[:ssl_security_level]
+      end
+
       # disable less secure options, when supported
       ssl_context_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
       if defined?(OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS)
@@ -27,18 +32,25 @@ module Excon
       if @data[:ssl_version]
         ssl_context.ssl_version = @data[:ssl_version]
       end
+      if @data[:ssl_min_version]
+        ssl_context.min_version = @data[:ssl_min_version]
+      end
+      if @data[:ssl_max_version]
+        ssl_context.max_version = @data[:ssl_max_version]
+      end
+
 
       if @data[:ssl_verify_peer]
         # turn verification on
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
 
-        if ca_file = @data[:ssl_ca_file] || ENV['SSL_CERT_FILE']
+        if (ca_file = @data[:ssl_ca_file] || ENV['SSL_CERT_FILE'])
           ssl_context.ca_file = ca_file
         end
-        if ca_path = @data[:ssl_ca_path] || ENV['SSL_CERT_DIR']
+        if (ca_path = @data[:ssl_ca_path] || ENV['SSL_CERT_DIR'])
           ssl_context.ca_path = ca_path
         end
-        if cert_store = @data[:ssl_cert_store]
+        if (cert_store = @data[:ssl_cert_store])
           ssl_context.cert_store = cert_store
         end
 
@@ -58,7 +70,7 @@ module Excon
           end
         end
 
-        if verify_callback = @data[:ssl_verify_callback]
+        if (verify_callback = @data[:ssl_verify_callback])
           ssl_context.verify_callback = verify_callback
         end
       else
@@ -66,6 +78,9 @@ module Excon
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = @data[:ssl_verify_hostname] if ssl_context.respond_to?(:verify_hostname=)
+
       if client_cert_data && client_key_data
         ssl_context.cert = OpenSSL::X509::Certificate.new client_cert_data
         if OpenSSL::PKey.respond_to? :read
@@ -94,13 +109,20 @@ module Excon
 
         request += "Proxy-Connection: Keep-Alive#{Excon::CR_NL}"
 
+        if @data[:ssl_proxy_headers]
+          request << Utils.headers_hash_to_s(@data[:ssl_proxy_headers])
+        end
+
         request += Excon::CR_NL
 
         # write out the proxy setup request
         @socket.write(request)
 
         # eat the proxy's connection response
-        Excon::Response.parse(self,  :expects => 200, :method => 'CONNECT')
+        response = Excon::Response.parse(self,  :expects => 200, :method => 'CONNECT')
+        if response[:response][:status] != 200
+          raise(Excon::Errors::ProxyConnectionError.new("proxy connection could not be established", request, response))
+        end
       end
 
       # convert Socket to OpenSSL::SSL::SSLSocket
@@ -132,18 +154,16 @@ module Excon
       if @data[:ssl_verify_peer]
         @socket.post_connection_check(@data[:ssl_verify_peer_host] || @data[:host])
       end
-
-      @socket
     end
 
     private
 
     def client_cert_data
-      @client_cert_data ||= if ccd = @data[:client_cert_data]
+      @client_cert_data ||= if (ccd = @data[:client_cert_data])
                               ccd
-                            elsif path = @data[:client_cert]
+                            elsif (path = @data[:client_cert])
                               File.read path
-                            elsif path = @data[:certificate_path]
+                            elsif (path = @data[:certificate_path])
                               warn ":certificate_path is no longer supported and will be deprecated. Please use :client_cert or :client_cert_data"
                               File.read path
                             end
@@ -156,11 +176,11 @@ module Excon
     end
 
     def client_key_data
-      @client_key_data ||= if ckd = @data[:client_key_data]
+      @client_key_data ||= if (ckd = @data[:client_key_data])
                              ckd
-                           elsif path = @data[:client_key]
+                           elsif (path = @data[:client_key])
                              File.read path
-                           elsif path = @data[:private_key_path]
+                           elsif (path = @data[:private_key_path])
                              warn ":private_key_path is no longer supported and will be deprecated. Please use :client_key or :client_key_data"
                              File.read path
                            end

data/lib/excon/test/plugin/server/exec.rb

@@ -5,12 +5,12 @@ module Excon
         module Exec
           def start(app_str = app)
             line = ''
-            open_process(app)
+            open_process(app_str)
             until line =~ /\Aready\Z/
               line = error.gets
               fatal_time = elapsed_time > timeout
               if fatal_time
-                msg = "executable #{app} has taken too long to start"
+                msg = "executable #{app_str} has taken too long to start"
                 raise msg
               end
             end

data/lib/excon/test/server.rb

@@ -72,7 +72,7 @@ module Excon
       end
       def dump_errors
         lines = error.read.split($/)
-        while line = lines.shift
+        while (line = lines.shift)
           case line
             when /(ERROR|Error)/
               unless line =~ /(null cert chain|did not return a certificate|SSL_read:: internal error)/

data/lib/excon/unix_socket.rb

@@ -20,6 +20,7 @@ module Excon
           begin
             @socket.connect_nonblock(sockaddr)
           rescue Errno::EISCONN
+            0 # same return as connect_nonblock success
           end
         end
       else

data/lib/excon/utils.rb

@@ -10,6 +10,18 @@ module Excon
     UNESCAPED = /([#{ Regexp.escape(CONTROL + ' ' + DELIMS + UNWISE + NONASCII) }])/
     ESCAPED   = /%([0-9a-fA-F]{2})/
 
+    def binary_encode(string)
+      if FORCE_ENC && string.encoding != Encoding::ASCII_8BIT
+        if string.frozen?
+          string.dup.force_encoding('BINARY')
+        else
+          string.force_encoding('BINARY')
+        end
+      else
+        string
+      end
+    end
+
     def connection_uri(datum = @data)
       unless datum
         raise ArgumentError, '`datum` must be given unless called on a Connection'
@@ -21,6 +33,30 @@ module Excon
       end
     end
 
+    # Redact sensitive info from provided data
+    def redact(datum)
+      datum = datum.dup
+      if datum.has_key?(:headers)
+        if datum[:headers].has_key?('Authorization') || datum[:headers].has_key?('Proxy-Authorization')
+          datum[:headers] = datum[:headers].dup
+        end
+        if datum[:headers].has_key?('Authorization')
+          datum[:headers]['Authorization'] = REDACTED
+        end
+        if datum[:headers].has_key?('Proxy-Authorization')
+          datum[:headers]['Proxy-Authorization'] = REDACTED
+        end
+      end
+      if datum.has_key?(:password)
+        datum[:password] = REDACTED
+      end
+      if datum.has_key?(:proxy) && datum[:proxy] && datum[:proxy].has_key?(:password)
+        datum[:proxy] = datum[:proxy].dup
+        datum[:proxy][:password] = REDACTED
+      end
+      datum
+    end
+
     def request_uri(datum)
       connection_uri(datum) + datum[:path] + query_string(datum)
     end
@@ -59,31 +95,48 @@ module Excon
     def split_header_value(str)
       return [] if str.nil?
       str = str.dup.strip
-      str.force_encoding('BINARY') if FORCE_ENC
-      str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+)
+      str = binary_encode(str)
+      str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",])+)
                     (?:,\s*|\Z)'xn).flatten
     end
 
     # Escapes HTTP reserved and unwise characters in +str+
     def escape_uri(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub(UNESCAPED) { "%%%02X" % $1[0].ord }
     end
 
     # Unescapes HTTP reserved and unwise characters in +str+
     def unescape_uri(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub(ESCAPED) { $1.hex.chr }
     end
 
     # Unescape form encoded values in +str+
     def unescape_form(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub!(/\+/, ' ')
       str.gsub(ESCAPED) { $1.hex.chr }
     end
+
+    # Performs validation on the passed header hash and returns a string representation of the headers
+    def headers_hash_to_s(headers)
+      headers_str = String.new
+      headers.each do |key, values|
+        if key.to_s.match(/[\r\n]/)
+          raise Excon::Errors::InvalidHeaderKey.new(key.to_s.inspect + ' contains forbidden "\r" or "\n"')
+        end
+        [values].flatten.each do |value|
+          if value.to_s.match(/[\r\n]/)
+            raise Excon::Errors::InvalidHeaderValue.new(value.to_s.inspect + ' contains forbidden "\r" or "\n"')
+          end
+          headers_str << key.to_s << ': ' << value.to_s << CR_NL
+        end
+      end
+      headers_str
+    end
   end
 end