excon 0.62.0 → 0.91.0

data/lib/excon/socket.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'resolv'
+
 module Excon
   class Socket
     include Utils
@@ -7,6 +9,23 @@ module Excon
 
     attr_accessor :data
 
+    # read/write drawn from https://github.com/ruby-amqp/bunny/commit/75d9dd79551b31a5dd3d1254c537bad471f108cf
+    CONNECT_RETRY_EXCEPTION_CLASSES = if defined?(IO::EINPROGRESSWaitWritable) # Ruby >= 2.1
+      [Errno::EINPROGRESS, IO::EINPROGRESSWaitWritable]
+    else # Ruby <= 2.0
+      [Errno::EINPROGRESS]
+    end
+    READ_RETRY_EXCEPTION_CLASSES = if defined?(IO::EAGAINWaitReadable) # Ruby >= 2.1
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable, IO::EAGAINWaitReadable, IO::EWOULDBLOCKWaitReadable]
+    else # Ruby <= 2.0
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable]
+    end
+    WRITE_RETRY_EXCEPTION_CLASSES = if defined?(IO::EAGAINWaitWritable) # Ruby >= 2.1
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable, IO::EAGAINWaitWritable, IO::EWOULDBLOCKWaitWritable]
+    else # Ruby <= 2.0
+      [Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable]
+    end
+
     def params
       Excon.display_warning('Excon::Socket#params is deprecated use Excon::Socket#data instead.')
       @data
@@ -43,18 +62,8 @@ module Excon
     def readline
       return legacy_readline if RUBY_VERSION.to_f <= 1.8_7
       buffer = String.new
-      begin
-        buffer << @socket.read_nonblock(1) while buffer[-1] != "\n"
-        buffer
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
-        select_with_timeout(@socket, :read) && retry
-      rescue OpenSSL::SSL::SSLError => error
-        if error.message == 'read would block'
-          select_with_timeout(@socket, :read) && retry
-        else
-          raise(error)
-        end
-      end
+      buffer << (read_nonblock(1) || raise(EOFError)) while buffer[-1] != "\n"
+      buffer
     end
 
     def legacy_readline
@@ -88,32 +97,30 @@ module Excon
     def connect
       @socket = nil
       exception = nil
+      hostname = @data[:hostname]
+      port = @port
+      family = @data[:family]
 
       if @data[:proxy]
-        family = @data[:proxy][:family] || ::Socket::Constants::AF_UNSPEC
-        args = [@data[:proxy][:hostname], @data[:proxy][:port], family, ::Socket::Constants::SOCK_STREAM]
-      else
-        family = @data[:family] || ::Socket::Constants::AF_UNSPEC
-        args = [@data[:hostname], @port, family, ::Socket::Constants::SOCK_STREAM]
+        hostname = @data[:proxy][:hostname]
+        port = @data[:proxy][:port]
+        family = @data[:proxy][:family]
       end
-      if RUBY_VERSION >= '1.9.2' && defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
-        args << nil << nil << false # no reverse lookup
-      end
-      addrinfo = ::Socket.getaddrinfo(*args)
 
-      addrinfo.each do |_, port, _, ip, a_family, s_type|
+      Resolv.each_address(hostname) do |ip|
         # already succeeded on previous addrinfo
         if @socket
           break
         end
-        
+
         @remote_ip = ip
+        @data[:remote_ip] = ip
 
         # nonblocking connect
         begin
           sockaddr = ::Socket.sockaddr_in(port, ip)
-
-          socket = ::Socket.new(a_family, s_type, 0)
+          addrinfo = Addrinfo.getaddrinfo(ip, port, family, :STREAM).first
+          socket = ::Socket.new(addrinfo.pfamily, addrinfo.socktype, addrinfo.protocol)
 
           if @data[:reuseaddr]
             socket.setsockopt(::Socket::Constants::SOL_SOCKET, ::Socket::Constants::SO_REUSEADDR, true)
@@ -128,7 +135,7 @@ module Excon
             socket.connect(sockaddr)
           end
           @socket = socket
-        rescue Errno::EINPROGRESS
+        rescue *CONNECT_RETRY_EXCEPTION_CLASSES
           select_with_timeout(socket, :connect_write)
           begin
             socket.connect_nonblock(sockaddr)
@@ -143,6 +150,8 @@ module Excon
         end
       end
 
+      exception ||= Resolv::ResolvError.new("no address for #{hostname}")
+
       # this will be our last encountered exception
       fail exception unless @socket
 
@@ -151,6 +160,17 @@ module Excon
                            ::Socket::TCP_NODELAY,
                            true)
       end
+
+      if @data[:keepalive]
+        if [:SOL_SOCKET, :SO_KEEPALIVE, :SOL_TCP, :TCP_KEEPIDLE, :TCP_KEEPINTVL, :TCP_KEEPCNT].all?{|c| ::Socket.const_defined? c}
+          @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true)
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPIDLE, @data[:keepalive][:time])
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPINTVL, @data[:keepalive][:intvl])
+          @socket.setsockopt(::Socket::SOL_TCP, ::Socket::TCP_KEEPCNT, @data[:keepalive][:probes])
+        else
+          Excon.display_warning('Excon::Socket keepalive was set, but is not supported by Ruby version.')
+        end
+      end
     end
 
     def read_nonblock(max_length)
@@ -170,7 +190,7 @@ module Excon
         else
           raise(error)
         end
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+      rescue *READ_RETRY_EXCEPTION_CLASSES
         if @read_buffer.empty?
           # if we didn't read anything, try again...
           select_with_timeout(@socket, :read) && retry
@@ -199,7 +219,7 @@ module Excon
       else
         raise(error)
       end
-    rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+    rescue *READ_RETRY_EXCEPTION_CLASSES
       if @read_buffer.empty?
         select_with_timeout(@socket, :read) && retry
       end
@@ -208,9 +228,7 @@ module Excon
     end
 
     def write_nonblock(data)
-      if FORCE_ENC
-        data.force_encoding('BINARY')
-      end
+      data = binary_encode(data)
       loop do
         written = nil
         begin
@@ -225,7 +243,7 @@ module Excon
           else
             raise error
           end
-        rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => error
+        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
           if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
             raise error
           else
@@ -246,7 +264,7 @@ module Excon
 
     def write_block(data)
       @socket.write(data)
-    rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => error
+    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
       if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
         raise error
       else

data/lib/excon/ssl_socket.rb

@@ -12,6 +12,11 @@ module Excon
       # create ssl context
       ssl_context = OpenSSL::SSL::SSLContext.new
 
+      # set the security level before setting other parameters affected by it
+      if @data[:ssl_security_level]
+        ssl_context.security_level = @data[:ssl_security_level]
+      end
+
       # disable less secure options, when supported
       ssl_context_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
       if defined?(OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS)
@@ -27,26 +32,35 @@ module Excon
       if @data[:ssl_version]
         ssl_context.ssl_version = @data[:ssl_version]
       end
+      if @data[:ssl_min_version]
+        ssl_context.min_version = @data[:ssl_min_version]
+      end
+      if @data[:ssl_max_version]
+        ssl_context.max_version = @data[:ssl_max_version]
+      end
+
 
       if @data[:ssl_verify_peer]
         # turn verification on
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
 
-        if ca_file = @data[:ssl_ca_file] || ENV['SSL_CERT_FILE']
+        if (ca_file = @data[:ssl_ca_file] || ENV['SSL_CERT_FILE'])
           ssl_context.ca_file = ca_file
         end
-        if ca_path = @data[:ssl_ca_path] || ENV['SSL_CERT_DIR']
+        if (ca_path = @data[:ssl_ca_path] || ENV['SSL_CERT_DIR'])
           ssl_context.ca_path = ca_path
         end
-        if cert_store = @data[:ssl_cert_store]
+        if (cert_store = @data[:ssl_cert_store])
           ssl_context.cert_store = cert_store
         end
 
-        # no defaults, fallback to bundled
-        unless ca_file || ca_path || cert_store
+        if cert_store.nil?
           ssl_context.cert_store = OpenSSL::X509::Store.new
           ssl_context.cert_store.set_default_paths
+        end
 
+        # no defaults, fallback to bundled
+        unless ca_file || ca_path || cert_store
           # workaround issue #257 (JRUBY-6970)
           ca_file = DEFAULT_CA_FILE
           ca_file = ca_file.gsub(/^jar:/, '') if ca_file =~ /^jar:file:\//
@@ -58,7 +72,7 @@ module Excon
           end
         end
 
-        if verify_callback = @data[:ssl_verify_callback]
+        if (verify_callback = @data[:ssl_verify_callback])
           ssl_context.verify_callback = verify_callback
         end
       else
@@ -66,6 +80,9 @@ module Excon
         ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = @data[:ssl_verify_hostname] if ssl_context.respond_to?(:verify_hostname=)
+
       if client_cert_data && client_key_data
         ssl_context.cert = OpenSSL::X509::Certificate.new client_cert_data
         if OpenSSL::PKey.respond_to? :read
@@ -94,13 +111,20 @@ module Excon
 
         request += "Proxy-Connection: Keep-Alive#{Excon::CR_NL}"
 
+        if @data[:ssl_proxy_headers]
+          request << Utils.headers_hash_to_s(@data[:ssl_proxy_headers])
+        end
+
         request += Excon::CR_NL
 
         # write out the proxy setup request
         @socket.write(request)
 
         # eat the proxy's connection response
-        Excon::Response.parse(self,  :expects => 200, :method => 'CONNECT')
+        response = Excon::Response.parse(self,  :expects => 200, :method => 'CONNECT')
+        if response[:response][:status] != 200
+          raise(Excon::Errors::ProxyConnectionError.new("proxy connection could not be established", request, response))
+        end
       end
 
       # convert Socket to OpenSSL::SSL::SSLSocket
@@ -132,18 +156,16 @@ module Excon
       if @data[:ssl_verify_peer]
         @socket.post_connection_check(@data[:ssl_verify_peer_host] || @data[:host])
       end
-
-      @socket
     end
 
     private
 
     def client_cert_data
-      @client_cert_data ||= if ccd = @data[:client_cert_data]
+      @client_cert_data ||= if (ccd = @data[:client_cert_data])
                               ccd
-                            elsif path = @data[:client_cert]
+                            elsif (path = @data[:client_cert])
                               File.read path
-                            elsif path = @data[:certificate_path]
+                            elsif (path = @data[:certificate_path])
                               warn ":certificate_path is no longer supported and will be deprecated. Please use :client_cert or :client_cert_data"
                               File.read path
                             end
@@ -156,11 +178,11 @@ module Excon
     end
 
     def client_key_data
-      @client_key_data ||= if ckd = @data[:client_key_data]
+      @client_key_data ||= if (ckd = @data[:client_key_data])
                              ckd
-                           elsif path = @data[:client_key]
+                           elsif (path = @data[:client_key])
                              File.read path
-                           elsif path = @data[:private_key_path]
+                           elsif (path = @data[:private_key_path])
                              warn ":private_key_path is no longer supported and will be deprecated. Please use :client_key or :client_key_data"
                              File.read path
                            end

data/lib/excon/test/plugin/server/exec.rb

@@ -4,13 +4,16 @@ module Excon
       module Server
         module Exec
           def start(app_str = app)
+            open_process(app_str)
+            process_stderr = ""
             line = ''
-            open_process(app)
             until line =~ /\Aready\Z/
               line = error.gets
+              raise process_stderr if line.nil?
+              process_stderr << line
               fatal_time = elapsed_time > timeout
               if fatal_time
-                msg = "executable #{app} has taken too long to start"
+                msg = "executable #{app_str} has taken too long to start"
                 raise msg
               end
             end

data/lib/excon/test/plugin/server/puma.rb

@@ -4,10 +4,13 @@ module Excon
       module Server
         module Puma
           def start(app_str = app, bind_uri = bind)
-            open_process('puma', '-b', bind_uri.to_s, app_str)
+            open_process(RbConfig.ruby, '-S', 'puma', '-b', bind_uri.to_s, app_str)
+            process_stderr = ""
             line = ''
             until line =~ /Use Ctrl-C to stop/
               line = read.gets
+              raise process_stderr if line.nil?
+              process_stderr << line
               fatal_time = elapsed_time > timeout
               raise 'puma server has taken too long to start' if fatal_time
             end

data/lib/excon/test/plugin/server/unicorn.rb

@@ -13,6 +13,8 @@ module Excon
               bind_str = "#{host}:#{bind_uri.port}"
             end
             args = [ 
+              RbConfig.ruby,
+              '-S',
               'unicorn', 
               '--no-default-middleware', 
               '-l',
@@ -20,9 +22,12 @@ module Excon
               app_str
             ]
             open_process(*args)
+            process_stderr = ''
             line = ''
             until line =~ /worker\=0 ready/
               line = error.gets
+              raise process_stderr if line.nil?
+              process_stderr << line
               fatal_time = elapsed_time > timeout
               raise 'unicorn server has taken too long to start' if fatal_time
             end

data/lib/excon/test/plugin/server/webrick.rb

@@ -7,10 +7,13 @@ module Excon
             bind_uri = URI.parse(bind_uri) unless bind_uri.is_a? URI::Generic
             host = bind_uri.host.gsub(/[\[\]]/, '')
             port = bind_uri.port.to_s
-            open_process('rackup', '-s', 'webrick', '--host', host, '--port', port, app_str)
+            open_process(RbConfig.ruby, '-S', 'rackup', '-s', 'webrick', '--host', host, '--port', port, app_str)
+            process_stderr = ""
             line = ''
             until line =~ /HTTPServer#start/
               line = error.gets
+              raise process_stderr if line.nil?
+              process_stderr << line
               fatal_time = elapsed_time > timeout
               raise 'webrick server has taken too long to start' if fatal_time
             end

data/lib/excon/test/server.rb

@@ -72,7 +72,7 @@ module Excon
       end
       def dump_errors
         lines = error.read.split($/)
-        while line = lines.shift
+        while (line = lines.shift)
           case line
             when /(ERROR|Error)/
               unless line =~ /(null cert chain|did not return a certificate|SSL_read:: internal error)/

data/lib/excon/unix_socket.rb

@@ -20,6 +20,7 @@ module Excon
           begin
             @socket.connect_nonblock(sockaddr)
           rescue Errno::EISCONN
+            0 # same return as connect_nonblock success
           end
         end
       else

data/lib/excon/utils.rb

@@ -10,6 +10,18 @@ module Excon
     UNESCAPED = /([#{ Regexp.escape(CONTROL + ' ' + DELIMS + UNWISE + NONASCII) }])/
     ESCAPED   = /%([0-9a-fA-F]{2})/
 
+    def binary_encode(string)
+      if FORCE_ENC && string.encoding != Encoding::ASCII_8BIT
+        if string.frozen?
+          string.dup.force_encoding('BINARY')
+        else
+          string.force_encoding('BINARY')
+        end
+      else
+        string
+      end
+    end
+
     def connection_uri(datum = @data)
       unless datum
         raise ArgumentError, '`datum` must be given unless called on a Connection'
@@ -21,6 +33,30 @@ module Excon
       end
     end
 
+    # Redact sensitive info from provided data
+    def redact(datum)
+      datum = datum.dup
+      if datum.has_key?(:headers)
+        if datum[:headers].has_key?('Authorization') || datum[:headers].has_key?('Proxy-Authorization')
+          datum[:headers] = datum[:headers].dup
+        end
+        if datum[:headers].has_key?('Authorization')
+          datum[:headers]['Authorization'] = REDACTED
+        end
+        if datum[:headers].has_key?('Proxy-Authorization')
+          datum[:headers]['Proxy-Authorization'] = REDACTED
+        end
+      end
+      if datum.has_key?(:password)
+        datum[:password] = REDACTED
+      end
+      if datum.has_key?(:proxy) && datum[:proxy] && datum[:proxy].has_key?(:password)
+        datum[:proxy] = datum[:proxy].dup
+        datum[:proxy][:password] = REDACTED
+      end
+      datum
+    end
+
     def request_uri(datum)
       connection_uri(datum) + datum[:path] + query_string(datum)
     end
@@ -59,31 +95,49 @@ module Excon
     def split_header_value(str)
       return [] if str.nil?
       str = str.dup.strip
-      str.force_encoding('BINARY') if FORCE_ENC
-      str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+)
+      str = binary_encode(str)
+      str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",])+)
                     (?:,\s*|\Z)'xn).flatten
     end
 
     # Escapes HTTP reserved and unwise characters in +str+
     def escape_uri(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub(UNESCAPED) { "%%%02X" % $1[0].ord }
     end
 
     # Unescapes HTTP reserved and unwise characters in +str+
     def unescape_uri(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub(ESCAPED) { $1.hex.chr }
     end
 
     # Unescape form encoded values in +str+
     def unescape_form(str)
       str = str.dup
-      str.force_encoding('BINARY') if FORCE_ENC
+      str = binary_encode(str)
       str.gsub!(/\+/, ' ')
       str.gsub(ESCAPED) { $1.hex.chr }
     end
+
+    # Performs validation on the passed header hash and returns a string representation of the headers
+    def headers_hash_to_s(headers)
+      headers_str = String.new
+      headers.each do |key, values|
+        if key.to_s.match(/[\r\n]/)
+          raise Excon::Errors::InvalidHeaderKey.new(key.to_s.inspect + ' contains forbidden "\r" or "\n"')
+        end
+        [values].flatten.each do |value|
+          if value.to_s.match(/[\r\n]/)
+            # Don't include the potentially sensitive header value (i.e. authorization token) in the message
+            raise Excon::Errors::InvalidHeaderValue.new(key.to_s + ' header value contains forbidden "\r" or "\n"')
+          end
+          headers_str << key.to_s << ': ' << value.to_s << CR_NL
+        end
+      end
+      headers_str
+    end
   end
 end

data/lib/excon/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Excon
-  VERSION = '0.62.0'
+  VERSION = '0.91.0'
 end

data/lib/excon.rb

@@ -23,11 +23,11 @@ require 'excon/middlewares/instrumentor'
 require 'excon/middlewares/mock'
 require 'excon/middlewares/response_parser'
 
+require 'excon/error'
 require 'excon/constants'
 require 'excon/utils'
 
 require 'excon/connection'
-require 'excon/error'
 require 'excon/headers'
 require 'excon/response'
 require 'excon/middlewares/decompress'
@@ -61,6 +61,14 @@ module Excon
       if $VERBOSE || ENV['EXCON_DEBUG']
         $stderr.puts "[excon][WARNING] #{warning}\n#{ caller.join("\n") }"
       end
+
+      if @raise_on_warnings
+        raise Error::Warning.new(warning)
+      end
+    end
+
+    def set_raise_on_warnings!(should_raise)
+      @raise_on_warnings = should_raise
     end
 
     # Status of mocking
@@ -105,9 +113,9 @@ module Excon
 
     # @see Connection#initialize
     # Initializes a new keep-alive session for a given remote host
-    #   @param [String] url The destination URL
-    #   @param [Hash<Symbol, >] params One or more option params to set on the Connection instance
-    #   @return [Connection] A new Excon::Connection instance
+    # @param [String] url The destination URL
+    # @param [Hash<Symbol, >] params One or more option params to set on the Connection instance
+    # @return [Connection] A new Excon::Connection instance
     def new(url, params = {})
       uri_parser = params[:uri_parser] || defaults[:uri_parser]
       uri = uri_parser.parse(url)
@@ -135,13 +143,13 @@ module Excon
     end
 
     # push an additional stub onto the list to check for mock requests
-    #   @param [Hash<Symbol, >] request params to match against, omitted params match all
-    #   @param [Hash<Symbol, >] response params to return from matched request or block to call with params
-    def stub(request_params = {}, response_params = nil)
-      if method = request_params.delete(:method)
+    # @param request_params [Hash<Symbol, >] request params to match against, omitted params match all
+    # @param response_params [Hash<Symbol, >] response params to return from matched request or block to call with params
+    def stub(request_params = {}, response_params = nil, &block)
+      if (method = request_params.delete(:method))
         request_params[:method] = method.to_s.downcase.to_sym
       end
-      if url = request_params.delete(:url)
+      if (url = request_params.delete(:url))
         uri = URI.parse(url)
         request_params = {
           :host              => uri.host,
@@ -167,7 +175,7 @@ module Excon
         if response_params
           raise(ArgumentError.new("stub requires either response_params OR a block"))
         else
-          stub = [request_params, Proc.new]
+          stub = [request_params, block]
         end
       elsif response_params
         stub = [request_params, response_params]
@@ -179,10 +187,10 @@ module Excon
     end
 
     # get a stub matching params or nil
-    #   @param [Hash<Symbol, >] request params to match against, omitted params match all
-    #   @return [Hash<Symbol, >] response params to return from matched request or block to call with params
+    # @param request_params [Hash<Symbol, >] request params to match against, omitted params match all
+    # @return [Hash<Symbol, >] response params to return from matched request or block to call with params
     def stub_for(request_params={})
-      if method = request_params.delete(:method)
+      if (method = request_params.delete(:method))
         request_params[:method] = method.to_s.downcase.to_sym
       end
       Excon.stubs.each do |stub, response_params|
@@ -190,7 +198,7 @@ module Excon
         headers_match = !stub.has_key?(:headers) || stub[:headers].keys.all? do |key|
           case value = stub[:headers][key]
           when Regexp
-            if match = value.match(request_params[:headers][key])
+            if (match = value.match(request_params[:headers][key]))
               captures[:headers][key] = match.captures
             end
             match
@@ -201,7 +209,7 @@ module Excon
         non_headers_match = (stub.keys - [:headers]).all? do |key|
           case value = stub[key]
           when Regexp
-            if match = value.match(request_params[key])
+            if (match = value.match(request_params[key]))
               captures[key] = match.captures
             end
             match
@@ -228,8 +236,8 @@ module Excon
     end
 
     # remove first/oldest stub matching request_params
-    #   @param [Hash<Symbol, >] request params to match against, omitted params match all
-    #   @return [Hash<Symbol, >] response params from deleted stub
+    # @param request_params [Hash<Symbol, >] request params to match against, omitted params match all
+    # @return [Hash<Symbol, >] response params from deleted stub
     def unstub(request_params = {})
       stub = stub_for(request_params)
       Excon.stubs.delete_at(Excon.stubs.index(stub))