excon 1.3.0 → 1.8.0

data/excon.gemspec

@@ -21,15 +21,12 @@ Gem::Specification.new do |s|
     'README.md',
     'excon.gemspec'
   ]
-  s.required_ruby_version = '>= 3.1.0'
+  s.required_ruby_version = '>= 3.3.0'
 
   s.add_dependency 'logger'
 
   s.add_development_dependency('activesupport')
   s.add_development_dependency('delorean')
-  s.add_development_dependency('eventmachine', '>= 1.0.4')
-  s.add_development_dependency('json', '>= 1.8.5')
-  s.add_development_dependency('open4')
   s.add_development_dependency('puma')
   s.add_development_dependency('rake')
   s.add_development_dependency('rdoc')

data/lib/excon/connection.rb

@@ -60,6 +60,8 @@ module Excon
     # @option params [Fixnum] :retry_interval Set how long to wait between retries. (Default 0)
     # @option params [Class] :instrumentor Responds to #instrument as in ActiveSupport::Notifications
     # @option params [String] :instrumentor_name Name prefix for #instrument events.  Defaults to 'excon'
+    # @option params [Resolv, nil] :resolv_resolver A ready to use +Resolv+ resolver instance.
+    # @option params [Class] :resolver_factory Const of the resolver factory.  Defaults to 'Excon::ResolverFactory'
     def initialize(params = {})
       @pid = Process.pid
       @data = Excon.defaults.dup
@@ -100,6 +102,10 @@ module Excon
           @socket_key = "#{@data[:scheme]}://#{@data[:socket]}"
         end
       else
+        if @data.key?(:host) && @data[:host].nil?
+          raise Excon::Error::InvalidParameter, "host is required for non-Unix connections; " \
+                                                "the URL may be malformed (e.g. missing \"//\" after the scheme)"
+        end
         @socket_key = "#{@data[:scheme]}://#{@data[:host]}#{port_string(@data)}"
       end
       reset
@@ -236,7 +242,11 @@ module Excon
         datum[:deadline] = Process.clock_gettime(Process::CLOCK_MONOTONIC) + datum[:timeout]
       end
 
-      datum[:headers] = @data[:headers].merge(datum[:headers] || {})
+      # header names are case-insensitive, so request headers replace defaults regardless of casing
+      request_headers = datum[:headers] || {}
+      datum[:headers] = @data[:headers].reject do |key, _|
+        request_headers.keys.any? { |k| k.to_s.casecmp?(key.to_s) }
+      end.merge(request_headers)
 
       validate_params(:request, params, datum[:middlewares])
       # If the user passed in new middleware, we want to validate that the original connection parameters
@@ -254,6 +264,10 @@ module Excon
       if datum[:scheme] == UNIX
         datum[:headers][host_key] ||= ''
       else
+        if datum[:host].nil?
+          raise Excon::Error::InvalidParameter, "host is required for non-Unix connections; " \
+                                                "the URL may be malformed (e.g. missing \"//\" after the scheme)"
+        end
         datum[:headers][host_key] ||= datum[:host] + port_string(datum)
       end
 
@@ -483,6 +497,13 @@ module Excon
       unix_proxy = datum[:proxy] ? datum[:proxy][:scheme] == UNIX : false
       sockets[@socket_key] ||= if datum[:scheme] == UNIX || unix_proxy
         Excon::UnixSocket.new(datum)
+      elsif datum[:socks5_proxy]
+        # SOCKS5 proxy - use appropriate socket based on target scheme
+        if datum[:ssl_uri_schemes].include?(datum[:scheme])
+          Excon::SOCKS5SSLSocket.new(datum)
+        else
+          Excon::SOCKS5Socket.new(datum)
+        end
       elsif datum[:ssl_uri_schemes].include?(datum[:scheme])
         Excon::SSLSocket.new(datum)
       else
@@ -491,22 +512,21 @@ module Excon
     end
 
     def sockets
-      @_excon_sockets ||= {}
-      @_excon_sockets.compare_by_identity
-
       if @pid != Process.pid
-        @_excon_sockets.clear # GC will take care of closing sockets
+        @_excon_sockets&.clear # GC will take care of closing sockets
         @pid = Process.pid
       end
 
       if @data[:thread_safe_sockets]
+        @_excon_sockets ||= ObjectSpace::WeakKeyMap.new
+
         # In a multi-threaded world, if the same connection is used by multiple
         # threads at the same time to connect to the same destination, they may
         # stomp on each other's sockets.  This ensures every thread gets their
         # own socket cache, within the context of a single connection.
         @_excon_sockets[Thread.current] ||= {}
       else
-        @_excon_sockets
+        @_excon_sockets ||= {}.compare_by_identity
       end
     end
 

data/lib/excon/constants.rb

@@ -60,6 +60,7 @@ module Excon
     read_timeout
     request_block
     resolv_resolver
+    resolver_factory
     response_block
     stubs
     timeout
@@ -97,6 +98,7 @@ module Excon
     proxy
     scheme
     socket
+    socks5_proxy
     ssl_ca_file
     ssl_ca_path
     ssl_cert_store
@@ -170,6 +172,7 @@ module Excon
     persistent: false,
     read_timeout: 60,
     resolv_resolver: nil,
+    resolver_factory: Excon::ResolverFactory,
     retry_errors: DEFAULT_RETRY_ERRORS,
     retry_limit: DEFAULT_RETRY_LIMIT,
     ssl_verify_peer: true,

data/lib/excon/error.rb

@@ -48,6 +48,7 @@ or:
 
     class InvalidHeaderKey < Error; end
     class InvalidHeaderValue < Error; end
+    class InvalidParameter < Error; end
     class Timeout < Error; end
     class ResponseParse < Error; end
 

data/lib/excon/middlewares/redirect_follower.rb

@@ -2,6 +2,19 @@
 module Excon
   module Middleware
     class RedirectFollower < Excon::Middleware::Base
+      REDACTION_HEADERS = %w[
+        Authorization
+        Cookie
+        Cookie2
+        Host
+        Proxy-Connection
+        Proxy-Authorization
+        X-API-Key
+        X-Auth-Token
+        X-CSRF-Token
+        X-Session-ID
+      ]
+
       def self.valid_parameter_keys
         [
           :redirects_remaining,
@@ -54,10 +67,9 @@ module Excon
               params[:headers].delete('Content-Length')
             end
             params[:headers] = datum[:headers].dup
-            params[:headers].delete('Authorization')
-            params[:headers].delete('Proxy-Connection')
-            params[:headers].delete('Proxy-Authorization')
-            params[:headers].delete('Host')
+            REDACTION_HEADERS.each do |header|
+              params[:headers].delete(header)
+            end
             params.merge!(
               :scheme     => uri.scheme || datum[:scheme],
               :host       => uri.host   || datum[:host],

data/lib/excon/resolver_factory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module Excon
+  # This factory produces new +resolv+ gem resolver instances. Users who wants
+  # to configure a custom resolver (varying settings for varying resolvers) can
+  # provide a custom resolver factory class and configure it globally on the
+  # Excon defaults:
+  #
+  #   Excon.defaults[:resolver_factory] = MyCustomResolverFactory
+  #
+  # Then you just need to provide a static method called +.create_resolver+
+  # which returns a new +Resolv+ instance. This allows the customization.
+  class ResolverFactory
+    # @return [Resolv] the new resolver instance
+    def self.create_resolver
+      Resolv.new
+    end
+  end
+end

data/lib/excon/socket.rb

@@ -52,7 +52,8 @@ module Excon
       @data = data
       @nonblock = data[:nonblock]
       @port ||= @data[:port] || 80
-      @read_buffer = String.new
+      @read_buffer = String.new(encoding: Encoding::BINARY)
+      @read_buffer_scratch = String.new(encoding: Encoding::BINARY)
       @read_offset = 0
       @eof = false
       @backend_eof = false
@@ -132,7 +133,7 @@ module Excon
         family = @data[:proxy][:family]
       end
 
-      resolver = @data[:resolv_resolver] || Resolv.new
+      resolver = @data[:resolv_resolver] || @data[:resolver_factory].create_resolver
 
       # Deprecated
       if @data[:dns_timeouts]
@@ -239,7 +240,7 @@ module Excon
               # Avoid allocating a new buffer string when the read buffer is empty
               @read_buffer = @socket.read_nonblock(max_length, @read_buffer)
             else
-              @read_buffer << @socket.read_nonblock(max_length - readable_bytes)
+              @read_buffer << @socket.read_nonblock(max_length - readable_bytes, @read_buffer_scratch)
             end
           end
         else
@@ -248,19 +249,15 @@ module Excon
               # Avoid allocating a new buffer string when the read buffer is empty
               @read_buffer = @socket.read_nonblock(@data[:chunk_size], @read_buffer)
             else
-              @read_buffer << @socket.read_nonblock(@data[:chunk_size])
+              @read_buffer << @socket.read_nonblock(@data[:chunk_size], @read_buffer_scratch)
             end
           end
         end
-      rescue OpenSSL::SSL::SSLError => error
-        if error.message == 'read would block'
-          if @read_buffer.empty?
-            select_with_timeout(@socket, :read) && retry
-          end
-        else
-          raise(error)
-        end
-      rescue *READ_RETRY_EXCEPTION_CLASSES
+      rescue OpenSSL::SSL::SSLError => e
+        raise(e) unless e.message == 'read would block'
+
+        select_with_timeout(@socket, :read) && retry if @read_buffer.empty?
+      rescue *READ_RETRY_EXCEPTION_CLASSES => e
         if @read_buffer.empty?
           # if we didn't read anything, try again...
           select_with_timeout(@socket, :read) && retry
@@ -299,12 +296,10 @@ module Excon
 
     def read_block(max_length)
       @socket.read(max_length)
-    rescue OpenSSL::SSL::SSLError => error
-      if error.message == 'read would block'
-        select_with_timeout(@socket, :read) && retry
-      else
-        raise(error)
-      end
+    rescue OpenSSL::SSL::SSLError => e
+      select_with_timeout(@socket, :read) && retry if e.message == 'read would block'
+
+      raise(e)
     rescue *READ_RETRY_EXCEPTION_CLASSES
       select_with_timeout(@socket, :read) && retry
     rescue EOFError
@@ -327,12 +322,10 @@ module Excon
           else
             raise error
           end
-        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
-          if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
-            raise error
-          else
-            select_with_timeout(@socket, :write) && retry
-          end
+        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => e
+          raise e if e.is_a?(OpenSSL::SSL::SSLError) && e.message != 'write would block'
+
+          select_with_timeout(@socket, :write) && retry
         end
 
         # Fast, common case.
@@ -348,12 +341,10 @@ module Excon
 
     def write_block(data)
       @socket.write(data)
-    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
-      if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
-        raise error
-      else
-        select_with_timeout(@socket, :write) && retry
-      end
+    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => e
+      raise e if e.is_a?(OpenSSL::SSL::SSLError) && e.message != 'write would block'
+
+      select_with_timeout(@socket, :write) && retry
     end
 
     def select_with_timeout(socket, type)
@@ -373,25 +364,19 @@ module Excon
       end
 
       select = case type
-      when :connect_read
-        IO.select([socket], nil, nil, timeout)
-      when :connect_write
-        IO.select(nil, [socket], nil, timeout)
-      when :read
-        IO.select([socket], nil, nil, timeout)
-      when :write
-        IO.select(nil, [socket], nil, timeout)
-      end
+               when :connect_read, :read
+                 IO.select([socket], nil, nil, timeout)
+               when :connect_write, :write
+                 IO.select(nil, [socket], nil, timeout)
+               end
 
-      select || raise(Excon::Errors::Timeout.new("#{timeout_kind} timeout reached"))
+      select || raise(Excon::Errors::Timeout.new, "#{timeout_kind} timeout reached")
     end
 
     def unpacked_sockaddr
       @unpacked_sockaddr ||= ::Socket.unpack_sockaddr_in(@socket.to_io.getsockname)
     rescue ArgumentError => e
-      unless e.message == 'not an AF_INET/AF_INET6 sockaddr'
-        raise
-      end
+      raise unless e.message == 'not an AF_INET/AF_INET6 sockaddr'
     end
 
     # Returns the remaining time in seconds until we reach the deadline for the request timeout.

data/lib/excon/socks5.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Excon
+  # SOCKS5 protocol implementation (RFC 1928, RFC 1929)
+  # Shared module for SOCKS5Socket and SOCKS5SSLSocket
+  module SOCKS5
+    SOCKS5_VERSION = 0x05
+    SOCKS5_RESERVED = 0x00
+
+    # Authentication methods
+    SOCKS5_NO_AUTH = 0x00
+    SOCKS5_AUTH_USERNAME_PASSWORD = 0x02
+    SOCKS5_NO_ACCEPTABLE_AUTH = 0xFF
+
+    # Commands
+    SOCKS5_CMD_CONNECT = 0x01
+
+    # Address types
+    SOCKS5_ATYP_IPV4 = 0x01
+    SOCKS5_ATYP_DOMAIN = 0x03
+    SOCKS5_ATYP_IPV6 = 0x04
+
+    # Reply codes
+    SOCKS5_SUCCESS = 0x00
+    SOCKS5_ERRORS = {
+      0x01 => 'General SOCKS server failure',
+      0x02 => 'Connection not allowed by ruleset',
+      0x03 => 'Network unreachable',
+      0x04 => 'Host unreachable',
+      0x05 => 'Connection refused',
+      0x06 => 'TTL expired',
+      0x07 => 'Command not supported',
+      0x08 => 'Address type not supported'
+    }.freeze
+
+    # Maximum hostname length per RFC 1928
+    MAX_HOSTNAME_LENGTH = 255
+
+    private
+
+    # Parse SOCKS5 proxy string into components
+    # @param proxy_string [String] Proxy specification in various formats
+    # @return [Array<String, String, String, String>] host, port, user, pass
+    def parse_socks5_proxy(proxy_string)
+      # Support formats:
+      #   host:port
+      #   user:pass@host:port
+      #   socks5://host:port
+      #   socks5://user:pass@host:port
+      proxy_string = proxy_string.to_s.sub(%r{^socks5://}, '')
+
+      user = nil
+      pass = nil
+
+      if proxy_string.include?('@')
+        auth, host_port = proxy_string.split('@', 2)
+        user, pass = auth.split(':', 2)
+      else
+        host_port = proxy_string
+      end
+
+      host, port = host_port.split(':', 2)
+      port ||= '1080'
+
+      [host, port, user, pass]
+    end
+
+    # Perform SOCKS5 authentication handshake
+    def socks5_authenticate
+      auth_methods = if @proxy_user && @proxy_pass
+        [SOCKS5_NO_AUTH, SOCKS5_AUTH_USERNAME_PASSWORD]
+      else
+        [SOCKS5_NO_AUTH]
+      end
+
+      greeting = [SOCKS5_VERSION, auth_methods.length, *auth_methods].pack('C*')
+      @socket.write(greeting)
+
+      response = socks5_read_exactly(2)
+      version, chosen_method = response.unpack('CC')
+
+      if version != SOCKS5_VERSION
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned invalid version: #{version}"))
+      end
+
+      case chosen_method
+      when SOCKS5_NO_AUTH
+        # No authentication required
+      when SOCKS5_AUTH_USERNAME_PASSWORD
+        unless @proxy_user && @proxy_pass
+          raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy requires authentication but no credentials provided'))
+        end
+        socks5_username_password_auth
+      when SOCKS5_NO_ACCEPTABLE_AUTH
+        raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy: no acceptable authentication methods'))
+      else
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy: unsupported authentication method #{chosen_method}"))
+      end
+    end
+
+    # RFC 1929: Username/Password Authentication
+    def socks5_username_password_auth
+      auth_request = [
+        0x01, # auth protocol version
+        @proxy_user.bytesize,
+        @proxy_user,
+        @proxy_pass.bytesize,
+        @proxy_pass
+      ].pack('CCA*CA*')
+
+      @socket.write(auth_request)
+
+      response = socks5_read_exactly(2)
+      _, status = response.unpack('CC')
+
+      unless status == 0x00
+        raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy authentication failed'))
+      end
+    end
+
+    # Request connection to target through SOCKS5 proxy
+    def socks5_connect(host, port)
+      if host.bytesize > MAX_HOSTNAME_LENGTH
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5: hostname exceeds maximum length of #{MAX_HOSTNAME_LENGTH} bytes"))
+      end
+
+      # Build CONNECT request with domain name (let proxy resolve DNS)
+      request = [SOCKS5_VERSION, SOCKS5_CMD_CONNECT, SOCKS5_RESERVED].pack('CCC')
+      request += [SOCKS5_ATYP_DOMAIN, host.bytesize, host].pack('CCA*')
+      request += [port.to_i].pack('n')
+
+      @socket.write(request)
+
+      response = socks5_read_exactly(4)
+      version, reply, _, atyp = response.unpack('CCCC')
+
+      if version != SOCKS5_VERSION
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned invalid version: #{version}"))
+      end
+
+      unless reply == SOCKS5_SUCCESS
+        error_msg = SOCKS5_ERRORS[reply] || "Unknown error (#{reply})"
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy connect failed: #{error_msg}"))
+      end
+
+      # Read and discard bound address (not needed for CONNECT)
+      socks5_read_bound_address(atyp)
+    end
+
+    def socks5_read_bound_address(atyp)
+      case atyp
+      when SOCKS5_ATYP_IPV4
+        socks5_read_exactly(4 + 2) # 4 bytes IP + 2 bytes port
+      when SOCKS5_ATYP_DOMAIN
+        domain_len = socks5_read_exactly(1).unpack1('C')
+        socks5_read_exactly(domain_len + 2)
+      when SOCKS5_ATYP_IPV6
+        socks5_read_exactly(16 + 2) # 16 bytes IP + 2 bytes port
+      else
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned unknown address type: #{atyp}"))
+      end
+    end
+
+    # Read exact number of bytes with timeout support
+    def socks5_read_exactly(nbytes)
+      data = ''.dup
+      deadline = @data[:read_timeout] ? Time.now + @data[:read_timeout] : nil
+
+      while data.bytesize < nbytes
+        if deadline
+          remaining = deadline - Time.now
+          if remaining <= 0
+            raise Excon::Error::Timeout.new('SOCKS5 read timeout')
+          end
+          ready = IO.select([@socket], nil, nil, remaining)
+          unless ready
+            raise Excon::Error::Timeout.new('SOCKS5 read timeout')
+          end
+        end
+
+        chunk = @socket.read_nonblock(nbytes - data.bytesize, exception: false)
+        case chunk
+        when :wait_readable
+          IO.select([@socket], nil, nil, deadline ? [deadline - Time.now, 0].max : nil)
+        when nil, ''
+          raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy connection closed unexpectedly'))
+        else
+          data << chunk
+        end
+      end
+      data
+    end
+  end
+end

data/lib/excon/socks5_socket.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Excon
+  class SOCKS5Socket < Socket
+    include SOCKS5
+
+    def initialize(data = {})
+      @socks5_proxy = data[:socks5_proxy]
+      @proxy_host, @proxy_port, @proxy_user, @proxy_pass = parse_socks5_proxy(@socks5_proxy)
+      super(data)
+    end
+
+    private
+
+    # Proxy-swap pattern: temporarily set @data[:proxy] to the SOCKS5 proxy
+    # so that Socket#connect routes the TCP connection there (inheriting DNS
+    # resolution, nonblock, retry, keepalive, reuseaddr, remote_ip tracking).
+    # After TCP is up, clear :proxy and run the SOCKS5 handshake.
+    def connect
+      @data[:proxy] = {
+        host:     @proxy_host,
+        hostname: @proxy_host,
+        port:     @proxy_port.to_i
+      }
+
+      begin
+        super
+      ensure
+        @data.delete(:proxy)
+      end
+
+      socks5_authenticate
+      socks5_connect(@data[:host], @data[:port])
+    end
+  end
+end

data/lib/excon/socks5_ssl_socket.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Excon
+  class SOCKS5SSLSocket < SSLSocket
+    include SOCKS5
+
+    def initialize(data = {})
+      @socks5_proxy = data[:socks5_proxy]
+      @proxy_host, @proxy_port, @proxy_user, @proxy_pass = parse_socks5_proxy(@socks5_proxy)
+      super(data)
+    end
+
+    private
+
+    # Proxy-swap pattern (same as SOCKS5Socket#connect).
+    #
+    # Call chain:
+    #   SOCKS5SSLSocket#initialize -> super (SSLSocket#initialize)
+    #     -> super (Socket#initialize) -> connect
+    #       -> SOCKS5SSLSocket#connect (this method)
+    #         -> super -> SSLSocket#connect -> Socket#connect (TCP to proxy)
+    #       -> SOCKS5 handshake on raw TCP socket
+    #     <- returns to SSLSocket#initialize
+    #     -> @data[:proxy] is nil, so HTTP CONNECT is skipped (line 111)
+    #     -> SSL wrapping on the SOCKS5-tunneled socket
+    def connect
+      @data[:proxy] = {
+        host:     @proxy_host,
+        hostname: @proxy_host,
+        port:     @proxy_port.to_i
+      }
+
+      begin
+        super
+      ensure
+        # Clear :proxy so SSLSocket#initialize skips HTTP CONNECT (line 111)
+        @data.delete(:proxy)
+      end
+
+      socks5_authenticate
+      socks5_connect(@data[:host], @data[:port])
+    end
+  end
+end

data/lib/excon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Excon
-  VERSION = '1.3.0'
+  VERSION = "1.8.0"
 end

data/lib/excon.rb

@@ -3,7 +3,7 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
-require 'cgi'
+require 'cgi/escape'
 require 'forwardable'
 require 'ipaddr'
 require 'openssl'
@@ -27,6 +27,7 @@ require 'excon/middlewares/mock'
 require 'excon/middlewares/response_parser'
 
 require 'excon/error'
+require 'excon/resolver_factory'
 require 'excon/constants'
 require 'excon/utils'
 
@@ -35,10 +36,12 @@ require 'excon/headers'
 require 'excon/response'
 require 'excon/middlewares/escape_path'
 require 'excon/middlewares/redirect_follower'
-require 'excon/middlewares/capture_cookies'
 require 'excon/pretty_printer'
 require 'excon/socket'
 require 'excon/ssl_socket'
+require 'excon/socks5'
+require 'excon/socks5_socket'
+require 'excon/socks5_ssl_socket'
 require 'excon/instrumentors/standard_instrumentor'
 require 'excon/instrumentors/logging_instrumentor'
 require 'excon/unix_socket'