excon 1.3.0 → 1.5.0

data/lib/excon/connection.rb

@@ -60,6 +60,8 @@ module Excon
     # @option params [Fixnum] :retry_interval Set how long to wait between retries. (Default 0)
     # @option params [Class] :instrumentor Responds to #instrument as in ActiveSupport::Notifications
     # @option params [String] :instrumentor_name Name prefix for #instrument events.  Defaults to 'excon'
+    # @option params [Resolv, nil] :resolv_resolver A ready to use +Resolv+ resolver instance.
+    # @option params [Class] :resolver_factory Const of the resolver factory.  Defaults to 'Excon::ResolverFactory'
     def initialize(params = {})
       @pid = Process.pid
       @data = Excon.defaults.dup
@@ -100,6 +102,10 @@ module Excon
           @socket_key = "#{@data[:scheme]}://#{@data[:socket]}"
         end
       else
+        if @data.key?(:host) && @data[:host].nil?
+          raise Excon::Error::InvalidParameter, "host is required for non-Unix connections; " \
+                                                "the URL may be malformed (e.g. missing \"//\" after the scheme)"
+        end
         @socket_key = "#{@data[:scheme]}://#{@data[:host]}#{port_string(@data)}"
       end
       reset
@@ -254,6 +260,10 @@ module Excon
       if datum[:scheme] == UNIX
         datum[:headers][host_key] ||= ''
       else
+        if datum[:host].nil?
+          raise Excon::Error::InvalidParameter, "host is required for non-Unix connections; " \
+                                                "the URL may be malformed (e.g. missing \"//\" after the scheme)"
+        end
         datum[:headers][host_key] ||= datum[:host] + port_string(datum)
       end
 
@@ -483,6 +493,13 @@ module Excon
       unix_proxy = datum[:proxy] ? datum[:proxy][:scheme] == UNIX : false
       sockets[@socket_key] ||= if datum[:scheme] == UNIX || unix_proxy
         Excon::UnixSocket.new(datum)
+      elsif datum[:socks5_proxy]
+        # SOCKS5 proxy - use appropriate socket based on target scheme
+        if datum[:ssl_uri_schemes].include?(datum[:scheme])
+          Excon::SOCKS5SSLSocket.new(datum)
+        else
+          Excon::SOCKS5Socket.new(datum)
+        end
       elsif datum[:ssl_uri_schemes].include?(datum[:scheme])
         Excon::SSLSocket.new(datum)
       else

data/lib/excon/constants.rb

@@ -60,6 +60,7 @@ module Excon
     read_timeout
     request_block
     resolv_resolver
+    resolver_factory
     response_block
     stubs
     timeout
@@ -97,6 +98,7 @@ module Excon
     proxy
     scheme
     socket
+    socks5_proxy
     ssl_ca_file
     ssl_ca_path
     ssl_cert_store
@@ -170,6 +172,7 @@ module Excon
     persistent: false,
     read_timeout: 60,
     resolv_resolver: nil,
+    resolver_factory: Excon::ResolverFactory,
     retry_errors: DEFAULT_RETRY_ERRORS,
     retry_limit: DEFAULT_RETRY_LIMIT,
     ssl_verify_peer: true,

data/lib/excon/error.rb

@@ -48,6 +48,7 @@ or:
 
     class InvalidHeaderKey < Error; end
     class InvalidHeaderValue < Error; end
+    class InvalidParameter < Error; end
     class Timeout < Error; end
     class ResponseParse < Error; end
 

data/lib/excon/middlewares/redirect_follower.rb

@@ -2,6 +2,19 @@
 module Excon
   module Middleware
     class RedirectFollower < Excon::Middleware::Base
+      REDACTION_HEADERS = %w[
+        Authorization
+        Cookie
+        Cookie2
+        Host
+        Proxy-Connection
+        Proxy-Authorization
+        X-API-Key
+        X-Auth-Token
+        X-CSRF-Token
+        X-Session-ID
+      ]
+
       def self.valid_parameter_keys
         [
           :redirects_remaining,
@@ -54,10 +67,9 @@ module Excon
               params[:headers].delete('Content-Length')
             end
             params[:headers] = datum[:headers].dup
-            params[:headers].delete('Authorization')
-            params[:headers].delete('Proxy-Connection')
-            params[:headers].delete('Proxy-Authorization')
-            params[:headers].delete('Host')
+            REDACTION_HEADERS.each do |header|
+              params[:headers].delete(header)
+            end
             params.merge!(
               :scheme     => uri.scheme || datum[:scheme],
               :host       => uri.host   || datum[:host],

data/lib/excon/resolver_factory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module Excon
+  # This factory produces new +resolv+ gem resolver instances. Users who wants
+  # to configure a custom resolver (varying settings for varying resolvers) can
+  # provide a custom resolver factory class and configure it globally on the
+  # Excon defaults:
+  #
+  #   Excon.defaults[:resolver_factory] = MyCustomResolverFactory
+  #
+  # Then you just need to provide a static method called +.create_resolver+
+  # which returns a new +Resolv+ instance. This allows the customization.
+  class ResolverFactory
+    # @return [Resolv] the new resolver instance
+    def self.create_resolver
+      Resolv.new
+    end
+  end
+end

data/lib/excon/socket.rb

@@ -132,7 +132,7 @@ module Excon
         family = @data[:proxy][:family]
       end
 
-      resolver = @data[:resolv_resolver] || Resolv.new
+      resolver = @data[:resolv_resolver] || @data[:resolver_factory].create_resolver
 
       # Deprecated
       if @data[:dns_timeouts]
@@ -252,15 +252,11 @@ module Excon
             end
           end
         end
-      rescue OpenSSL::SSL::SSLError => error
-        if error.message == 'read would block'
-          if @read_buffer.empty?
-            select_with_timeout(@socket, :read) && retry
-          end
-        else
-          raise(error)
-        end
-      rescue *READ_RETRY_EXCEPTION_CLASSES
+      rescue OpenSSL::SSL::SSLError => e
+        raise(e) unless e.message == 'read would block'
+
+        select_with_timeout(@socket, :read) && retry if @read_buffer.empty?
+      rescue *READ_RETRY_EXCEPTION_CLASSES => e
         if @read_buffer.empty?
           # if we didn't read anything, try again...
           select_with_timeout(@socket, :read) && retry
@@ -299,12 +295,10 @@ module Excon
 
     def read_block(max_length)
       @socket.read(max_length)
-    rescue OpenSSL::SSL::SSLError => error
-      if error.message == 'read would block'
-        select_with_timeout(@socket, :read) && retry
-      else
-        raise(error)
-      end
+    rescue OpenSSL::SSL::SSLError => e
+      select_with_timeout(@socket, :read) && retry if e.message == 'read would block'
+
+      raise(error)
     rescue *READ_RETRY_EXCEPTION_CLASSES
       select_with_timeout(@socket, :read) && retry
     rescue EOFError
@@ -327,12 +321,10 @@ module Excon
           else
             raise error
           end
-        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
-          if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
-            raise error
-          else
-            select_with_timeout(@socket, :write) && retry
-          end
+        rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => e
+          raise e if error.is_a?(OpenSSL::SSL::SSLError) && e.message != 'write would block'
+
+          select_with_timeout(@socket, :write) && retry
         end
 
         # Fast, common case.
@@ -348,12 +340,10 @@ module Excon
 
     def write_block(data)
       @socket.write(data)
-    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => error
-      if error.is_a?(OpenSSL::SSL::SSLError) && error.message != 'write would block'
-        raise error
-      else
-        select_with_timeout(@socket, :write) && retry
-      end
+    rescue OpenSSL::SSL::SSLError, *WRITE_RETRY_EXCEPTION_CLASSES => e
+      raise e if e.is_a?(OpenSSL::SSL::SSLError) && e.message != 'write would block'
+
+      select_with_timeout(@socket, :write) && retry
     end
 
     def select_with_timeout(socket, type)
@@ -373,25 +363,19 @@ module Excon
       end
 
       select = case type
-      when :connect_read
-        IO.select([socket], nil, nil, timeout)
-      when :connect_write
-        IO.select(nil, [socket], nil, timeout)
-      when :read
-        IO.select([socket], nil, nil, timeout)
-      when :write
-        IO.select(nil, [socket], nil, timeout)
-      end
+               when :connect_read, :read
+                 IO.select([socket], nil, nil, timeout)
+               when :connect_write, :write
+                 IO.select(nil, [socket], nil, timeout)
+               end
 
-      select || raise(Excon::Errors::Timeout.new("#{timeout_kind} timeout reached"))
+      select || raise(Excon::Errors::Timeout.new, "#{timeout_kind} timeout reached")
     end
 
     def unpacked_sockaddr
       @unpacked_sockaddr ||= ::Socket.unpack_sockaddr_in(@socket.to_io.getsockname)
     rescue ArgumentError => e
-      unless e.message == 'not an AF_INET/AF_INET6 sockaddr'
-        raise
-      end
+      raise unless e.message == 'not an AF_INET/AF_INET6 sockaddr'
     end
 
     # Returns the remaining time in seconds until we reach the deadline for the request timeout.

data/lib/excon/socks5.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Excon
+  # SOCKS5 protocol implementation (RFC 1928, RFC 1929)
+  # Shared module for SOCKS5Socket and SOCKS5SSLSocket
+  module SOCKS5
+    SOCKS5_VERSION = 0x05
+    SOCKS5_RESERVED = 0x00
+
+    # Authentication methods
+    SOCKS5_NO_AUTH = 0x00
+    SOCKS5_AUTH_USERNAME_PASSWORD = 0x02
+    SOCKS5_NO_ACCEPTABLE_AUTH = 0xFF
+
+    # Commands
+    SOCKS5_CMD_CONNECT = 0x01
+
+    # Address types
+    SOCKS5_ATYP_IPV4 = 0x01
+    SOCKS5_ATYP_DOMAIN = 0x03
+    SOCKS5_ATYP_IPV6 = 0x04
+
+    # Reply codes
+    SOCKS5_SUCCESS = 0x00
+    SOCKS5_ERRORS = {
+      0x01 => 'General SOCKS server failure',
+      0x02 => 'Connection not allowed by ruleset',
+      0x03 => 'Network unreachable',
+      0x04 => 'Host unreachable',
+      0x05 => 'Connection refused',
+      0x06 => 'TTL expired',
+      0x07 => 'Command not supported',
+      0x08 => 'Address type not supported'
+    }.freeze
+
+    # Maximum hostname length per RFC 1928
+    MAX_HOSTNAME_LENGTH = 255
+
+    private
+
+    # Parse SOCKS5 proxy string into components
+    # @param proxy_string [String] Proxy specification in various formats
+    # @return [Array<String, String, String, String>] host, port, user, pass
+    def parse_socks5_proxy(proxy_string)
+      # Support formats:
+      #   host:port
+      #   user:pass@host:port
+      #   socks5://host:port
+      #   socks5://user:pass@host:port
+      proxy_string = proxy_string.to_s.sub(%r{^socks5://}, '')
+
+      user = nil
+      pass = nil
+
+      if proxy_string.include?('@')
+        auth, host_port = proxy_string.split('@', 2)
+        user, pass = auth.split(':', 2)
+      else
+        host_port = proxy_string
+      end
+
+      host, port = host_port.split(':', 2)
+      port ||= '1080'
+
+      [host, port, user, pass]
+    end
+
+    # Perform SOCKS5 authentication handshake
+    def socks5_authenticate
+      auth_methods = if @proxy_user && @proxy_pass
+        [SOCKS5_NO_AUTH, SOCKS5_AUTH_USERNAME_PASSWORD]
+      else
+        [SOCKS5_NO_AUTH]
+      end
+
+      greeting = [SOCKS5_VERSION, auth_methods.length, *auth_methods].pack('C*')
+      @socket.write(greeting)
+
+      response = socks5_read_exactly(2)
+      version, chosen_method = response.unpack('CC')
+
+      if version != SOCKS5_VERSION
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned invalid version: #{version}"))
+      end
+
+      case chosen_method
+      when SOCKS5_NO_AUTH
+        # No authentication required
+      when SOCKS5_AUTH_USERNAME_PASSWORD
+        unless @proxy_user && @proxy_pass
+          raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy requires authentication but no credentials provided'))
+        end
+        socks5_username_password_auth
+      when SOCKS5_NO_ACCEPTABLE_AUTH
+        raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy: no acceptable authentication methods'))
+      else
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy: unsupported authentication method #{chosen_method}"))
+      end
+    end
+
+    # RFC 1929: Username/Password Authentication
+    def socks5_username_password_auth
+      auth_request = [
+        0x01, # auth protocol version
+        @proxy_user.bytesize,
+        @proxy_user,
+        @proxy_pass.bytesize,
+        @proxy_pass
+      ].pack('CCA*CA*')
+
+      @socket.write(auth_request)
+
+      response = socks5_read_exactly(2)
+      _, status = response.unpack('CC')
+
+      unless status == 0x00
+        raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy authentication failed'))
+      end
+    end
+
+    # Request connection to target through SOCKS5 proxy
+    def socks5_connect(host, port)
+      if host.bytesize > MAX_HOSTNAME_LENGTH
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5: hostname exceeds maximum length of #{MAX_HOSTNAME_LENGTH} bytes"))
+      end
+
+      # Build CONNECT request with domain name (let proxy resolve DNS)
+      request = [SOCKS5_VERSION, SOCKS5_CMD_CONNECT, SOCKS5_RESERVED].pack('CCC')
+      request += [SOCKS5_ATYP_DOMAIN, host.bytesize, host].pack('CCA*')
+      request += [port.to_i].pack('n')
+
+      @socket.write(request)
+
+      response = socks5_read_exactly(4)
+      version, reply, _, atyp = response.unpack('CCCC')
+
+      if version != SOCKS5_VERSION
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned invalid version: #{version}"))
+      end
+
+      unless reply == SOCKS5_SUCCESS
+        error_msg = SOCKS5_ERRORS[reply] || "Unknown error (#{reply})"
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy connect failed: #{error_msg}"))
+      end
+
+      # Read and discard bound address (not needed for CONNECT)
+      socks5_read_bound_address(atyp)
+    end
+
+    def socks5_read_bound_address(atyp)
+      case atyp
+      when SOCKS5_ATYP_IPV4
+        socks5_read_exactly(4 + 2) # 4 bytes IP + 2 bytes port
+      when SOCKS5_ATYP_DOMAIN
+        domain_len = socks5_read_exactly(1).unpack1('C')
+        socks5_read_exactly(domain_len + 2)
+      when SOCKS5_ATYP_IPV6
+        socks5_read_exactly(16 + 2) # 16 bytes IP + 2 bytes port
+      else
+        raise Excon::Error::Socket.new(Exception.new("SOCKS5 proxy returned unknown address type: #{atyp}"))
+      end
+    end
+
+    # Read exact number of bytes with timeout support
+    def socks5_read_exactly(nbytes)
+      data = ''.dup
+      deadline = @data[:read_timeout] ? Time.now + @data[:read_timeout] : nil
+
+      while data.bytesize < nbytes
+        if deadline
+          remaining = deadline - Time.now
+          if remaining <= 0
+            raise Excon::Error::Timeout.new('SOCKS5 read timeout')
+          end
+          ready = IO.select([@socket], nil, nil, remaining)
+          unless ready
+            raise Excon::Error::Timeout.new('SOCKS5 read timeout')
+          end
+        end
+
+        chunk = @socket.read_nonblock(nbytes - data.bytesize, exception: false)
+        case chunk
+        when :wait_readable
+          IO.select([@socket], nil, nil, deadline ? [deadline - Time.now, 0].max : nil)
+        when nil, ''
+          raise Excon::Error::Socket.new(Exception.new('SOCKS5 proxy connection closed unexpectedly'))
+        else
+          data << chunk
+        end
+      end
+      data
+    end
+  end
+end

data/lib/excon/socks5_socket.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Excon
+  class SOCKS5Socket < Socket
+    include SOCKS5
+
+    def initialize(data = {})
+      @socks5_proxy = data[:socks5_proxy]
+      @proxy_host, @proxy_port, @proxy_user, @proxy_pass = parse_socks5_proxy(@socks5_proxy)
+      super(data)
+    end
+
+    private
+
+    # Proxy-swap pattern: temporarily set @data[:proxy] to the SOCKS5 proxy
+    # so that Socket#connect routes the TCP connection there (inheriting DNS
+    # resolution, nonblock, retry, keepalive, reuseaddr, remote_ip tracking).
+    # After TCP is up, clear :proxy and run the SOCKS5 handshake.
+    def connect
+      @data[:proxy] = {
+        host:     @proxy_host,
+        hostname: @proxy_host,
+        port:     @proxy_port.to_i
+      }
+
+      begin
+        super
+      ensure
+        @data.delete(:proxy)
+      end
+
+      socks5_authenticate
+      socks5_connect(@data[:host], @data[:port])
+    end
+  end
+end

data/lib/excon/socks5_ssl_socket.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Excon
+  class SOCKS5SSLSocket < SSLSocket
+    include SOCKS5
+
+    def initialize(data = {})
+      @socks5_proxy = data[:socks5_proxy]
+      @proxy_host, @proxy_port, @proxy_user, @proxy_pass = parse_socks5_proxy(@socks5_proxy)
+      super(data)
+    end
+
+    private
+
+    # Proxy-swap pattern (same as SOCKS5Socket#connect).
+    #
+    # Call chain:
+    #   SOCKS5SSLSocket#initialize -> super (SSLSocket#initialize)
+    #     -> super (Socket#initialize) -> connect
+    #       -> SOCKS5SSLSocket#connect (this method)
+    #         -> super -> SSLSocket#connect -> Socket#connect (TCP to proxy)
+    #       -> SOCKS5 handshake on raw TCP socket
+    #     <- returns to SSLSocket#initialize
+    #     -> @data[:proxy] is nil, so HTTP CONNECT is skipped (line 111)
+    #     -> SSL wrapping on the SOCKS5-tunneled socket
+    def connect
+      @data[:proxy] = {
+        host:     @proxy_host,
+        hostname: @proxy_host,
+        port:     @proxy_port.to_i
+      }
+
+      begin
+        super
+      ensure
+        # Clear :proxy so SSLSocket#initialize skips HTTP CONNECT (line 111)
+        @data.delete(:proxy)
+      end
+
+      socks5_authenticate
+      socks5_connect(@data[:host], @data[:port])
+    end
+  end
+end

data/lib/excon/test/plugin/server/webrick.rb

@@ -10,7 +10,7 @@ module Excon
             open_process(RbConfig.ruby, '-S', 'rackup', '-s', 'webrick', '--host', host, '--port', port, app_str)
             process_stderr = ""
             line = ''
-            until line.include?('HTTPServer#start')
+            until line.include?('Server#start')
               line = error.gets
               raise process_stderr if line.nil?
               process_stderr << line

data/lib/excon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Excon
-  VERSION = '1.3.0'
+  VERSION = '1.5.0'
 end

data/lib/excon.rb

@@ -3,7 +3,7 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
-require 'cgi'
+require 'cgi/escape'
 require 'forwardable'
 require 'ipaddr'
 require 'openssl'
@@ -27,6 +27,7 @@ require 'excon/middlewares/mock'
 require 'excon/middlewares/response_parser'
 
 require 'excon/error'
+require 'excon/resolver_factory'
 require 'excon/constants'
 require 'excon/utils'
 
@@ -35,10 +36,12 @@ require 'excon/headers'
 require 'excon/response'
 require 'excon/middlewares/escape_path'
 require 'excon/middlewares/redirect_follower'
-require 'excon/middlewares/capture_cookies'
 require 'excon/pretty_printer'
 require 'excon/socket'
 require 'excon/ssl_socket'
+require 'excon/socks5'
+require 'excon/socks5_socket'
+require 'excon/socks5_ssl_socket'
 require 'excon/instrumentors/standard_instrumentor'
 require 'excon/instrumentors/logging_instrumentor'
 require 'excon/unix_socket'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: excon
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - dpiddy (Dan Peterson)
@@ -231,7 +231,6 @@ files:
 - lib/excon/instrumentors/logging_instrumentor.rb
 - lib/excon/instrumentors/standard_instrumentor.rb
 - lib/excon/middlewares/base.rb
-- lib/excon/middlewares/capture_cookies.rb
 - lib/excon/middlewares/decompress.rb
 - lib/excon/middlewares/escape_path.rb
 - lib/excon/middlewares/expects.rb
@@ -241,8 +240,12 @@ files:
 - lib/excon/middlewares/redirect_follower.rb
 - lib/excon/middlewares/response_parser.rb
 - lib/excon/pretty_printer.rb
+- lib/excon/resolver_factory.rb
 - lib/excon/response.rb
 - lib/excon/socket.rb
+- lib/excon/socks5.rb
+- lib/excon/socks5_socket.rb
+- lib/excon/socks5_ssl_socket.rb
 - lib/excon/ssl_socket.rb
 - lib/excon/test/plugin/server/exec.rb
 - lib/excon/test/plugin/server/puma.rb
@@ -278,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: speed, persistence, http(s)
 test_files: []

data/lib/excon/middlewares/capture_cookies.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-module Excon
-  module Middleware
-    class CaptureCookies < Excon::Middleware::Base
-
-      def extract_cookies_from_set_cookie(set_cookie)
-        set_cookie.split(',').map { |full| full.split(';').first.strip }.join('; ')
-      end
-
-      def get_header(datum, header)
-        _, header_value = datum[:response][:headers].detect do |key, _|
-          key.casecmp?(header)
-        end
-        header_value
-      end
-
-      def response_call(datum)
-        cookie = get_header(datum, 'Set-Cookie')
-        if cookie
-          cookie = extract_cookies_from_set_cookie(cookie)
-          unless datum[:headers].key?("Cookie")
-            datum[:headers]["Cookie"] = cookie
-          else
-            original_cookies = datum[:headers]["Cookie"]
-            datum[:headers]["Cookie"] = original_cookies.empty? ? cookie : [original_cookies,cookie].join('; ')
-          end
-        end
-        super(datum)
-      end
-    end
-  end
-end