httpx-patched 1.6.2.1

data/lib/httpx/plugins/proxy/socks5.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class Socks5Error < ProxyError; end
+
+  module Plugins
+    module Proxy
+      module Socks5
+        VERSION = 5
+        NOAUTH = 0
+        PASSWD = 2
+        NONE   = 0xff
+        CONNECT = 1
+        IPV4 = 1
+        DOMAIN = 3
+        IPV6 = 4
+        SUCCESS = 0
+
+        Error = Socks5Error
+
+        class << self
+          def load_dependencies(*)
+            require_relative "../auth/socks5"
+          end
+
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
+          end
+        end
+
+        module ConnectionMethods
+          def call
+            super
+
+            return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
+
+            case @state
+            when :connecting,
+                 :negotiating,
+                 :authenticating
+              consume
+            end
+          end
+
+          def connecting?
+            super || @state == :authenticating || @state == :negotiating
+          end
+
+          def interests
+            if @state == :connecting || @state == :authenticating || @state == :negotiating
+              return @write_buffer.empty? ? :r : :w
+            end
+
+            super
+          end
+
+          private
+
+          def handle_transition(nextstate)
+            return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
+
+            case nextstate
+            when :connecting
+              return unless @state == :idle
+
+              @io.connect
+              return unless @io.connected?
+
+              @write_buffer << Packet.negotiate(@options.proxy)
+              __socks5_proxy_connect
+            when :authenticating
+              return unless @state == :connecting
+
+              @write_buffer << Packet.authenticate(@options.proxy)
+            when :negotiating
+              return unless @state == :connecting || @state == :authenticating
+
+              req = @pending.first
+              request_uri = req.uri
+              @write_buffer << Packet.connect(request_uri)
+            when :connected
+              return unless @state == :negotiating
+
+              @parser = nil
+            end
+            log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
+            super
+          end
+
+          def __socks5_proxy_connect
+            @parser = SocksParser.new(@write_buffer, @options)
+            @parser.on(:packet, &method(:__socks5_on_packet))
+            transition(:negotiating)
+          end
+
+          def __socks5_on_packet(packet)
+            case @state
+            when :connecting
+              version, method = packet.unpack("CC")
+              __socks5_check_version(version)
+              case method
+              when PASSWD
+                transition(:authenticating)
+                nil
+              when NONE
+                __on_socks5_error("no supported authorization methods")
+              else
+                transition(:negotiating)
+              end
+            when :authenticating
+              _, status = packet.unpack("CC")
+              return transition(:negotiating) if status == SUCCESS
+
+              __on_socks5_error("socks authentication error: #{status}")
+            when :negotiating
+              version, reply, = packet.unpack("CC")
+              __socks5_check_version(version)
+              __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
+              req = @pending.first
+              request_uri = req.uri
+              @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
+              transition(:connected)
+              throw(:called)
+            end
+          end
+
+          def __socks5_check_version(version)
+            __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
+          end
+
+          def __on_socks5_error(message)
+            ex = Error.new(message)
+            ex.set_backtrace(caller)
+            on_error(ex)
+            throw(:called)
+          end
+        end
+
+        class SocksParser
+          include HTTPX::Callbacks
+
+          def initialize(buffer, options)
+            @buffer = buffer
+            @options = options
+          end
+
+          def close; end
+
+          def consume(*); end
+
+          def empty?
+            true
+          end
+
+          def <<(packet)
+            emit(:packet, packet)
+          end
+        end
+
+        module Packet
+          module_function
+
+          def negotiate(parameters)
+            methods = [NOAUTH]
+            methods << PASSWD if parameters.can_authenticate?
+            methods.unshift(methods.size)
+            methods.unshift(VERSION)
+            methods.pack("C*")
+          end
+
+          def authenticate(parameters)
+            parameters.authenticate
+          end
+
+          def connect(uri)
+            packet = [VERSION, CONNECT, 0].pack("C*")
+            begin
+              ip = IPAddr.new(uri.host)
+
+              ipcode = ip.ipv6? ? IPV6 : IPV4
+
+              packet << [ipcode].pack("C") << ip.hton
+            rescue IPAddr::InvalidAddressError
+              packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
+            end
+            packet << [uri.port].pack("n")
+            packet
+          end
+        end
+      end
+    end
+    register_plugin :"proxy/socks5", Proxy::Socks5
+  end
+end

data/lib/httpx/plugins/proxy/ssh.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "httpx/plugins/proxy"
+
+module HTTPX
+  module Plugins
+    module Proxy
+      module SSH
+        class << self
+          def load_dependencies(*)
+            require "net/ssh/gateway"
+          end
+        end
+
+        module OptionsMethods
+          private
+
+          def option_proxy(value)
+            Hash[value]
+          end
+        end
+
+        module InstanceMethods
+          def request(*args, **options)
+            raise ArgumentError, "must perform at least one request" if args.empty?
+
+            requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+            request = requests.first or return super
+
+            request_options = request.options
+
+            return super unless request_options.proxy
+
+            ssh_options = request_options.proxy
+            ssh_uris = ssh_options.delete(:uri)
+            ssh_uri = URI.parse(ssh_uris.shift)
+
+            return super unless ssh_uri.scheme == "ssh"
+
+            ssh_username = ssh_options.delete(:username)
+            ssh_options[:port] ||= ssh_uri.port || 22
+            if request_options.debug
+              ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
+            end
+
+            request_uri = URI(requests.first.uri)
+            @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
+            begin
+              @_gateway.open(request_uri.host, request_uri.port) do |local_port|
+                io = build_gateway_socket(local_port, request_uri, request_options)
+                super(*args, **options.merge(io: io))
+              end
+            ensure
+              @_gateway.shutdown!
+            end
+          end
+
+          private
+
+          def build_gateway_socket(port, request_uri, options)
+            case request_uri.scheme
+            when "https"
+              ctx = OpenSSL::SSL::SSLContext.new
+              ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
+              ctx.set_params(ctx_options) unless ctx_options.empty?
+              sock = TCPSocket.open("localhost", port)
+              io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+              io.hostname = request_uri.host
+              io.sync_close = true
+              io.connect
+              io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+              io
+            when "http"
+              TCPSocket.open("localhost", port)
+            else
+              raise TypeError, "unexpected scheme: #{request_uri.scheme}"
+            end
+          end
+        end
+
+        module ConnectionMethods
+          # should not coalesce connections here, as the IP is the IP of the proxy
+          def coalescable?(*)
+            return super unless @options.proxy
+
+            false
+          end
+        end
+      end
+    end
+    register_plugin :"proxy/ssh", Proxy::SSH
+  end
+end

data/lib/httpx/plugins/proxy.rb

@@ -0,0 +1,349 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class ProxyError < ConnectionError; end
+
+  module Plugins
+    #
+    # This plugin adds support for proxies. It ships with support for:
+    #
+    # * HTTP proxies
+    # * HTTPS proxies
+    # * Socks4/4a proxies
+    # * Socks5 proxies
+    #
+    # https://gitlab.com/os85/httpx/wikis/Proxy
+    #
+    module Proxy
+      class ProxyConnectionError < ProxyError; end
+
+      PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
+
+      class << self
+        def configure(klass)
+          klass.plugin(:"proxy/http")
+          klass.plugin(:"proxy/socks4")
+          klass.plugin(:"proxy/socks5")
+        end
+
+        def extra_options(options)
+          options.merge(supported_proxy_protocols: [])
+        end
+
+        def subplugins
+          {
+            retries: ProxyRetries,
+          }
+        end
+      end
+
+      class Parameters
+        attr_reader :uri, :username, :password, :scheme, :no_proxy
+
+        def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
+          @no_proxy = Array(no_proxy) if no_proxy
+          @uris = Array(uri)
+          uri = @uris.first
+
+          @username = username
+          @password = password
+
+          @ns = 0
+
+          if uri
+            @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
+            @username ||= @uri.user
+            @password ||= @uri.password
+          end
+
+          @scheme = scheme
+
+          return unless @uri && @username && @password
+
+          @authenticator = nil
+          @scheme ||= infer_default_auth_scheme(@uri)
+
+          return unless @scheme
+
+          @authenticator = load_authenticator(@scheme, @username, @password, **extra)
+        end
+
+        def shift
+          # TODO: this operation must be synchronized
+          @ns += 1
+          @uri = @uris[@ns]
+
+          return unless @uri
+
+          @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
+
+          scheme = infer_default_auth_scheme(@uri)
+
+          return unless scheme != @scheme
+
+          @scheme = scheme
+          @username = username || @uri.user
+          @password = password || @uri.password
+          @authenticator = load_authenticator(scheme, @username, @password)
+        end
+
+        def can_authenticate?(*args)
+          return false unless @authenticator
+
+          @authenticator.can_authenticate?(*args)
+        end
+
+        def authenticate(*args)
+          return unless @authenticator
+
+          @authenticator.authenticate(*args)
+        end
+
+        def ==(other)
+          case other
+          when Parameters
+            @uri == other.uri &&
+              @username == other.username &&
+              @password == other.password &&
+              @scheme == other.scheme
+          when URI::Generic, String
+            proxy_uri = @uri.dup
+            proxy_uri.user = @username
+            proxy_uri.password = @password
+            other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
+            proxy_uri == other_uri
+          else
+            super
+          end
+        end
+
+        private
+
+        def infer_default_auth_scheme(uri)
+          case uri.scheme
+          when "socks5"
+            uri.scheme
+          when "http", "https"
+            "basic"
+          end
+        end
+
+        def load_authenticator(scheme, username, password, **extra)
+          auth_scheme = scheme.to_s.capitalize
+
+          require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+
+          Authentication.const_get(auth_scheme).new(username, password, **extra)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
+      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
+      module OptionsMethods
+        private
+
+        def option_proxy(value)
+          value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
+        end
+
+        def option_supported_proxy_protocols(value)
+          raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
+
+          value.map(&:to_s)
+        end
+      end
+
+      module InstanceMethods
+        def find_connection(request_uri, selector, options)
+          return super unless options.respond_to?(:proxy)
+
+          if (next_proxy = request_uri.find_proxy)
+            return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
+          end
+
+          proxy = options.proxy
+
+          return super unless proxy
+
+          next_proxy = proxy.uri
+
+          raise ProxyError, "Failed to connect to proxy" unless next_proxy
+
+          raise ProxyError,
+                "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
+
+          if (no_proxy = proxy.no_proxy)
+            no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
+
+            # TODO: setting proxy to nil leaks the connection object in the pool
+            return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
+                                                                                                          next_proxy.port, no_proxy)
+          end
+
+          super(request_uri, selector, options.merge(proxy: proxy))
+        end
+
+        private
+
+        def fetch_response(request, selector, options)
+          response = request.response # in case it goes wrong later
+
+          begin
+            response = super
+
+            if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
+              options.proxy.shift
+
+              # return last error response if no more proxies to try
+              return response if options.proxy.uri.nil?
+
+              log { "failed connecting to proxy, trying next..." }
+              request.transition(:idle)
+              send_request(request, selector, options)
+              return
+            end
+            response
+          rescue ProxyError
+            # may happen if coupled with retries, and there are no more proxies to try, in which case
+            # it'll end up here
+            response
+          end
+        end
+
+        def proxy_error?(_request, response, options)
+          return false unless options.proxy
+
+          error = response.error
+          case error
+          when NativeResolveError
+            proxy_uri = URI(options.proxy.uri)
+
+            unresolved_host = error.host
+
+            # failed resolving proxy domain
+            unresolved_host == proxy_uri.host
+          when ResolveError
+            proxy_uri = URI(options.proxy.uri)
+
+            error.message.end_with?(proxy_uri.to_s)
+          when ProxyConnectionError
+            # timeout errors connecting to proxy
+            true
+          else
+            false
+          end
+        end
+      end
+
+      module ConnectionMethods
+        using URIExtensions
+
+        def initialize(*)
+          super
+          return unless @options.proxy
+
+          # redefining the connection origin as the proxy's URI,
+          # as this will be used as the tcp peer ip.
+          @proxy_uri = URI(@options.proxy.uri)
+        end
+
+        def peer
+          @proxy_uri || super
+        end
+
+        def connecting?
+          return super unless @options.proxy
+
+          super || @state == :connecting || @state == :connected
+        end
+
+        def call
+          super
+
+          return unless @options.proxy
+
+          case @state
+          when :connecting
+            consume
+          end
+        rescue *PROXY_ERRORS => e
+          if connecting?
+            error = ProxyConnectionError.new(e.message)
+            error.set_backtrace(e.backtrace)
+            raise error
+          end
+
+          raise e
+        end
+
+        def reset
+          return super unless @options.proxy
+
+          @state = :open
+
+          super
+          # emit(:close)
+        end
+
+        private
+
+        def initialize_type(uri, options)
+          return super unless options.proxy
+
+          "tcp"
+        end
+
+        def connect
+          return super unless @options.proxy
+
+          case @state
+          when :idle
+            transition(:connecting)
+          when :connected
+            transition(:open)
+          end
+        end
+
+        def handle_transition(nextstate)
+          return super unless @options.proxy
+
+          case nextstate
+          when :closing
+            # this is a hack so that we can use the super method
+            # and it'll think that the current state is open
+            @state = :open if @state == :connecting
+          end
+          super
+        end
+
+        def purge_after_closed
+          super
+          @io = @io.proxy_io if @io.respond_to?(:proxy_io)
+        end
+      end
+
+      module ProxyRetries
+        module InstanceMethods
+          def retryable_error?(ex)
+            super || ex.is_a?(ProxyConnectionError)
+          end
+        end
+      end
+    end
+    register_plugin :proxy, Proxy
+  end
+
+  class ProxySSL < SSL
+    attr_reader :proxy_io
+
+    def initialize(tcp, request_uri, options)
+      @proxy_io = tcp
+      @io = tcp.to_io
+      super(request_uri, tcp.addresses, options)
+      @hostname = request_uri.host
+      @state = :connected
+    end
+  end
+end

data/lib/httpx/plugins/push_promise.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for HTTP/2 Push responses.
+    #
+    # In order to benefit from this, requests are sent one at a time, so that
+    # no push responses are received after corresponding request has been sent.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Push
+    #
+    module PushPromise
+      def self.extra_options(options)
+        options.merge(http2_settings: { settings_enable_push: 1 },
+                      max_concurrent_requests: 1)
+      end
+
+      module ResponseMethods
+        def pushed?
+          @__pushed
+        end
+
+        def mark_as_pushed!
+          @__pushed = true
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        def promise_headers
+          @promise_headers ||= {}
+        end
+
+        def on_promise(parser, stream)
+          stream.on(:promise_headers) do |h|
+            __on_promise_request(parser, stream, h)
+          end
+          stream.on(:headers) do |h|
+            __on_promise_response(parser, stream, h)
+          end
+        end
+
+        def __on_promise_request(parser, stream, h)
+          log(level: 1, color: :yellow) do
+            # :nocov:
+            h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
+            # :nocov:
+          end
+          headers = @options.headers_class.new(h)
+          path = headers[":path"]
+          authority = headers[":authority"]
+
+          request = parser.pending.find { |r| r.authority == authority && r.path == path }
+          if request
+            request.merge_headers(headers)
+            promise_headers[stream] = request
+            parser.pending.delete(request)
+            parser.streams[request] = stream
+            request.transition(:done)
+          else
+            stream.refuse
+          end
+        end
+
+        def __on_promise_response(parser, stream, h)
+          request = promise_headers.delete(stream)
+          return unless request
+
+          parser.__send__(:on_stream_headers, stream, request, h)
+          response = request.response
+          response.mark_as_pushed!
+          stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
+          stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
+        end
+      end
+    end
+    register_plugin(:push_promise, PushPromise)
+  end
+end