httpx 1.8.4 → 1.9.1

data/lib/httpx/plugins/proxy/http.rb

@@ -10,19 +10,57 @@ module HTTPX
           end
         end
 
+        # adds support for the following options:
+        #
+        # :proxy_headers :: hash of HTTP headers to send while connecting to the proxy.
+        # :enable_proxy_tunnel :: when <tt>true</tt>, it forces the establishment of the CONNECT proxy
+        #                         tunnel, in cases where it wouldn't otherwise normally establish it
+        #                         (i.e. if the request is an http:// request).
+        # :connect_request_class :: class of the request object used for the CONNECT tunnel request.
+        module OptionsMethods
+          def extend_with_plugin_classes(pl)
+            super
+
+            @connect_request_class = @request_class.dup
+            Options::SET_TEMPORARY_NAME[@connect_request_class, pl]
+            @connect_request_class.__send__(:include, ConnectRequestMethods)
+          end
+
+          private
+
+          def option_connect_request_class(val)
+            val
+          end
+
+          def option_proxy_headers(value)
+            value = value.dup if value.frozen?
+
+            headers_class.new(value)
+          end
+
+          def option_enable_proxy_tunnel(value) # rubocop:disable Naming/PredicateMethod
+            !!value
+          end
+        end
+
         module InstanceMethods
+          # returns a new session with the proxy authentication scheme set to <tt>"basic"</tt>, for HTTP Basic Auth.
           def with_proxy_basic_auth(opts)
             with(proxy: opts.merge(scheme: "basic"))
           end
 
+          # returns a new session with the proxy authentication scheme set to <tt>"digest"</tt>, for HTTP Digest Auth.
           def with_proxy_digest_auth(opts)
             with(proxy: opts.merge(scheme: "digest"))
           end
 
+          # returns a new session with the proxy authentication scheme set to <tt>"ntlm"</tt>, for NTLM Auth.
           def with_proxy_ntlm_auth(opts)
             with(proxy: opts.merge(scheme: "ntlm"))
           end
 
+          private
+
           def fetch_response(request, selector, options)
             response = super
 
@@ -51,17 +89,21 @@ module HTTPX
             super
           end
 
-          def force_close(*)
+          def handle_connect_error(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
               @parser.reset
-              @inflight -= @parser.pending.size
-              @parser.pending.clear
             end
 
             super
           end
 
+          def on_error(error, *)
+            error = error.to_connection_error if error.is_a?(RequestTimeoutError) && connecting?
+
+            super
+          end
+
           private
 
           def purge_after_closed
@@ -83,12 +125,12 @@ module HTTPX
               @tunnel_parser || begin
                 if @parser
                   # in case this happens, requests from @parser need to be reenqueued for delivery.
-                  enqueue_pending_requests_from_parser(@parser)
+                  @parser.reset_requests
                   @parser = nil
                 end
 
                 @tunnel_parser = @parser = parser =
-                                   parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                                   parser_type(@io.protocol).new(@write_buffer, @pending, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
@@ -96,11 +138,12 @@ module HTTPX
 
                   reset
                 end
+                parser.on(:error, &method(:on_parser_error))
                 parser.on(:reset) do
-                  if parser.pending.empty? && parser.empty?
+                  if parser.empty?
                     reset
                   else
-                    enqueue_pending_requests_from_parser(parser)
+                    parser.reset_requests
 
                     initial_state = @state
 
@@ -121,7 +164,7 @@ module HTTPX
                     if initial_state == :connecting
                       parser = tunnel_parser
                       parser.reset
-                      if @pending.first.is_a?(ConnectRequest)
+                      if @pending.first.is_a?(@options.connect_request_class)
                         connect_request = @pending.shift # this happened when reenqueing
                       end
                     end
@@ -129,14 +172,11 @@ module HTTPX
                     idling
 
                     @tunnel_parser = @parser = parser
-                    if connect_request
-                      @inflight += 1
-                      parser.send(connect_request)
-                    end
+                    parser.send(connect_request) if connect_request
                     transition(:connecting)
                   end
                 end
-                __http_proxy_connect(parser)
+                __http_proxy_connect
               end
               return if @state == :connected
             when :connected
@@ -155,27 +195,29 @@ module HTTPX
             super
           end
 
-          def __http_proxy_connect(parser)
+          def __http_proxy_connect
             req = @pending.first
-            if req && req.uri.scheme == "https"
+            enable_proxy_tunnel = (
+              req.options.enable_proxy_tunnel.nil? && req && req.uri.scheme == "https"
+            ) || req.options.enable_proxy_tunnel
+
+            if enable_proxy_tunnel
               # if the first request after CONNECT is to an https address, it is assumed that
               # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
               # and therefore, will share the connection.
               #
-              connect_request = ConnectRequest.new(req.uri, @options)
-              @inflight += 1
-              parser.send(connect_request)
+              connect_request = @options.connect_request_class.new(req.uri, @options)
+              send(connect_request)
             else
               handle_transition(:connected)
             end
           end
 
           def __http_on_connect(request, response)
-            @inflight -= 1
             if response.is_a?(Response) && response.status == 200
               req = @pending.first
               request_uri = req.uri
-              @io = ProxySSL.new(@io, request_uri, @options)
+              @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
               transition(:connected)
               throw(:called)
             elsif response.is_a?(Response) &&
@@ -185,11 +227,9 @@ module HTTPX
 
               request.transition(:idle)
               request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-              @parser.send(request)
-              @inflight += 1
+              send(request)
             else
-              pending = @pending + @parser.pending
-              while (req = pending.shift)
+              while (req = @pending.shift)
                 response.finish!
                 req.response = response
                 req.emit_response(response)
@@ -219,10 +259,11 @@ module HTTPX
           end
         end
 
-        class ConnectRequest < Request
+        module ConnectRequestMethods
           def initialize(uri, options)
             super("CONNECT", uri, options)
-            @headers.delete("accept")
+            @headers = options.headers_class.new
+            merge_headers(options.proxy_headers) if options.proxy_headers
           end
 
           def path

data/lib/httpx/plugins/proxy/socks5.rb

@@ -42,10 +42,6 @@ module HTTPX
             end
           end
 
-          def connecting?
-            super || @state == :authenticating || @state == :negotiating
-          end
-
           def interests
             if @state == :connecting || @state == :authenticating || @state == :negotiating
               return @write_buffer.empty? ? :r : :w
@@ -56,6 +52,10 @@ module HTTPX
 
           private
 
+          def proxy_connecting?
+            super || @state == :authenticating || @state == :negotiating
+          end
+
           def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
 

data/lib/httpx/plugins/proxy/ssh.rb

@@ -21,31 +21,50 @@ module HTTPX
         end
 
         module InstanceMethods
+          def initialize(*)
+            super
+            @_gateway = nil
+          end
+
           def request(*args, **options)
             raise ArgumentError, "must perform at least one request" if args.empty?
 
             requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
 
-            request = requests.first or return super
+            request = requests.first
+
+            return super unless request
 
             request_options = request.options
 
             return super unless request_options.proxy
 
-            ssh_options = request_options.proxy
-            ssh_uris = ssh_options.delete(:uri)
-            ssh_uri = URI.parse(ssh_uris.shift)
+            ssh_options = request_options.proxy.dup
+            ssh_uris = Array(ssh_options.delete(:uri))
+
+            proxy_ns = 0
+
+            begin
+              ssh_uri = URI(ssh_uris[proxy_ns])
+
+              return super unless ssh_uri.scheme == "ssh"
+
+              ssh_username = ssh_options.delete(:username)
+              ssh_options[:port] ||= ssh_uri.port || 22
+              if request_options.debug
+                ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
+              end
 
-            return super unless ssh_uri.scheme == "ssh"
+              request_uri = URI(requests.first.uri)
+              @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
+            rescue StandardError => e
+              log { "failed connecting to ssh proxy, trying next..." }
+              proxy_ns += 1
+              raise e if proxy_ns >= ssh_uris.size
 
-            ssh_username = ssh_options.delete(:username)
-            ssh_options[:port] ||= ssh_uri.port || 22
-            if request_options.debug
-              ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
+              retry
             end
 
-            request_uri = URI(requests.first.uri)
-            @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)

data/lib/httpx/plugins/proxy.rb

@@ -11,7 +11,7 @@ module HTTPX
     #
     # * HTTP proxies
     # * HTTPS proxies
-    # * Socks4/4a proxies
+    # * Socks4 proxies (also supports Socks4a)
     # * Socks5 proxies
     #
     # https://gitlab.com/os85/httpx/wikis/Proxy
@@ -19,6 +19,7 @@ module HTTPX
     module Proxy
       class ProxyConnectionError < ProxyError; end
 
+      # :nodoc:
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
       class << self
@@ -39,21 +40,24 @@ module HTTPX
         end
       end
 
+      # Manages the proxy parameters and the proxy authentication flow.
       class Parameters
         attr_reader :uri, :username, :password, :scheme, :no_proxy
 
-        def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
+        def initialize(uri: nil, ns: 0, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
           @no_proxy = Array(no_proxy) if no_proxy
           @uris = Array(uri)
-          uri = @uris.first
+          uri = @uris[ns]
 
           @username = username
           @password = password
 
-          @ns = 0
+          @initial_credentials = @username && @password
+
+          @ns = ns
 
           if uri
-            @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
+            @uri = URI(uri)
             @username ||= @uri.user
             @password ||= @uri.password
           end
@@ -74,22 +78,15 @@ module HTTPX
         end
 
         def shift
-          # TODO: this operation must be synchronized
-          @ns += 1
-          @uri = @uris[@ns]
-
-          return unless @uri
-
-          @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
-
-          scheme = infer_default_auth_scheme(@uri)
-
-          return unless scheme != @scheme
-
-          @scheme = scheme
-          @username = username || @uri.user
-          @password = password || @uri.password
-          @authenticator = load_authenticator(scheme, @username, @password)
+          return if @ns >= @uris.size - 1
+
+          Parameters.new(
+            uri: @uris,
+            ns: @ns + 1,
+            username: (@username if @initial_credentials),
+            password: (@password if @initial_credentials),
+            no_proxy: @no_proxy
+          )
         end
 
         def can_authenticate?(...)
@@ -144,8 +141,26 @@ module HTTPX
 
       # adds support for the following options:
       #
-      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
-      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
+      # :proxy :: hash of proxy options, where a URI an be passed containing most of the requred information,
+      #           or instead the different proxy options can be passed independently.
+      #
+      # The following proxy options are supported:
+      #
+      # :uri :: proxy origin URI, or list of URIs; in the case of the latter, any failed attempt to connect to a list item
+      #         will retry for the next item of the list, until all elements are exhausted or it times out meanwhile.
+      #         <tt>:username</tt> and <tt>:password</tt> will be inferred from it when available. The URI scheme will identify
+      #         the proxy type.  (ex: <tt>"http://proxi-uri:8980"</tt>, <tt>"http://user:pass@proxy-uri:8181"</tt>,
+      #         <tt>"socks5://user:pass@socks-proxy-uri:10300"</tt>, <tt>["http://proxi-uri:8980", "http://user:pass@proxy-uri:8181"]</tt>).
+      # :scheme :: identifies the authentication scheme to use with the proxy, if necessary; defaults to <tt>"basic"</tt>, for basic auth,
+      #            can also be <tt>"digest"</tt> or <tt>"ntml"</tt>). Your recommended to use session methods like HTTP::InstanceMethods#with_proxy_basic_auth
+      #            to set these instead.
+      # :username :: the username to use when authenticating with the proxy
+      # :password :: the password to use when authenticating with the proxy
+      # :no_proxy :: A list of hostnames (or regex patterns) for which the proxy should be bypassed
+      #              (ex: <tt>["example.com", "localhost", /.*\.google\.com/]</tt>). Can also receive ["*"] as a catch-all.
+      #
+      # if you systems setting environment variables like `HTTP_PROXY` or `NO_PROXY` though, you won't need to set
+      # most of the above.
       module OptionsMethods
         private
 
@@ -161,25 +176,31 @@ module HTTPX
       end
 
       module InstanceMethods
-        def find_connection(request_uri, selector, options)
-          return super unless options.respond_to?(:proxy)
+        def initialize(*)
+          super
+          @proxy_parameters = @options.proxy
+        end
+
+        def initialize_dup(other)
+          @proxy_parameters = other.instance_variable_get(:@proxy_parameters)
+          super
+        end
 
+        def find_connection(request_uri, selector, options)
           if (next_proxy = request_uri.find_proxy)
             return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
           end
 
-          proxy = options.proxy
-
-          return super unless proxy
+          return super unless @proxy_parameters
 
-          next_proxy = proxy.uri
+          next_proxy = @proxy_parameters.uri
 
           raise ProxyError, "Failed to connect to proxy" unless next_proxy
 
           raise ProxyError,
                 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
-          if (no_proxy = proxy.no_proxy)
+          if (no_proxy = @proxy_parameters.no_proxy)
             no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
             unless no_proxy != "*" && # NO_PROXY=* bypasses proxy use
@@ -189,7 +210,7 @@ module HTTPX
             end
           end
 
-          super(request_uri, selector, options.merge(proxy: proxy))
+          super(request_uri, selector, options.merge(proxy: @proxy_parameters))
         end
 
         private
@@ -201,10 +222,10 @@ module HTTPX
             response = super
 
             if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
-              options.proxy.shift
+              @proxy_parameters = options.proxy.shift
 
               # return last error response if no more proxies to try
-              return response if options.proxy.uri.nil?
+              return response unless @proxy_parameters
 
               log { "failed connecting to proxy, trying next..." }
               request.transition(:idle)
@@ -222,22 +243,22 @@ module HTTPX
           end
         end
 
-        def proxy_error?(_request, response, options)
-          return false unless options.proxy
+        def proxy_error?(_request, response, _options)
+          proxy_parameters = @proxy_parameters
+
+          return false unless proxy_parameters
 
           error = response.error
           case error
           when NativeResolveError
-            proxy_uri = URI(options.proxy.uri)
+            proxy_uri = proxy_parameters.uri
 
             unresolved_host = error.host
 
             # failed resolving proxy domain
             unresolved_host == proxy_uri.host
           when ResolveError
-            proxy_uri = URI(options.proxy.uri)
-
-            error.message.end_with?(proxy_uri.to_s)
+            error.message.end_with?(proxy_parameters.uri.to_s)
           when ProxyConnectionError
             # timeout errors connecting to proxy
             true
@@ -256,7 +277,7 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          @proxy_uri = URI(@options.proxy.uri)
+          @proxy_uri = @options.proxy.uri
         end
 
         def peer
@@ -266,7 +287,7 @@ module HTTPX
         def connecting?
           return super unless @options.proxy
 
-          super || @state == :connecting || @state == :connected
+          super || proxy_connecting?
         end
 
         def call
@@ -288,15 +309,6 @@ module HTTPX
           raise e
         end
 
-        def reset
-          return super unless @options.proxy
-
-          @state = :open
-
-          super
-          # emit(:close)
-        end
-
         private
 
         def initialize_type(uri, options)
@@ -305,6 +317,10 @@ module HTTPX
           "tcp"
         end
 
+        def proxy_connecting?
+          @state == :connecting || @state == :connected
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -312,7 +328,7 @@ module HTTPX
           when :idle
             transition(:connecting)
           when :connected
-            transition(:open)
+            super
           end
         end
 
@@ -321,9 +337,7 @@ module HTTPX
 
           case nextstate
           when :closing
-            # this is a hack so that we can use the super method
-            # and it'll think that the current state is open
-            @state = :open if @state == :connecting
+            @parser = nil if proxy_connecting?
           end
           super
         end

data/lib/httpx/plugins/push_promise.rb

@@ -12,8 +12,10 @@ module HTTPX
     #
     module PushPromise
       def self.extra_options(options)
-        options.merge(http2_settings: { settings_enable_push: 1 },
-                      max_concurrent_requests: 1)
+        options.merge(
+          http2_settings: { settings_enable_push: 1 },
+          max_concurrent_requests: 1
+        )
       end
 
       module ResponseMethods

data/lib/httpx/plugins/query.rb

@@ -14,6 +14,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # sends a QUERY request for the given +uri+, using +options+ for each.
         def query(*uri, **options)
           request("QUERY", uri, **options)
         end

data/lib/httpx/plugins/retries.rb

@@ -165,7 +165,7 @@ module HTTPX
             # retry-after must be calculated before prepare_to_retry, as it relies on
             # state changed byit.
             retry_after = when_to_retry(request, response, options)
-            prepare_to_retry(request, response)
+            prepare_to_retry(request, response, retry_after)
 
             if retry_after&.positive?
               retry_start = Utils.now
@@ -207,7 +207,7 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
-        def prepare_to_retry(request, response)
+        def prepare_to_retry(request, response, _retry_after)
           request.retries -= 1 unless can_reconnect?(request, response)
           request.transition(:idle)
         end

data/lib/httpx/plugins/server_sent_events.rb

@@ -40,6 +40,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # sends a stream request.
         def request(*, **options)
           options[:stream] = true if options[:event_stream]
 

data/lib/httpx/plugins/ssrf_filter.rb

@@ -142,7 +142,7 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs.reject! do |ipaddr|
+          addrs = addrs.reject do |ipaddr|
             ipaddr = ipaddr.address
             next false if @options.safe_private_ranges&.any? { |r| r.include?(ipaddr) }
 

data/lib/httpx/plugins/stream.rb

@@ -125,12 +125,12 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
-      STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
+      STREAM_REQUEST_OPTIONS = { timeout: { operation_timeout: 60 }.freeze }.freeze
 
       def self.extra_options(options)
         options.merge(
           stream: false,
-          timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
+          timeout: { operation_timeout: 60 },
           stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
         )
       end
@@ -140,14 +140,6 @@ module HTTPX
       # :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
       # :stream_response_class :: Class used to build the stream response object.
       module OptionsMethods
-        def option_stream(val)
-          val
-        end
-
-        def option_stream_response_class(value)
-          value
-        end
-
         def extend_with_plugin_classes(pl)
           return super unless defined?(pl::StreamResponseMethods)
 
@@ -157,6 +149,16 @@ module HTTPX
 
           super
         end
+
+        private
+
+        def option_stream(val)
+          val
+        end
+
+        def option_stream_response_class(value)
+          value
+        end
       end
 
       module InstanceMethods

data/lib/httpx/plugins/stream_bidi.rb

@@ -158,6 +158,10 @@ module HTTPX
           @closed ? :closed : :open
         end
 
+        def closed_for_select?
+          @closed
+        end
+
         # noop
         def log(**, &_); end