httpx 1.8.3 → 1.9.0

data/lib/httpx/plugins/hedging.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for request hedging, i.e. a latency-mitigation strategy that involves
+    # sending requests through multiple connections and returning the fastest response.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Hedging#hedging
+    #
+    module Hedging
+      # :hedge_after:: seconds after which, if the request didn't complete yet, it'll be sent on a different connection
+      module OptionsMethods
+        private
+
+        def option_hedge_after(value)
+          if value.respond_to?(:call)
+            value1 = value
+            value1 = value1.method(:call) unless value1.respond_to?(:arity)
+
+            # allow ->(*) arity as well, which is < 0
+            raise TypeError, "`:hedge_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
+              1, 2
+            )
+
+          else
+            value = Float(value)
+            raise TypeError, "`:hedge_after` must be positive" unless value.positive?
+          end
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def select_connection(connection, selector)
+          connection.try_yield_hedge_connection do |hedge_conn|
+            super.tap do
+              select_connection(hedge_conn, selector) if hedge_conn
+            end
+          end
+        end
+
+        private
+
+        def do_request(request, connection, selector)
+          super.tap do
+            # forego hedging if the connection is already open, or if
+            # the connection is scheduled for DNS resolution, as pending requests
+            # will be hedged afterwards.
+            next if connection.open? || !connection.addresses?
+
+            hedge_request!(request, connection, selector)
+          end
+        end
+
+        # creates the hedge connection before a connection initiates the connection handshake.
+        # This excludes connections which are already open.
+        def on_resolver_connection(connection, selector)
+          super.tap do
+            next if connection.open? || connection.hedge_connection
+
+            connection.pending.each do |request|
+              hedge_request!(request, connection, selector)
+            end
+
+            on_resolver_connection(connection.hedge_connection, selector) if connection.hedge_connection
+          end
+        end
+
+        def hedge_connection(connection)
+          connection.hedge_connection || begin
+            new_connection = connection.dup
+            new_connection.log { "cloning hedge connection of connection##{connection.object_id}..." }
+            connection.hedge_connection = new_connection
+            new_connection
+          end
+        end
+
+        def hedge_request!(request, connection, selector)
+          hedge_after = when_to_hedge(request)
+
+          if hedge_after&.positive?
+            # hedge after interval
+            cb = selector.after(hedge_after) do
+              next if request.response
+
+              request.hedge_request = hedge_request = request.dup
+              hedge_conn = hedge_connection(connection)
+              hedge_conn.send(hedge_request)
+              select_connection(hedge_conn, selector)
+            end
+
+            # forego hedging if request is over
+            request.once(:response) do
+              cb.cancel
+            end
+          else
+            request.hedge_request = hedge_request = request.dup
+            hedge_connection(connection).send(hedge_request)
+          end
+        end
+
+        def when_to_hedge(request)
+          hedge_after = request.options.hedge_after
+
+          return unless hedge_after
+
+          hedge_after = hedge_after.call(request) if hedge_after.respond_to?(:call)
+
+          hedge_after
+        end
+
+        # hedge connections should not deactivate, instead they should terminate.
+        # this ensures that persistent hedge connections aren't treated as persistent,
+        # and close instead of deactivating.
+        def deactivate(selector, *)
+          hedge_connections = selector.each_connection.select(&:is_hedge_connection)
+
+          return super unless hedge_connections.any?
+
+          selector_close(selector, hedge_connections)
+
+          super
+        end
+
+        # do not check hedge connections back in
+        def can_checkin?(connection)
+          super && !connection.is_hedge_connection
+        end
+      end
+
+      module RequestMethods
+        # TODO: would be nice to get rid of this
+        attr_reader :connection
+
+        attr_reader :hedge_request
+
+        def initialize(*)
+          super
+          @is_hedge_request = false
+        end
+
+        def hedge_request=(request)
+          @hedge_request = request
+
+          return unless request
+
+          @is_hedge_request = !request.hedge_request.nil?
+
+          return if @is_hedge_request
+
+          request.hedge_request = self
+        end
+
+        def response=(response)
+          return super unless @hedge_request # rubocop:disable Lint/ReturnInVoidContext
+
+          prev_response = @response
+
+          begin
+            super
+          ensure
+            @response = prev_response if prev_response
+          end
+        end
+
+        def emit_response(response)
+          hedge_request = @hedge_request
+
+          return super unless hedge_request
+
+          return super unless hedge_request &&
+                              (
+                                hedge_request.response.nil? ||
+                                !hedge_request.response.finished?
+                              )
+
+          # if this response is for a hedge request, and the main request
+          # didn't complete yet, then we replace it.
+
+          super.tap do
+            hedge_request.response = response
+
+            # should cancel any pending HTTP/2 stream; there's no way to
+            # cancel an HTTP/1 request, so that one will run to completion.
+            hedge_request.emit(:refuse)
+          end
+        ensure
+          self.hedge_request = nil
+        end
+      end
+
+      module ResponseMethods
+        def finish!
+          connection = @request.connection
+
+          super
+
+          return unless connection && (hedge_connection = connection.hedge_connection)
+
+          # disconnect connection with its hedge, so that objects do not leak after
+          # they're not needed.
+          connection.hedge_connection = hedge_connection.hedge_connection = nil
+
+          # remarking itself as the main connection, so that it's not discarded when
+          # going back to the pool.
+          connection.is_hedge_connection = false
+          hedge_connection.is_hedge_connection = true
+        end
+      end
+
+      module ConnectionMethods
+        attr_reader :hedge_connection
+
+        attr_accessor :is_hedge_connection
+
+        def initialize(*)
+          super
+          @is_hedge_connection = false
+        end
+
+        def addresses=(addrs)
+          try_yield_hedge_connection do |hedge_conn|
+            super.tap do
+              hedge_conn.addresses = addrs if hedge_conn
+            end
+          end
+        end
+
+        def hedge_connection=(connection)
+          @hedge_connection = connection
+
+          return unless connection
+
+          @is_hedge_connection = !connection.hedge_connection.nil?
+
+          return if @is_hedge_connection
+
+          connection.hedge_connection = self
+        end
+
+        def try_yield_hedge_connection
+          return yield(nil) unless (hedge_conn = @hedge_connection)
+
+          @hedge_connection = nil
+
+          begin
+            hedge_conn.try_yield_hedge_connection do
+              yield(hedge_conn)
+            end
+          ensure
+            @hedge_connection = hedge_conn
+          end
+        end
+
+        # connections should not merge with their hedge connections
+        def mergeable?(connection)
+          super && connection.hedge_connection != self
+        end
+
+        def send(request)
+          super.tap do
+            if @hedge_connection && !request.hedge_request
+              request.hedge_request = hedge_request = request.dup
+              @hedge_connection.send(hedge_request)
+            end
+          end
+        end
+
+        private
+
+        def send_request_to_parser(request)
+          # if original request was in the queue, but has since been finished
+          # in the hedge connection.
+          return if request.response&.finished?
+
+          super
+        end
+      end
+    end
+    register_plugin(:hedging, Hedging)
+  end
+end

data/lib/httpx/plugins/ntlm_auth.rb

@@ -28,6 +28,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a new session which authenticates requests using NTLM auth.
         def ntlm_auth(user, password, domain = nil)
           with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
         end

data/lib/httpx/plugins/ntlm_v2_auth.rb

@@ -54,6 +54,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a new session which authenticates requests using NTLM v2 auth.
         def ntlm_auth(user, password, domain = nil)
           with(ntlm: Authenticator.new(user, password, domain: domain))
         end

data/lib/httpx/plugins/oauth.rb

@@ -323,7 +323,7 @@ module HTTPX
         module InstanceMethods
           private
 
-          def prepare_to_retry(_request, response)
+          def prepare_to_retry(*)
             @oauth_session.reset! if @oauth_session
 
             super

data/lib/httpx/plugins/proxy/http.rb

@@ -10,19 +10,44 @@ module HTTPX
           end
         end
 
+        # adds support for the following options:
+        #
+        # :proxy_headers :: hash of HTTP headers to send while connecting to the proxy.
+        # :enable_proxy_tunnel :: when <tt>true</tt>, it forces the establishment of the CONNECT proxy
+        #                         tunnel, in cases where it wouldn't otherwise normally establish it
+        #                         (i.e. if the request is an http:// request).
+        module OptionsMethods
+          private
+
+          def option_proxy_headers(value)
+            value = value.dup if value.frozen?
+
+            headers_class.new(value)
+          end
+
+          def option_enable_proxy_tunnel(value) # rubocop:disable Naming/PredicateMethod
+            !!value
+          end
+        end
+
         module InstanceMethods
+          # returns a new session with the proxy authentication scheme set to <tt>"basic"</tt>, for HTTP Basic Auth.
           def with_proxy_basic_auth(opts)
             with(proxy: opts.merge(scheme: "basic"))
           end
 
+          # returns a new session with the proxy authentication scheme set to <tt>"digest"</tt>, for HTTP Digest Auth.
           def with_proxy_digest_auth(opts)
             with(proxy: opts.merge(scheme: "digest"))
           end
 
+          # returns a new session with the proxy authentication scheme set to <tt>"ntlm"</tt>, for NTLM Auth.
           def with_proxy_ntlm_auth(opts)
             with(proxy: opts.merge(scheme: "ntlm"))
           end
 
+          private
+
           def fetch_response(request, selector, options)
             response = super
 
@@ -46,7 +71,12 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def force_close(*)
+          def initialize(*)
+            @tunnel_parser = nil
+            super
+          end
+
+          def handle_connect_error(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
               @parser.reset
@@ -57,8 +87,20 @@ module HTTPX
             super
           end
 
+          def on_error(error, *)
+            error = error.to_connection_error if error.is_a?(RequestTimeoutError) && connecting?
+
+            super
+          end
+
           private
 
+          def purge_after_closed
+            super
+
+            @tunnel_parser = nil
+          end
+
           def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "http"
 
@@ -69,8 +111,15 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser || begin
-                @parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+              @tunnel_parser || begin
+                if @parser
+                  # in case this happens, requests from @parser need to be reenqueued for delivery.
+                  enqueue_pending_requests_from_parser(@parser)
+                  @parser = nil
+                end
+
+                @tunnel_parser = @parser = parser =
+                                   parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
@@ -86,6 +135,9 @@ module HTTPX
 
                     initial_state = @state
 
+                    # because #reset will nillify @parser...
+                    tunnel_parser = @parser
+
                     reset
 
                     if @pending.empty?
@@ -98,8 +150,8 @@ module HTTPX
                     connect_request = parser = nil
 
                     if initial_state == :connecting
-                      parser = @parser
-                      @parser.reset
+                      parser = tunnel_parser
+                      parser.reset
                       if @pending.first.is_a?(ConnectRequest)
                         connect_request = @pending.shift # this happened when reenqueing
                       end
@@ -107,7 +159,7 @@ module HTTPX
 
                     idling
 
-                    @parser = parser
+                    @tunnel_parser = @parser = parser
                     if connect_request
                       @inflight += 1
                       parser.send(connect_request)
@@ -115,7 +167,7 @@ module HTTPX
                     transition(:connecting)
                   end
                 end
-                __http_proxy_connect(parser)
+                __http_proxy_connect
               end
               return if @state == :connected
             when :connected
@@ -124,7 +176,7 @@ module HTTPX
               case @state
               when :connecting
                 parser = @parser
-                @parser = nil
+                @tunnel_parser = @parser = nil
                 parser.close
               when :idle
                 @parser.callbacks.clear
@@ -134,16 +186,19 @@ module HTTPX
             super
           end
 
-          def __http_proxy_connect(parser)
+          def __http_proxy_connect
             req = @pending.first
-            if req && req.uri.scheme == "https"
+            enable_proxy_tunnel = (
+              req.options.enable_proxy_tunnel.nil? && req && req.uri.scheme == "https"
+            ) || req.options.enable_proxy_tunnel
+
+            if enable_proxy_tunnel
               # if the first request after CONNECT is to an https address, it is assumed that
               # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
               # and therefore, will share the connection.
               #
               connect_request = ConnectRequest.new(req.uri, @options)
-              @inflight += 1
-              parser.send(connect_request)
+              send(connect_request)
             else
               handle_transition(:connected)
             end
@@ -154,7 +209,7 @@ module HTTPX
             if response.is_a?(Response) && response.status == 200
               req = @pending.first
               request_uri = req.uri
-              @io = ProxySSL.new(@io, request_uri, @options)
+              @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
               transition(:connected)
               throw(:called)
             elsif response.is_a?(Response) &&
@@ -164,8 +219,7 @@ module HTTPX
 
               request.transition(:idle)
               request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-              @parser.send(request)
-              @inflight += 1
+              send(request)
             else
               pending = @pending + @parser.pending
               while (req = pending.shift)
@@ -201,7 +255,8 @@ module HTTPX
         class ConnectRequest < Request
           def initialize(uri, options)
             super("CONNECT", uri, options)
-            @headers.delete("accept")
+            @headers = options.headers_class.new
+            merge_headers(options.proxy_headers) if options.proxy_headers
           end
 
           def path

data/lib/httpx/plugins/proxy/ssh.rb

@@ -21,31 +21,50 @@ module HTTPX
         end
 
         module InstanceMethods
+          def initialize(*)
+            super
+            @_gateway = nil
+          end
+
           def request(*args, **options)
             raise ArgumentError, "must perform at least one request" if args.empty?
 
             requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
 
-            request = requests.first or return super
+            request = requests.first
+
+            return super unless request
 
             request_options = request.options
 
             return super unless request_options.proxy
 
-            ssh_options = request_options.proxy
-            ssh_uris = ssh_options.delete(:uri)
-            ssh_uri = URI.parse(ssh_uris.shift)
+            ssh_options = request_options.proxy.dup
+            ssh_uris = Array(ssh_options.delete(:uri))
+
+            proxy_ns = 0
+
+            begin
+              ssh_uri = URI(ssh_uris[proxy_ns])
+
+              return super unless ssh_uri.scheme == "ssh"
+
+              ssh_username = ssh_options.delete(:username)
+              ssh_options[:port] ||= ssh_uri.port || 22
+              if request_options.debug
+                ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
+              end
 
-            return super unless ssh_uri.scheme == "ssh"
+              request_uri = URI(requests.first.uri)
+              @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
+            rescue StandardError => e
+              log { "failed connecting to ssh proxy, trying next..." }
+              proxy_ns += 1
+              raise e if proxy_ns >= ssh_uris.size
 
-            ssh_username = ssh_options.delete(:username)
-            ssh_options[:port] ||= ssh_uri.port || 22
-            if request_options.debug
-              ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
+              retry
             end
 
-            request_uri = URI(requests.first.uri)
-            @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)