httpx 1.2.6 → 1.4.4

data/lib/httpx/plugins/follow_redirects.rb

@@ -4,12 +4,17 @@ module HTTPX
   InsecureRedirectError = Class.new(Error)
   module Plugins
     #
-    # This plugin adds support for following redirect (status 30X) responses.
+    # This plugin adds support for automatically following redirect (status 30X) responses.
     #
-    # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
-    # will return the last redirect response. It will **not** raise an exception.
+    # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
+    # after which it will return the last redirect response. It will **not** raise an exception.
     #
-    # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    #
+    # It doesn't propagate authorization related headers to requests redirecting to different origins
+    # (see *allow_auth_to_other_origins*) to override.
+    #
+    # It allows customization of when to redirect via the *redirect_on* callback option).
     #
     # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
     #
@@ -20,6 +25,14 @@ module HTTPX
 
       using URIExtensions
 
+      # adds support for the following options:
+      #
+      # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
+      # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
+      #                               (defaults to <tt>false</tt>).
+      # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
+      #                                 (defaults to <tt>false</tt>).
+      # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
       module OptionsMethods
         def option_max_redirects(value)
           num = Integer(value)
@@ -44,15 +57,16 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a session with the *max_redirects* option set to +n+
         def max_redirects(n)
           with(max_redirects: n.to_i)
         end
 
         private
 
-        def fetch_response(request, connections, options)
+        def fetch_response(request, selector, options)
           redirect_request = request.redirect_request
-          response = super(redirect_request, connections, options)
+          response = super(redirect_request, selector, options)
           return unless response
 
           max_redirects = redirect_request.max_redirects
@@ -71,40 +85,40 @@ module HTTPX
           # build redirect request
           request_body = redirect_request.body
           redirect_method = "GET"
+          redirect_params = {}
 
           if response.status == 305 && options.respond_to?(:proxy)
             request_body.rewind
             # The requested resource MUST be accessed through the proxy given by
             # the Location field. The Location field gives the URI of the proxy.
-            retry_options = options.merge(headers: redirect_request.headers,
-                                          proxy: { uri: redirect_uri },
-                                          body: request_body,
-                                          max_redirects: max_redirects - 1)
+            redirect_options = options.merge(headers: redirect_request.headers,
+                                             proxy: { uri: redirect_uri },
+                                             max_redirects: max_redirects - 1)
+
+            redirect_params[:body] = request_body
             redirect_uri = redirect_request.uri
-            options = retry_options
+            options = redirect_options
           else
             redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
-
-            retry_opts = Hash[options].merge(max_redirects: max_redirects - 1)
+            redirect_opts = Hash[options]
+            redirect_params[:max_redirects] = max_redirects - 1
 
             unless request_body.empty?
               if response.status == 307
                 # The method and the body of the original request are reused to perform the redirected request.
                 redirect_method = redirect_request.verb
                 request_body.rewind
-                retry_opts[:body] = request_body
+                redirect_params[:body] = request_body
               else
                 # redirects are **ALWAYS** GET, so remove body-related headers
                 REQUEST_BODY_HEADERS.each do |h|
                   redirect_headers.delete(h)
                 end
-                retry_opts.delete(:body)
+                redirect_params[:body] = nil
               end
             end
 
-            retry_opts[:headers] = redirect_headers.to_h
-
-            retry_options = options.class.new(retry_opts)
+            options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
           end
 
           redirect_uri = Utils.to_uri(redirect_uri)
@@ -114,34 +128,42 @@ module HTTPX
              redirect_uri.scheme == "http"
             error = InsecureRedirectError.new(redirect_uri.to_s)
             error.set_backtrace(caller)
-            return ErrorResponse.new(request, error, options)
+            return ErrorResponse.new(request, error)
           end
 
-          retry_request = build_request(redirect_method, redirect_uri, retry_options)
+          retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
 
           request.redirect_request = retry_request
 
-          retry_after = response.headers["retry-after"]
+          redirect_after = response.headers["retry-after"]
 
-          if retry_after
+          if redirect_after
             # Servers send the "Retry-After" header field to indicate how long the
             # user agent ought to wait before making a follow-up request.
             # When sent with any 3xx (Redirection) response, Retry-After indicates
             # the minimum time that the user agent is asked to wait before issuing
             # the redirected request.
             #
-            retry_after = Utils.parse_retry_after(retry_after)
-
-            log { "redirecting after #{retry_after} secs..." }
-            pool.after(retry_after) do
-              send_request(retry_request, connections, options)
+            redirect_after = Utils.parse_retry_after(redirect_after)
+
+            retry_start = Utils.now
+            log { "redirecting after #{redirect_after} secs..." }
+            selector.after(redirect_after) do
+              if request.response
+                # request has terminated abruptly meanwhile
+                retry_request.emit(:response, request.response)
+              else
+                log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                send_request(retry_request, selector, options)
+              end
             end
           else
-            send_request(retry_request, connections, options)
+            send_request(retry_request, selector, options)
           end
           nil
         end
 
+        # :nodoc:
         def redirect_request_headers(original_uri, redirect_uri, headers, options)
           headers = headers.dup
 
@@ -149,14 +171,14 @@ module HTTPX
 
           return headers unless headers.key?("authorization")
 
-          unless original_uri.origin == redirect_uri.origin
-            headers = headers.dup
-            headers.delete("authorization")
-          end
+          return headers if original_uri.origin == redirect_uri.origin
+
+          headers.delete("authorization")
 
           headers
         end
 
+        # :nodoc:
         def __get_location_from_response(response)
           # @type var location_uri: http_uri
           location_uri = URI(response.headers["location"])
@@ -166,12 +188,15 @@ module HTTPX
       end
 
       module RequestMethods
+        # returns the top-most original HTTPX::Request from the redirect chain
         attr_accessor :root_request
 
+        # returns the follow-up redirect request, or itself
         def redirect_request
           @redirect_request || self
         end
 
+        # sets the follow-up redirect request
         def redirect_request=(req)
           @redirect_request = req
           req.root_request = @root_request || self
@@ -179,7 +204,7 @@ module HTTPX
         end
 
         def response
-          return super unless @redirect_request
+          return super unless @redirect_request && @response.nil?
 
           @redirect_request.response
         end
@@ -188,6 +213,16 @@ module HTTPX
           @options.max_redirects || MAX_REDIRECTS
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def set_request_request_timeout(request)
+          return unless request.root_request.nil?
+
+          super
+        end
+      end
     end
     register_plugin :follow_redirects, FollowRedirects
   end

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -29,6 +29,8 @@ module HTTPX
 
           buf = outbuf if outbuf
 
+          buf = buf.b if buf.frozen?
+
           buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
           buf
         end

data/lib/httpx/plugins/grpc.rb

@@ -110,10 +110,10 @@ module HTTPX
       end
 
       module RequestBodyMethods
-        def initialize(headers, _)
+        def initialize(*, **)
           super
 
-          if (compression = headers["grpc-encoding"])
+          if (compression = @headers["grpc-encoding"])
             deflater_body = self.class.initialize_deflater_body(@body, compression)
             @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
           else

data/lib/httpx/plugins/h2c.rb

@@ -25,26 +25,6 @@ module HTTPX
         end
       end
 
-      module InstanceMethods
-        def send_requests(*requests)
-          upgrade_request, *remainder = requests
-
-          return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
-
-          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
-
-          return super if connection && connection.upgrade_protocol == "h2c"
-
-          # build upgrade request
-          upgrade_request.headers.add("connection", "upgrade")
-          upgrade_request.headers.add("connection", "http2-settings")
-          upgrade_request.headers["upgrade"] = "h2c"
-          upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
-
-          super(upgrade_request, *remainder)
-        end
-      end
-
       class H2CParser < Connection::HTTP2
         def upgrade(request, response)
           # skip checks, it is assumed that this is the first
@@ -65,6 +45,29 @@ module HTTPX
       module ConnectionMethods
         using URIExtensions
 
+        def initialize(*)
+          super
+          @h2c_handshake = false
+        end
+
+        def send(request)
+          return super if @h2c_handshake
+
+          return super unless VALID_H2C_VERBS.include?(request.verb) && request.scheme == "http"
+
+          return super if @upgrade_protocol == "h2c"
+
+          @h2c_handshake = true
+
+          # build upgrade request
+          request.headers.add("connection", "upgrade")
+          request.headers.add("connection", "http2-settings")
+          request.headers["upgrade"] = "h2c"
+          request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
+
+          super
+        end
+
         def upgrade_to_h2c(request, response)
           prev_parser = @parser
 

data/lib/httpx/plugins/internal_telemetry.rb

@@ -13,6 +13,12 @@ module HTTPX
     # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
     # using the monotonic clock.
     module InternalTelemetry
+      DEBUG_LEVEL = 3
+
+      def self.extra_options(options)
+        options.merge(debug_level: 3)
+      end
+
       module TrackTimeMethods
         private
 
@@ -28,7 +34,19 @@ module HTTPX
           after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
           # $http_init_time = after_time
           elapsed = after_time - prev_time
-          warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
+          # klass = self.class
+
+          # until (class_name = klass.name)
+          #   klass = klass.superclass
+          # end
+          log(
+            level: DEBUG_LEVEL,
+            color: :red,
+            debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
+            debug: nil
+          ) do
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+          end
         end
       end
 
@@ -76,10 +94,19 @@ module HTTPX
           meter_elapsed_time("Session -> response") if response
           response
         end
+
+        def coalesce_connections(conn1, conn2, selector, *)
+          result = super
+
+          meter_elapsed_time("Connection##{conn2.object_id} coalescing to Connection##{conn1.object_id}") if result
+
+          result
+        end
       end
 
       module RequestMethods
         def self.included(klass)
+          klass.prepend Loggable
           klass.prepend TrackTimeMethods
           super
         end
@@ -103,6 +130,26 @@ module HTTPX
           meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
         end
       end
+
+      module PoolMethods
+        def self.included(klass)
+          klass.prepend Loggable
+          klass.prepend TrackTimeMethods
+          super
+        end
+
+        def checkout_connection(request_uri, options)
+          super.tap do |connection|
+            meter_elapsed_time("Pool##{object_id}: checked out connection for Connection##{connection.object_id}[#{connection.origin}]}")
+          end
+        end
+
+        def checkin_connection(connection)
+          super.tap do
+            meter_elapsed_time("Pool##{object_id}: checked in connection for Connection##{connection.object_id}[#{connection.origin}]}")
+          end
+        end
+      end
     end
     register_plugin :internal_telemetry, InternalTelemetry
   end

data/lib/httpx/plugins/oauth.rb

@@ -155,7 +155,7 @@ module HTTPX
           with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
         end
 
-        def build_request(*, _)
+        def build_request(*)
           request = super
 
           return request if request.headers.key?("authorization")

data/lib/httpx/plugins/persistent.rb

@@ -30,6 +30,22 @@ module HTTPX
       def self.extra_options(options)
         options.merge(persistent: true)
       end
+
+      module InstanceMethods
+        private
+
+        def get_current_selector
+          super(&nil) || begin
+            return unless block_given?
+
+            default = yield
+
+            set_current_selector(default)
+
+            default
+          end
+        end
+      end
     end
     register_plugin :persistent, Persistent
   end

data/lib/httpx/plugins/proxy/http.rb

@@ -23,24 +23,19 @@ module HTTPX
             with(proxy: opts.merge(scheme: "ntlm"))
           end
 
-          def fetch_response(request, connections, options)
+          def fetch_response(request, selector, options)
             response = super
 
             if response &&
                response.is_a?(Response) &&
                response.status == 407 &&
                !request.headers.key?("proxy-authorization") &&
-               response.headers.key?("proxy-authenticate")
-
-              connection = find_connection(request, connections, options)
-
-              if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
-                request.transition(:idle)
-                request.headers["proxy-authorization"] =
-                  connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-                send_request(request, connections)
-                return
-              end
+               response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+              request.transition(:idle)
+              request.headers["proxy-authorization"] =
+                options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              send_request(request, selector, options)
+              return
             end
 
             response
@@ -69,7 +64,14 @@ module HTTPX
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
-                parser.on(:close) { transition(:closing) }
+                parser.on(:close) do |force|
+                  next unless @parser
+
+                  if force
+                    reset
+                    emit(:terminate)
+                  end
+                end
                 parser.on(:reset) do
                   if parser.empty?
                     reset
@@ -90,8 +92,9 @@ module HTTPX
 
               case @state
               when :connecting
-                @parser.close
+                parser = @parser
                 @parser = nil
+                parser.close
               when :idle
                 @parser.callbacks.clear
                 set_parser_callbacks(@parser)
@@ -163,8 +166,8 @@ module HTTPX
         end
 
         class ConnectRequest < Request
-          def initialize(uri, _options)
-            super("CONNECT", uri, {})
+          def initialize(uri, options)
+            super("CONNECT", uri, options)
             @headers.delete("accept")
           end
 

data/lib/httpx/plugins/proxy/socks4.rb

@@ -89,7 +89,7 @@ module HTTPX
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
+            @options = options
           end
 
           def close; end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -141,7 +141,7 @@ module HTTPX
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
+            @options = options
           end
 
           def close; end