httpx 0.20.0 → 1.3.1

data/lib/httpx/plugins/{digest_authentication.rb → digest_auth.rb}

@@ -3,9 +3,9 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Digest Auth (https://tools.ietf.org/html/rfc7616)
+    # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#authentication
+    # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #
     module DigestAuth
       DigestError = Class.new(Error)
@@ -16,36 +16,38 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require_relative "authentication/digest"
+          require_relative "auth/digest"
         end
       end
 
+      # adds support for the following options:
+      #
+      # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
       module OptionsMethods
         def option_digest(value)
-          raise TypeError, ":digest must be a Digest" unless value.is_a?(Authentication::Digest)
+          raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
 
           value
         end
       end
 
       module InstanceMethods
-        def digest_authentication(user, password)
-          with(digest: Authentication::Digest.new(user, password))
+        def digest_auth(user, password, hashed: false)
+          with(digest: Authentication::Digest.new(user, password, hashed: hashed))
         end
 
-        alias_method :digest_auth, :digest_authentication
+        private
 
         def send_requests(*requests)
           requests.flat_map do |request|
             digest = request.options.digest
 
-            unless digest
-              super(request)
-              next
-            end
+            next super(request) unless digest
 
             probe_response = wrap { super(request).first }
 
+            return probe_response unless probe_response.is_a?(Response)
+
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
               request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
@@ -58,6 +60,6 @@ module HTTPX
       end
     end
 
-    register_plugin :digest_authentication, DigestAuth
+    register_plugin :digest_auth, DigestAuth
   end
 end

data/lib/httpx/plugins/expect.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Expect#expect
+    # https://gitlab.com/os85/httpx/wikis/Expect#expect
     #
     module Expect
       EXPECT_TIMEOUT = 2
@@ -20,9 +20,14 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
+      #                    before retrying without the Expect header (defaults to <tt>2</tt>).
+      # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
       module OptionsMethods
         def option_expect_timeout(value)
-          seconds = Integer(value)
+          seconds = Float(value)
           raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
 
           seconds
@@ -50,7 +55,8 @@ module HTTPX
         end
 
         def response=(response)
-          if response && response.status == 100 &&
+          if response.is_a?(Response) &&
+             response.status == 100 &&
              !@headers.key?("expect") &&
              (@state == :body || @state == :done)
 
@@ -74,14 +80,16 @@ module HTTPX
 
           return unless request.headers["expect"] == "100-continue"
 
-          request.once(:expect) do
-            @timers.after(request.options.expect_timeout) do
-              # expect timeout expired
-              if request.state == :expect && !request.expects?
-                Expect.no_expect_store << request.origin
-                request.headers.delete("expect")
-                consume
-              end
+          expect_timeout = request.options.expect_timeout
+
+          return if expect_timeout.nil? || expect_timeout.infinite?
+
+          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+            # expect timeout expired
+            if request.state == :expect && !request.expects?
+              Expect.no_expect_store << request.origin
+              request.headers.delete("expect")
+              consume
             end
           end
         end
@@ -92,12 +100,11 @@ module HTTPX
           response = @responses.delete(request)
           return unless response
 
-          if response.status == 417 && request.headers.key?("expect")
+          if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
             response.close
             request.headers.delete("expect")
             request.transition(:idle)
-            connection = find_connection(request, connections, options)
-            connection.send(request)
+            send_request(request, connections, options)
             return
           end
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -4,19 +4,35 @@ module HTTPX
   InsecureRedirectError = Class.new(Error)
   module Plugins
     #
-    # This plugin adds support for following redirect (status 30X) responses.
+    # This plugin adds support for automatically following redirect (status 30X) responses.
     #
-    # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
-    # will return the last redirect response. It will **not** raise an exception.
+    # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
+    # after which it will return the last redirect response. It will **not** raise an exception.
     #
-    # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Follow-Redirects
+    # It doesn't propagate authorization related headers to requests redirecting to different origins
+    # (see *allow_auth_to_other_origins*) to override.
+    #
+    # It allows customization of when to redirect via the *redirect_on* callback option).
+    #
+    # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
     #
     module FollowRedirects
       MAX_REDIRECTS = 3
       REDIRECT_STATUS = (300..399).freeze
-
+      REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
+
+      using URIExtensions
+
+      # adds support for the following options:
+      #
+      # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
+      # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
+      #                               (defaults to <tt>false</tt>).
+      # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
+      #                                 (defaults to <tt>false</tt>).
+      # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
       module OptionsMethods
         def option_max_redirects(value)
           num = Integer(value)
@@ -28,9 +44,20 @@ module HTTPX
         def option_follow_insecure_redirects(value)
           value
         end
+
+        def option_allow_auth_to_other_origins(value)
+          value
+        end
+
+        def option_redirect_on(value)
+          raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
+
+          value
+        end
       end
 
       module InstanceMethods
+        # returns a session with the *max_redirects* option set to +n+
         def max_redirects(n)
           with(max_redirects: n.to_i)
         end
@@ -44,68 +71,117 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
+          return response unless response.is_a?(Response)
           return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
-          retry_request = build_redirect_request(redirect_request, response, options)
+          redirect_uri = __get_location_from_response(response)
 
-          request.redirect_request = retry_request
+          if options.redirect_on
+            redirect_allowed = options.redirect_on.call(redirect_uri)
+            return response unless redirect_allowed
+          end
+
+          # build redirect request
+          request_body = redirect_request.body
+          redirect_method = "GET"
+          redirect_params = {}
+
+          if response.status == 305 && options.respond_to?(:proxy)
+            request_body.rewind
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            redirect_options = options.merge(headers: redirect_request.headers,
+                                             proxy: { uri: redirect_uri },
+                                             max_redirects: max_redirects - 1)
+
+            redirect_params[:body] = request_body
+            redirect_uri = redirect_request.uri
+            options = redirect_options
+          else
+            redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
+            redirect_opts = Hash[options]
+            redirect_params[:max_redirects] = max_redirects - 1
+
+            unless request_body.empty?
+              if response.status == 307
+                # The method and the body of the original request are reused to perform the redirected request.
+                redirect_method = redirect_request.verb
+                request_body.rewind
+                redirect_params[:body] = request_body
+              else
+                # redirects are **ALWAYS** GET, so remove body-related headers
+                REQUEST_BODY_HEADERS.each do |h|
+                  redirect_headers.delete(h)
+                end
+                redirect_params[:body] = nil
+              end
+            end
+
+            options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
+          end
+
+          redirect_uri = Utils.to_uri(redirect_uri)
 
           if !options.follow_insecure_redirects &&
              response.uri.scheme == "https" &&
-             retry_request.uri.scheme == "http"
-            error = InsecureRedirectError.new(retry_request.uri.to_s)
+             redirect_uri.scheme == "http"
+            error = InsecureRedirectError.new(redirect_uri.to_s)
             error.set_backtrace(caller)
-            return ErrorResponse.new(request, error, options)
+            return ErrorResponse.new(request, error)
           end
 
-          retry_after = response.headers["retry-after"]
+          retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
 
-          if retry_after
+          request.redirect_request = retry_request
+
+          redirect_after = response.headers["retry-after"]
+
+          if redirect_after
             # Servers send the "Retry-After" header field to indicate how long the
             # user agent ought to wait before making a follow-up request.
             # When sent with any 3xx (Redirection) response, Retry-After indicates
             # the minimum time that the user agent is asked to wait before issuing
             # the redirected request.
             #
-            retry_after = Utils.parse_retry_after(retry_after)
+            redirect_after = Utils.parse_retry_after(redirect_after)
+
+            log { "redirecting after #{redirect_after} secs..." }
 
-            log { "redirecting after #{retry_after} secs..." }
-            pool.after(retry_after) do
-              connection = find_connection(retry_request, connections, options)
-              connection.send(retry_request)
+            deactivate_connection(request, connections, options)
+
+            pool.after(redirect_after) do
+              if request.response
+                # request has terminated abruptly meanwhile
+                retry_request.emit(:response, request.response)
+              else
+                send_request(retry_request, connections, options)
+              end
             end
           else
-            connection = find_connection(retry_request, connections, options)
-            connection.send(retry_request)
+            send_request(retry_request, connections, options)
           end
           nil
         end
 
-        def build_redirect_request(request, response, options)
-          redirect_uri = __get_location_from_response(response)
-          max_redirects = request.max_redirects
+        # :nodoc:
+        def redirect_request_headers(original_uri, redirect_uri, headers, options)
+          headers = headers.dup
 
-          if response.status == 305 && options.respond_to?(:proxy)
-            # The requested resource MUST be accessed through the proxy given by
-            # the Location field. The Location field gives the URI of the proxy.
-            retry_options = options.merge(headers: request.headers,
-                                          proxy: { uri: redirect_uri },
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
-            redirect_uri = request.url
-          else
+          return headers if options.allow_auth_to_other_origins
 
-            # redirects are **ALWAYS** GET
-            retry_options = options.merge(headers: request.headers,
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
-          end
+          return headers unless headers.key?("authorization")
+
+          return headers if original_uri.origin == redirect_uri.origin
+
+          headers.delete("authorization")
 
-          build_request(:get, redirect_uri, retry_options)
+          headers
         end
 
+        # :nodoc:
         def __get_location_from_response(response)
+          # @type var location_uri: http_uri
           location_uri = URI(response.headers["location"])
           location_uri = response.uri.merge(location_uri) if location_uri.relative?
           location_uri
@@ -113,18 +189,41 @@ module HTTPX
       end
 
       module RequestMethods
-        def self.included(klass)
-          klass.__send__(:attr_writer, :redirect_request)
-        end
+        # returns the top-most original HTTPX::Request from the redirect chain
+        attr_accessor :root_request
 
+        # returns the follow-up redirect request, or itself
         def redirect_request
           @redirect_request || self
         end
 
+        # sets the follow-up redirect request
+        def redirect_request=(req)
+          @redirect_request = req
+          req.root_request = @root_request || self
+          @response = nil
+        end
+
+        def response
+          return super unless @redirect_request && @response.nil?
+
+          @redirect_request.response
+        end
+
         def max_redirects
           @options.max_redirects || MAX_REDIRECTS
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def set_request_request_timeout(request)
+          return unless request.root_request.nil?
+
+          super
+        end
+      end
     end
     register_plugin :follow_redirects, FollowRedirects
   end

data/lib/httpx/plugins/grpc/call.rb

@@ -11,6 +11,7 @@ module HTTPX
           @response = response
           @decoder = ->(z) { z }
           @consumed = false
+          @grpc_response = nil
         end
 
         def inspect
@@ -34,9 +35,7 @@ module HTTPX
         private
 
         def grpc_response
-          return @grpc_response if defined?(@grpc_response)
-
-          @grpc_response = if @response.respond_to?(:each)
+          @grpc_response ||= if @response.respond_to?(:each)
             Enumerator.new do |y|
               Message.stream(@response).each do |message|
                 y << @decoder.call(message)

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder
+    module GRPCEncoding
+      class Deflater
+        extend Forwardable
+
+        attr_reader :content_type
+
+        def initialize(body, compressed:)
+          @content_type = body.content_type
+          @body = BodyReader.new(body)
+          @compressed = compressed
+        end
+
+        def bytesize
+          return @body.bytesize if @body.respond_to?(:bytesize)
+
+          Float::INFINITY
+        end
+
+        def read(length = nil, outbuf = nil)
+          buf = @body.read(length, outbuf)
+
+          return unless buf
+
+          compressed_flag = @compressed ? 1 : 0
+
+          buf = outbuf if outbuf
+
+          buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
+          buf
+        end
+      end
+
+      class Inflater
+        def initialize(response)
+          @response = response
+          @grpc_encodings = nil
+        end
+
+        def call(message, &blk)
+          data = "".b
+
+          until message.empty?
+            compressed, size = message.unpack("CL>")
+
+            encoded_data = message.byteslice(5..size + 5 - 1)
+
+            if compressed == 1
+              grpc_encodings.reverse_each do |encoding|
+                decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
+                encoded_data = decoder.call(encoded_data)
+
+                blk.call(encoded_data) if blk
+
+                data << encoded_data
+              end
+            else
+              blk.call(encoded_data) if blk
+
+              data << encoded_data
+            end
+
+            message = message.byteslice((size + 5)..-1)
+          end
+
+          data
+        end
+
+        private
+
+        def grpc_encodings
+          @grpc_encodings ||= @response.headers.get("grpc-encoding")
+        end
+      end
+
+      def self.encode(*args, **kwargs)
+        Deflater.new(*args, **kwargs)
+      end
+
+      def self.decode(response)
+        Inflater.new(response)
+      end
+    end
+  end
+end

data/lib/httpx/plugins/grpc/message.rb

@@ -12,55 +12,25 @@ module HTTPX
         # decodes a unary grpc response
         def unary(response)
           verify_status(response)
-          decode(response.to_s, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders)
+
+          decoder = Transcoder::GRPCEncoding.decode(response)
+
+          decoder.call(response.to_s)
         end
 
         # lazy decodes a grpc stream response
         def stream(response, &block)
           return enum_for(__method__, response) unless block
 
+          decoder = Transcoder::GRPCEncoding.decode(response)
+
           response.each do |frame|
-            decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
+            decoder.call(frame, &block)
           end
 
           verify_status(response)
         end
 
-        # encodes a single grpc message
-        def encode(bytes, deflater:)
-          if deflater
-            compressed_flag = 1
-            bytes = deflater.deflate(StringIO.new(bytes))
-          else
-            compressed_flag = 0
-          end
-
-          "".b << [compressed_flag, bytes.bytesize].pack("CL>") << bytes.to_s
-        end
-
-        # decodes a single grpc message
-        def decode(message, encodings:, encoders:)
-          until message.empty?
-
-            compressed, size = message.unpack("CL>")
-
-            data = message.byteslice(5..size + 5 - 1)
-            if compressed == 1
-              encodings.reverse_each do |algo|
-                inflater = encoders.registry(algo).inflater(size)
-                data = inflater.inflate(data)
-                size = data.bytesize
-              end
-            end
-
-            return data unless block_given?
-
-            yield data
-
-            message = message.byteslice((5 + size)..-1)
-          end
-        end
-
         def cancel(request)
           request.emit(:refuse, :client_cancellation)
         end