httpx 0.21.0 → 1.2.1

data/lib/httpx/plugins/{digest_authentication.rb → digest_auth.rb}

@@ -3,9 +3,9 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Digest Auth (https://tools.ietf.org/html/rfc7616)
+    # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#authentication
+    # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #
     module DigestAuth
       DigestError = Class.new(Error)
@@ -16,36 +16,35 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require_relative "authentication/digest"
+          require_relative "auth/digest"
         end
       end
 
       module OptionsMethods
         def option_digest(value)
-          raise TypeError, ":digest must be a Digest" unless value.is_a?(Authentication::Digest)
+          raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
 
           value
         end
       end
 
       module InstanceMethods
-        def digest_authentication(user, password)
-          with(digest: Authentication::Digest.new(user, password))
+        def digest_auth(user, password, hashed: false)
+          with(digest: Authentication::Digest.new(user, password, hashed: hashed))
         end
 
-        alias_method :digest_auth, :digest_authentication
+        private
 
         def send_requests(*requests)
           requests.flat_map do |request|
             digest = request.options.digest
 
-            unless digest
-              super(request)
-              next
-            end
+            next super(request) unless digest
 
             probe_response = wrap { super(request).first }
 
+            return probe_response unless probe_response.is_a?(Response)
+
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
               request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
@@ -58,6 +57,6 @@ module HTTPX
       end
     end
 
-    register_plugin :digest_authentication, DigestAuth
+    register_plugin :digest_auth, DigestAuth
   end
 end

data/lib/httpx/plugins/expect.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Expect#expect
+    # https://gitlab.com/os85/httpx/wikis/Expect#expect
     #
     module Expect
       EXPECT_TIMEOUT = 2
@@ -50,7 +50,8 @@ module HTTPX
         end
 
         def response=(response)
-          if response && response.status == 100 &&
+          if response.is_a?(Response) &&
+             response.status == 100 &&
              !@headers.key?("expect") &&
              (@state == :body || @state == :done)
 
@@ -74,14 +75,16 @@ module HTTPX
 
           return unless request.headers["expect"] == "100-continue"
 
-          request.once(:expect) do
-            @timers.after(request.options.expect_timeout) do
-              # expect timeout expired
-              if request.state == :expect && !request.expects?
-                Expect.no_expect_store << request.origin
-                request.headers.delete("expect")
-                consume
-              end
+          expect_timeout = request.options.expect_timeout
+
+          return if expect_timeout.nil? || expect_timeout.infinite?
+
+          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+            # expect timeout expired
+            if request.state == :expect && !request.expects?
+              Expect.no_expect_store << request.origin
+              request.headers.delete("expect")
+              consume
             end
           end
         end
@@ -92,12 +95,11 @@ module HTTPX
           response = @responses.delete(request)
           return unless response
 
-          if response.status == 417 && request.headers.key?("expect")
+          if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
             response.close
             request.headers.delete("expect")
             request.transition(:idle)
-            connection = find_connection(request, connections, options)
-            connection.send(request)
+            send_request(request, connections, options)
             return
           end
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -11,12 +11,14 @@ module HTTPX
     #
     # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Follow-Redirects
+    # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
     #
     module FollowRedirects
       MAX_REDIRECTS = 3
       REDIRECT_STATUS = (300..399).freeze
 
+      using URIExtensions
+
       module OptionsMethods
         def option_max_redirects(value)
           num = Integer(value)
@@ -28,6 +30,16 @@ module HTTPX
         def option_follow_insecure_redirects(value)
           value
         end
+
+        def option_allow_auth_to_other_origins(value)
+          value
+        end
+
+        def option_redirect_on(value)
+          raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
+
+          value
+        end
       end
 
       module InstanceMethods
@@ -44,21 +56,53 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
+          return response unless response.is_a?(Response)
           return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
-          retry_request = build_redirect_request(redirect_request, response, options)
+          # build redirect request
+          redirect_uri = __get_location_from_response(response)
 
-          request.redirect_request = retry_request
+          if options.redirect_on
+            redirect_allowed = options.redirect_on.call(redirect_uri)
+            return response unless redirect_allowed
+          end
+
+          if response.status == 305 && options.respond_to?(:proxy)
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            retry_options = options.merge(headers: redirect_request.headers,
+                                          proxy: { uri: redirect_uri },
+                                          body: redirect_request.body,
+                                          max_redirects: max_redirects - 1)
+            redirect_uri = redirect_request.uri
+            options = retry_options
+          else
+            redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
+
+            # redirects are **ALWAYS** GET
+            retry_opts = Hash[options].merge(
+              headers: redirect_headers.to_h,
+              body: redirect_request.body,
+              max_redirects: max_redirects - 1
+            )
+            retry_options = options.class.new(retry_opts)
+          end
+
+          redirect_uri = Utils.to_uri(redirect_uri)
 
           if !options.follow_insecure_redirects &&
              response.uri.scheme == "https" &&
-             retry_request.uri.scheme == "http"
-            error = InsecureRedirectError.new(retry_request.uri.to_s)
+             redirect_uri.scheme == "http"
+            error = InsecureRedirectError.new(redirect_uri.to_s)
             error.set_backtrace(caller)
             return ErrorResponse.new(request, error, options)
           end
 
+          retry_request = build_request("GET", redirect_uri, retry_options)
+
+          request.redirect_request = retry_request
+
           retry_after = response.headers["retry-after"]
 
           if retry_after
@@ -72,37 +116,25 @@ module HTTPX
 
             log { "redirecting after #{retry_after} secs..." }
             pool.after(retry_after) do
-              connection = find_connection(retry_request, connections, options)
-              connection.send(retry_request)
+              send_request(retry_request, connections, options)
             end
           else
-            connection = find_connection(retry_request, connections, options)
-            connection.send(retry_request)
+            send_request(retry_request, connections, options)
           end
           nil
         end
 
-        def build_redirect_request(request, response, options)
-          redirect_uri = __get_location_from_response(response)
-          max_redirects = request.max_redirects
+        def redirect_request_headers(original_uri, redirect_uri, headers, options)
+          return headers if options.allow_auth_to_other_origins
 
-          if response.status == 305 && options.respond_to?(:proxy)
-            # The requested resource MUST be accessed through the proxy given by
-            # the Location field. The Location field gives the URI of the proxy.
-            retry_options = options.merge(headers: request.headers,
-                                          proxy: { uri: redirect_uri },
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
-            redirect_uri = request.url
-          else
+          return headers unless headers.key?("authorization")
 
-            # redirects are **ALWAYS** GET
-            retry_options = options.merge(headers: request.headers,
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
+          unless original_uri.origin == redirect_uri.origin
+            headers = headers.dup
+            headers.delete("authorization")
           end
 
-          build_request(:get, redirect_uri, retry_options)
+          headers
         end
 
         def __get_location_from_response(response)
@@ -113,14 +145,24 @@ module HTTPX
       end
 
       module RequestMethods
-        def self.included(klass)
-          klass.__send__(:attr_writer, :redirect_request)
-        end
+        attr_accessor :root_request
 
         def redirect_request
           @redirect_request || self
         end
 
+        def redirect_request=(req)
+          @redirect_request = req
+          req.root_request = @root_request || self
+          @response = nil
+        end
+
+        def response
+          return super unless @redirect_request
+
+          @redirect_request.response
+        end
+
         def max_redirects
           @options.max_redirects || MAX_REDIRECTS
         end

data/lib/httpx/plugins/grpc/call.rb

@@ -11,6 +11,7 @@ module HTTPX
           @response = response
           @decoder = ->(z) { z }
           @consumed = false
+          @grpc_response = nil
         end
 
         def inspect
@@ -34,9 +35,7 @@ module HTTPX
         private
 
         def grpc_response
-          return @grpc_response if defined?(@grpc_response)
-
-          @grpc_response = if @response.respond_to?(:each)
+          @grpc_response ||= if @response.respond_to?(:each)
             Enumerator.new do |y|
               Message.stream(@response).each do |message|
                 y << @decoder.call(message)

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder
+    module GRPCEncoding
+      class Deflater
+        extend Forwardable
+
+        attr_reader :content_type
+
+        def initialize(body, compressed:)
+          @content_type = body.content_type
+          @body = BodyReader.new(body)
+          @compressed = compressed
+        end
+
+        def bytesize
+          return @body.bytesize if @body.respond_to?(:bytesize)
+
+          Float::INFINITY
+        end
+
+        def read(length = nil, outbuf = nil)
+          buf = @body.read(length, outbuf)
+
+          return unless buf
+
+          compressed_flag = @compressed ? 1 : 0
+
+          buf = outbuf if outbuf
+
+          buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
+          buf
+        end
+      end
+
+      class Inflater
+        def initialize(response)
+          @response = response
+          @grpc_encodings = nil
+        end
+
+        def call(message, &blk)
+          data = "".b
+
+          until message.empty?
+            compressed, size = message.unpack("CL>")
+
+            encoded_data = message.byteslice(5..size + 5 - 1)
+
+            if compressed == 1
+              grpc_encodings.reverse_each do |encoding|
+                decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
+                encoded_data = decoder.call(encoded_data)
+
+                blk.call(encoded_data) if blk
+
+                data << encoded_data
+              end
+            else
+              blk.call(encoded_data) if blk
+
+              data << encoded_data
+            end
+
+            message = message.byteslice((size + 5)..-1)
+          end
+
+          data
+        end
+
+        private
+
+        def grpc_encodings
+          @grpc_encodings ||= @response.headers.get("grpc-encoding")
+        end
+      end
+
+      def self.encode(*args, **kwargs)
+        Deflater.new(*args, **kwargs)
+      end
+
+      def self.decode(response)
+        Inflater.new(response)
+      end
+    end
+  end
+end

data/lib/httpx/plugins/grpc/message.rb

@@ -12,55 +12,25 @@ module HTTPX
         # decodes a unary grpc response
         def unary(response)
           verify_status(response)
-          decode(response.to_s, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders)
+
+          decoder = Transcoder::GRPCEncoding.decode(response)
+
+          decoder.call(response.to_s)
         end
 
         # lazy decodes a grpc stream response
         def stream(response, &block)
           return enum_for(__method__, response) unless block
 
+          decoder = Transcoder::GRPCEncoding.decode(response)
+
           response.each do |frame|
-            decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
+            decoder.call(frame, &block)
           end
 
           verify_status(response)
         end
 
-        # encodes a single grpc message
-        def encode(bytes, deflater:)
-          if deflater
-            compressed_flag = 1
-            bytes = deflater.deflate(StringIO.new(bytes))
-          else
-            compressed_flag = 0
-          end
-
-          "".b << [compressed_flag, bytes.bytesize].pack("CL>") << bytes.to_s
-        end
-
-        # decodes a single grpc message
-        def decode(message, encodings:, encoders:)
-          until message.empty?
-
-            compressed, size = message.unpack("CL>")
-
-            data = message.byteslice(5..size + 5 - 1)
-            if compressed == 1
-              encodings.reverse_each do |algo|
-                inflater = encoders.registry(algo).inflater(size)
-                data = inflater.inflate(data)
-                size = data.bytesize
-              end
-            end
-
-            return data unless block_given?
-
-            yield data
-
-            message = message.byteslice((5 + size)..-1)
-          end
-        end
-
         def cancel(request)
           request.emit(:refuse, :client_cancellation)
         end

data/lib/httpx/plugins/grpc.rb

@@ -16,7 +16,7 @@ module HTTPX
     #
     # This plugin adds DSL to build GRPC interfaces.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/GRPC
+    # https://gitlab.com/os85/httpx/wikis/GRPC
     #
     module GRPC
       unless String.method_defined?(:underscore)
@@ -49,20 +49,19 @@ module HTTPX
       class << self
         def load_dependencies(*)
           require "stringio"
+          require "httpx/plugins/grpc/grpc_encoding"
           require "httpx/plugins/grpc/message"
           require "httpx/plugins/grpc/call"
         end
 
         def configure(klass)
           klass.plugin(:persistent)
-          klass.plugin(:compression)
           klass.plugin(:stream)
         end
 
         def extra_options(options)
           options.merge(
             fallback_protocol: "h2",
-            http2_settings: { wait_for_handshake: false },
             grpc_rpcs: {}.freeze,
             grpc_compression: false,
             grpc_deadline: DEADLINE
@@ -108,9 +107,18 @@ module HTTPX
           @trailing_metadata = Hash[trailers]
           super
         end
+      end
+
+      module RequestBodyMethods
+        def initialize(headers, _)
+          super
 
-        def encoders
-          @options.encodings
+          if (compression = headers["grpc-encoding"])
+            deflater_body = self.class.initialize_deflater_body(@body, compression)
+            @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
+          else
+            @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
+          end
         end
       end
 
@@ -141,17 +149,29 @@ module HTTPX
             deadline: @options.grpc_deadline,
           }.merge(opts)
 
+          local_rpc_name = rpc_name.underscore
+
           session_class = Class.new(self.class) do
+            # define rpc method with ruby style name
             class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-              def #{rpc_name}(input, **opts)              # def grpc_action(input, **opts)
-                rpc_execute("#{rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
-              end                                         # end
+              def #{local_rpc_name}(input, **opts)              # def grpc_action(input, **opts)
+                rpc_execute("#{local_rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+              end                                               # end
             OUT
+
+            # define rpc method with original name
+            unless local_rpc_name == rpc_name
+              class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+                def #{rpc_name}(input, **opts)                    # def grpcAction(input, **opts)
+                  rpc_execute("#{local_rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+                end                                               # end
+              OUT
+            end
           end
 
           session_class.new(@options.merge(
                               grpc_rpcs: @options.grpc_rpcs.merge(
-                                rpc_name.underscore => [rpc_name, input, output, rpc_opts]
+                                local_rpc_name => [rpc_name, input, output, rpc_opts]
                               ).freeze
                             ))
         end
@@ -195,7 +215,7 @@ module HTTPX
                     **opts)
           grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
           response = request(grpc_request, **opts)
-          response.raise_for_status
+          response.raise_for_status unless opts[:stream]
           GRPC::Call.new(response)
         end
 
@@ -233,7 +253,7 @@ module HTTPX
           uri.path = rpc_method
 
           headers = HEADERS.merge(
-            "grpc-accept-encoding" => ["identity", *@options.encodings.registry.keys]
+            "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
           )
           unless deadline == Float::INFINITY
             # convert to milliseconds
@@ -241,30 +261,16 @@ module HTTPX
             headers["grpc-timeout"] = "#{deadline}m"
           end
 
-          headers = headers.merge(metadata) if metadata
+          headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
 
           # prepare compressor
-          deflater = nil
           compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
 
-          if compression
-            headers["grpc-encoding"] = compression
-            deflater = @options.encodings.registry(compression).deflater
-          end
-
-          headers.merge!(@options.call_credentials.call) if @options.call_credentials
+          headers["grpc-encoding"] = compression if compression
 
-          body = if input.respond_to?(:each)
-            Enumerator.new do |y|
-              input.each do |message|
-                y << Message.encode(message, deflater: deflater)
-              end
-            end
-          else
-            Message.encode(input, deflater: deflater)
-          end
+          headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
-          build_request(:post, uri, headers: headers, body: body)
+          build_request("POST", uri, headers: headers, body: input)
         end
       end
     end

data/lib/httpx/plugins/h2c.rb

@@ -4,21 +4,16 @@ module HTTPX
   module Plugins
     #
     # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-    # (https://tools.ietf.org/html/rfc7540#section-3.2)
+    # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2c
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
     #
     module H2C
-      VALID_H2C_VERBS = %i[get options head].freeze
+      VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
 
       class << self
-        def load_dependencies(*)
-          require "base64"
-        end
-
-        def configure(klass)
+        def load_dependencies(klass)
           klass.plugin(:upgrade)
-          klass.default_options.upgrade_handlers.register "h2c", self
         end
 
         def call(connection, request, response)
@@ -26,7 +21,7 @@ module HTTPX
         end
 
         def extra_options(options)
-          options.merge(max_concurrent_requests: 1)
+          options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
         end
       end
 
@@ -38,7 +33,7 @@ module HTTPX
 
           connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
-          return super if connection && connection.upgrade_protocol == :h2c
+          return super if connection && connection.upgrade_protocol == "h2c"
 
           # build upgrade request
           upgrade_request.headers.add("connection", "upgrade")
@@ -78,22 +73,34 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
-          @parser = H2CParser.new(@write_buffer, parser_options)
+          @parser = H2CParser.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @inflight += 1
           @parser.upgrade(request, response)
-          @upgrade_protocol = :h2c
-
-          if request.options.max_concurrent_requests != @options.max_concurrent_requests
-            @options = @options.merge(max_concurrent_requests: nil)
-          end
+          @upgrade_protocol = "h2c"
 
           prev_parser.requests.each do |req|
             req.transition(:idle)
             send(req)
           end
         end
+
+        private
+
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
+
+          max_concurrent_requests = parser.max_concurrent_requests
+
+          return if max_concurrent_requests == 1
+
+          parser.max_concurrent_requests = 1
+          request.once(:response) do
+            parser.max_concurrent_requests = max_concurrent_requests
+          end
+        end
       end
     end
     register_plugin(:h2c, H2C)