httpx 0.21.0 → 1.2.1

data/lib/httpx/request/body.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module HTTPX
+  # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
+  class Request::Body < SimpleDelegator
+    class << self
+      def new(_, options)
+        return options.body if options.body.is_a?(self)
+
+        super
+      end
+    end
+
+    # inits the instance with the request +headers+ and +options+, which contain the payload definition.
+    def initialize(headers, options)
+      @headers = headers
+
+      # forego compression in the Range request case
+      if @headers.key?("range")
+        @headers.delete("accept-encoding")
+      else
+        @headers["accept-encoding"] ||= options.supported_compression_formats
+      end
+
+      initialize_body(options)
+
+      return if @body.nil?
+
+      @headers["content-type"] ||= @body.content_type
+      @headers["content-length"] = @body.bytesize unless unbounded_body?
+      super(@body)
+    end
+
+    # consumes and yields the request payload in chunks.
+    def each(&block)
+      return enum_for(__method__) unless block
+      return if @body.nil?
+
+      body = stream(@body)
+      if body.respond_to?(:read)
+        ::IO.copy_stream(body, ProcIO.new(block))
+      elsif body.respond_to?(:each)
+        body.each(&block)
+      else
+        block[body.to_s]
+      end
+    end
+
+    # if the +@body+ is rewindable, it rewinnds it.
+    def rewind
+      return if empty?
+
+      @body.rewind if @body.respond_to?(:rewind)
+    end
+
+    # return +true+ if the +body+ has been fully drained (or does nnot exist).
+    def empty?
+      return true if @body.nil?
+      return false if chunked?
+
+      @body.bytesize.zero?
+    end
+
+    # returns the +@body+ payload size in bytes.
+    def bytesize
+      return 0 if @body.nil?
+
+      @body.bytesize
+    end
+
+    # sets the body to yield using chunked trannsfer encoding format.
+    def stream(body)
+      return body unless chunked?
+
+      Transcoder::Chunker.encode(body.enum_for(:each))
+    end
+
+    # returns whether the body yields infinitely.
+    def unbounded_body?
+      return @unbounded_body if defined?(@unbounded_body)
+
+      @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
+    end
+
+    # returns whether the chunked transfer encoding header is set.
+    def chunked?
+      @headers["transfer-encoding"] == "chunked"
+    end
+
+    # sets the chunked transfer encoding header.
+    def chunk!
+      @headers.add("transfer-encoding", "chunked")
+    end
+
+    # :nocov:
+    def inspect
+      "#<HTTPX::Request::Body:#{object_id} " \
+        "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
+    end
+    # :nocov:
+
+    private
+
+    # wraps the given body with the appropriate encoder.
+    #
+    #   ..., json: { foo: "bar" }) #=> json encoder
+    #   ..., form: { foo: "bar" }) #=> form urlencoded encoder
+    #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { body: "bla") }) #=> raw data encoder
+    def initialize_body(options)
+      @body = if options.body
+        Transcoder::Body.encode(options.body)
+      elsif options.form
+        Transcoder::Form.encode(options.form)
+      elsif options.json
+        Transcoder::JSON.encode(options.json)
+      elsif options.xml
+        Transcoder::Xml.encode(options.xml)
+      end
+
+      return unless @body && options.compress_request_body && @headers.key?("content-encoding")
+
+      @headers.get("content-encoding").each do |encoding|
+        @body = self.class.initialize_deflater_body(@body, encoding)
+      end
+    end
+
+    class << self
+      # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
+      def initialize_deflater_body(body, encoding)
+        case encoding
+        when "gzip"
+          Transcoder::GZIP.encode(body)
+        when "deflate"
+          Transcoder::Deflate.encode(body)
+        when "identity"
+          body
+        else
+          body
+        end
+      end
+    end
+  end
+
+  # Wrapper yielder which can be used with functions which expect an IO writer.
+  class ProcIO
+    def initialize(block)
+      @block = block
+    end
+
+    # Implementation the IO write protocol, which yield the given chunk to +@block+.
+    def write(data)
+      @block.call(data.dup)
+      data.bytesize
+    end
+  end
+end

data/lib/httpx/request.rb

@@ -4,22 +4,52 @@ require "delegate"
 require "forwardable"
 
 module HTTPX
+  # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
+  # as well as maintaining the state machine which manages streaming the request onto the wire.
   class Request
     extend Forwardable
     include Callbacks
     using URIExtensions
 
+    # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
+    # the upcased string HTTP verb for this request.
+    attr_reader :verb
 
-    # Exception raised during enumerable body writes
+    # the absolute URI object for this request.
+    attr_reader :uri
+
+    # an HTTPX::Headers object containing the request HTTP headers.
+    attr_reader :headers
+
+    # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
+    attr_reader :body
+
+    # a symbol describing which frame is currently being flushed.
+    attr_reader :state
+
+    # an HTTPX::Options object containing request options.
+    attr_reader :options
+
+    # the corresponding HTTPX::Response object, when there is one.
+    attr_reader :response
+
+    # Exception raised during enumerable body writes.
     attr_reader :drain_error
 
+    # The IP address from the peer server.
+    attr_accessor :peer_address
+
+    attr_writer :persistent
+
+    # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
+    # initializes the instance with the given +verb+, an absolute or relative +uri+, and the
+    # request options.
     def initialize(verb, uri, options = {})
-      @verb    = verb.to_s.downcase.to_sym
+      @verb    = verb.to_s.upcase
       @options = Options.new(options)
       @uri     = Utils.to_uri(uri)
       if @uri.relative?
@@ -37,20 +67,30 @@ module HTTPX
 
       @body = @options.request_body_class.new(@headers, @options)
       @state = :idle
+      @response = nil
+      @peer_address = nil
+      @persistent = @options.persistent
     end
 
+    # the read timeout defied for this requet.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
+    # the write timeout defied for this requet.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
+    # the request timeout defied for this requet.
     def request_timeout
       @options.timeout[:request_timeout]
     end
 
+    def persistent?
+      @persistent
+    end
+
     def trailers?
       defined?(@trailers)
     end
@@ -59,40 +99,45 @@ module HTTPX
       @trailers ||= @options.headers_class.new
     end
 
+    # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
     def interests
       return :r if @state == :done || @state == :expect
 
       :w
     end
 
-    if RUBY_VERSION < "2.2"
-      URIParser = URI::DEFAULT_PARSER
-
-      def initialize_with_escape(verb, uri, options = {})
-        initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
-      end
-      alias_method :initialize_without_escape, :initialize
-      alias_method :initialize, :initialize_with_escape
-    end
-
     def merge_headers(h)
       @headers = @headers.merge(h)
     end
 
+    # the URI scheme of the request +uri+.
     def scheme
       @uri.scheme
     end
 
+    # sets the +response+ on this request.
     def response=(response)
       return unless response
 
-      if response.is_a?(Response) && response.status == 100
-        @informational_status = response.status
-        return
+      if response.is_a?(Response) && response.status < 200
+        # deal with informational responses
+
+        if response.status == 100 && @headers.key?("expect")
+          @informational_status = response.status
+          return
+        end
+
+        # 103 Early Hints advertises resources in document to browsers.
+        # not very relevant for an HTTP client, discard.
+        return if response.status >= 103
       end
+
       @response = response
+
+      emit(:response_started, response)
     end
 
+    # returnns the URI path of the request +uri+.
     def path
       path = uri.path.dup
       path =  +"" if path.nil?
@@ -101,33 +146,48 @@ module HTTPX
       path
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI authority of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
     def authority
       @uri.authority
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI origin of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
     def origin
       @uri.origin
     end
 
+    # returs the URI query string of the request (when available).
+    #
+    #   session.build_request("GET", "https://search.com").query #=> ""
+    #   session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
+    #   session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
+    #   session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
     def query
       return @query if defined?(@query)
 
       query = []
       if (q = @options.params)
-        query << Transcoder.registry("form").encode(q)
+        query << Transcoder::Form.encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")
     end
 
+    # consumes and returns the next available chunk of request body that can be sent
     def drain_body
       return nil if @body.nil?
 
       @drainer ||= @body.each
-      chunk = @drainer.next
-      chunk.dup
+      chunk = @drainer.next.dup
+
+      emit(:body_chunk, chunk)
+      chunk
     rescue StopIteration
       nil
     rescue StandardError => e
@@ -138,101 +198,14 @@ module HTTPX
     # :nocov:
     def inspect
       "#<HTTPX::Request:#{object_id} " \
-        "#{@verb.to_s.upcase} " \
+        "#{@verb} " \
         "#{uri} " \
         "@headers=#{@headers} " \
         "@body=#{@body}>"
     end
     # :nocov:
 
-    class Body < SimpleDelegator
-      class << self
-        def new(_, options)
-          return options.body if options.body.is_a?(self)
-
-          super
-        end
-      end
-
-      def initialize(headers, options)
-        @headers = headers
-        @body = if options.body
-          Transcoder.registry("body").encode(options.body)
-        elsif options.form
-          Transcoder.registry("form").encode(options.form)
-        elsif options.json
-          Transcoder.registry("json").encode(options.json)
-        elsif options.xml
-          Transcoder.registry("xml").encode(options.xml)
-        end
-        return if @body.nil?
-
-        @headers["content-type"] ||= @body.content_type
-        @headers["content-length"] = @body.bytesize unless unbounded_body?
-        super(@body)
-      end
-
-      def each(&block)
-        return enum_for(__method__) unless block
-        return if @body.nil?
-
-        body = stream(@body)
-        if body.respond_to?(:read)
-          ::IO.copy_stream(body, ProcIO.new(block))
-        elsif body.respond_to?(:each)
-          body.each(&block)
-        else
-          block[body.to_s]
-        end
-      end
-
-      def rewind
-        return if empty?
-
-        @body.rewind if @body.respond_to?(:rewind)
-      end
-
-      def empty?
-        return true if @body.nil?
-        return false if chunked?
-
-        @body.bytesize.zero?
-      end
-
-      def bytesize
-        return 0 if @body.nil?
-
-        @body.bytesize
-      end
-
-      def stream(body)
-        encoded = body
-        encoded = Transcoder.registry("chunker").encode(body.enum_for(:each)) if chunked?
-        encoded
-      end
-
-      def unbounded_body?
-        return @unbounded_body if defined?(@unbounded_body)
-
-        @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
-      end
-
-      def chunked?
-        @headers["transfer-encoding"] == "chunked"
-      end
-
-      def chunk!
-        @headers.add("transfer-encoding", "chunked")
-      end
-
-      # :nocov:
-      def inspect
-        "#<HTTPX::Request::Body:#{object_id} " \
-          "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
-      end
-      # :nocov:
-    end
-
+    # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
     def transition(nextstate)
       case nextstate
       when :idle
@@ -267,19 +240,11 @@ module HTTPX
       nil
     end
 
+    # whether the request supports the 100-continue handshake and already processed the 100 response.
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
-
-    class ProcIO
-      def initialize(block)
-        @block = block
-      end
-
-      def write(data)
-        @block.call(data.dup)
-        data.bytesize
-      end
-    end
   end
 end
+
+require_relative "request/body"

data/lib/httpx/resolver/https.rb

@@ -4,12 +4,12 @@ require "resolv"
 require "uri"
 require "cgi"
 require "forwardable"
+require "httpx/base64"
 
 module HTTPX
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
     using URIExtensions
-    using StringExtensions
 
     module DNSExtensions
       refine Resolv::DNS do
@@ -27,7 +27,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
 
     def initialize(_, options)
       super
@@ -50,6 +50,7 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       end
 
@@ -67,9 +68,10 @@ module HTTPX
     def resolver_connection
       @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
-        connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
+        connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
         @pool.init_connection(connection, @options)
-        emit_addresses(connection, @family, @uri_addresses)
+        # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
+        emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
         @building_connection = false
         connection
       end
@@ -103,7 +105,7 @@ module HTTPX
         resolver_connection.send(request)
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
-        @queries.delete(hostname)
+        reset_hostname(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
     end
@@ -112,7 +114,7 @@ module HTTPX
       response.raise_for_status
     rescue StandardError => e
       hostname = @requests.delete(request)
-      connection = @queries.delete(hostname)
+      connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
@@ -127,17 +129,40 @@ module HTTPX
     end
 
     def parse(request, response)
-      begin
-        answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError => e
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, connection.origin.host, e)
-        return
+      code, result = decode_response_body(response)
+
+      case code
+      when :ok
+        parse_addresses(result, request)
+      when :no_domain_found
+        # Indicates no such domain was found.
+
+        host = @requests.delete(request)
+        connection = reset_hostname(host, reset_candidates: false)
+
+        unless @queries.value?(connection)
+          emit_resolve_error(connection)
+          return
+        end
+
+        resolve
+      when :dns_error
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+
+        emit_resolve_error(connection)
+      when :decode_error
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+        emit_resolve_error(connection, connection.origin.host, result)
       end
-      if answers.nil? || answers.empty?
+    end
+
+    def parse_addresses(answers, request)
+      if answers.empty?
+        # no address found, eliminate candidates
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
         emit_resolve_error(connection)
         return
 
@@ -148,7 +173,7 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                @queries.delete(address["name"])
+                reset_hostname(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
                 else
@@ -165,7 +190,7 @@ module HTTPX
           next if addresses.empty?
 
           hostname.delete_suffix!(".") if hostname.end_with?(".")
-          connection = @queries.delete(hostname)
+          connection = reset_hostname(hostname, reset_candidates: false)
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
@@ -210,5 +235,17 @@ module HTTPX
         raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      connection = @queries.delete(hostname)
+
+      return connection unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+
+      connection
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -46,14 +46,15 @@ module HTTPX
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
       return unless addresses
 
-      addresses = addresses.group_by(&:family)
+      addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
+        # try to match the resolver by family. However, there are cases where that's not possible, as when
+        # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
+        resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
 
-      @resolvers.each do |resolver|
-        addrs = addresses[resolver.family]
+        next unless resolver # this should ever happen
 
-        next if !addrs || addrs.empty?
-
-        resolver.emit_addresses(connection, resolver.family, addrs)
+        # it does not matter which resolver it is, as early-resolve code is shared.
+        resolver.emit_addresses(connection, family, addrs, true)
       end
     end
 
@@ -64,12 +65,7 @@ module HTTPX
     end
 
     def on_resolver_error(connection, error)
-      @errors[connection] << error
-
-      return unless @errors[connection].size >= @resolvers.size
-
-      errors = @errors.delete(connection)
-      emit(:error, connection, errors.first)
+      emit(:error, connection, error)
     end
 
     def on_resolver_close(resolver)