httpx 0.22.4 → 0.23.0

data/lib/httpx/resolver/https.rb

@@ -104,7 +104,7 @@ module HTTPX
         resolver_connection.send(request)
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
-        @queries.delete(hostname)
+        reset_hostname(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
     end
@@ -113,7 +113,7 @@ module HTTPX
       response.raise_for_status
     rescue StandardError => e
       hostname = @requests.delete(request)
-      connection = @queries.delete(hostname)
+      connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
@@ -128,30 +128,35 @@ module HTTPX
     end
 
     def parse(request, response)
-      begin
-        answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError => e
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, connection.origin.host, e)
-        return
-      end
+      code, result = decode_response_body(response)
 
-      if answers.nil?
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
         # Indicates no such domain was found.
 
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
 
-        emit_resolve_error(connection) unless @queries.value?(connection)
-      elsif answers.empty?
-        # no address found, eliminate candidates
+        emit_resolve_error(connection)
+      when :dns_error
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
+        emit_resolve_error(connection)
+      when :decode_error
+        host, connection = @queries.first
+        reset_hostname(host)
+        emit_resolve_error(connection, connection.origin.host, result)
+      end
+    end
 
+    def parse_addresses(answers)
+      if answers.empty?
+        # no address found, eliminate candidates
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
         emit_resolve_error(connection)
         return
 
@@ -162,7 +167,7 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                @queries.delete(address["name"])
+                reset_hostname(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
                 else
@@ -179,7 +184,7 @@ module HTTPX
           next if addresses.empty?
 
           hostname.delete_suffix!(".") if hostname.end_with?(".")
-          connection = @queries.delete(hostname)
+          connection = reset_hostname(hostname, reset_candidates: false)
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
@@ -224,5 +229,17 @@ module HTTPX
         raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      connection = @queries.delete(hostname)
+
+      return connection unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+
+      connection
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -64,12 +64,7 @@ module HTTPX
     end
 
     def on_resolver_error(connection, error)
-      @errors[connection] << error
-
-      return unless @errors[connection].size >= @resolvers.size
-
-      errors = @errors.delete(connection)
-      emit(:error, connection, errors.first)
+      emit(:error, connection, error)
     end
 
     def on_resolver_close(resolver)

data/lib/httpx/resolver/native.rb

@@ -33,6 +33,7 @@ module HTTPX
       super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @socket_type = @resolver_options.fetch(:socket_type, :udp)
       @nameserver = Array(@resolver_options[:nameserver]) if @resolver_options[:nameserver]
       @ndots = @resolver_options[:ndots]
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
@@ -156,7 +157,7 @@ module HTTPX
       else
 
         @timeouts.delete(host)
-        @queries.delete(h)
+        reset_hostname(h, reset_candidates: false)
 
         return unless @queries.empty?
 
@@ -169,10 +170,49 @@ module HTTPX
 
     def dread(wsize = @resolver_options[:packet_size])
       loop do
+        wsize = @large_packet.capacity if @large_packet
+
         siz = @io.read(wsize, @read_buffer)
-        return unless siz && siz.positive?
 
-        parse(@read_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
+
+        if @socket_type == :tcp
+          # packet may be incomplete, need to keep draining from the socket
+          if @large_packet
+            # large packet buffer already exists, continue pumping
+            @large_packet << @read_buffer
+
+            next unless @large_packet.full?
+
+            parse(@large_packet.to_s)
+
+            @socket_type = @resolver_options.fetch(:socket_type, :udp)
+            @large_packet = nil
+            transition(:closed)
+            return
+          else
+            size = @read_buffer[0, 2].unpack1("n")
+
+            if size > @read_buffer.bytesize
+              # only do buffer logic if it's worth it, and the whole packet isn't here already
+              @large_packet = Buffer.new(size)
+              @large_packet << @read_buffer.byteslice(2..-1)
+
+              next
+            else
+              parse(@read_buffer)
+            end
+          end
+        else # udp
+          parse(@read_buffer)
+        end
+
         return if @state == :closed
       end
     end
@@ -182,41 +222,63 @@ module HTTPX
         return if @write_buffer.empty?
 
         siz = @io.write(@write_buffer)
-        return unless siz && siz.positive?
+
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
 
         return if @state == :closed
       end
     end
 
     def parse(buffer)
-      begin
-        addresses = Resolver.decode_dns_answer(buffer)
-      rescue Resolv::DNS::DecodeError => e
+      code, result = Resolver.decode_dns_answer(buffer)
+
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
+        # Indicates no such domain was found.
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname)
+
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
+        raise NativeResolveError.new(connection, connection.origin.host, "name or service not known (#{hostname})")
+      when :message_truncated
+        # TODO: what to do if it's already tcp??
+        return if @socket_type == :tcp
+
+        @socket_type = :tcp
+
+        hostname, _ = @queries.first
+        reset_hostname(hostname)
+        transition(:closed)
+      when :dns_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
         ex.set_backtrace(e.backtrace)
         raise ex
-      end
-
-      if addresses.nil?
-        # Indicates no such domain was found.
+      when :decode_error
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex.set_backtrace(result.backtrace)
+        raise ex
+      end
+    end
 
-        unless @queries.value?(connection)
-          @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host)
-        end
-      elsif addresses.empty?
+    def parse_addresses(addresses)
+      if addresses.empty?
         # no address found, eliminate candidates
         _, connection = @queries.first
-        candidates = @queries.select { |_, conn| connection == conn }.keys
-        @queries.delete_if { |hs, _| candidates.include?(hs) }
-        @timeouts.delete_if { |hs, _| candidates.include?(hs) }
+        reset_hostname(hostname)
         @connections.delete(connection)
         raise NativeResolveError.new(connection, connection.origin.host)
       else
@@ -226,20 +288,21 @@ module HTTPX
         connection = @queries.delete(name)
 
         unless connection
+          orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.keys.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
-          return unless name
+          unless name
+            @timeouts.delete(orig_name)
+            return
+          end
 
           address["name"] = name
           connection = @queries.delete(name)
         end
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
-
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -265,6 +328,7 @@ module HTTPX
 
     def resolve(connection = @connections.first, hostname = nil)
       raise Error, "no URI to resolve" unless connection
+
       return unless @write_buffer.empty?
 
       hostname ||= @queries.key(connection)
@@ -281,12 +345,19 @@ module HTTPX
       end
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
-        @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
+        @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         emit_resolve_error(connection, hostname, e)
       end
     end
 
+    def encode_dns_query(hostname)
+      message_id = Resolver.generate_id
+      msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
+      msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      msg
+    end
+
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
@@ -294,18 +365,25 @@ module HTTPX
       name_parts = name.scan(/[^.]+/)
       candidates = [name] if @ndots <= name_parts.size - 1
       candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
-      candidates << name unless candidates.include?(name)
+      fname = "#{name}."
+      candidates << fname unless candidates.include?(fname)
 
       candidates
     end
 
     def build_socket
-      return if @io
-
       ip, port = @nameserver[@ns_index]
       port ||= DNS_PORT
-      log { "resolver: server: #{ip}:#{port}..." }
-      @io = UDP.new(ip, port, @options)
+
+      case @socket_type
+      when :udp
+        log { "resolver: server: udp://#{ip}:#{port}..." }
+        UDP.new(ip, port, @options)
+      when :tcp
+        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        origin = URI("tcp://#{ip}:#{port}")
+        TCP.new(origin, [ip], @options)
+      end
     end
 
     def transition(nextstate)
@@ -319,7 +397,7 @@ module HTTPX
       when :open
         return unless @state == :idle
 
-        build_socket
+        @io ||= build_socket
 
         @io.connect
         return unless @io.connected?
@@ -346,5 +424,19 @@ module HTTPX
         end
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      @timeouts.delete(hostname)
+      connection = @queries.delete(hostname)
+      @timeouts.delete(hostname)
+
+      return unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
+    end
   end
 end

data/lib/httpx/resolver.rb

@@ -5,8 +5,6 @@ require "ipaddr"
 
 module HTTPX
   module Resolver
-    extend Registry
-
     RESOLVE_TIMEOUT = 5
 
     require "httpx/resolver/resolver"
@@ -15,10 +13,6 @@ module HTTPX
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    register :system, System
-    register :native, Native
-    register :https,  HTTPS
-
     @lookup_mutex = Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }
 
@@ -28,6 +22,18 @@ module HTTPX
 
     module_function
 
+    def resolver_for(resolver_type)
+      case resolver_type
+      when :native then Native
+      when :system then System
+      when :https then HTTPS
+      else
+        return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
+
+        raise Error, "unsupported resolver type (#{resolver_type})"
+      end
+    end
+
     def nolookup_resolve(hostname)
       ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
     end
@@ -98,24 +104,29 @@ module HTTPX
       @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
-    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A)
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
       Resolv::DNS::Message.new.tap do |query|
-        query.id = generate_id
+        query.id = message_id
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
     end
 
     def decode_dns_answer(payload)
-      message = Resolv::DNS::Message.decode(payload)
+      begin
+        message = Resolv::DNS::Message.decode(payload)
+      rescue Resolv::DNS::DecodeError => e
+        return :decode_error, e
+      end
 
       # no domain was found
-      return if message.rcode == Resolv::DNS::RCode::NXDomain
+      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
 
-      addresses = []
+      return :message_truncated if message.tc == 1
 
-      # TODO: raise an "other dns OtherResolvError" type of error
-      return addresses if message.rcode != Resolv::DNS::RCode::NoError
+      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+
+      addresses = []
 
       message.each_answer do |question, _, value|
         case value
@@ -134,7 +145,8 @@ module HTTPX
           }
         end
       end
-      addresses
+
+      [:ok, addresses]
     end
   end
 end

data/lib/httpx/response.rb

@@ -56,12 +56,12 @@ module HTTPX
     end
 
     def bodyless?
-      @request.verb == :head ||
+      @request.verb == "HEAD" ||
         no_data?
     end
 
     def complete?
-      bodyless? || (@request.verb == :connect && @status == 200)
+      bodyless? || (@request.verb == "CONNECT" && @status == 200)
     end
 
     # :nocov:
@@ -87,32 +87,27 @@ module HTTPX
     end
 
     def json(*args)
-      decode("json", *args)
+      decode(Transcoder::JSON, *args)
     end
 
     def form
-      decode("form")
+      decode(Transcoder::Form)
     end
 
     def xml
-      decode("xml")
+      decode(Transcoder::Xml)
     end
 
     private
 
-    def decode(format, *args)
+    def decode(transcoder, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
-      transcoder = Transcoder.registry(format)
-
-      raise Error, "no decoder available for \"#{format}\"" unless transcoder.respond_to?(:decode)
 
       decoder = transcoder.decode(self)
 
-      raise Error, "no decoder available for \"#{format}\"" unless decoder
+      raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
 
       decoder.call(self, *args)
-    rescue Registry::Error
-      raise Error, "no decoder available for \"#{format}\""
     end
 
     def no_data?
@@ -203,10 +198,8 @@ module HTTPX
           rescue ArgumentError # ex: unknown encoding name - utf
             content
           end
-        when nil
-          "".b
         else
-          @buffer
+          "".b
         end
       end
       alias_method :to_str, :to_s
@@ -334,12 +327,13 @@ module HTTPX
     include Loggable
     extend Forwardable
 
-    attr_reader :request, :error
+    attr_reader :request, :response, :error
 
     def_delegator :@request, :uri
 
     def initialize(request, error, options)
       @request = request
+      @response = request.response if request.response.is_a?(Response)
       @error = error
       @options = Options.new(options)
       log_exception(@error)
@@ -361,6 +355,10 @@ module HTTPX
       end
     end
 
+    def close
+      @response.close if @response.respond_to?(:close)
+    end
+
     def finished?
       true
     end

data/lib/httpx/session.rb

@@ -203,6 +203,7 @@ module HTTPX
     end
 
     def receive_requests(requests, connections)
+      # @type var responses: Array[response]
       responses = []
 
       begin

data/lib/httpx/transcoder/body.rb

@@ -55,5 +55,4 @@ module HTTPX::Transcoder
       Encoder.new(body)
     end
   end
-  register "body", Body
 end

data/lib/httpx/transcoder/chunker.rb

@@ -112,5 +112,4 @@ module HTTPX::Transcoder
       Encoder.new(chunks)
     end
   end
-  register "chunker", Chunker
 end

data/lib/httpx/transcoder/form.rb

@@ -55,5 +55,4 @@ module HTTPX::Transcoder
       Decoder
     end
   end
-  register "form", Form
 end

data/lib/httpx/transcoder/json.rb

@@ -56,5 +56,4 @@ module HTTPX::Transcoder
     end
     # rubocop:enable Style/SingleLineMethods
   end
-  register "json", JSON
 end

data/lib/httpx/transcoder/xml.rb

@@ -51,5 +51,4 @@ module HTTPX::Transcoder
       end
     end
   end
-  register "xml", Xml
 end

data/lib/httpx/transcoder.rb

@@ -2,8 +2,6 @@
 
 module HTTPX
   module Transcoder
-    extend Registry
-
     using RegexpExtensions unless Regexp.method_defined?(:match?)
 
     module_function

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.22.4"
+  VERSION = "0.23.0"
 end

data/lib/httpx.rb

@@ -11,7 +11,6 @@ require "httpx/domain_name"
 require "httpx/altsvc"
 require "httpx/callbacks"
 require "httpx/loggable"
-require "httpx/registry"
 require "httpx/transcoder"
 require "httpx/timers"
 require "httpx/pool"

data/sig/buffer.rbs

@@ -11,6 +11,7 @@ module HTTPX
 
     def full?: () -> bool
     def shift!: (Integer) -> void
+    def capacity: () -> Integer
 
     # delegated
     def <<: (string data) -> String

data/sig/chainable.rbs

@@ -2,9 +2,9 @@ module HTTPX
   module Chainable
     def request: (*Request, **untyped) -> Array[response]
                | (Request, **untyped) -> response
-               | (verb | string, uri | [uri], **untyped) -> response
-               | (Array[[verb | string, uri] | [verb | string, uri, options]], **untyped) -> Array[response]
-               | (verb | string, _Each[uri | [uri, options]], **untyped) -> Array[response]
+               | (verb, uri | [uri], **untyped) -> response
+               | (Array[[verb, uri] | [verb, uri, options]], **untyped) -> Array[response]
+               | (verb, _Each[uri | [uri, options]], **untyped) -> Array[response]
 
     def accept: (String) -> Session
     def wrap: () { (Session) -> void } -> void