httpx 0.22.5 → 0.23.0

data/lib/httpx/resolver/native.rb

@@ -33,6 +33,7 @@ module HTTPX
       super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @socket_type = @resolver_options.fetch(:socket_type, :udp)
       @nameserver = Array(@resolver_options[:nameserver]) if @resolver_options[:nameserver]
       @ndots = @resolver_options[:ndots]
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
@@ -156,7 +157,7 @@ module HTTPX
       else
 
         @timeouts.delete(host)
-        @queries.delete(h)
+        reset_hostname(h, reset_candidates: false)
 
         return unless @queries.empty?
 
@@ -169,10 +170,49 @@ module HTTPX
 
     def dread(wsize = @resolver_options[:packet_size])
       loop do
+        wsize = @large_packet.capacity if @large_packet
+
         siz = @io.read(wsize, @read_buffer)
-        return unless siz && siz.positive?
 
-        parse(@read_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
+
+        if @socket_type == :tcp
+          # packet may be incomplete, need to keep draining from the socket
+          if @large_packet
+            # large packet buffer already exists, continue pumping
+            @large_packet << @read_buffer
+
+            next unless @large_packet.full?
+
+            parse(@large_packet.to_s)
+
+            @socket_type = @resolver_options.fetch(:socket_type, :udp)
+            @large_packet = nil
+            transition(:closed)
+            return
+          else
+            size = @read_buffer[0, 2].unpack1("n")
+
+            if size > @read_buffer.bytesize
+              # only do buffer logic if it's worth it, and the whole packet isn't here already
+              @large_packet = Buffer.new(size)
+              @large_packet << @read_buffer.byteslice(2..-1)
+
+              next
+            else
+              parse(@read_buffer)
+            end
+          end
+        else # udp
+          parse(@read_buffer)
+        end
+
         return if @state == :closed
       end
     end
@@ -182,41 +222,63 @@ module HTTPX
         return if @write_buffer.empty?
 
         siz = @io.write(@write_buffer)
-        return unless siz && siz.positive?
+
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
 
         return if @state == :closed
       end
     end
 
     def parse(buffer)
-      begin
-        addresses = Resolver.decode_dns_answer(buffer)
-      rescue Resolv::DNS::DecodeError => e
+      code, result = Resolver.decode_dns_answer(buffer)
+
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
+        # Indicates no such domain was found.
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname)
+
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
+        raise NativeResolveError.new(connection, connection.origin.host, "name or service not known (#{hostname})")
+      when :message_truncated
+        # TODO: what to do if it's already tcp??
+        return if @socket_type == :tcp
+
+        @socket_type = :tcp
+
+        hostname, _ = @queries.first
+        reset_hostname(hostname)
+        transition(:closed)
+      when :dns_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
         ex.set_backtrace(e.backtrace)
         raise ex
-      end
-
-      if addresses.nil?
-        # Indicates no such domain was found.
+      when :decode_error
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex.set_backtrace(result.backtrace)
+        raise ex
+      end
+    end
 
-        unless @queries.value?(connection)
-          @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host)
-        end
-      elsif addresses.empty?
+    def parse_addresses(addresses)
+      if addresses.empty?
         # no address found, eliminate candidates
         _, connection = @queries.first
-        candidates = @queries.select { |_, conn| connection == conn }.keys
-        @queries.delete_if { |hs, _| candidates.include?(hs) }
-        @timeouts.delete_if { |hs, _| candidates.include?(hs) }
+        reset_hostname(hostname)
         @connections.delete(connection)
         raise NativeResolveError.new(connection, connection.origin.host)
       else
@@ -226,20 +288,21 @@ module HTTPX
         connection = @queries.delete(name)
 
         unless connection
+          orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.keys.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
-          return unless name
+          unless name
+            @timeouts.delete(orig_name)
+            return
+          end
 
           address["name"] = name
           connection = @queries.delete(name)
         end
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
-
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -265,6 +328,7 @@ module HTTPX
 
     def resolve(connection = @connections.first, hostname = nil)
       raise Error, "no URI to resolve" unless connection
+
       return unless @write_buffer.empty?
 
       hostname ||= @queries.key(connection)
@@ -281,12 +345,19 @@ module HTTPX
       end
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
-        @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
+        @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         emit_resolve_error(connection, hostname, e)
       end
     end
 
+    def encode_dns_query(hostname)
+      message_id = Resolver.generate_id
+      msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
+      msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      msg
+    end
+
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
@@ -294,18 +365,25 @@ module HTTPX
       name_parts = name.scan(/[^.]+/)
       candidates = [name] if @ndots <= name_parts.size - 1
       candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
-      candidates << name unless candidates.include?(name)
+      fname = "#{name}."
+      candidates << fname unless candidates.include?(fname)
 
       candidates
     end
 
     def build_socket
-      return if @io
-
       ip, port = @nameserver[@ns_index]
       port ||= DNS_PORT
-      log { "resolver: server: #{ip}:#{port}..." }
-      @io = UDP.new(ip, port, @options)
+
+      case @socket_type
+      when :udp
+        log { "resolver: server: udp://#{ip}:#{port}..." }
+        UDP.new(ip, port, @options)
+      when :tcp
+        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        origin = URI("tcp://#{ip}:#{port}")
+        TCP.new(origin, [ip], @options)
+      end
     end
 
     def transition(nextstate)
@@ -319,7 +397,7 @@ module HTTPX
       when :open
         return unless @state == :idle
 
-        build_socket
+        @io ||= build_socket
 
         @io.connect
         return unless @io.connected?
@@ -346,5 +424,19 @@ module HTTPX
         end
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      @timeouts.delete(hostname)
+      connection = @queries.delete(hostname)
+      @timeouts.delete(hostname)
+
+      return unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
+    end
   end
 end

data/lib/httpx/resolver.rb

@@ -5,8 +5,6 @@ require "ipaddr"
 
 module HTTPX
   module Resolver
-    extend Registry
-
     RESOLVE_TIMEOUT = 5
 
     require "httpx/resolver/resolver"
@@ -15,10 +13,6 @@ module HTTPX
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    register :system, System
-    register :native, Native
-    register :https,  HTTPS
-
     @lookup_mutex = Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }
 
@@ -28,6 +22,18 @@ module HTTPX
 
     module_function
 
+    def resolver_for(resolver_type)
+      case resolver_type
+      when :native then Native
+      when :system then System
+      when :https then HTTPS
+      else
+        return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
+
+        raise Error, "unsupported resolver type (#{resolver_type})"
+      end
+    end
+
     def nolookup_resolve(hostname)
       ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
     end
@@ -98,24 +104,29 @@ module HTTPX
       @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
-    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A)
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
       Resolv::DNS::Message.new.tap do |query|
-        query.id = generate_id
+        query.id = message_id
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
     end
 
     def decode_dns_answer(payload)
-      message = Resolv::DNS::Message.decode(payload)
+      begin
+        message = Resolv::DNS::Message.decode(payload)
+      rescue Resolv::DNS::DecodeError => e
+        return :decode_error, e
+      end
 
       # no domain was found
-      return if message.rcode == Resolv::DNS::RCode::NXDomain
+      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
 
-      addresses = []
+      return :message_truncated if message.tc == 1
 
-      # TODO: raise an "other dns OtherResolvError" type of error
-      return addresses if message.rcode != Resolv::DNS::RCode::NoError
+      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+
+      addresses = []
 
       message.each_answer do |question, _, value|
         case value
@@ -134,7 +145,8 @@ module HTTPX
           }
         end
       end
-      addresses
+
+      [:ok, addresses]
     end
   end
 end

data/lib/httpx/response.rb

@@ -56,12 +56,12 @@ module HTTPX
     end
 
     def bodyless?
-      @request.verb == :head ||
+      @request.verb == "HEAD" ||
         no_data?
     end
 
     def complete?
-      bodyless? || (@request.verb == :connect && @status == 200)
+      bodyless? || (@request.verb == "CONNECT" && @status == 200)
     end
 
     # :nocov:
@@ -87,32 +87,27 @@ module HTTPX
     end
 
     def json(*args)
-      decode("json", *args)
+      decode(Transcoder::JSON, *args)
     end
 
     def form
-      decode("form")
+      decode(Transcoder::Form)
     end
 
     def xml
-      decode("xml")
+      decode(Transcoder::Xml)
     end
 
     private
 
-    def decode(format, *args)
+    def decode(transcoder, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
-      transcoder = Transcoder.registry(format)
-
-      raise Error, "no decoder available for \"#{format}\"" unless transcoder.respond_to?(:decode)
 
       decoder = transcoder.decode(self)
 
-      raise Error, "no decoder available for \"#{format}\"" unless decoder
+      raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
 
       decoder.call(self, *args)
-    rescue Registry::Error
-      raise Error, "no decoder available for \"#{format}\""
     end
 
     def no_data?
@@ -203,10 +198,8 @@ module HTTPX
           rescue ArgumentError # ex: unknown encoding name - utf
             content
           end
-        when nil
-          "".b
         else
-          @buffer
+          "".b
         end
       end
       alias_method :to_str, :to_s
@@ -334,12 +327,13 @@ module HTTPX
     include Loggable
     extend Forwardable
 
-    attr_reader :request, :error
+    attr_reader :request, :response, :error
 
     def_delegator :@request, :uri
 
     def initialize(request, error, options)
       @request = request
+      @response = request.response if request.response.is_a?(Response)
       @error = error
       @options = Options.new(options)
       log_exception(@error)
@@ -361,6 +355,10 @@ module HTTPX
       end
     end
 
+    def close
+      @response.close if @response.respond_to?(:close)
+    end
+
     def finished?
       true
     end

data/lib/httpx/session.rb

@@ -203,6 +203,7 @@ module HTTPX
     end
 
     def receive_requests(requests, connections)
+      # @type var responses: Array[response]
       responses = []
 
       begin

data/lib/httpx/transcoder/body.rb

@@ -55,5 +55,4 @@ module HTTPX::Transcoder
       Encoder.new(body)
     end
   end
-  register "body", Body
 end

data/lib/httpx/transcoder/chunker.rb

@@ -112,5 +112,4 @@ module HTTPX::Transcoder
       Encoder.new(chunks)
     end
   end
-  register "chunker", Chunker
 end

data/lib/httpx/transcoder/form.rb

@@ -55,5 +55,4 @@ module HTTPX::Transcoder
       Decoder
     end
   end
-  register "form", Form
 end

data/lib/httpx/transcoder/json.rb

@@ -56,5 +56,4 @@ module HTTPX::Transcoder
     end
     # rubocop:enable Style/SingleLineMethods
   end
-  register "json", JSON
 end

data/lib/httpx/transcoder/xml.rb

@@ -51,5 +51,4 @@ module HTTPX::Transcoder
       end
     end
   end
-  register "xml", Xml
 end

data/lib/httpx/transcoder.rb

@@ -2,8 +2,6 @@
 
 module HTTPX
   module Transcoder
-    extend Registry
-
     using RegexpExtensions unless Regexp.method_defined?(:match?)
 
     module_function

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.22.5"
+  VERSION = "0.23.0"
 end

data/lib/httpx.rb

@@ -11,7 +11,6 @@ require "httpx/domain_name"
 require "httpx/altsvc"
 require "httpx/callbacks"
 require "httpx/loggable"
-require "httpx/registry"
 require "httpx/transcoder"
 require "httpx/timers"
 require "httpx/pool"

data/sig/buffer.rbs

@@ -11,6 +11,7 @@ module HTTPX
 
     def full?: () -> bool
     def shift!: (Integer) -> void
+    def capacity: () -> Integer
 
     # delegated
     def <<: (string data) -> String

data/sig/chainable.rbs

@@ -2,9 +2,9 @@ module HTTPX
   module Chainable
     def request: (*Request, **untyped) -> Array[response]
                | (Request, **untyped) -> response
-               | (verb | string, uri | [uri], **untyped) -> response
-               | (Array[[verb | string, uri] | [verb | string, uri, options]], **untyped) -> Array[response]
-               | (verb | string, _Each[uri | [uri, options]], **untyped) -> Array[response]
+               | (verb, uri | [uri], **untyped) -> response
+               | (Array[[verb, uri] | [verb, uri, options]], **untyped) -> Array[response]
+               | (verb, _Each[uri | [uri, options]], **untyped) -> Array[response]
 
     def accept: (String) -> Session
     def wrap: () { (Session) -> void } -> void

data/sig/connection.rbs

@@ -17,9 +17,7 @@ module HTTPX
     extend Forwardable
     include Loggable
     include Callbacks
-    include HTTPX::Registry[String, Class]
 
-    BUFFER_SIZE: Integer
 
     attr_reader type: io_type
     attr_reader origin: URI::Generic
@@ -36,7 +34,13 @@ module HTTPX
     @write_buffer: Buffer
     @inflight: Integer
     @keep_alive_timeout: Numeric?
+    @timeout: Numeric?
+    @current_timeout: Numeric?
     @total_timeout: Numeric?
+    @io: TCP | SSL | UNIX
+    @parser: HTTP1 | HTTP2 | _Parser
+    @connected_at: Float
+    @response_received_at: Float
 
     def addresses: () -> Array[ipaddr]?
 
@@ -77,6 +81,8 @@ module HTTPX
 
     def deactivate: () -> void
 
+    def open?: () -> bool
+
     def raise_timeout_error: (Numeric interval) -> void
 
     private
@@ -91,17 +97,20 @@ module HTTPX
 
     def send_pending: () -> void
 
-    def parser: () -> _Parser
+    def parser: () -> (HTTP1 | HTTP2 | _Parser)
 
     def send_request_to_parser: (Request request) -> void
 
-    def build_parser: () -> _Parser
-                    | (String) -> _Parser
+    def build_parser: (?String protocol) -> (HTTP1 | HTTP2)
 
-    def set_parser_callbacks: (_Parser) -> void
+    def set_parser_callbacks: (HTTP1 | HTTP2 parser) -> void
 
     def transition: (Symbol) -> void
 
+    def handle_transition: (Symbol) -> void
+
+    def build_socket: (?Array[ipaddr]? addrs) -> (TCP | SSL | UNIX)
+
     def on_error: (HTTPX::TimeoutError | Error | StandardError) -> void
 
     def handle_error: (StandardError) -> void
@@ -113,5 +122,7 @@ module HTTPX
     def write_timeout_callback: (Request request, Numeric write_timeout) -> void
 
     def read_timeout_callback: (Request request, Numeric read_timeout, ?singleton(RequestTimeoutError) error_type) -> void
+
+    def self.parser_type: (String protocol) -> (singleton(HTTP1) | singleton(HTTP2))
   end
 end

data/sig/errors.rbs

@@ -2,6 +2,12 @@ module HTTPX
   class Error < StandardError
   end
 
+  class UnsupportedSchemeError < Error
+  end
+
+  class ConnectionError < Error
+  end
+
   class TimeoutError < Error
     attr_reader timeout: Numeric
 
@@ -55,4 +61,7 @@ module HTTPX
 
     def initialize: (Connection connection, String hostname, ?String message) -> untyped
   end
+
+  class MisdirectedRequestError < HTTPError
+  end
 end

data/sig/httpx.rbs

@@ -8,9 +8,9 @@ module HTTPX
   type uri = URI::HTTP | URI::HTTPS | string
   type generic_uri = String | URI::Generic
 
-  type verb = :options | :get | :head | :post | :put | :delete | :trace | :connect |
-              :propfind | :proppatch | :mkcol | :copy | :move | :lock | :unlock | :orderpatch |
-              :acl | :report | :patch | :search
+  type verb = "OPTIONS" | "GET" | "HEAD" | "POST" | "PUT" | "DELETE" | "TRACE" | "CONNECT" |
+              "PROPFIND" | "PROPPATCH" | "MKCOL" | "COPY" | "MOVE" | "LOCK" | "UNLOCK" | "ORDERPATCH" |
+              "ACL" | "REPORT" | "PATCH" | "SEARCH"
 
   type ip_family = Integer #Socket::AF_INET6 | Socket::AF_INET
 

data/sig/io/ssl.rbs

@@ -0,0 +1,17 @@
+module HTTPX
+  IPRegex: Regexp
+
+  class TLSError < OpenSSL::SSL::SSLError
+  end
+
+  class SSL < TCP
+    TLS_OPTIONS: Hash[Symbol, untyped]
+
+    def can_verify_peer?: () -> bool
+
+    def verify_hostname: (String host) -> bool
+
+    # :nocov:
+    def try_ssl_connect: () -> void
+  end
+end