httpx 0.18.4 → 0.19.0

data/lib/httpx/resolver/resolver.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "resolv"
+require "ipaddr"
+
+module HTTPX
+  class Resolver::Resolver
+    include Callbacks
+    include Loggable
+
+    RECORD_TYPES = {
+      Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
+      Socket::AF_INET => Resolv::DNS::Resource::IN::A,
+    }.freeze
+
+    FAMILY_TYPES = {
+      Resolv::DNS::Resource::IN::AAAA => "AAAA",
+      Resolv::DNS::Resource::IN::A => "A",
+    }.freeze
+
+    class << self
+      def multi?
+        true
+      end
+    end
+
+    attr_reader :family
+
+    attr_writer :pool
+
+    def initialize(family, options)
+      @family = family
+      @record_type = RECORD_TYPES[family]
+      @options = Options.new(options)
+    end
+
+    def close; end
+
+    def closed?
+      true
+    end
+
+    def empty?
+      true
+    end
+
+    def emit_addresses(connection, family, addresses)
+      addresses.map! do |address|
+        address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
+      end
+      log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
+      if !connection.io &&
+         connection.options.ip_families.size > 1 &&
+         family == Socket::AF_INET &&
+         addresses.first.to_s != connection.origin.host.to_s
+        log { "resolver: A response, applying resolution delay..." }
+        @pool.after(0.05) do
+          connection.addresses = addresses
+          emit(:resolve, connection)
+        end
+      else
+        connection.addresses = addresses
+        emit(:resolve, connection)
+      end
+    end
+
+    private
+
+    def early_resolve(connection, hostname: connection.origin.host)
+      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+
+      return unless addresses
+
+      addresses.select! { |addr| addr.family == @family }
+
+      emit_addresses(connection, @family, addresses)
+    end
+
+    def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
+      emit(:error, connection, resolve_error(hostname, ex))
+    end
+
+    def resolve_error(hostname, ex = nil)
+      return ex if ex.is_a?(ResolveError)
+
+      message = ex ? ex.message : "Can't resolve #{hostname}"
+      error = ResolveError.new(message)
+      error.set_backtrace(ex ? ex.backtrace : caller)
+      error
+    end
+  end
+end

data/lib/httpx/resolver/system.rb

@@ -4,48 +4,204 @@ require "forwardable"
 require "resolv"
 
 module HTTPX
-  class Resolver::System
-    include Resolver::ResolverMixin
+  class Resolver::System < Resolver::Resolver
+    using URIExtensions
+    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
                      Resolv::DNS::EncodeError,
                      Resolv::DNS::DecodeError].freeze
 
+    DONE = 1
+    ERROR = 2
+
+    class << self
+      def multi?
+        false
+      end
+    end
+
     attr_reader :state
 
+    def_delegator :@connections, :empty?
+
     def initialize(options)
-      @options = Options.new(options)
+      super(nil, options)
       @resolver_options = @options.resolver_options
-      @state = :idle
       resolv_options = @resolver_options.dup
-      timeouts = resolv_options.delete(:timeouts)
+      timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
+      @_timeouts = Array(timeouts)
+      @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
       resolv_options.delete(:cache)
-      @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
-      @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
+      @connections = []
+      @queries = []
+      @ips = []
+      @pipe_mutex = Thread::Mutex.new
+      @state = :idle
+    end
+
+    def resolvers
+      return enum_for(__method__) unless block_given?
+
+      yield self
+    end
+
+    def connections
+      EMPTY
+    end
+
+    def close
+      transition(:closed)
     end
 
     def closed?
-      true
+      @state == :closed
+    end
+
+    def to_io
+      @pipe_read.to_io
+    end
+
+    def call
+      case @state
+      when :open
+        consume
+      end
+      nil
     end
 
-    def empty?
-      true
+    def interests
+      return if @queries.empty?
+
+      :r
+    end
+
+    def timeout
+      return unless @queries.empty?
+
+      _, connection = @queries.first
+
+      @timeouts[connection.origin.host].first
     end
 
     def <<(connection)
+      @connections << connection
+      resolve
+    end
+
+    private
+
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        @timeouts.clear
+      when :open
+        return unless @state == :idle
+
+        @pipe_read, @pipe_write = ::IO.pipe
+      when :closed
+        return unless @state == :open
+
+        @pipe_write.close
+        @pipe_read.close
+      end
+      @state = nextstate
+    end
+
+    def consume
+      return if @connections.empty?
+
+      while @pipe_read.ready? && (event = @pipe_read.getbyte)
+        case event
+        when DONE
+          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+          @queries.delete(pair)
+
+          family, connection = pair
+          emit_addresses(connection, family, addrs)
+        when ERROR
+          *pair, error = @pipe_mutex.synchronize { @ips.pop }
+          @queries.delete(pair)
+
+          family, connection = pair
+          emit_resolve_error(connection, connection.origin.host, error)
+        end
+
+        @connections.delete(connection) if @queries.empty?
+      end
+
+      return emit(:close, self) if @connections.empty?
+
+      resolve
+    end
+
+    def resolve(connection = @connections.first)
+      raise Error, "no URI to resolve" unless connection
+      return unless @queries.empty?
+
       hostname = connection.origin.host
-      addresses = connection.addresses ||
-                  ip_resolve(hostname) ||
-                  system_resolve(hostname) ||
-                  @resolver.getaddresses(hostname)
-      throw(:resolve_error, resolve_error(hostname)) if addresses.empty?
+      scheme = connection.origin.scheme
+      log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+      transition(:open)
 
-      emit_addresses(connection, addresses)
-    rescue Errno::EHOSTUNREACH, *RESOLV_ERRORS => e
-      emit_resolve_error(connection, hostname, e)
+      connection.options.ip_families.each do |family|
+        @queries << [family, connection]
+      end
+      async_resolve(connection, hostname, scheme)
+      consume
     end
 
-    def uncache(*); end
+    def async_resolve(connection, hostname, scheme)
+      families = connection.options.ip_families
+      log { "resolver: query for #{hostname}" }
+      resolve_timeout = @timeouts[connection.origin.host].first
+
+      Thread.start do
+        Thread.current.report_on_exception = false
+        begin
+          addrs = if resolve_timeout
+            Timeout.timeout(resolve_timeout) do
+              __addrinfo_resolve(hostname, scheme)
+            end
+          else
+            __addrinfo_resolve(hostname, scheme)
+          end
+          addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
+          families.each do |family|
+            addresses = addrs[family]
+            next unless addresses
+
+            addresses.map!(&:ip_address)
+            addresses.uniq!
+            @pipe_mutex.synchronize do
+              @ips.unshift([family, connection, addresses])
+              @pipe_write.putc(DONE) unless @pipe_write.closed?
+            end
+          end
+        rescue Timeout::Error => e
+          ex = ResolveTimeoutError.new(resolve_timeout, e.message)
+          ex.set_backtrace(ex.backtrace)
+          @pipe_mutex.synchronize do
+            families.each do |family|
+              @ips.unshift([family, connection, ex])
+              @pipe_write.putc(ERROR) unless @pipe_write.closed?
+            end
+          end
+        rescue StandardError => e
+          @pipe_mutex.synchronize do
+            families.each do |family|
+              @ips.unshift([family, connection, e])
+              @pipe_write.putc(ERROR) unless @pipe_write.closed?
+            end
+          end
+        end
+      end
+    end
+
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
   end
 end

data/lib/httpx/resolver.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "resolv"
+require "ipaddr"
 
 module HTTPX
   module Resolver
@@ -8,10 +9,11 @@ module HTTPX
 
     RESOLVE_TIMEOUT = 5
 
-    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
     require "httpx/resolver/https"
+    require "httpx/resolver/multi"
 
     register :system, System
     register :native, Native
@@ -22,9 +24,27 @@ module HTTPX
 
     @identifier_mutex = Mutex.new
     @identifier = 1
+    @system_resolver = Resolv::Hosts.new
 
     module_function
 
+    def nolookup_resolve(hostname)
+      ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
+    end
+
+    def ip_resolve(hostname)
+      [IPAddr.new(hostname)]
+    rescue ArgumentError
+    end
+
+    def system_resolve(hostname)
+      ips = @system_resolver.getaddresses(hostname)
+      return if ips.empty?
+
+      ips.map { |ip| IPAddr.new(ip) }
+    rescue IOError
+    end
+
     def cached_lookup(hostname)
       now = Utils.now
       @lookup_mutex.synchronize do
@@ -32,25 +52,31 @@ module HTTPX
       end
     end
 
-    def cached_lookup_set(hostname, entries)
+    def cached_lookup_set(hostname, family, entries)
       now = Utils.now
       entries.each do |entry|
         entry["TTL"] += now
       end
       @lookup_mutex.synchronize do
-        @lookups[hostname] += entries
+        case family
+        when Socket::AF_INET6
+          @lookups[hostname].concat(entries)
+        when Socket::AF_INET
+          @lookups[hostname].unshift(*entries)
+        end
         entries.each do |entry|
-          @lookups[entry["name"]] << entry if entry["name"] != hostname
+          next unless entry["name"] != hostname
+
+          case family
+          when Socket::AF_INET6
+            @lookups[entry["name"]] << entry
+          when Socket::AF_INET
+            @lookups[entry["name"]].unshift(entry)
+          end
         end
       end
     end
 
-    def uncache(hostname)
-      @lookup_mutex.synchronize do
-        @lookups.delete(hostname)
-      end
-    end
-
     # do not use directly!
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
@@ -62,7 +88,7 @@ module HTTPX
         if address.key?("alias")
           lookup(address["alias"], ttl)
         else
-          address["data"]
+          IPAddr.new(address["data"])
         end
       end
       ips unless ips.empty?

data/lib/httpx/response.rb

@@ -173,7 +173,7 @@ module HTTPX
           rescue ArgumentError
             @buffer.string
           end
-        when Tempfile, File
+        when Tempfile
           rewind
           content = _with_same_buffer_pos { @buffer.read }
           begin
@@ -253,6 +253,7 @@ module HTTPX
             @buffer = StringIO.new("".b)
           end
         when :memory
+          # @type ivar @buffer: StringIO | Tempfile
           if @length > @threshold_size
             aux = @buffer
             @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
@@ -272,11 +273,12 @@ module HTTPX
       def _with_same_buffer_pos
         return yield unless @buffer && @buffer.respond_to?(:pos)
 
+        # @type ivar @buffer: StringIO | Tempfile
         current_pos = @buffer.pos
         @buffer.rewind
         begin
           yield
-        rescue StandardError
+        ensure
           @buffer.pos = current_pos
         end
       end

data/lib/httpx/session.rb

@@ -207,7 +207,7 @@ module HTTPX
 
           return responses unless request
 
-          pool.next_tick until (response = fetch_response(request, connections, request.options))
+          catch(:coalesced) { pool.next_tick } until (response = fetch_response(request, connections, request.options))
 
           responses << response
           requests.shift
@@ -309,18 +309,4 @@ module HTTPX
       # :nocov:
     end
   end
-
-  unless ENV.grep(/https?_proxy$/i).empty?
-    proxy_session = plugin(:proxy)
-    ::HTTPX.send(:remove_const, :Session)
-    ::HTTPX.send(:const_set, :Session, proxy_session.class)
-  end
-
-  # :nocov:
-  if Session.default_options.debug_level > 2
-    proxy_session = plugin(:internal_telemetry)
-    ::HTTPX.send(:remove_const, :Session)
-    ::HTTPX.send(:const_set, :Session, proxy_session.class)
-  end
-  # :nocov:
 end

data/lib/httpx/session_extensions.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module HTTPX
+  unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
+    proxy_session = plugin(:proxy)
+    remove_const(:Session)
+    const_set(:Session, proxy_session.class)
+
+    # redefine the default options static var, which needs to
+    # refresh options_class
+    options = proxy_session.class.default_options.to_hash
+    options.freeze
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    Options.send(:const_set, :DEFAULT_OPTIONS, options)
+    $VERBOSE = original_verbosity
+  end
+
+  # :nocov:
+  if Session.default_options.debug_level > 2
+    proxy_session = plugin(:internal_telemetry)
+    remove_const(:Session)
+    const_set(:Session, proxy_session.class)
+  end
+  # :nocov:
+end

data/lib/httpx/timers.rb

@@ -64,7 +64,7 @@ module HTTPX
       end
 
       def to_f
-        @interval
+        Float(@interval)
       end
 
       def <<(callback)

data/lib/httpx/transcoder/chunker.rb

@@ -40,7 +40,6 @@ module HTTPX::Transcoder
 
       def initialize(buffer, trailers = false)
         @buffer = buffer
-        @chunk_length = nil
         @chunk_buffer = "".b
         @finished = false
         @state = :length

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.18.4"
+  VERSION = "0.19.0"
 end

data/lib/httpx.rb

@@ -25,6 +25,8 @@ require "mutex_m"
 # Top-Level Namespace
 #
 module HTTPX
+  EMPTY = [].freeze
+
   # All plugins should be stored under this module/namespace. Can register and load
   # plugins.
   #
@@ -64,3 +66,4 @@ module HTTPX
 end
 
 require "httpx/session"
+require "httpx/session_extensions"

data/sig/connection/http1.rbs

@@ -67,8 +67,6 @@ module HTTPX
 
     def set_protocol_headers: (Request) -> _Each[[String, String]]
 
-    def headline_uri: (Request) -> String
-
     def handle: (Request request) -> void
 
     def join_headers: (Request request) -> void

data/sig/connection/http2.rbs

@@ -46,8 +46,6 @@ module HTTPX
 
     def send_pending: () -> void
 
-    def headline_uri: (Request) -> String
-
     def set_protocol_headers: (Request) -> _Each[[String, String]]
 
     def handle: (Request request, HTTP2Next::Stream stream) -> void
@@ -56,6 +54,8 @@ module HTTPX
 
     def handle_stream: (HTTP2Next::Stream stream, Request request) -> void
 
+    def join_headline: (Request request) -> String
+
     def join_headers: (HTTP2Next::Stream stream, Request request) -> void
 
     def join_trailers: (HTTP2Next::Stream stream, Request request) -> void

data/sig/connection.rbs

@@ -22,6 +22,7 @@ module HTTPX
     BUFFER_SIZE: Integer
 
     attr_reader origin: URI::Generic
+    attr_reader origins: Array[String]
     attr_reader state: Symbol
     attr_reader pending: Array[Request]
     attr_reader options: Options

data/sig/errors.rbs

@@ -26,6 +26,14 @@ module HTTPX
   class ResolveError < Error
   end
 
+  class HTTPError < Error
+    attr_reader response: Response
+
+    private
+
+    def initialize: (Response response) -> void
+  end
+
   class NativeResolveError < ResolveError
     attr_reader connection: Connection
     attr_reader host: String

data/sig/headers.rbs

@@ -2,8 +2,6 @@ module HTTPX
   class Headers
     include _ToS
 
-    EMPTY: Array[untyped]
-
     @headers: Hash[String, Array[String]]
 
     def self.new: (?untyped headers) -> instance

data/sig/httpx.rbs

@@ -1,6 +1,8 @@
 module HTTPX
   extend Chainable
 
+  EMPTY: Array[untyped]
+
   VERSION: String
 
   type uri = URI::HTTP | URI::HTTPS | string
@@ -10,6 +12,8 @@ module HTTPX
               :propfind | :proppatch | :mkcol | :copy | :move | :lock | :unlock | :orderpatch |
               :acl | :report | :patch | :search
 
+  type ip_family = Integer #Socket::AF_INET6 | Socket::AF_INET
+
   module Plugins
     def self?.load_plugin: (Symbol) -> Module
 

data/sig/options.rbs

@@ -83,18 +83,16 @@ module HTTPX
 
     attr_reader response_body_class: singleton(Response::Body)
 
-    attr_reader ssl: Hash[Symbol, untyped]
+    attr_reader resolver_class: Symbol | Class
 
-    # request_class response_class headers_class request_body_class
-    # response_body_class connection_class
-    # resolver_class resolver_options
+    attr_reader ssl: Hash[Symbol, untyped]
 
     # io
     type io_option = _ToIO | Hash[String, _ToIO]
     attr_reader io: io_option?
 
     # fallback_protocol
-    attr_reader fallback_protocol: String?
+    attr_reader fallback_protocol: String
 
     # debug
     attr_reader debug: _IOLogger?
@@ -103,10 +101,13 @@ module HTTPX
     attr_reader debug_level: Integer
 
     # persistent
-    attr_reader persistent: bool?
+    attr_reader persistent: bool
 
     # resolver_options
-    attr_reader resolver_options: Hash[Symbol, untyped]?
+    attr_reader resolver_options: Hash[Symbol, untyped]
+
+    # ip_families
+    attr_reader ip_families: Array[ip_family]
 
     def ==: (untyped other) -> bool
     def merge: (_ToHash[Symbol, untyped] other) -> instance
@@ -117,6 +118,8 @@ module HTTPX
     REQUEST_IVARS: Array[Symbol]
 
     def initialize: (?options options) -> untyped
+
+    def __initialize__: (?options options) -> untyped
   end
 
   type options = Options | Hash[Symbol, untyped]