httpx 0.18.7 → 0.19.3

data/lib/httpx/resolver/system.rb

@@ -4,48 +4,204 @@ require "forwardable"
 require "resolv"
 
 module HTTPX
-  class Resolver::System
-    include Resolver::ResolverMixin
+  class Resolver::System < Resolver::Resolver
+    using URIExtensions
+    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
                      Resolv::DNS::EncodeError,
                      Resolv::DNS::DecodeError].freeze
 
+    DONE = 1
+    ERROR = 2
+
+    class << self
+      def multi?
+        false
+      end
+    end
+
     attr_reader :state
 
+    def_delegator :@connections, :empty?
+
     def initialize(options)
-      @options = HTTPX::Options.new(options)
+      super(nil, options)
       @resolver_options = @options.resolver_options
-      @state = :idle
       resolv_options = @resolver_options.dup
-      timeouts = resolv_options.delete(:timeouts)
+      timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
+      @_timeouts = Array(timeouts)
+      @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
       resolv_options.delete(:cache)
-      @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
-      @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
+      @connections = []
+      @queries = []
+      @ips = []
+      @pipe_mutex = Thread::Mutex.new
+      @state = :idle
+    end
+
+    def resolvers
+      return enum_for(__method__) unless block_given?
+
+      yield self
+    end
+
+    def connections
+      EMPTY
+    end
+
+    def close
+      transition(:closed)
     end
 
     def closed?
-      true
+      @state == :closed
+    end
+
+    def to_io
+      @pipe_read.to_io
+    end
+
+    def call
+      case @state
+      when :open
+        consume
+      end
+      nil
     end
 
-    def empty?
-      true
+    def interests
+      return if @queries.empty?
+
+      :r
+    end
+
+    def timeout
+      return unless @queries.empty?
+
+      _, connection = @queries.first
+
+      @timeouts[connection.origin.host].first
     end
 
     def <<(connection)
+      @connections << connection
+      resolve
+    end
+
+    private
+
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        @timeouts.clear
+      when :open
+        return unless @state == :idle
+
+        @pipe_read, @pipe_write = ::IO.pipe
+      when :closed
+        return unless @state == :open
+
+        @pipe_write.close
+        @pipe_read.close
+      end
+      @state = nextstate
+    end
+
+    def consume
+      return if @connections.empty?
+
+      while @pipe_read.ready? && (event = @pipe_read.getbyte)
+        case event
+        when DONE
+          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+          @queries.delete(pair)
+
+          family, connection = pair
+          emit_addresses(connection, family, addrs)
+        when ERROR
+          *pair, error = @pipe_mutex.synchronize { @ips.pop }
+          @queries.delete(pair)
+
+          family, connection = pair
+          emit_resolve_error(connection, connection.origin.host, error)
+        end
+
+        @connections.delete(connection) if @queries.empty?
+      end
+
+      return emit(:close, self) if @connections.empty?
+
+      resolve
+    end
+
+    def resolve(connection = @connections.first)
+      raise Error, "no URI to resolve" unless connection
+      return unless @queries.empty?
+
       hostname = connection.origin.host
-      addresses = connection.addresses ||
-                  ip_resolve(hostname) ||
-                  system_resolve(hostname) ||
-                  @resolver.getaddresses(hostname)
-      throw(:resolve_error, resolve_error(hostname)) if addresses.empty?
+      scheme = connection.origin.scheme
+      log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+      transition(:open)
 
-      emit_addresses(connection, addresses)
-    rescue Errno::EHOSTUNREACH, *RESOLV_ERRORS => e
-      emit_resolve_error(connection, hostname, e)
+      connection.options.ip_families.each do |family|
+        @queries << [family, connection]
+      end
+      async_resolve(connection, hostname, scheme)
+      consume
     end
 
-    def uncache(*); end
+    def async_resolve(connection, hostname, scheme)
+      families = connection.options.ip_families
+      log { "resolver: query for #{hostname}" }
+      resolve_timeout = @timeouts[connection.origin.host].first
+
+      Thread.start do
+        Thread.current.report_on_exception = false
+        begin
+          addrs = if resolve_timeout
+            Timeout.timeout(resolve_timeout) do
+              __addrinfo_resolve(hostname, scheme)
+            end
+          else
+            __addrinfo_resolve(hostname, scheme)
+          end
+          addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
+          families.each do |family|
+            addresses = addrs[family]
+            next unless addresses
+
+            addresses.map!(&:ip_address)
+            addresses.uniq!
+            @pipe_mutex.synchronize do
+              @ips.unshift([family, connection, addresses])
+              @pipe_write.putc(DONE) unless @pipe_write.closed?
+            end
+          end
+        rescue Timeout::Error => e
+          ex = ResolveTimeoutError.new(resolve_timeout, e.message)
+          ex.set_backtrace(ex.backtrace)
+          @pipe_mutex.synchronize do
+            families.each do |family|
+              @ips.unshift([family, connection, ex])
+              @pipe_write.putc(ERROR) unless @pipe_write.closed?
+            end
+          end
+        rescue StandardError => e
+          @pipe_mutex.synchronize do
+            families.each do |family|
+              @ips.unshift([family, connection, e])
+              @pipe_write.putc(ERROR) unless @pipe_write.closed?
+            end
+          end
+        end
+      end
+    end
+
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
   end
 end

data/lib/httpx/resolver.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "resolv"
+require "ipaddr"
 
 module HTTPX
   module Resolver
@@ -8,10 +9,11 @@ module HTTPX
 
     RESOLVE_TIMEOUT = 5
 
-    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
     require "httpx/resolver/https"
+    require "httpx/resolver/multi"
 
     register :system, System
     register :native, Native
@@ -22,9 +24,27 @@ module HTTPX
 
     @identifier_mutex = Mutex.new
     @identifier = 1
+    @system_resolver = Resolv::Hosts.new
 
     module_function
 
+    def nolookup_resolve(hostname)
+      ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
+    end
+
+    def ip_resolve(hostname)
+      [IPAddr.new(hostname)]
+    rescue ArgumentError
+    end
+
+    def system_resolve(hostname)
+      ips = @system_resolver.getaddresses(hostname)
+      return if ips.empty?
+
+      ips.map { |ip| IPAddr.new(ip) }
+    rescue IOError
+    end
+
     def cached_lookup(hostname)
       now = Utils.now
       @lookup_mutex.synchronize do
@@ -32,25 +52,31 @@ module HTTPX
       end
     end
 
-    def cached_lookup_set(hostname, entries)
+    def cached_lookup_set(hostname, family, entries)
       now = Utils.now
       entries.each do |entry|
         entry["TTL"] += now
       end
       @lookup_mutex.synchronize do
-        @lookups[hostname] += entries
+        case family
+        when Socket::AF_INET6
+          @lookups[hostname].concat(entries)
+        when Socket::AF_INET
+          @lookups[hostname].unshift(*entries)
+        end
         entries.each do |entry|
-          @lookups[entry["name"]] << entry if entry["name"] != hostname
+          next unless entry["name"] != hostname
+
+          case family
+          when Socket::AF_INET6
+            @lookups[entry["name"]] << entry
+          when Socket::AF_INET
+            @lookups[entry["name"]].unshift(entry)
+          end
         end
       end
     end
 
-    def uncache(hostname)
-      @lookup_mutex.synchronize do
-        @lookups.delete(hostname)
-      end
-    end
-
     # do not use directly!
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
@@ -62,7 +88,7 @@ module HTTPX
         if address.key?("alias")
           lookup(address["alias"], ttl)
         else
-          address["data"]
+          IPAddr.new(address["data"])
         end
       end
       ips unless ips.empty?

data/lib/httpx/response.rb

@@ -173,7 +173,7 @@ module HTTPX
           rescue ArgumentError
             @buffer.string
           end
-        when Tempfile, File
+        when Tempfile
           rewind
           content = _with_same_buffer_pos { @buffer.read }
           begin
@@ -253,6 +253,7 @@ module HTTPX
             @buffer = StringIO.new("".b)
           end
         when :memory
+          # @type ivar @buffer: StringIO | Tempfile
           if @length > @threshold_size
             aux = @buffer
             @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
@@ -272,11 +273,12 @@ module HTTPX
       def _with_same_buffer_pos
         return yield unless @buffer && @buffer.respond_to?(:pos)
 
+        # @type ivar @buffer: StringIO | Tempfile
         current_pos = @buffer.pos
         @buffer.rewind
         begin
           yield
-        rescue StandardError
+        ensure
           @buffer.pos = current_pos
         end
       end

data/lib/httpx/session_extensions.rb

@@ -5,8 +5,15 @@ module HTTPX
     proxy_session = plugin(:proxy)
     remove_const(:Session)
     const_set(:Session, proxy_session.class)
-    remove_const(:Options)
-    const_set(:Options, proxy_session.class.default_options.class)
+
+    # redefine the default options static var, which needs to
+    # refresh options_class
+    options = proxy_session.class.default_options.to_hash
+    options.freeze
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    Options.send(:const_set, :DEFAULT_OPTIONS, options)
+    $VERBOSE = original_verbosity
   end
 
   # :nocov:

data/lib/httpx/timers.rb

@@ -64,7 +64,7 @@ module HTTPX
       end
 
       def to_f
-        @interval
+        Float(@interval)
       end
 
       def <<(callback)

data/lib/httpx/transcoder/chunker.rb

@@ -40,7 +40,6 @@ module HTTPX::Transcoder
 
       def initialize(buffer, trailers = false)
         @buffer = buffer
-        @chunk_length = nil
         @chunk_buffer = "".b
         @finished = false
         @state = :length

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.18.7"
+  VERSION = "0.19.3"
 end

data/lib/httpx.rb

@@ -25,6 +25,8 @@ require "mutex_m"
 # Top-Level Namespace
 #
 module HTTPX
+  EMPTY = [].freeze
+
   # All plugins should be stored under this module/namespace. Can register and load
   # plugins.
   #

data/sig/errors.rbs

@@ -26,6 +26,14 @@ module HTTPX
   class ResolveError < Error
   end
 
+  class HTTPError < Error
+    attr_reader response: Response
+
+    private
+
+    def initialize: (Response response) -> void
+  end
+
   class NativeResolveError < ResolveError
     attr_reader connection: Connection
     attr_reader host: String

data/sig/headers.rbs

@@ -2,8 +2,6 @@ module HTTPX
   class Headers
     include _ToS
 
-    EMPTY: Array[untyped]
-
     @headers: Hash[String, Array[String]]
 
     def self.new: (?untyped headers) -> instance

data/sig/httpx.rbs

@@ -1,6 +1,8 @@
 module HTTPX
   extend Chainable
 
+  EMPTY: Array[untyped]
+
   VERSION: String
 
   type uri = URI::HTTP | URI::HTTPS | string
@@ -10,6 +12,8 @@ module HTTPX
               :propfind | :proppatch | :mkcol | :copy | :move | :lock | :unlock | :orderpatch |
               :acl | :report | :patch | :search
 
+  type ip_family = Integer #Socket::AF_INET6 | Socket::AF_INET
+
   module Plugins
     def self?.load_plugin: (Symbol) -> Module
 

data/sig/options.rbs

@@ -83,18 +83,16 @@ module HTTPX
 
     attr_reader response_body_class: singleton(Response::Body)
 
-    attr_reader ssl: Hash[Symbol, untyped]
+    attr_reader resolver_class: Symbol | Class
 
-    # request_class response_class headers_class request_body_class
-    # response_body_class connection_class
-    # resolver_class resolver_options
+    attr_reader ssl: Hash[Symbol, untyped]
 
     # io
     type io_option = _ToIO | Hash[String, _ToIO]
     attr_reader io: io_option?
 
     # fallback_protocol
-    attr_reader fallback_protocol: String?
+    attr_reader fallback_protocol: String
 
     # debug
     attr_reader debug: _IOLogger?
@@ -103,10 +101,13 @@ module HTTPX
     attr_reader debug_level: Integer
 
     # persistent
-    attr_reader persistent: bool?
+    attr_reader persistent: bool
 
     # resolver_options
-    attr_reader resolver_options: Hash[Symbol, untyped]?
+    attr_reader resolver_options: Hash[Symbol, untyped]
+
+    # ip_families
+    attr_reader ip_families: Array[ip_family]
 
     def ==: (untyped other) -> bool
     def merge: (_ToHash[Symbol, untyped] other) -> instance
@@ -117,6 +118,8 @@ module HTTPX
     REQUEST_IVARS: Array[Symbol]
 
     def initialize: (?options options) -> untyped
+
+    def __initialize__: (?options options) -> untyped
   end
 
   type options = Options | Hash[Symbol, untyped]

data/sig/parser/http1.rbs

@@ -1,5 +1,8 @@
 module HTTPX
   module Parser
+    class Error < HTTPX::Error
+    end
+
     type parsed_headers = Hash[String, Array[String]]
 
     interface _HTTP1Events
@@ -14,16 +17,22 @@ module HTTPX
     class HTTP1
       VERSIONS: Array[String]
 
-      def <<: (String chunk) -> void
+      attr_reader status_code: Integer
+      attr_reader http_version: [Integer, Integer]
+      attr_reader headers: parsed_headers
 
-      def headers: () -> parsed_headers
+      @observer: _HTTP1Events
+      @state: Symbol
+      @buffer: String
+      @content_type: String?
+      @content_length: Integer
+      @_has_trailers: bool
+      @upgrade: bool
 
-      def http_version: () -> Array[1 | 0]
+      def <<: (String chunk) -> void
 
       def reset!: () -> void
 
-      def status_code: () -> Integer
-
       def upgrade?: () -> bool
 
       def upgrade_data: () -> String

data/sig/pool.rbs

@@ -1,14 +1,22 @@
 module HTTPX
   class Pool
+    type resolver_manager = Resolver::Multi | Resolver::System
+
+    @resolvers: Hash[Class, resolver_manager]
+    @timers: Timers
+    @selector: Selector
+    @connections: Array[Connection]
+    @connected_connections: Integer
+
     def empty?: () -> void
 
     def next_tick: () -> void
 
-    def close: (*Array[Connection]) -> void
+    def close: (?Array[Connection] connections) -> void
 
-    def init_connection: (Connection, Options) -> void
+    def init_connection: (Connection connection, Options options) -> void
 
-    def find_connection: (generic_uri, Options) -> Connection?
+    def find_connection: (URI::Generic uri, Options options) -> Connection?
 
     def deactivate: (*Array[Connection]) -> void
 
@@ -22,20 +30,20 @@ module HTTPX
 
     def on_resolver_error: (Connection, StandardError) -> void
 
-    def on_resolver_close: (resolver) -> void
+    def on_resolver_close: (Resolver::Resolver) -> void
 
     def register_connection: (Connection) -> void
 
     def unregister_connection: (Connection) -> void
 
-    def select_connection: (resolver | Connection connection) -> void
+    def select_connection: (Resolver::Resolver | Connection connection) -> void
 
-    def deselect_connection: (resolver | Connection connection) -> void
+    def deselect_connection: (Resolver::Resolver | Connection connection) -> void
 
-    def coalesce_connections: (Connection, Connection) -> void
+    def coalesce_connections: (Connection coalescable, Connection coalescing) -> void
 
-    def next_timeout: () -> Numeric?
+    def next_timeout: () -> (Integer | Float | nil)
 
-    def find_resolver_for: (Connection) -> resolver
+    def find_resolver_for: (Connection) { (Resolver::Resolver resolver) -> void } -> resolver_manager
   end
 end

data/sig/registry.rbs

@@ -1,4 +1,7 @@
 module HTTPX::Registry[T, V]
+  class Error < HTTPX::Error
+  end
+
   # type registrable = Symbol | String | Class
 
   def self.registry: (T tag) -> Class

data/sig/request.rbs

@@ -15,6 +15,11 @@ module HTTPX
     attr_reader response: response?
     attr_reader drain_error: StandardError?
 
+    @trailers: Headers?
+    @informational_status: Integer?
+    @query: String?
+    @drainer: Enumerator[String, void]?
+
     def initialize: (Symbol | String, generic_uri, ?options) -> untyped
 
     def interests: () -> (:r | :w)
@@ -46,6 +51,10 @@ module HTTPX
     def trailers?: () -> boolish
 
     class Body
+      @headers: Headers
+      @body: body_encoder?
+      @unbounded_body: bool
+
       def initialize: (Headers, Options) -> untyped
       def each: () { (String) -> void } -> void
               | () -> Enumerable[String]
@@ -61,6 +70,8 @@ module HTTPX
     end
 
     class ProcIO
+      @block: ^(String) -> void
+
       def initialize: (^(String) -> void) -> untyped
 
       def write: (String data) -> Integer

data/sig/resolver/https.rbs

@@ -1,49 +1,37 @@
 module HTTPX
   module Resolver
-    class HTTPS
-      include ResolverMixin
-      include _ToIO
+    class HTTPS < Resolver
+      NAMESERVER: String
 
+      DEFAULTS: Hash[Symbol, untyped]
+      FAMILY_TYPES: Hash[singleton(Resolv::DNS::Resource), String]
+
+      @family: ip_family
       @options: Options
-      @resolver_options: Hash[Symbol, untyped]
-      @_record_types: Hash[String, Hash["A" | "AAAA", dns_resource]]
-      @queries: Hash[String, Connection]
       @requests: Hash[Request, Connection]
       @connections: Array[Connection]
-      @uri: URI::HTTPS
+      @uri: URI::Generic
       @uri_addresses: Array[String]?
+      @resolver: Resolv::DNS
+      @resolver_connection: Connection
 
-      def <<: (Connection) -> void
-
-      def timeout: () -> Numeric?
-
-      def closed?: () -> bool
-
-      def empty?: () -> bool
+      attr_writer pool: Pool
 
-      def close: () -> void
-
-      def call: () -> void
-
-      def interests: () -> io_interests?
+      def <<: (Connection) -> void
 
       private
 
-      def initialize: (options) -> untyped
-
-      def pool: () -> Pool
+      def initialize: (ip_family family, options options) -> void
 
       def resolver_connection: () -> Connection
 
-      def resolve: (Connection, String hostname) -> void
-                 | (Connection) -> void
-                 | () -> void
+      def resolve: (?Connection connection, ?String? hostname) -> void
 
       def on_response: (Request, response) -> void
 
-      def response: (Response) -> void
+      def parse: (Response response) -> void
 
-      def build_request: (String hostname, "A" | "AAAA") -> Request
+      def build_request: (String hostname) -> Request
 
       def decode_response_body: (Response) -> Array[dns_result]
     end