httpx 0.10.0 → 0.11.2

data/lib/httpx/resolver/https.rb

@@ -21,6 +21,7 @@ module HTTPX
     DEFAULTS = {
       uri: NAMESERVER,
       use_get: false,
+      record_types: RECORD_TYPES.keys,
     }.freeze
 
     def_delegator :@connections, :empty?
@@ -29,27 +30,29 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @queries = {}
       @requests = {}
       @connections = []
-      @uri = URI(@resolver_options.uri)
+      @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
+      @resolver = Resolv::DNS.new
+      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
     end
 
     def <<(connection)
       return if @uri.origin == connection.origin.to_s
 
-      @uri_addresses ||= Resolv.getaddresses(@uri.host)
+      @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}")
+        ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
-      else
-        early_resolve(connection) || resolve(connection)
+        throw(:resolve_error, ex)
       end
+
+      early_resolve(connection) || resolve(connection)
     end
 
     def timeout
@@ -70,12 +73,6 @@ module HTTPX
 
     private
 
-    def connect
-      return if @queries.empty?
-
-      resolver_connection.__send__(__method__)
-    end
-
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
@@ -100,10 +97,12 @@ module HTTPX
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
       end
-      type = @_record_types[hostname].first
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
+        request.on(:response, &method(:on_response).curry(2)[request])
+        request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)
         @queries[hostname] = connection
@@ -118,9 +117,7 @@ module HTTPX
     rescue StandardError => e
       connection = @requests[request]
       hostname = @queries.key(connection)
-      error = ResolveError.new("Can't resolve #{hostname}: #{e.message}")
-      error.set_backtrace(e.backtrace)
-      emit(:error, connection, error)
+      emit_resolve_error(connection, hostname, e)
     else
       parse(response)
     ensure
@@ -143,7 +140,7 @@ module HTTPX
           return
         end
       end
-      if answers.empty?
+      if answers.nil? || answers.empty?
         host, connection = @queries.first
         @_record_types[host].shift
         if @_record_types[host].empty?
@@ -177,7 +174,7 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
-          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -191,7 +188,7 @@ module HTTPX
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
 
-      if @resolver_options.use_get
+      if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", type]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
@@ -202,8 +199,6 @@ module HTTPX
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
-      request.on(:response, &method(:on_response).curry[request])
-      request.on(:promise, &method(:on_promise))
       request
     end
 

data/lib/httpx/resolver/native.rb

@@ -9,18 +9,16 @@ module HTTPX
     include Resolver::ResolverMixin
     using URIExtensions
 
-    RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
       "A" => Resolv::DNS::Resource::IN::A,
       "AAAA" => Resolv::DNS::Resource::IN::AAAA,
     }.freeze
 
-    # :nocov:
     DEFAULTS = if RUBY_VERSION < "2.2"
       {
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     else
@@ -28,7 +26,7 @@ module HTTPX
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     end
@@ -44,7 +42,6 @@ module HTTPX
         false
       end
     end if DEFAULTS[:nameserver]
-    # :nocov:
 
     DNS_PORT = 53
 
@@ -53,15 +50,15 @@ module HTTPX
     def initialize(options)
       @options = Options.new(options)
       @ns_index = 0
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @nameserver = @resolver_options.nameserver
-      @_timeouts = Array(@resolver_options.timeouts)
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @nameserver = @resolver_options[:nameserver]
+      @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @connections = []
       @queries = {}
       @read_buffer = "".b
-      @write_buffer = Buffer.new(@resolver_options.packet_size)
+      @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
     end
 
@@ -111,9 +108,9 @@ module HTTPX
       return if early_resolve(connection)
 
       if @nameserver.nil?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}: no nameserver")
+        ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
+        throw(:resolve_error, ex)
       else
         @connections << connection
         resolve
@@ -150,21 +147,26 @@ module HTTPX
           queries[h] = connection
           next
         end
+
         @timeouts[host].shift
         if @timeouts[host].empty?
           @timeouts.delete(host)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, host)
+          # This loop_time passed to the exception is bogus. Ideally we would pass the total
+          # resolve timeout, including from the previous retries.
+          raise ResolveTimeoutError.new(loop_time, "Timed out")
+          # raise NativeResolveError.new(connection, host)
         else
+          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
           connections << connection
-          log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
+          queries[h] = connection
         end
       end
       @queries = queries
       connections.each { |ch| resolve(ch) }
     end
 
-    def dread(wsize = @resolver_options.packet_size)
+    def dread(wsize = @resolver_options[:packet_size])
       loop do
         siz = @io.read(wsize, @read_buffer)
         return unless siz && siz.positive?
@@ -199,13 +201,14 @@ module HTTPX
         end
       end
 
-      if addresses.empty?
+      if addresses.nil? || addresses.empty?
         hostname, connection = @queries.first
         @_record_types[hostname].shift
         if @_record_types[hostname].empty?
           @queries.delete(hostname)
           @_record_types.delete(hostname)
           @connections.delete(connection)
+
           raise NativeResolveError.new(connection, hostname)
         end
       else
@@ -223,7 +226,7 @@ module HTTPX
           end
         else
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -243,7 +246,7 @@ module HTTPX
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
       end
       @queries[hostname] = connection
-      type = @_record_types[hostname].first
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
@@ -280,7 +283,7 @@ module HTTPX
         @io.connect
         return unless @io.connected?
 
-        resolve if @queries.empty?
+        resolve if @queries.empty? && !@connections.empty?
       when :closed
         return unless @state == :open
 

data/lib/httpx/resolver/resolver_mixin.rb

@@ -38,7 +38,7 @@ module HTTPX
       def early_resolve(connection, hostname: connection.origin.host)
         addresses = connection.addresses ||
                     ip_resolve(hostname) ||
-                    (@resolver_options.cache && Resolver.cached_lookup(hostname)) ||
+                    (@resolver_options[:cache] && Resolver.cached_lookup(hostname)) ||
                     system_resolve(hostname)
         return unless addresses
 
@@ -57,11 +57,13 @@ module HTTPX
         ips.map { |ip| IPAddr.new(ip) }
       end
 
-      def emit_resolve_error(connection, hostname, ex = nil)
+      def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
         emit(:error, connection, resolve_error(hostname, ex))
       end
 
       def resolve_error(hostname, ex = nil)
+        return ex if ex.is_a?(ResolveError)
+
         message = ex ? ex.message : "Can't resolve #{hostname}"
         error = ResolveError.new(message)
         error.set_backtrace(ex ? ex.backtrace : caller)

data/lib/httpx/resolver/system.rb

@@ -14,13 +14,13 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(@options.resolver_options)
+      @resolver_options = @options.resolver_options
       @state = :idle
-      resolv_options = @resolver_options.to_h
+      resolv_options = @resolver_options.dup
       timeouts = resolv_options.delete(:timeouts)
       resolv_options.delete(:cache)
       @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
-      @resolver.timeouts = timeouts if timeouts
+      @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
     end
 
     def closed?

data/lib/httpx/selector.rb

@@ -4,19 +4,14 @@ require "io/wait"
 
 module IOExtensions
   refine IO do
-    def wait(timeout = nil, mode = :read)
-      case mode
-      when :read
-        wait_readable(timeout)
-      when :write
-        wait_writable(timeout)
-      when :read_write
-        r, w = IO.select([self], [self], nil, timeout)
-
-        return unless r || w
-
-        self
-      end
+    # provides a fallback for rubies where IO#wait isn't implemented,
+    # but IO#wait_readable and IO#wait_writable are.
+    def wait(timeout = nil, _mode = :read_write)
+      r, w = IO.select([self], [self], nil, timeout)
+
+      return unless r || w
+
+      self
     end
   end
 end
@@ -122,6 +117,7 @@ class HTTPX::Selector
     yield io
   rescue IOError, SystemCallError
     @selectables.reject!(&:closed?)
+    raise unless @selectables.empty?
   end
 
   def select(interval, &block)

data/lib/httpx/session.rb

@@ -41,7 +41,7 @@ module HTTPX
     def build_request(verb, uri, options = EMPTY_HASH)
       rklass = @options.request_class
       request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
-      request.on(:response, &method(:on_response).curry[request])
+      request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
       request
     end
@@ -77,10 +77,16 @@ module HTTPX
     end
 
     def set_connection_callbacks(connection, connections, options)
-      connection.on(:uncoalesce) do |uncoalesced_uri|
-        other_connection = build_connection(uncoalesced_uri, options)
+      connection.on(:misdirected) do |misdirected_request|
+        other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
+        other_connection.merge(connection)
+        catch(:coalesced) do
+          pool.init_connection(other_connection, options)
+        end
+        set_connection_callbacks(other_connection, connections, options)
         connections << other_connection
-        connection.unmerge(other_connection)
+        misdirected_request.transition(:idle)
+        other_connection.send(misdirected_request)
       end
       connection.on(:altsvc) do |alt_origin, origin, alt_params|
         other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
@@ -130,23 +136,20 @@ module HTTPX
     def build_requests(*args, options)
       request_options = @options.merge(options)
 
-      requests = case args.size
-                 when 1
-                   reqs = args.first
-                   reqs.map do |verb, uri, opts = EMPTY_HASH|
-                     build_request(verb, uri, request_options.merge(opts))
-                   end
-                 when 2
-                   verb, uris = args
-                   if uris.respond_to?(:each)
-                     uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
-                       build_request(verb, uri, request_options.merge(opts))
-                     end
-                   else
-                     [build_request(verb, uris, request_options)]
-                   end
-                 else
-                   raise ArgumentError, "unsupported number of arguments"
+      requests = if args.size == 1
+        reqs = args.first
+        reqs.map do |verb, uri, opts = EMPTY_HASH|
+          build_request(verb, uri, request_options.merge(opts))
+        end
+      else
+        verb, uris = args
+        if uris.respond_to?(:each)
+          uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
+            build_request(verb, uri, request_options.merge(opts))
+          end
+        else
+          [build_request(verb, uris, request_options)]
+        end
       end
       raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
 

data/lib/httpx/transcoder.rb

@@ -3,6 +3,26 @@
 module HTTPX
   module Transcoder
     extend Registry
+
+    def self.normalize_keys(key, value, cond = nil, &block)
+      if (cond && cond.call(value))
+        block.call(key.to_s, value)
+      elsif value.respond_to?(:to_ary)
+        if value.empty?
+          block.call("#{key}[]")
+        else
+          value.to_ary.each do |element|
+            normalize_keys("#{key}[]", element, cond, &block)
+          end
+        end
+      elsif value.respond_to?(:to_hash)
+        value.to_hash.each do |child_key, child_value|
+          normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
+        end
+      else
+        block.call(key.to_s, value)
+      end
+    end
   end
 end
 

data/lib/httpx/transcoder/form.rb

@@ -12,10 +12,18 @@ module HTTPX::Transcoder
 
       def_delegator :@raw, :to_s
 
+      def_delegator :@raw, :to_str
+
       def_delegator :@raw, :bytesize
 
       def initialize(form)
-        @raw = URI.encode_www_form(form)
+        @raw = form.each_with_object("".b) do |(key, val), buf|
+          HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
+            buf << "&" unless buf.empty?
+            buf << URI.encode_www_form_component(k)
+            buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
+          end
+        end
       end
 
       def content_type