httpx 0.8.0 → 0.10.1

data/lib/httpx/pool.rb

@@ -41,13 +41,18 @@ module HTTPX
       @connections.each do |connection|
         connection.emit(:error, e)
       end
+    rescue Exception # rubocop:disable Lint/RescueException
+      @connections.each(&:reset)
+      raise
     end
 
     def close(connections = @connections)
+      return if connections.empty?
+
       @timers.cancel
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
-      next_tick until connections.none? { |c| @connections.include?(c) }
+      next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
       end if @connections.empty?
@@ -103,10 +108,10 @@ module HTTPX
       end
     end
 
-    def on_resolver_error(ch, error)
-      ch.emit(:error, error)
+    def on_resolver_error(connection, error)
+      connection.emit(:error, error)
       # must remove connection by hand, hasn't been started yet
-      unregister_connection(ch)
+      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -139,12 +144,12 @@ module HTTPX
       @connected_connections -= 1
     end
 
-    def coalesce_connections(ch1, ch2)
-      if ch1.coalescable?(ch2)
-        ch1.merge(ch2)
-        @connections.delete(ch2)
+    def coalesce_connections(conn1, conn2)
+      if conn1.coalescable?(conn2)
+        conn1.merge(conn2)
+        @connections.delete(conn2)
       else
-        register_connection(ch2)
+        register_connection(conn2)
       end
     end
 
@@ -163,15 +168,7 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      # rubocop: disable Layout/RescueEnsureAlignment
-      rescue ArgumentError
-        # this block is here because of an error which happens on CI from time to time
-        warn "tried resolver: #{resolver_type}"
-        warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
-        warn "new: #{resolver_type.method(:new).source_location}"
-        raise
       end
-      # rubocop: enable Layout/RescueEnsureAlignment
     end
   end
 end

data/lib/httpx/request.rb

@@ -33,11 +33,7 @@ module HTTPX
 
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state
-
-    attr_reader :options, :response
-
-    def_delegator :@body, :<<
+    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
 
     def_delegator :@body, :empty?
 
@@ -45,7 +41,7 @@ module HTTPX
 
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
-      @uri     = URI(uri.to_s)
+      @uri     = Utils.uri(uri)
       @options = Options.new(options)
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -64,17 +60,15 @@ module HTTPX
       :w
     end
 
-    # :nocov:
     if RUBY_VERSION < "2.2"
-      # rubocop: disable Lint/UriEscapeUnescape:
+      URIParser = URI::DEFAULT_PARSER
+
       def initialize_with_escape(verb, uri, options = {})
-        initialize_without_escape(verb, URI.escape(uri.to_s), options)
+        initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
       end
       alias_method :initialize_without_escape, :initialize
       alias_method :initialize, :initialize_with_escape
-      # rubocop: enable Lint/UriEscapeUnescape:
     end
-    # :nocov:
 
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -112,7 +106,7 @@ module HTTPX
 
       query = []
       if (q = @options.params)
-        query << URI.encode_www_form(q)
+        query << Transcoder.registry("form").encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")
@@ -180,19 +174,13 @@ module HTTPX
         return true if @body.nil?
         return false if chunked?
 
-        bytesize.zero?
+        @body.bytesize.zero?
       end
 
       def bytesize
         return 0 if @body.nil?
 
-        if @body.respond_to?(:bytesize)
-          @body.bytesize
-        elsif @body.respond_to?(:size)
-          @body.size
-        else
-          raise Error, "cannot determine size of body: #{@body.inspect}"
-        end
+        @body.bytesize
       end
 
       def stream(body)

data/lib/httpx/resolver.rb

@@ -1,19 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
+require "httpx/resolver/resolver_mixin"
+require "httpx/resolver/system"
+require "httpx/resolver/native"
+require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
-    autoload :ResolverMixin, "httpx/resolver/resolver_mixin"
-    autoload :System, "httpx/resolver/system"
-    autoload :Native, "httpx/resolver/native"
-    autoload :HTTPS, "httpx/resolver/https"
-
     extend Registry
 
-    register :system, :System
-    register :native, :Native
-    register :https,  :HTTPS
+    register :system, System
+    register :native, Native
+    register :https,  HTTPS
 
     @lookup_mutex = Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }
@@ -102,5 +101,3 @@ module HTTPX
     end
   end
 end
-
-require "httpx/resolver/options"

data/lib/httpx/resolver/https.rb

@@ -21,6 +21,7 @@ module HTTPX
     DEFAULTS = {
       uri: NAMESERVER,
       use_get: false,
+      record_types: RECORD_TYPES.keys,
     }.freeze
 
     def_delegator :@connections, :empty?
@@ -29,12 +30,12 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @queries = {}
       @requests = {}
       @connections = []
-      @uri = URI(@resolver_options.uri)
+      @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
     end
 
@@ -44,12 +45,12 @@ module HTTPX
       @uri_addresses ||= Resolv.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}")
+        ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
-      else
-        early_resolve(connection) || resolve(connection)
+        throw(:resolve_error, ex)
       end
+
+      early_resolve(connection) || resolve(connection)
     end
 
     def timeout
@@ -70,12 +71,6 @@ module HTTPX
 
     private
 
-    def connect
-      return if @queries.empty?
-
-      resolver_connection.__send__(__method__)
-    end
-
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
@@ -94,11 +89,18 @@ module HTTPX
     def resolve(connection = @connections.first, hostname = nil)
       return if @building_connection
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
-      type = @_record_types[hostname].first
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
+        request.on(:response, &method(:on_response).curry[request])
+        request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)
         @queries[hostname] = connection
@@ -113,9 +115,7 @@ module HTTPX
     rescue StandardError => e
       connection = @requests[request]
       hostname = @queries.key(connection)
-      error = ResolveError.new("Can't resolve #{hostname}: #{e.message}")
-      error.set_backtrace(e.backtrace)
-      emit(:error, connection, error)
+      emit_resolve_error(connection, hostname, e)
     else
       parse(response)
     ensure
@@ -138,7 +138,7 @@ module HTTPX
           return
         end
       end
-      if answers.empty?
+      if answers.nil? || answers.empty?
         host, connection = @queries.first
         @_record_types[host].shift
         if @_record_types[host].empty?
@@ -172,7 +172,7 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
-          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -186,7 +186,7 @@ module HTTPX
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
 
-      if @resolver_options.use_get
+      if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", type]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
@@ -197,8 +197,6 @@ module HTTPX
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
-      request.on(:response, &method(:on_response).curry[request])
-      request.on(:promise, &method(:on_promise))
       request
     end
 
@@ -206,7 +204,7 @@ module HTTPX
       case response.headers["content-type"]
       when "application/dns-json",
            "application/json",
-           %r{^application\/x\-javascript} # because google...
+           %r{^application/x-javascript} # because google...
         payload = JSON.parse(response.to_s)
         payload["Answer"]
       when "application/dns-udpwireformat",

data/lib/httpx/resolver/native.rb

@@ -7,6 +7,7 @@ module HTTPX
   class Resolver::Native
     extend Forwardable
     include Resolver::ResolverMixin
+    using URIExtensions
 
     RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
@@ -14,7 +15,6 @@ module HTTPX
       "AAAA" => Resolv::DNS::Resource::IN::AAAA,
     }.freeze
 
-    # :nocov:
     DEFAULTS = if RUBY_VERSION < "2.2"
       {
         **Resolv::DNS::Config.default_config_hash,
@@ -43,7 +43,6 @@ module HTTPX
         false
       end
     end if DEFAULTS[:nameserver]
-    # :nocov:
 
     DNS_PORT = 53
 
@@ -52,15 +51,15 @@ module HTTPX
     def initialize(options)
       @options = Options.new(options)
       @ns_index = 0
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @nameserver = @resolver_options.nameserver
-      @_timeouts = Array(@resolver_options.timeouts)
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @nameserver = @resolver_options[:nameserver]
+      @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @connections = []
       @queries = {}
       @read_buffer = "".b
-      @write_buffer = Buffer.new(@resolver_options.packet_size)
+      @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
     end
 
@@ -110,9 +109,9 @@ module HTTPX
       return if early_resolve(connection)
 
       if @nameserver.nil?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}: no nameserver")
+        ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
+        throw(:resolve_error, ex)
       else
         @connections << connection
         resolve
@@ -163,12 +162,11 @@ module HTTPX
       connections.each { |ch| resolve(ch) }
     end
 
-    def dread(wsize = @resolver_options.packet_size)
+    def dread(wsize = @resolver_options[:packet_size])
       loop do
         siz = @io.read(wsize, @read_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: READ: #{siz} bytes..." }
         parse(@read_buffer)
         return if @state == :closed
       end
@@ -181,7 +179,6 @@ module HTTPX
         siz = @io.write(@write_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: WRITE: #{siz} bytes..." }
         return if @state == :closed
       end
     end
@@ -200,13 +197,14 @@ module HTTPX
         end
       end
 
-      if addresses.empty?
+      if addresses.nil? || addresses.empty?
         hostname, connection = @queries.first
         @_record_types[hostname].shift
         if @_record_types[hostname].empty?
           @queries.delete(hostname)
           @_record_types.delete(hostname)
           @connections.delete(connection)
+
           raise NativeResolveError.new(connection, hostname)
         end
       else
@@ -224,7 +222,7 @@ module HTTPX
           end
         else
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -237,9 +235,14 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @write_buffer.empty?
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
       @queries[hostname] = connection
-      type = @_record_types[hostname].first
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])

data/lib/httpx/resolver/resolver_mixin.rb

@@ -38,7 +38,7 @@ module HTTPX
       def early_resolve(connection, hostname: connection.origin.host)
         addresses = connection.addresses ||
                     ip_resolve(hostname) ||
-                    (@resolver_options.cache && Resolver.cached_lookup(hostname)) ||
+                    (@resolver_options[:cache] && Resolver.cached_lookup(hostname)) ||
                     system_resolve(hostname)
         return unless addresses
 
@@ -57,11 +57,13 @@ module HTTPX
         ips.map { |ip| IPAddr.new(ip) }
       end
 
-      def emit_resolve_error(connection, hostname, ex = nil)
+      def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
         emit(:error, connection, resolve_error(hostname, ex))
       end
 
       def resolve_error(hostname, ex = nil)
+        return ex if ex.is_a?(ResolveError)
+
         message = ex ? ex.message : "Can't resolve #{hostname}"
         error = ResolveError.new(message)
         error.set_backtrace(ex ? ex.backtrace : caller)

data/lib/httpx/resolver/system.rb

@@ -14,9 +14,9 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(@options.resolver_options)
+      @resolver_options = @options.resolver_options
       @state = :idle
-      resolv_options = @resolver_options.to_h
+      resolv_options = @resolver_options.dup
       timeouts = resolv_options.delete(:timeouts)
       resolv_options.delete(:cache)
       @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)