httpx 0.9.0 → 0.11.1

data/lib/httpx/request.rb

@@ -33,11 +33,7 @@ module HTTPX
 
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state
-
-    attr_reader :options, :response
-
-    def_delegator :@body, :<<
+    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
 
     def_delegator :@body, :empty?
 
@@ -45,7 +41,7 @@ module HTTPX
 
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
-      @uri     = URI(uri.to_s)
+      @uri     = Utils.uri(uri)
       @options = Options.new(options)
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -64,17 +60,15 @@ module HTTPX
       :w
     end
 
-    # :nocov:
     if RUBY_VERSION < "2.2"
-      # rubocop: disable Lint/UriEscapeUnescape:
+      URIParser = URI::DEFAULT_PARSER
+
       def initialize_with_escape(verb, uri, options = {})
-        initialize_without_escape(verb, URI.escape(uri.to_s), options)
+        initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
       end
       alias_method :initialize_without_escape, :initialize
       alias_method :initialize, :initialize_with_escape
-      # rubocop: enable Lint/UriEscapeUnescape:
     end
-    # :nocov:
 
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -87,6 +81,10 @@ module HTTPX
     def response=(response)
       return unless response
 
+      if response.status == 100
+        @informational_status = response.status
+        return
+      end
       @response = response
     end
 
@@ -112,7 +110,7 @@ module HTTPX
 
       query = []
       if (q = @options.params)
-        query << URI.encode_www_form(q)
+        query << Transcoder.registry("form").encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")
@@ -176,23 +174,23 @@ module HTTPX
         end
       end
 
+      def rewind
+        return if empty?
+
+        @body.rewind if @body.respond_to?(:rewind)
+      end
+
       def empty?
         return true if @body.nil?
         return false if chunked?
 
-        bytesize.zero?
+        @body.bytesize.zero?
       end
 
       def bytesize
         return 0 if @body.nil?
 
-        if @body.respond_to?(:bytesize)
-          @body.bytesize
-        elsif @body.respond_to?(:size)
-          @body.size
-        else
-          raise Error, "cannot determine size of body: #{@body.inspect}"
-        end
+        @body.bytesize
       end
 
       def stream(body)
@@ -224,6 +222,7 @@ module HTTPX
     def transition(nextstate)
       case nextstate
       when :idle
+        @body.rewind
         @response = nil
         @drainer = nil
       when :headers
@@ -233,15 +232,15 @@ module HTTPX
                       @state == :expect
 
         if @headers.key?("expect")
-          unless @response
-            @state = :expect
-            return
-          end
-
-          case @response.status
-          when 100
-            # deallocate
-            @response = nil
+          if @informational_status && @informational_status == 100
+            # check for 100 Continue response, and deallocate the var
+            # if @informational_status == 100
+            #   @response = nil
+            # end
+          else
+            return if @state == :expect # do not re-set it
+
+            nextstate = :expect
           end
         end
       when :done
@@ -253,8 +252,7 @@ module HTTPX
     end
 
     def expects?
-      @headers["expect"] == "100-continue" &&
-        @response && @response.status == 100
+      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
 
     class ProcIO

data/lib/httpx/resolver.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "httpx/resolver/resolver_mixin"
-require "httpx/resolver/system"
-require "httpx/resolver/native"
-require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
     extend Registry
 
+    RESOLVE_TIMEOUT = 5
+
+    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/system"
+    require "httpx/resolver/native"
+    require "httpx/resolver/https"
+
     register :system, System
     register :native, Native
     register :https,  HTTPS
@@ -101,5 +104,3 @@ module HTTPX
     end
   end
 end
-
-require "httpx/resolver/options"

data/lib/httpx/resolver/https.rb

@@ -21,6 +21,7 @@ module HTTPX
     DEFAULTS = {
       uri: NAMESERVER,
       use_get: false,
+      record_types: RECORD_TYPES.keys,
     }.freeze
 
     def_delegator :@connections, :empty?
@@ -29,27 +30,29 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @queries = {}
       @requests = {}
       @connections = []
-      @uri = URI(@resolver_options.uri)
+      @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
+      @resolver = Resolv::DNS.new
+      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
     end
 
     def <<(connection)
       return if @uri.origin == connection.origin.to_s
 
-      @uri_addresses ||= Resolv.getaddresses(@uri.host)
+      @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}")
+        ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
-      else
-        early_resolve(connection) || resolve(connection)
+        throw(:resolve_error, ex)
       end
+
+      early_resolve(connection) || resolve(connection)
     end
 
     def timeout
@@ -70,12 +73,6 @@ module HTTPX
 
     private
 
-    def connect
-      return if @queries.empty?
-
-      resolver_connection.__send__(__method__)
-    end
-
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
@@ -94,11 +91,18 @@ module HTTPX
     def resolve(connection = @connections.first, hostname = nil)
       return if @building_connection
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
-      type = @_record_types[hostname].first
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
+        request.on(:response, &method(:on_response).curry(2)[request])
+        request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)
         @queries[hostname] = connection
@@ -113,9 +117,7 @@ module HTTPX
     rescue StandardError => e
       connection = @requests[request]
       hostname = @queries.key(connection)
-      error = ResolveError.new("Can't resolve #{hostname}: #{e.message}")
-      error.set_backtrace(e.backtrace)
-      emit(:error, connection, error)
+      emit_resolve_error(connection, hostname, e)
     else
       parse(response)
     ensure
@@ -138,7 +140,7 @@ module HTTPX
           return
         end
       end
-      if answers.empty?
+      if answers.nil? || answers.empty?
         host, connection = @queries.first
         @_record_types[host].shift
         if @_record_types[host].empty?
@@ -172,7 +174,7 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
-          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -186,7 +188,7 @@ module HTTPX
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
 
-      if @resolver_options.use_get
+      if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", type]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
@@ -197,8 +199,6 @@ module HTTPX
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
-      request.on(:response, &method(:on_response).curry[request])
-      request.on(:promise, &method(:on_promise))
       request
     end
 
@@ -206,7 +206,7 @@ module HTTPX
       case response.headers["content-type"]
       when "application/dns-json",
            "application/json",
-           %r{^application\/x\-javascript} # because google...
+           %r{^application/x-javascript} # because google...
         payload = JSON.parse(response.to_s)
         payload["Answer"]
       when "application/dns-udpwireformat",

data/lib/httpx/resolver/native.rb

@@ -7,19 +7,18 @@ module HTTPX
   class Resolver::Native
     extend Forwardable
     include Resolver::ResolverMixin
+    using URIExtensions
 
-    RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
       "A" => Resolv::DNS::Resource::IN::A,
       "AAAA" => Resolv::DNS::Resource::IN::AAAA,
     }.freeze
 
-    # :nocov:
     DEFAULTS = if RUBY_VERSION < "2.2"
       {
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     else
@@ -27,7 +26,7 @@ module HTTPX
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     end
@@ -43,7 +42,6 @@ module HTTPX
         false
       end
     end if DEFAULTS[:nameserver]
-    # :nocov:
 
     DNS_PORT = 53
 
@@ -52,15 +50,15 @@ module HTTPX
     def initialize(options)
       @options = Options.new(options)
       @ns_index = 0
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @nameserver = @resolver_options.nameserver
-      @_timeouts = Array(@resolver_options.timeouts)
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @nameserver = @resolver_options[:nameserver]
+      @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @connections = []
       @queries = {}
       @read_buffer = "".b
-      @write_buffer = Buffer.new(@resolver_options.packet_size)
+      @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
     end
 
@@ -110,9 +108,9 @@ module HTTPX
       return if early_resolve(connection)
 
       if @nameserver.nil?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}: no nameserver")
+        ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
+        throw(:resolve_error, ex)
       else
         @connections << connection
         resolve
@@ -149,26 +147,30 @@ module HTTPX
           queries[h] = connection
           next
         end
+
         @timeouts[host].shift
         if @timeouts[host].empty?
           @timeouts.delete(host)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, host)
+          # This loop_time passed to the exception is bogus. Ideally we would pass the total
+          # resolve timeout, including from the previous retries.
+          raise ResolveTimeoutError.new(loop_time, "Timed out")
+          # raise NativeResolveError.new(connection, host)
         else
+          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
           connections << connection
-          log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
+          queries[h] = connection
         end
       end
       @queries = queries
       connections.each { |ch| resolve(ch) }
     end
 
-    def dread(wsize = @resolver_options.packet_size)
+    def dread(wsize = @resolver_options[:packet_size])
       loop do
         siz = @io.read(wsize, @read_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: READ: #{siz} bytes..." }
         parse(@read_buffer)
         return if @state == :closed
       end
@@ -181,7 +183,6 @@ module HTTPX
         siz = @io.write(@write_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: WRITE: #{siz} bytes..." }
         return if @state == :closed
       end
     end
@@ -200,13 +201,14 @@ module HTTPX
         end
       end
 
-      if addresses.empty?
+      if addresses.nil? || addresses.empty?
         hostname, connection = @queries.first
         @_record_types[hostname].shift
         if @_record_types[hostname].empty?
           @queries.delete(hostname)
           @_record_types.delete(hostname)
           @connections.delete(connection)
+
           raise NativeResolveError.new(connection, hostname)
         end
       else
@@ -224,7 +226,7 @@ module HTTPX
           end
         else
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -237,9 +239,14 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @write_buffer.empty?
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
       @queries[hostname] = connection
-      type = @_record_types[hostname].first
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
@@ -276,7 +283,7 @@ module HTTPX
         @io.connect
         return unless @io.connected?
 
-        resolve if @queries.empty?
+        resolve if @queries.empty? && !@connections.empty?
       when :closed
         return unless @state == :open