httpx 0.20.0 → 1.3.1

data/lib/httpx/resolver/resolver.rb

@@ -8,6 +8,8 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    using ArrayExtensions::Intersect
+
     RECORD_TYPES = {
       Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
       Socket::AF_INET => Resolv::DNS::Resource::IN::A,
@@ -31,11 +33,13 @@ module HTTPX
     def initialize(family, options)
       @family = family
       @record_type = RECORD_TYPES[family]
-      @options = Options.new(options)
+      @options = options
     end
 
     def close; end
 
+    alias_method :terminate, :close
+
     def closed?
       true
     end
@@ -44,11 +48,15 @@ module HTTPX
       true
     end
 
-    def emit_addresses(connection, family, addresses)
+    def emit_addresses(connection, family, addresses, early_resolve = false)
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
       end
-      log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
+
+      # double emission check, but allow early resolution to work
+      return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
+
+      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
          connection.options.ip_families.size > 1 &&
@@ -56,17 +64,34 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          connection.addresses = addresses
-          emit(:resolve, connection)
+          unless connection.state == :closed ||
+                 # double emission check
+                 (connection.addresses && addresses.intersect?(connection.addresses))
+            emit_resolved_connection(connection, addresses, early_resolve)
+          end
         end
       else
-        connection.addresses = addresses
-        emit(:resolve, connection)
+        emit_resolved_connection(connection, addresses, early_resolve)
       end
     end
 
     private
 
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        emit(:resolve, connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_reset
+          throw(:resolve_error, e)
+        else
+          emit(:error, connection, e)
+        end
+      end
+    end
+
     def early_resolve(connection, hostname: connection.origin.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
@@ -76,7 +101,7 @@ module HTTPX
 
       return if addresses.empty?
 
-      emit_addresses(connection, @family, addresses)
+      emit_addresses(connection, @family, addresses, true)
     end
 
     def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
@@ -84,7 +109,7 @@ module HTTPX
     end
 
     def resolve_error(hostname, ex = nil)
-      return ex if ex.is_a?(ResolveError)
+      return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
 
       message = ex ? ex.message : "Can't resolve #{hostname}"
       error = ResolveError.new(message)

data/lib/httpx/resolver/system.rb

@@ -92,6 +92,12 @@ module HTTPX
       resolve
     end
 
+    def handle_socket_timeout(interval)
+      error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
+
     private
 
     def transition(nextstate)
@@ -121,7 +127,7 @@ module HTTPX
           @queries.delete(pair)
 
           family, connection = pair
-          emit_addresses(connection, family, addrs)
+          catch(:coalesced) { emit_addresses(connection, family, addrs) }
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
@@ -158,12 +164,14 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      resolve_timeout = @timeouts[connection.origin.host].first
+      timeouts = @timeouts[connection.origin.host]
+      resolve_timeout = timeouts.first
 
       Thread.start do
         Thread.current.report_on_exception = false
         begin
           addrs = if resolve_timeout
+
             Timeout.timeout(resolve_timeout) do
               __addrinfo_resolve(hostname, scheme)
             end
@@ -182,16 +190,13 @@ module HTTPX
               @pipe_write.putc(DONE) unless @pipe_write.closed?
             end
           end
-        rescue Timeout::Error => e
-          ex = ResolveTimeoutError.new(resolve_timeout, e.message)
-          ex.set_backtrace(ex.backtrace)
-          @pipe_mutex.synchronize do
-            families.each do |family|
-              @ips.unshift([family, connection, ex])
-              @pipe_write.putc(ERROR) unless @pipe_write.closed?
-            end
-          end
         rescue StandardError => e
+          if e.is_a?(Timeout::Error)
+            timeouts.shift
+            retry unless timeouts.empty?
+            e = ResolveTimeoutError.new(resolve_timeout, e.message)
+            e.set_backtrace(e.backtrace)
+          end
           @pipe_mutex.synchronize do
             families.each do |family|
               @ips.unshift([family, connection, e])

data/lib/httpx/resolver.rb

@@ -5,9 +5,7 @@ require "ipaddr"
 
 module HTTPX
   module Resolver
-    extend Registry
-
-    RESOLVE_TIMEOUT = 5
+    RESOLVE_TIMEOUT = [2, 3].freeze
 
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
@@ -15,19 +13,27 @@ module HTTPX
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    register :system, System
-    register :native, Native
-    register :https,  HTTPS
-
-    @lookup_mutex = Mutex.new
+    @lookup_mutex = Thread::Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }
 
-    @identifier_mutex = Mutex.new
+    @identifier_mutex = Thread::Mutex.new
     @identifier = 1
     @system_resolver = Resolv::Hosts.new
 
     module_function
 
+    def resolver_for(resolver_type)
+      case resolver_type
+      when :native then Native
+      when :system then System
+      when :https then HTTPS
+      else
+        return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
+
+        raise Error, "unsupported resolver type (#{resolver_type})"
+      end
+    end
+
     def nolookup_resolve(hostname)
       ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
     end
@@ -81,16 +87,18 @@ module HTTPX
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
 
-      @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = @lookups[hostname] = @lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
-      ips = @lookups[hostname].flat_map do |address|
+
+      ips = entries.flat_map do |address|
         if address.key?("alias")
           lookup(address["alias"], ttl)
         else
           IPAddr.new(address["data"])
         end
-      end
+      end.compact
+
       ips unless ips.empty?
     end
 
@@ -98,17 +106,30 @@ module HTTPX
       @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
-    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A)
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
       Resolv::DNS::Message.new.tap do |query|
-        query.id = generate_id
+        query.id = message_id
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
     end
 
     def decode_dns_answer(payload)
-      message = Resolv::DNS::Message.decode(payload)
+      begin
+        message = Resolv::DNS::Message.decode(payload)
+      rescue Resolv::DNS::DecodeError => e
+        return :decode_error, e
+      end
+
+      # no domain was found
+      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
+
+      return :message_truncated if message.tc == 1
+
+      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+
       addresses = []
+
       message.each_answer do |question, _, value|
         case value
         when Resolv::DNS::Resource::IN::CNAME
@@ -126,7 +147,8 @@ module HTTPX
           }
         end
       end
-      addresses
+
+      [:ok, addresses]
     end
   end
 end

data/lib/httpx/response/body.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module HTTPX
+  # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
+  # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
+  # and can be iterated over (via #each, which yields the payload in chunks).
+  class Response::Body
+    # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
+    attr_reader :encoding
+
+    # Array of encodings contained in the response "content-encoding" header.
+    attr_reader :encodings
+
+    # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
+    def initialize(response, options)
+      @response = response
+      @headers = response.headers
+      @options = options
+      @window_size = options.window_size
+      @encoding = response.content_type.charset || Encoding::BINARY
+      @encodings = []
+      @length = 0
+      @buffer = nil
+      @reader = nil
+      @state = :idle
+      initialize_inflaters
+    end
+
+    def initialize_dup(other)
+      super
+
+      @buffer = other.instance_variable_get(:@buffer).dup
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    # write the response payload +chunk+ into the buffer. Inflates the chunk when required
+    # and supported.
+    def write(chunk)
+      return if @state == :closed
+
+      return 0 if chunk.empty?
+
+      chunk = decode_chunk(chunk)
+
+      size = chunk.bytesize
+      @length += size
+      transition(:open)
+      @buffer.write(chunk)
+
+      @response.emit(:chunk_received, chunk)
+      size
+    end
+
+    # reads a chunk from the payload (implementation of the IO reader protocol).
+    def read(*args)
+      return unless @buffer
+
+      unless @reader
+        rewind
+        @reader = @buffer
+      end
+
+      @reader.read(*args)
+    end
+
+    # size of the decoded response payload. May differ from "content-length" header if
+    # response was encoded over-the-wire.
+    def bytesize
+      @length
+    end
+
+    # yields the payload in chunks.
+    def each
+      return enum_for(__method__) unless block_given?
+
+      begin
+        if @buffer
+          rewind
+          while (chunk = @buffer.read(@window_size))
+            yield(chunk.force_encoding(@encoding))
+          end
+        end
+      ensure
+        close
+      end
+    end
+
+    # returns the declared filename in the "contennt-disposition" header, when present.
+    def filename
+      return unless @headers.key?("content-disposition")
+
+      Utils.get_filename(@headers["content-disposition"])
+    end
+
+    # returns the full response payload as a string.
+    def to_s
+      return "".b unless @buffer
+
+      @buffer.to_s
+    end
+
+    alias_method :to_str, :to_s
+
+    # whether the payload is empty.
+    def empty?
+      @length.zero?
+    end
+
+    # copies the payload to +dest+.
+    #
+    #   body.copy_to("path/to/file")
+    #   body.copy_to(Pathname.new("path/to/file"))
+    #   body.copy_to(File.new("path/to/file"))
+    def copy_to(dest)
+      return unless @buffer
+
+      rewind
+
+      if dest.respond_to?(:path) && @buffer.respond_to?(:path)
+        FileUtils.mv(@buffer.path, dest.path)
+      else
+        ::IO.copy_stream(@buffer, dest)
+      end
+    end
+
+    # closes/cleans the buffer, resets everything
+    def close
+      if @buffer
+        @buffer.close
+        @buffer = nil
+      end
+      @length = 0
+      transition(:closed)
+    end
+
+    def ==(other)
+      object_id == other.object_id || begin
+        if other.respond_to?(:read)
+          _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
+        else
+          to_s == other.to_s
+        end
+      end
+    end
+
+    # :nocov:
+    def inspect
+      "#<HTTPX::Response::Body:#{object_id} " \
+        "@state=#{@state} " \
+        "@length=#{@length}>"
+    end
+    # :nocov:
+
+    # rewinds the response payload buffer.
+    def rewind
+      return unless @buffer
+
+      # in case there's some reading going on
+      @reader = nil
+
+      @buffer.rewind
+    end
+
+    private
+
+    # prepares inflaters for the advertised encodings in "content-encoding" header.
+    def initialize_inflaters
+      @inflaters = nil
+
+      return unless @headers.key?("content-encoding")
+
+      return unless @options.decompress_response_body
+
+      @inflaters = @headers.get("content-encoding").filter_map do |encoding|
+        next if encoding == "identity"
+
+        inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
+
+        # do not uncompress if there is no decoder available. In fact, we can't reliably
+        # continue decompressing beyond that, so ignore.
+        break unless inflater
+
+        @encodings << encoding
+        inflater
+      end
+    end
+
+    # passes the +chunk+ through all inflaters to decode it.
+    def decode_chunk(chunk)
+      @inflaters.reverse_each do |inflater|
+        chunk = inflater.call(chunk)
+      end if @inflaters
+
+      chunk
+    end
+
+    # tries transitioning the body STM to the +nextstate+.
+    def transition(nextstate)
+      case nextstate
+      when :open
+        return unless @state == :idle
+
+        @buffer = Response::Buffer.new(
+          threshold_size: @options.body_threshold_size,
+          bytesize: @length,
+          encoding: @encoding
+        )
+      when :closed
+        return if @state == :closed
+      end
+
+      @state = nextstate
+    end
+
+    def _with_same_buffer_pos # :nodoc:
+      return yield unless @buffer && @buffer.respond_to?(:pos)
+
+      # @type ivar @buffer: StringIO | Tempfile
+      current_pos = @buffer.pos
+      @buffer.rewind
+      begin
+        yield
+      ensure
+        @buffer.pos = current_pos
+      end
+    end
+
+    class << self
+      def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
+        case encoding
+        when "gzip"
+          Transcoder::GZIP.decode(response, **kwargs)
+        when "deflate"
+          Transcoder::Deflate.decode(response, **kwargs)
+        end
+      end
+    end
+  end
+end

data/lib/httpx/response/buffer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "stringio"
+require "tempfile"
+
+module HTTPX
+  # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
+  class Response::Buffer < SimpleDelegator
+    # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
+    # the initial +bytesize+, and the +encoding+.
+    def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
+      @threshold_size = threshold_size
+      @bytesize = bytesize
+      @encoding = encoding
+      @buffer = StringIO.new("".b)
+      super(@buffer)
+    end
+
+    def initialize_dup(other)
+      super
+
+      @buffer = other.instance_variable_get(:@buffer).dup
+    end
+
+    # size in bytes of the buffered content.
+    def size
+      @bytesize
+    end
+
+    # writes the +chunk+ into the buffer.
+    def write(chunk)
+      @bytesize += chunk.bytesize
+      try_upgrade_buffer
+      @buffer.write(chunk)
+    end
+
+    # returns the buffered content as a string.
+    def to_s
+      case @buffer
+      when StringIO
+        begin
+          @buffer.string.force_encoding(@encoding)
+        rescue ArgumentError
+          @buffer.string
+        end
+      when Tempfile
+        rewind
+        content = _with_same_buffer_pos { @buffer.read }
+        begin
+          content.force_encoding(@encoding)
+        rescue ArgumentError # ex: unknown encoding name - utf
+          content
+        end
+      end
+    end
+
+    # closes the buffer.
+    def close
+      @buffer.close
+      @buffer.unlink if @buffer.respond_to?(:unlink)
+    end
+
+    private
+
+    # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
+    # has been reached.
+    def try_upgrade_buffer
+      return unless @bytesize > @threshold_size
+
+      return if @buffer.is_a?(Tempfile)
+
+      aux = @buffer
+
+      @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+
+      if aux
+        aux.rewind
+        ::IO.copy_stream(aux, @buffer)
+        aux.close
+      end
+
+      __setobj__(@buffer)
+    end
+
+    def _with_same_buffer_pos # :nodoc:
+      current_pos = @buffer.pos
+      @buffer.rewind
+      begin
+        yield
+      ensure
+        @buffer.pos = current_pos
+      end
+    end
+  end
+end