httpx-patched 1.6.2.1

data/lib/httpx/resolver/system.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require "resolv"
+
+module HTTPX
+  # Implementation of a synchronous name resolver which relies on the system resolver,
+  # which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
+  #
+  # Its main advantage is relying on the reference implementation for name resolution
+  # across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
+  # disadvantage is the inability to set timeouts / check socket for readiness events,
+  # hence why it relies on using the Timeout module, which poses a lot of problems for
+  # the selector loop, specially when network is unstable.
+  #
+  class Resolver::System < Resolver::Resolver
+    using URIExtensions
+
+    RESOLV_ERRORS = [Resolv::ResolvError,
+                     Resolv::DNS::Requester::RequestError,
+                     Resolv::DNS::EncodeError,
+                     Resolv::DNS::DecodeError].freeze
+
+    DONE = 1
+    ERROR = 2
+
+    class << self
+      def multi?
+        false
+      end
+    end
+
+    attr_reader :state
+
+    def initialize(options)
+      super(0, options)
+      @resolver_options = @options.resolver_options
+      resolv_options = @resolver_options.dup
+      timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
+      @_timeouts = Array(timeouts)
+      @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
+      resolv_options.delete(:cache)
+      @queries = []
+      @ips = []
+      @pipe_mutex = Thread::Mutex.new
+      @state = :idle
+    end
+
+    def resolvers
+      return enum_for(__method__) unless block_given?
+
+      yield self
+    end
+
+    def multi
+      self
+    end
+
+    def empty?
+      true
+    end
+
+    def close
+      transition(:closed)
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    def to_io
+      @pipe_read.to_io
+    end
+
+    def call
+      case @state
+      when :open
+        consume
+      end
+      nil
+    end
+
+    def interests
+      return if @queries.empty?
+
+      :r
+    end
+
+    def timeout
+      return unless @queries.empty?
+
+      _, connection = @queries.first
+
+      return unless connection
+
+      @timeouts[connection.peer.host].first
+    end
+
+    def <<(connection)
+      @connections << connection
+      resolve
+    end
+
+    def early_resolve(connection, **)
+      self << connection
+      true
+    end
+
+    def handle_socket_timeout(interval)
+      error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      @queries.each do |host, connection|
+        @connections.delete(connection)
+        emit_resolve_error(connection, host, error)
+      end
+
+      while (connection = @connections.shift)
+        emit_resolve_error(connection, connection.peer.host, error)
+      end
+    end
+
+    private
+
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        @timeouts.clear
+      when :open
+        return unless @state == :idle
+
+        @pipe_read, @pipe_write = IO.pipe
+      when :closed
+        return unless @state == :open
+
+        @pipe_write.close
+        @pipe_read.close
+      end
+      @state = nextstate
+    end
+
+    def consume
+      return if @connections.empty?
+
+      if @pipe_read.wait_readable
+        event = @pipe_read.getbyte
+
+        case event
+        when DONE
+          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+          if pair
+            @queries.delete(pair)
+            family, connection = pair
+            @connections.delete(connection)
+
+            catch(:coalesced) { emit_addresses(connection, family, addrs) }
+          end
+        when ERROR
+          *pair, error = @pipe_mutex.synchronize { @ips.pop }
+          if pair && error
+            @queries.delete(pair)
+            @connections.delete(connection)
+
+            _, connection = pair
+            emit_resolve_error(connection, connection.peer.host, error)
+          end
+        end
+      end
+
+      return emit(:close, self) if @connections.empty?
+
+      resolve
+    end
+
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.first
+
+      raise Error, "no URI to resolve" unless connection
+
+      return unless @queries.empty?
+
+      hostname ||= connection.peer.host
+      scheme = connection.origin.scheme
+      log do
+        "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+      end if connection.peer.non_ascii_hostname
+
+      transition(:open)
+
+      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+
+      ip_families.each do |family|
+        @queries << [family, connection]
+      end
+      async_resolve(connection, hostname, scheme)
+      consume
+    end
+
+    def async_resolve(connection, hostname, scheme)
+      families = connection.options.ip_families || Resolver.supported_ip_families
+      log { "resolver: query for #{hostname}" }
+      timeouts = @timeouts[connection.peer.host]
+      resolve_timeout = timeouts.first
+
+      Thread.start do
+        Thread.current.report_on_exception = false
+        begin
+          addrs = if resolve_timeout
+
+            Timeout.timeout(resolve_timeout) do
+              __addrinfo_resolve(hostname, scheme)
+            end
+          else
+            __addrinfo_resolve(hostname, scheme)
+          end
+          addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
+          families.each do |family|
+            addresses = addrs[family]
+            next unless addresses
+
+            addresses.map!(&:ip_address)
+            addresses.uniq!
+            @pipe_mutex.synchronize do
+              @ips.unshift([family, connection, addresses])
+              @pipe_write.putc(DONE) unless @pipe_write.closed?
+            end
+          end
+        rescue StandardError => e
+          if e.is_a?(Timeout::Error)
+            timeouts.shift
+            retry unless timeouts.empty?
+            e = ResolveTimeoutError.new(resolve_timeout, e.message)
+            e.set_backtrace(e.backtrace)
+          end
+          @pipe_mutex.synchronize do
+            families.each do |family|
+              @ips.unshift([family, connection, e])
+              @pipe_write.putc(ERROR) unless @pipe_write.closed?
+            end
+          end
+        end
+      end
+    end
+
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
+
+    def emit_connection_error(_, error)
+      throw(:resolve_error, error)
+    end
+
+    def close_resolver(resolver); end
+  end
+end

data/lib/httpx/resolver.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "socket"
+require "resolv"
+
+module HTTPX
+  module Resolver
+    RESOLVE_TIMEOUT = [2, 3].freeze
+
+    require "httpx/resolver/entry"
+    require "httpx/resolver/resolver"
+    require "httpx/resolver/system"
+    require "httpx/resolver/native"
+    require "httpx/resolver/https"
+    require "httpx/resolver/multi"
+
+    @lookup_mutex = Thread::Mutex.new
+    @lookups = Hash.new { |h, k| h[k] = [] }
+
+    @identifier_mutex = Thread::Mutex.new
+    @identifier = 1
+    @hosts_resolver = Resolv::Hosts.new
+
+    module_function
+
+    def supported_ip_families
+      @supported_ip_families ||= begin
+        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
+        list = Socket.ip_address_list
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
+
+    def resolver_for(resolver_type, options)
+      case resolver_type
+      when Symbol
+        meth = :"resolver_#{resolver_type}_class"
+
+        return options.__send__(meth) if options.respond_to?(meth)
+      when Class
+        return resolver_type if resolver_type < Resolver
+      end
+
+      raise Error, "unsupported resolver type (#{resolver_type})"
+    end
+
+    def nolookup_resolve(hostname)
+      ip_resolve(hostname) || cached_lookup(hostname) || hosts_resolve(hostname)
+    end
+
+    # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
+    def ip_resolve(hostname)
+      [Entry.new(hostname)]
+    rescue ArgumentError
+    end
+
+    # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
+    # found, or there is no hosts file.
+    def hosts_resolve(hostname)
+      ips = @hosts_resolver.getaddresses(hostname)
+      return if ips.empty?
+
+      ips.map { |ip| Entry.new(ip) }
+    rescue IOError
+    end
+
+    def cached_lookup(hostname)
+      now = Utils.now
+      lookup_synchronize do |lookups|
+        lookup(hostname, lookups, now)
+      end
+    end
+
+    def cached_lookup_set(hostname, family, entries)
+      lookup_synchronize do |lookups|
+        case family
+        when Socket::AF_INET6
+          lookups[hostname].concat(entries)
+        when Socket::AF_INET
+          lookups[hostname].unshift(*entries)
+        end
+        entries.each do |entry|
+          next unless entry["name"] != hostname
+
+          case family
+          when Socket::AF_INET6
+            lookups[entry["name"]] << entry
+          when Socket::AF_INET
+            lookups[entry["name"]].unshift(entry)
+          end
+        end
+      end
+    end
+
+    def cached_lookup_evict(hostname, ip)
+      ip = ip.to_s
+
+      lookup_synchronize do |lookups|
+        entries = lookups[hostname]
+
+        return unless entries
+
+        lookups.delete_if { |entry| entry["data"] == ip }
+      end
+    end
+
+    # do not use directly!
+    def lookup(hostname, lookups, ttl)
+      return unless lookups.key?(hostname)
+
+      entries = lookups[hostname] = lookups[hostname].select do |address|
+        address["TTL"] > ttl
+      end
+
+      ips = entries.flat_map do |address|
+        if (als = address["alias"])
+          lookup(als, lookups, ttl)
+        else
+          Entry.new(address["data"], address["TTL"])
+        end
+      end.compact
+
+      ips unless ips.empty?
+    end
+
+    def generate_id
+      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+    end
+
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
+      Resolv::DNS::Message.new(message_id).tap do |query|
+        query.rd = 1
+        query.add_question(hostname, type)
+      end.encode
+    end
+
+    def decode_dns_answer(payload)
+      begin
+        message = Resolv::DNS::Message.decode(payload)
+      rescue Resolv::DNS::DecodeError => e
+        return :decode_error, e
+      end
+
+      # no domain was found
+      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
+
+      return :message_truncated if message.tc == 1
+
+      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+
+      addresses = []
+
+      now = Utils.now
+      message.each_answer do |question, _, value|
+        case value
+        when Resolv::DNS::Resource::IN::CNAME
+          addresses << {
+            "name" => question.to_s,
+            "TTL" => (now + value.ttl),
+            "alias" => value.name.to_s,
+          }
+        when Resolv::DNS::Resource::IN::A,
+             Resolv::DNS::Resource::IN::AAAA
+          addresses << {
+            "name" => question.to_s,
+            "TTL" => (now + value.ttl),
+            "data" => value.address.to_s,
+          }
+        end
+      end
+
+      [:ok, addresses]
+    end
+
+    def lookup_synchronize
+      @lookup_mutex.synchronize { yield(@lookups) }
+    end
+
+    def id_synchronize(&block)
+      @identifier_mutex.synchronize(&block)
+    end
+  end
+end

data/lib/httpx/response/body.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module HTTPX
+  # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
+  # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
+  # and can be iterated over (via #each, which yields the payload in chunks).
+  class Response::Body
+    # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
+    attr_reader :encoding
+
+    # Array of encodings contained in the response "content-encoding" header.
+    attr_reader :encodings
+
+    attr_reader :buffer
+    protected :buffer
+
+    # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
+    def initialize(response, options)
+      @response = response
+      @headers = response.headers
+      @options = options
+      @window_size = options.window_size
+      @encodings = []
+      @length = 0
+      @buffer = nil
+      @reader = nil
+      @state = :idle
+
+      # initialize response encoding
+      @encoding = if (enc = response.content_type.charset)
+        begin
+          Encoding.find(enc)
+        rescue ArgumentError
+          Encoding::BINARY
+        end
+      else
+        Encoding::BINARY
+      end
+
+      initialize_inflaters
+    end
+
+    def initialize_dup(other)
+      super
+
+      @buffer = other.instance_variable_get(:@buffer).dup
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    # write the response payload +chunk+ into the buffer. Inflates the chunk when required
+    # and supported.
+    def write(chunk)
+      return if @state == :closed
+
+      return 0 if chunk.empty?
+
+      chunk = decode_chunk(chunk)
+
+      size = chunk.bytesize
+      @length += size
+      transition(:open)
+      @buffer.write(chunk)
+
+      @response.emit(:chunk_received, chunk)
+      size
+    end
+
+    # reads a chunk from the payload (implementation of the IO reader protocol).
+    def read(*args)
+      return unless @buffer
+
+      unless @reader
+        rewind
+        @reader = @buffer
+      end
+
+      @reader.read(*args)
+    end
+
+    # size of the decoded response payload. May differ from "content-length" header if
+    # response was encoded over-the-wire.
+    def bytesize
+      @length
+    end
+
+    # yields the payload in chunks.
+    def each
+      return enum_for(__method__) unless block_given?
+
+      begin
+        if @buffer
+          rewind
+          while (chunk = @buffer.read(@window_size))
+            yield(chunk.force_encoding(@encoding))
+          end
+        end
+      ensure
+        close
+      end
+    end
+
+    # returns the declared filename in the "contennt-disposition" header, when present.
+    def filename
+      return unless @headers.key?("content-disposition")
+
+      Utils.get_filename(@headers["content-disposition"])
+    end
+
+    # returns the full response payload as a string.
+    def to_s
+      return "".b unless @buffer
+
+      @buffer.to_s
+    end
+
+    alias_method :to_str, :to_s
+
+    # whether the payload is empty.
+    def empty?
+      @length.zero?
+    end
+
+    # copies the payload to +dest+.
+    #
+    #   body.copy_to("path/to/file")
+    #   body.copy_to(Pathname.new("path/to/file"))
+    #   body.copy_to(File.new("path/to/file"))
+    def copy_to(dest)
+      return unless @buffer
+
+      rewind
+
+      if dest.respond_to?(:path) && @buffer.respond_to?(:path)
+        FileUtils.mv(@buffer.path, dest.path)
+      else
+        IO.copy_stream(@buffer, dest)
+      end
+    end
+
+    # closes/cleans the buffer, resets everything
+    def close
+      if @buffer
+        @buffer.close
+        @buffer = nil
+      end
+      @length = 0
+      transition(:closed)
+    end
+
+    def ==(other)
+      super || case other
+               when Response::Body
+                 @buffer == other.buffer
+               else
+                 @buffer = other
+               end
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "@state=#{@state} " \
+        "@length=#{@length}>"
+    end
+    # :nocov:
+
+    # rewinds the response payload buffer.
+    def rewind
+      return unless @buffer
+
+      # in case there's some reading going on
+      @reader = nil
+
+      @buffer.rewind
+    end
+
+    private
+
+    # prepares inflaters for the advertised encodings in "content-encoding" header.
+    def initialize_inflaters
+      @inflaters = nil
+
+      return unless @headers.key?("content-encoding")
+
+      return unless @options.decompress_response_body
+
+      @inflaters = @headers.get("content-encoding").filter_map do |encoding|
+        next if encoding == "identity"
+
+        inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
+
+        # do not uncompress if there is no decoder available. In fact, we can't reliably
+        # continue decompressing beyond that, so ignore.
+        break unless inflater
+
+        @encodings << encoding
+        inflater
+      end
+    end
+
+    # passes the +chunk+ through all inflaters to decode it.
+    def decode_chunk(chunk)
+      @inflaters.reverse_each do |inflater|
+        chunk = inflater.call(chunk)
+      end if @inflaters
+
+      chunk
+    end
+
+    # tries transitioning the body STM to the +nextstate+.
+    def transition(nextstate)
+      case nextstate
+      when :open
+        return unless @state == :idle
+
+        @buffer = Response::Buffer.new(
+          threshold_size: @options.body_threshold_size,
+          bytesize: @length,
+          encoding: @encoding
+        )
+      when :closed
+        return if @state == :closed
+      end
+
+      @state = nextstate
+    end
+
+    class << self
+      def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
+        case encoding
+        when "gzip"
+          Transcoder::GZIP.decode(response, **kwargs)
+        when "deflate"
+          Transcoder::Deflate.decode(response, **kwargs)
+        end
+      end
+    end
+  end
+end