httpx 0.1.0 → 0.2.0

data/lib/httpx/resolver/native.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "resolv"
+
+module HTTPX
+  class Resolver::Native
+    extend Forwardable
+    include Resolver::ResolverMixin
+
+    RESOLVE_TIMEOUT = 5
+    RECORD_TYPES = {
+      "A" => Resolv::DNS::Resource::IN::A,
+      "AAAA" => Resolv::DNS::Resource::IN::AAAA,
+    }.freeze
+
+    DEFAULTS = if RUBY_VERSION < "2.2"
+      {
+        **Resolv::DNS::Config.default_config_hash,
+        packet_size: 512,
+        timeouts: RESOLVE_TIMEOUT,
+        record_types: RECORD_TYPES.keys,
+      }.freeze
+    else
+      {
+        nameserver: nil,
+        **Resolv::DNS::Config.default_config_hash,
+        packet_size: 512,
+        timeouts: RESOLVE_TIMEOUT,
+        record_types: RECORD_TYPES.keys,
+      }.freeze
+    end
+
+    DNS_PORT = 53
+
+    def_delegator :@channels, :empty?
+
+    def initialize(_, options)
+      @options = Options.new(options)
+      @ns_index = 0
+      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
+      @nameserver = @resolver_options.nameserver
+      @_timeouts = Array(@resolver_options.timeouts)
+      @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
+      @channels = []
+      @queries = {}
+      @read_buffer = Buffer.new(@resolver_options.packet_size)
+      @write_buffer = Buffer.new(@resolver_options.packet_size)
+      @state = :idle
+    end
+
+    def close
+      transition(:closed)
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    def to_io
+      case @state
+      when :idle
+        transition(:open)
+      when :closed
+        transition(:idle)
+        transition(:open)
+      end
+      resolve if @queries.empty?
+      @io.to_io
+    end
+
+    def call
+      case @state
+      when :open
+        consume
+      end
+      nil
+    rescue Errno::EHOSTUNREACH => e
+      @ns_index += 1
+      if @ns_index < @nameserver.size
+        transition(:idle)
+      else
+        ex = ResolvError.new(e.message)
+        ex.set_backtrace(e.backtrace)
+        raise ex
+      end
+    end
+
+    def interests
+      readable = !@read_buffer.full?
+      writable = !@write_buffer.empty?
+      if readable
+        writable ? :rw : :r
+      else
+        writable ? :w : :r
+      end
+    end
+
+    def <<(channel)
+      return if early_resolve(channel)
+      if @nameserver.nil?
+        ex = ResolveError.new("Can't resolve #{channel.uri.host}")
+        ex.set_backtrace(caller)
+        emit(:error, channel, ex)
+      else
+        @channels << channel
+      end
+    end
+
+    def timeout
+      @start_timeout = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      hosts = @queries.keys
+      @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
+    end
+
+    private
+
+    def consume
+      dread
+      do_retry
+      dwrite
+    end
+
+    def do_retry
+      return if @queries.empty?
+      loop_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_timeout
+      channels = []
+      queries = {}
+      while (query = @queries.shift)
+        h, channel = query
+        host = channel.uri.host
+        timeout = (@timeouts[host][0] -= loop_time)
+        unless timeout.negative?
+          queries[h] = channel
+          next
+        end
+        @timeouts[host].shift
+        if @timeouts[host].empty?
+          @timeouts.delete(host)
+          emit_resolve_error(channel, host)
+          return
+        else
+          channels << channel
+          log(label: "resolver: ") do
+            "timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..."
+          end
+        end
+      end
+      @queries = queries
+      channels.each { |ch| resolve(ch) }
+    end
+
+    def dread(wsize = @read_buffer.limit)
+      loop do
+        siz = @io.read(wsize, @read_buffer)
+        unless siz
+          emit(:close)
+          return
+        end
+        return if siz.zero?
+        log(label: "resolver: ") { "READ: #{siz} bytes..." }
+        parse(@read_buffer.to_s)
+      end
+    end
+
+    def dwrite
+      loop do
+        return if @write_buffer.empty?
+        siz = @io.write(@write_buffer)
+        unless siz
+          emit(:close)
+          return
+        end
+        log(label: "resolver: ") { "WRITE: #{siz} bytes..." }
+        return if siz.zero?
+      end
+    end
+
+    def parse(buffer)
+      addresses = Resolver.decode_dns_answer(buffer)
+      if addresses.empty?
+        hostname, channel = @queries.first
+        if @_record_types[hostname].empty?
+          emit_resolve_error(channel, hostname)
+          return
+        end
+      else
+        address = addresses.first
+        channel = @queries.delete(address["name"])
+        return unless channel # probably a retried query for which there's an answer
+        if address.key?("alias") # CNAME
+          if early_resolve(channel, hostname: address["alias"])
+            @channels.delete(channel)
+          else
+            resolve(channel, address["alias"])
+            @queries.delete(address["name"])
+            return
+          end
+        else
+          @channels.delete(channel)
+          Resolver.cached_lookup_set(channel.uri.host, addresses)
+          emit_addresses(channel, addresses.map { |addr| addr["data"] })
+        end
+      end
+      return emit(:close) if @channels.empty?
+      resolve
+    end
+
+    def resolve(channel = @channels.first, hostname = nil)
+      raise Error, "no URI to resolve" unless channel
+      return unless @write_buffer.empty?
+      hostname = hostname || @queries.key(channel) || channel.uri.host
+      @queries[hostname] = channel
+      type = @_record_types[hostname].shift
+      log(label: "resolver: ") { "query #{type} for #{hostname}" }
+      @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
+    end
+
+    def build_socket
+      return if @io
+      ip, port = @nameserver[@ns_index]
+      port ||= DNS_PORT
+      uri = URI::Generic.build(scheme: "udp", port: port)
+      uri.hostname = ip
+      type = IO.registry(uri.scheme)
+      log(label: "resolver: ") { "server: #{uri}..." }
+      @io = type.new(uri, [IPAddr.new(ip)], @options)
+    end
+
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        if @io
+          @io.close
+          @io = nil
+        end
+        @timeouts.clear
+      when :open
+        return unless @state == :idle
+        build_socket
+        @io.connect
+        return unless @io.connected?
+      when :closed
+        return unless @state == :open
+        @io.close if @io
+      end
+      @state = nextstate
+    end
+  end
+end

data/lib/httpx/resolver/options.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class Resolver::Options
+    def initialize(options = {})
+      @options = options
+    end
+
+    def method_missing(m, *args, &block)
+      if @options.key?(m)
+        @options[m]
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(m)
+      @options.key?(m) || super
+    end
+
+    def to_h
+      @options
+    end
+  end
+end

data/lib/httpx/resolver/resolver_mixin.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "resolv"
+require "ipaddr"
+
+module HTTPX
+  module Resolver
+    module ResolverMixin
+      include Callbacks
+      include Loggable
+
+      CHECK_IF_IP = proc do |name|
+        begin
+          IPAddr.new(name)
+          true
+        rescue ArgumentError
+          false
+        end
+      end
+
+      def uncache(channel)
+        hostname = hostname || @queries.key(channel) || channel.uri.host
+        Resolver.uncache(hostname)
+      end
+
+      private
+
+      def emit_addresses(channel, addresses)
+        addresses.map! do |address|
+          address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
+        end
+        log(label: "resolver: ") { "answer #{channel.uri.host}: #{addresses.inspect}" }
+        channel.addresses = addresses
+        emit(:resolve, channel, addresses)
+      end
+
+      def early_resolve(channel, hostname: channel.uri.host)
+        addresses = ip_resolve(hostname) || Resolver.cached_lookup(hostname) || system_resolve(hostname)
+        return unless addresses
+        emit_addresses(channel, addresses)
+      end
+
+      def ip_resolve(hostname)
+        [hostname] if CHECK_IF_IP[hostname]
+      end
+
+      def system_resolve(hostname)
+        @system_resolver ||= Resolv::Hosts.new
+        ips = @system_resolver.getaddresses(hostname)
+        return if ips.empty?
+        ips.map { |ip| IPAddr.new(ip) }
+      end
+
+      def emit_resolve_error(channel, hostname)
+        error = ResolveError.new("Can't resolve #{hostname}")
+        error.set_backtrace(caller)
+        emit(:error, channel, error)
+      end
+    end
+  end
+end

data/lib/httpx/resolver/system.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "resolv"
+
+module HTTPX
+  class Resolver::System
+    include Resolver::ResolverMixin
+
+    def initialize(_, options)
+      @options = Options.new(options)
+      roptions = @options.resolver_options
+      @state = :idle
+      @resolver = Resolv::DNS.new(roptions.nil? ? nil : roptions)
+      @resolver.timeouts = roptions[:timeouts] if roptions
+    end
+
+    def closed?
+      true
+    end
+
+    def empty?
+      true
+    end
+
+    def <<(channel)
+      hostname = channel.uri.host
+      addresses = ip_resolve(hostname) || system_resolve(hostname) || @resolver.getaddresses(hostname)
+      addresses.empty? ? emit_resolve_error(channel, hostname) : emit_addresses(channel, addresses)
+    end
+
+    def uncache(*); end
+  end
+end

data/lib/httpx/resolver.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Resolver
+    autoload :ResolverMixin, "httpx/resolver/resolver_mixin"
+    autoload :System, "httpx/resolver/system"
+    autoload :Native, "httpx/resolver/native"
+    autoload :HTTPS, "httpx/resolver/https"
+
+    extend Registry
+
+    register :system, :System
+    register :native, :Native
+    register :https,  :HTTPS
+
+    @lookup_mutex = Mutex.new
+    @lookups = Hash.new { |h, k| h[k] = [] }
+
+    @identifier_mutex = Mutex.new
+    @identifier = 1
+
+    module_function
+
+    def cached_lookup(hostname)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @lookup_mutex.synchronize do
+        lookup(hostname, now)
+      end
+    end
+
+    def cached_lookup_set(hostname, entries)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      entries.each do |entry|
+        entry["TTL"] += now
+      end
+      @lookup_mutex.synchronize do
+        @lookups[hostname] += entries
+        entries.each do |entry|
+          @lookups[entry["name"]] << entry if entry["name"] != hostname
+        end
+      end
+    end
+
+    def uncache(hostname)
+      @lookup_mutex.synchronize do
+        @lookups.delete(hostname)
+      end
+    end
+
+    # do not use directly!
+    def lookup(hostname, ttl)
+      return unless @lookups.key?(hostname)
+      @lookups[hostname] = @lookups[hostname].select do |address|
+        address["TTL"] > ttl
+      end
+      ips = @lookups[hostname].flat_map do |address|
+        if address.key?("alias")
+          lookup(address["alias"], ttl)
+        else
+          address["data"]
+        end
+      end
+      ips unless ips.empty?
+    end
+
+    def generate_id
+      @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+    end
+
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A)
+      Resolv::DNS::Message.new.tap do |query|
+        query.id = generate_id
+        query.rd = 1
+        query.add_question(hostname, type)
+      end.encode
+    end
+
+    def decode_dns_answer(payload)
+      message = Resolv::DNS::Message.decode(payload)
+      addresses = []
+      message.each_answer do |question, _, value|
+        case value
+        when Resolv::DNS::Resource::IN::CNAME
+          addresses << {
+            "name" => question.to_s,
+            "TTL" => value.ttl,
+            "alias" => value.name.to_s,
+          }
+        when Resolv::DNS::Resource::IN::A,
+             Resolv::DNS::Resource::IN::AAAA
+          addresses << {
+            "name" => question.to_s,
+            "TTL"  => value.ttl,
+            "data" => value.address.to_s,
+          }
+        end
+      end
+      addresses
+    end
+  end
+end
+
+require "httpx/resolver/options"