httpx 0.1.0 → 0.2.0

data/lib/httpx/errors.rb

@@ -5,6 +5,8 @@ module HTTPX
 
   TimeoutError = Class.new(Error)
 
+  ResolveError = Class.new(Error)
+
   HTTPError = Class.new(Error) do
     attr_reader :response
 
@@ -17,4 +19,6 @@ module HTTPX
       @response.status
     end
   end
+
+  MisdirectedRequestError = Class.new(HTTPError)
 end

data/lib/httpx/io/ssl.rb

@@ -10,7 +10,7 @@ module HTTPX
       {}
     end
 
-    def initialize(_, options)
+    def initialize(_, _, options)
       @ctx = OpenSSL::SSL::SSLContext.new
       ctx_options = TLS_OPTIONS.merge(options.ssl)
       @ctx.set_params(ctx_options) unless ctx_options.empty?
@@ -28,6 +28,12 @@ module HTTPX
       super
     end
 
+    def verify_hostname(host)
+      return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
+      return false if @io.peer_cert.nil?
+      OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
+    end
+
     def close
       super
       # allow reconnections
@@ -103,5 +109,17 @@ module HTTPX
       end
       do_transition(nextstate)
     end
+
+    def log_transition_state(nextstate)
+      return super unless nextstate == :negotiated
+      server_cert = @io.peer_cert
+      "SSL connection using #{@io.ssl_version} / #{@io.cipher.first}\n" \
+        "ALPN, server accepted to use #{protocol}\n" \
+        "Server certificate:\n" \
+        " subject: #{server_cert.subject}\n" \
+        " start date: #{server_cert.not_before}\n" \
+        " start date: #{server_cert.not_after}\n" \
+        " issuer: #{server_cert.issuer}"
+    end
   end
 end

data/lib/httpx/io/tcp.rb

@@ -9,18 +9,22 @@ module HTTPX
 
     attr_reader :ip, :port
 
+    attr_reader :addresses
+
     alias_method :host, :ip
 
-    def initialize(uri, options)
+    def initialize(uri, addresses, options)
       @state = :idle
       @hostname = uri.host
+      @addresses = addresses
+      @ip_index = @addresses.size - 1
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
       @port = uri.port
       if @options.io
         @io = case @options.io
               when Hash
-                @ip = Resolv.getaddress(@hostname)
+                @ip = @addresses[@ip_index]
                 @options.io[@ip] || @options.io["#{@ip}:#{@port}"]
               else
                 @ip = @hostname
@@ -31,7 +35,7 @@ module HTTPX
           @state = :connected
         end
       else
-        @ip = Resolv.getaddress(@hostname)
+        @ip = @addresses[@ip_index]
       end
       @io ||= build_socket
     end
@@ -55,10 +59,14 @@ module HTTPX
           transition(:idle)
           @io = build_socket
         end
-        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip))
+        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
       rescue Errno::EISCONN
       end
       transition(:connected)
+    rescue Errno::EHOSTUNREACH => e
+      raise e if @ip_index <= 0
+      @ip_index -= 1
+      retry
     rescue Errno::EINPROGRESS,
            Errno::EALREADY,
            ::IO::WaitReadable
@@ -125,8 +133,7 @@ module HTTPX
     private
 
     def build_socket
-      addr = IPAddr.new(@ip)
-      Socket.new(addr.family, :STREAM, 0)
+      Socket.new(@ip.family, :STREAM, 0)
     end
 
     def transition(nextstate)
@@ -141,8 +148,19 @@ module HTTPX
     end
 
     def do_transition(nextstate)
-      log(level: 1, label: "#{inspect}: ") { nextstate.to_s }
+      log(level: 1) do
+        log_transition_state(nextstate)
+      end
       @state = nextstate
     end
+
+    def log_transition_state(nextstate)
+      case nextstate
+      when :connected
+        "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
+      else
+        "#{@ip}:#{@port} #{@state} -> #{nextstate}"
+      end
+    end
   end
 end

data/lib/httpx/io/udp.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "socket"
+require "ipaddr"
+
+module HTTPX
+  class UDP
+    include Loggable
+
+    def initialize(uri, _, _)
+      ip = IPAddr.new(uri.host)
+      @host = ip.to_s
+      @port = uri.port
+      @io = UDPSocket.new(ip.family)
+    end
+
+    def to_io
+      @io.to_io
+    end
+
+    def connect; end
+
+    def connected?
+      true
+    end
+
+    def close
+      @io.close
+    end
+
+    def write(buffer)
+      siz = @io.send(buffer, 0, @host, @port)
+      buffer.slice!(0, siz)
+      siz
+    end
+
+    if RUBY_VERSION < "2.3"
+      def read(size, buffer)
+        data, _ = @io.recvfrom_nonblock(size)
+        buffer.replace(data)
+        buffer.bytesize
+      rescue ::IO::WaitReadable
+        0
+      rescue IOError
+      end
+    else
+      def read(size, buffer)
+        ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
+        return 0 if ret == :wait_readable
+        return if ret.nil?
+        buffer.bytesize
+      rescue IOError
+      end
+    end
+  end
+end

data/lib/httpx/io/unix.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module HTTPX
@@ -6,8 +8,9 @@ module HTTPX
 
     def_delegator :@uri, :port, :scheme
 
-    def initialize(uri, options)
+    def initialize(uri, addresses, options)
       @uri = uri
+      @addresses = addresses
       @state = :idle
       @options = Options.new(options)
       @path = @options.transport_options[:path]

data/lib/httpx/io.rb

@@ -4,12 +4,14 @@ require "socket"
 require "httpx/io/tcp"
 require "httpx/io/ssl"
 require "httpx/io/unix"
+require "httpx/io/udp"
 
 module HTTPX
   module IO
     extend Registry
     register "tcp", TCP
     register "ssl", SSL
+    register "udp", UDP
     register "unix", HTTPX::UNIX
   end
 end

data/lib/httpx/options.rb

@@ -55,6 +55,7 @@ module HTTPX
         :response_body_class      => Class.new(Response::Body),
         :transport                => nil,
         :transport_options        => nil,
+        :resolver_class           => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
       }
 
       defaults.merge!(options)
@@ -86,7 +87,7 @@ module HTTPX
 
     def_option(:transport) do |tr|
       transport = tr.to_s
-      raise Error, "#{transport} is an unsupported transport type" unless IO.registry.keys.include?(transport)
+      raise Error, "#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
       self.transport = transport
     end
 
@@ -94,7 +95,7 @@ module HTTPX
       params form json body
       follow ssl http2_settings
       request_class response_class headers_class request_body_class response_body_class
-      io fallback_protocol debug debug_level transport_options
+      io fallback_protocol debug debug_level transport_options resolver_class resolver_options
     ].each do |method_name|
       def_option(method_name)
     end

data/lib/httpx/plugins/proxy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "resolv"
+require "ipaddr"
 require "forwardable"
 
 module HTTPX
@@ -49,21 +51,22 @@ module HTTPX
           uri = URI(request.uri)
           proxy = proxy_params(uri)
           return super unless proxy
-          @connection.find_channel(proxy) || begin
-            channel = build_proxy_channel(proxy, **options)
-            set_channel_callbacks(channel)
-            channel
-          end
+          @connection.find_channel(proxy) || build_channel(proxy, options)
+        end
+
+        def build_channel(proxy, options)
+          channel = build_proxy_channel(proxy, **options)
+          set_channel_callbacks(channel, options)
+          channel
         end
 
         def build_proxy_channel(proxy, **options)
           parameters = Parameters.new(**proxy)
           uri = parameters.uri
           log { "proxy: #{uri}" }
-          io = TCP.new(uri, @options)
           proxy_type = Parameters.registry(parameters.uri.scheme)
-          channel = proxy_type.new(io, parameters, @options.merge(options), &method(:on_response))
-          @connection.__send__(:register_channel, channel)
+          channel = proxy_type.new("tcp", uri, parameters, @options.merge(options), &method(:on_response))
+          @connection.__send__(:resolve_channel, channel)
           channel
         end
 
@@ -102,8 +105,8 @@ module HTTPX
   end
 
   class ProxyChannel < Channel
-    def initialize(io, parameters, options, &blk)
-      super(io, options, &blk)
+    def initialize(type, uri, parameters, options, &blk)
+      super(type, uri, options, &blk)
       @parameters = parameters
     end
 
@@ -144,7 +147,7 @@ module HTTPX
   class ProxySSL < SSL
     def initialize(tcp, request_uri, options)
       @io = tcp.to_io
-      super(tcp, options)
+      super(request_uri, tcp.addresses, options)
       @hostname = request_uri.host
       @state = :connected
     end

data/lib/httpx/resolver/https.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "uri"
+require "cgi"
+require "forwardable"
+
+module HTTPX
+  class Resolver::HTTPS
+    extend Forwardable
+    include Resolver::ResolverMixin
+
+    NAMESERVER = "https://1.1.1.1/dns-query"
+
+    RECORD_TYPES = {
+      "A" => Resolv::DNS::Resource::IN::A,
+      "AAAA" => Resolv::DNS::Resource::IN::AAAA,
+    }.freeze
+
+    DEFAULTS = {
+      uri: NAMESERVER,
+      use_get: false,
+    }.freeze
+
+    def_delegator :@channels, :empty?
+
+    def_delegators :@resolver_channel, :to_io, :call, :interests, :close
+
+    def initialize(connection, options)
+      @connection = connection
+      @options = Options.new(options)
+      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
+      @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
+      @queries = {}
+      @requests = {}
+      @channels = []
+      @uri = URI(@resolver_options.uri)
+      @uri_addresses = nil
+    end
+
+    def <<(channel)
+      @uri_addresses ||= Resolv.getaddresses(@uri.host)
+      if @uri_addresses.empty?
+        ex = ResolveError.new("Can't resolve #{channel.uri.host}")
+        ex.set_backtrace(caller)
+        emit(:error, channel, ex)
+      else
+        early_resolve(channel) || resolve(channel)
+      end
+    end
+
+    def timeout
+      timeout = @options.timeout
+      timeout.timeout
+    end
+
+    def closed?
+      return true unless @resolver_channel
+      resolver_channel.closed?
+    end
+
+    private
+
+    def resolver_channel
+      @resolver_channel ||= find_channel(@uri, @options)
+    end
+
+    def resolve(channel = @channels.first, hostname = nil)
+      return if @building_channel
+      hostname = hostname || @queries.key(channel) || channel.uri.host
+      type = @_record_types[hostname].shift
+      log(label: "resolver: ") { "query #{type} for #{hostname}" }
+      request = build_request(hostname, type)
+      @requests[request] = channel
+      resolver_channel.send(request)
+      @queries[hostname] = channel
+      @channels << channel
+    end
+
+    def find_channel(_request, **options)
+      @connection.find_channel(@uri) || begin
+        @building_channel = true
+        channel = @connection.build_channel(@uri, **options)
+        emit_addresses(channel, @uri_addresses)
+        set_channel_callbacks(channel)
+        @building_channel = false
+        channel
+      end
+    end
+
+    def set_channel_callbacks(channel)
+      channel.on(:response, &method(:on_response))
+      channel.on(:promise, &method(:on_response))
+    end
+
+    def on_response(request, response)
+      response.raise_for_status
+    rescue Error => ex
+      channel = @requests[request]
+      hostname = @queries.key(channel)
+      error = ResolveError.new("Can't resolve #{hostname}: #{ex.message}")
+      error.set_backtrace(ex.backtrace)
+      emit(:error, channel, error)
+    else
+      parse(response)
+    ensure
+      @requests.delete(request)
+    end
+
+    def parse(response)
+      answers = decode_response_body(response)
+      if answers.empty?
+        host, channel = @queries.first
+        if @_record_types[host].empty?
+          emit_resolve_error(channel, host)
+          return
+        end
+      else
+        answers = answers.group_by { |answer| answer["name"] }
+        answers.each do |hostname, addresses|
+          addresses = addresses.flat_map do |address|
+            if address.key?("alias")
+              alias_address = answers[address["alias"]]
+              if alias_address.nil?
+                channel = @queries[hostname]
+                @queries.delete(address["name"])
+                resolve(channel, address["alias"])
+                return # rubocop:disable Lint/NonLocalExitFromIterator
+              else
+                alias_address
+              end
+            else
+              address
+            end
+          end.compact
+          next if addresses.empty?
+          hostname = hostname[0..-2] if hostname.end_with?(".")
+          channel = @queries.delete(hostname)
+          next unless channel # probably a retried query for which there's an answer
+          @channels.delete(channel)
+          Resolver.cached_lookup_set(hostname, addresses)
+          emit_addresses(channel, addresses.map { |addr| addr["data"] })
+        end
+      end
+      return if @channels.empty?
+      resolve
+    end
+
+    def build_request(hostname, type)
+      uri = @uri.dup
+      rklass = @options.request_class
+      if @resolver_options.use_get
+        params = URI.decode_www_form(uri.query.to_s)
+        params << ["type", type]
+        params << ["name", CGI.escape(hostname)]
+        uri.query = URI.encode_www_form(params)
+        request = rklass.new("GET", uri, @options)
+      else
+        payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
+        request = rklass.new("POST", uri, @options.merge(body: [payload]))
+        request.headers["content-type"] = "application/dns-message"
+        request.headers["accept"] = "application/dns-message"
+      end
+      request
+    end
+
+    def decode_response_body(response)
+      case response.headers["content-type"]
+      when "application/dns-json",
+           "application/json",
+           %r{^application\/x\-javascript} # because google...
+        payload = JSON.parse(response.to_s)
+        payload["Answer"]
+      when "application/dns-udpwireformat",
+           "application/dns-message"
+        Resolver.decode_dns_answer(response.to_s)
+
+        # TODO: what about the rest?
+      end
+    end
+  end
+end