httpx 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34654a12cf778aeffd39429a869bbfb62dc487369ed50fdfc8b14c512e141c91
-  data.tar.gz: 055b3f79a33ecf43982264f2af5d61631d290aab74c7b63d30fc82501ba366c3
+  metadata.gz: dd416a44c18c7d00a8257aa0bdb1532c670ef1b9a6b467d21600014544c03df6
+  data.tar.gz: 3b7dce3446530daa9acb1b66b2844ffc88fe50a95e225750d1d83ec052dd4a4b
 SHA512:
-  metadata.gz: b9cc0fb05f09c33d2b18f7aa3b46313240744ecb0a8474ffb5345afc2e533768b987609eabd8f1461c00f64815610dcd83e68f340933ce1695ee3efcbc6653a2
-  data.tar.gz: 21bc5f92e8c8c639a2b0f0885d7652805a099f44a89900da8336993c1e033359245666204d2adccc9fbfb18e49c6dd883fc09ffac5c56b194df056a8cda4b0de
+  metadata.gz: 46995eff536a340ec6d70743b650dd7dcb41fcd146308a3ff863980c1b74e8454fa45838b39a94c14a83ca6c6c6d9d8715c51a7bab4a4a969cae0cbf6154d3e8
+  data.tar.gz: 4199024dbaf25064ddb778cabcb8cbd33508ba1715742b774626deda7007ae627aca6523da4c8bbd410b67e7a431754d67f6deb50820953a4420128866319c31

data/README.md

@@ -90,6 +90,10 @@ It means that it loads the bare minimum to perform requests, and the user has to
 
 It also means that it ships with the minimum amount of dependencies.
 
+### DNS-over-HTTPS
+
+`HTTPX` ships with custom DNS resolver implementations, including a DNS-over-HTTPS resolver.
+
 ## Easy to test
 
 The test suite runs against [httpbin proxied over nghttp2](https://nghttp2.org/httpbin/), so there are no mocking/stubbing false positives. The test suite uses [minitest](https://github.com/seattlerb/minitest), but its matchers usage is (almost) limited to `#assert` (`assert` is all you need).

data/lib/httpx/buffer.rb

@@ -22,6 +22,8 @@ module HTTPX
 
     def_delegator :@buffer, :replace
 
+    attr_reader :limit
+
     def initialize(limit)
       @buffer = "".b
       @limit = limit

data/lib/httpx/chainable.rb

@@ -32,8 +32,8 @@ module HTTPX
     end
     alias_method :plugins, :plugin
 
-    def with(options)
-      branch(default_options.merge(options))
+    def with(options, &blk)
+      branch(default_options.merge(options), &blk)
     end
 
     private
@@ -43,9 +43,9 @@ module HTTPX
     end
 
     # :nodoc:
-    def branch(options)
-      return self.class.new(options) if is_a?(Client)
-      Client.new(options)
+    def branch(options, &blk)
+      return self.class.new(options, &blk) if is_a?(Client)
+      Client.new(options, &blk)
     end
   end
 end

data/lib/httpx/channel/http2.rb

@@ -7,6 +7,12 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    Error = Class.new(Error) do
+      def initialize(id, code)
+        super("stream #{id} closed with error: #{code}")
+      end
+    end
+
     attr_reader :streams, :pending
 
     def initialize(buffer, options)
@@ -157,10 +163,18 @@ module HTTPX
     def on_stream_close(stream, request, error)
       return handle(request, stream) if request.expects?
       if error
-        emit(:error, request, error)
+        ex = Error.new(stream.id, error)
+        ex.set_backtrace(caller)
+        emit(:error, request, ex)
       else
         response = request.response
-        emit(:response, request, response)
+        if response.status == 421
+          ex = MisdirectedRequestError.new(response)
+          ex.set_backtrace(caller)
+          emit(:error, request, ex)
+        else
+          emit(:response, request, response)
+        end
       end
       log(level: 2, label: "#{stream.id}: ") { "closing stream" }
 
@@ -177,7 +191,12 @@ module HTTPX
                                   @connection.remote_settings[:settings_max_concurrent_streams]].min
     end
 
-    def on_close(*)
+    def on_close(_last_frame, error, _payload)
+      if error
+        ex = Error.new(0, error)
+        ex.set_backtrace(caller)
+        emit(:error, request, ex)
+      end
       return unless @connection.state == :closed && @connection.active_stream_count.zero?
       emit(:close)
     end

data/lib/httpx/channel.rb

@@ -50,8 +50,7 @@ module HTTPX
             raise Error, "#{uri}: #{uri.scheme}: unrecognized channel"
           end
         end
-        io = IO.registry(type).new(uri, options)
-        new(io, options)
+        new(type, uri, options)
       end
     end
 
@@ -59,8 +58,12 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    def initialize(io, options)
-      @io = io
+    attr_reader :uri, :state
+
+    def initialize(type, uri, options)
+      @type = type
+      @uri = uri
+      @hostnames = [@uri.host]
       @options = Options.new(options)
       @window_size = @options.window_size
       @read_buffer = Buffer.new(BUFFER_SIZE)
@@ -70,17 +73,54 @@ module HTTPX
       on(:error) { |ex| on_error(ex) }
     end
 
+    def addresses=(addrs)
+      @io = IO.registry(@type).new(@uri, addrs, @options)
+    end
+
+    def mergeable?(addresses)
+      return false if @state == :closing || !@io
+      !(@io.addresses & addresses).empty?
+    end
+
+    # coalescable channels need to be mergeable!
+    # but internally, #mergeable? is called before #coalescable?
+    def coalescable?(channel)
+      if @io.protocol == "h2" && @uri.scheme == "https"
+        @io.verify_hostname(channel.uri.host)
+      else
+        @uri.host == channel.uri.host &&
+          @uri.port == channel.uri.port &&
+          @uri.scheme == channel.uri.scheme
+      end
+    end
+
+    def merge(channel)
+      @hostnames += channel.instance_variable_get(:@hostnames)
+      pending = channel.instance_variable_get(:@pending)
+      pending.each do |req, args|
+        send(req, args)
+      end
+    end
+
+    def unmerge(channel)
+      @hostnames -= channel.instance_variable_get(:@hostnames)
+      [@parser.pending, @pending].each do |pending|
+        pending.reject! do |request|
+          request.uri == channel.uri && begin
+            request.transition(:idle)
+            channel.send(request)
+            true
+          end
+        end
+      end
+    end
+
     def match?(uri)
       return false if @state == :closing
-      ips = begin
-        Resolv.getaddresses(uri.host)
-      rescue StandardError
-        [uri.host]
-      end
 
-      ips.include?(@io.ip) &&
-        uri.port == @io.port &&
-        uri.scheme == @io.scheme
+      @hostnames.include?(uri.host) &&
+        uri.port == @uri.port &&
+        uri.scheme == @uri.scheme
     end
 
     def interests
@@ -114,7 +154,9 @@ module HTTPX
     end
 
     def send(request, **args)
-      if @parser && !@write_buffer.full?
+      if @error_response
+        emit(:response, request, @error_response)
+      elsif @parser && !@write_buffer.full?
         parser.send(request, **args)
       else
         @pending << [request, args]
@@ -211,8 +253,13 @@ module HTTPX
         end
       end
       parser.on(:error) do |request, ex|
-        response = ErrorResponse.new(ex, @options)
-        emit(:response, request, response)
+        case ex
+        when MisdirectedRequestError
+          emit(:uncoalesce, request.uri)
+        else
+          response = ErrorResponse.new(ex, @options)
+          emit(:response, request, response)
+        end
       end
       parser
     end
@@ -220,12 +267,14 @@ module HTTPX
     def transition(nextstate)
       case nextstate
       # when :idle
-
+      when :idle
+        @error_response = nil
       when :open
         return if @state == :closed
         @io.connect
         return unless @io.connected?
         send_pending
+        emit(:open)
       when :closing
         return unless @state == :open
       when :closed
@@ -235,9 +284,14 @@ module HTTPX
         @read_buffer.clear
       end
       @state = nextstate
+    rescue Errno::EHOSTUNREACH
+      # at this point, all addresses from the IO object have failed
+      reset
+      emit(:unreachable)
+      throw(:jump_tick)
     rescue Errno::ECONNREFUSED,
-           Errno::ENETUNREACH,
            Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
            OpenSSL::SSL::SSLError => e
       # connect errors, exit gracefully
       handle_error(e)
@@ -251,10 +305,10 @@ module HTTPX
     end
 
     def handle_error(e)
-      parser.handle_error(e) if parser.respond_to?(:handle_error)
-      response = ErrorResponse.new(e, @options)
+      parser.handle_error(e) if @parser && parser.respond_to?(:handle_error)
+      @error_response = ErrorResponse.new(e, @options)
       @pending.each do |request, _|
-        emit(:response, request, response)
+        emit(:response, request, @error_response)
       end
     end
   end

data/lib/httpx/client.rb

@@ -56,37 +56,45 @@ module HTTPX
 
     def find_channel(request, **options)
       uri = URI(request.uri)
-      @connection.find_channel(uri) || begin
-        channel = @connection.build_channel(uri, **options)
-        set_channel_callbacks(channel)
-        channel
-      end
+      @connection.find_channel(uri) || build_channel(uri, options)
     end
 
-    def set_channel_callbacks(channel)
+    def set_channel_callbacks(channel, options)
       channel.on(:response, &method(:on_response))
       channel.on(:promise, &method(:on_promise))
+      channel.on(:uncoalesce) do |uncoalesced_uri|
+        other_channel = build_channel(uncoalesced_uri, options)
+        channel.unmerge(other_channel)
+      end
+    end
+
+    def build_channel(uri, options)
+      channel = @connection.build_channel(uri, **options)
+      set_channel_callbacks(channel, options)
+      channel
     end
 
     def __build_reqs(*args, **options)
-      case args.size
-      when 1
-        reqs = args.first
-        reqs.map do |verb, uri|
-          __build_req(verb, uri, options)
-        end
-      when 2, 3
-        verb, uris = args
-        if uris.respond_to?(:each)
-          uris.map do |uri|
-            __build_req(verb, uri, options)
-          end
-        else
-          [__build_req(verb, uris, options)]
-        end
-      else
-        raise ArgumentError, "unsupported number of arguments"
+      requests = case args.size
+                 when 1
+                   reqs = args.first
+                   reqs.map do |verb, uri|
+                     __build_req(verb, uri, options)
+                   end
+                 when 2, 3
+                   verb, uris = args
+                   if uris.respond_to?(:each)
+                     uris.map do |uri|
+                       __build_req(verb, uri, options)
+                     end
+                   else
+                     [__build_req(verb, uris, options)]
+                   end
+                 else
+                   raise ArgumentError, "unsupported number of arguments"
       end
+      raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
+      requests
     end
 
     def __send_reqs(*requests, **options)
@@ -165,5 +173,7 @@ module HTTPX
         self
       end
     end
+
+    plugin(:proxy) unless ENV.grep(/https?_proxy$/i).empty?
   end
 end

data/lib/httpx/connection.rb

@@ -2,14 +2,21 @@
 
 require "httpx/selector"
 require "httpx/channel"
+require "httpx/resolver"
 
 module HTTPX
   class Connection
     def initialize(options)
       @options = Options.new(options)
       @timeout = options.timeout
+      resolver_type = @options.resolver_class
+      resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
       @selector = Selector.new
       @channels = []
+      @resolver = resolver_type.new(self, @options)
+      @resolver.on(:resolve, &method(:on_resolver_channel))
+      @resolver.on(:error, &method(:on_resolver_error))
+      @resolver.on(:close, &method(:on_resolver_close))
     end
 
     def running?
@@ -17,12 +24,13 @@ module HTTPX
     end
 
     def next_tick
-      timeout = @timeout.timeout
-      @selector.select(timeout) do |monitor|
-        if (channel = monitor.value)
-          channel.call
+      catch(:jump_tick) do
+        @selector.select(next_timeout) do |monitor|
+          if (channel = monitor.value)
+            channel.call
+          end
+          monitor.interests = channel.interests
         end
-        monitor.interests = channel.interests
       end
     rescue TimeoutError,
            Errno::ECONNRESET,
@@ -34,13 +42,18 @@ module HTTPX
     end
 
     def close
+      @resolver.close unless @resolver.closed?
       @channels.each(&:close)
       next_tick until @channels.empty?
     end
 
     def build_channel(uri, **options)
       channel = Channel.by(uri, @options.merge(options))
-      register_channel(channel)
+      resolve_channel(channel)
+      channel.once(:unreachable) do
+        @resolver.uncache(channel)
+        resolve_channel(channel)
+      end
       channel
     end
 
@@ -56,14 +69,69 @@ module HTTPX
 
     private
 
+    def resolve_channel(channel)
+      @channels << channel unless @channels.include?(channel)
+      @resolver << channel
+      return if @resolver.empty?
+      @_resolver_monitor ||= begin # rubocop:disable Naming/MemoizedInstanceVariableName
+        monitor = @selector.register(@resolver, :w)
+        monitor.value = @resolver
+        monitor
+      end
+    end
+
+    def on_resolver_channel(channel, addresses)
+      found_channel = @channels.find do |ch|
+        ch != channel && ch.mergeable?(addresses)
+      end
+      return register_channel(channel) unless found_channel
+      if found_channel.state == :open
+        coalesce_channels(found_channel, channel)
+      else
+        found_channel.once(:open) do
+          coalesce_channels(found_channel, channel)
+        end
+      end
+    end
+
+    def on_resolver_error(ch, error)
+      ch.emit(:error, error)
+      # must remove channel by hand, hasn't been started yet
+      unregister_channel(ch)
+    end
+
+    def on_resolver_close
+      @selector.deregister(@resolver)
+      @_resolver_monitor = nil
+      @resolver.close unless @resolver.closed?
+    end
+
     def register_channel(channel)
       monitor = @selector.register(channel, :w)
       monitor.value = channel
       channel.on(:close) do
-        @channels.delete(channel)
-        @selector.deregister(channel)
+        unregister_channel(channel)
+      end
+    end
+
+    def unregister_channel(channel)
+      @channels.delete(channel)
+      @selector.deregister(channel)
+    end
+
+    def next_timeout
+      timeout = @timeout.timeout # force log time
+      return (@resolver.timeout || timeout) unless @resolver.closed?
+      timeout
+    end
+
+    def coalesce_channels(ch1, ch2)
+      if ch1.coalescable?(ch2)
+        ch1.merge(ch2)
+        @channels.delete(ch2)
+      else
+        register_channel(ch2)
       end
-      @channels << channel
     end
   end
 end