httpx 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0f44b6fc0f644a3f70b58a7051e963c5f74f9a5cb8ce6ccde0f6d2aa8909b14
-  data.tar.gz: d341c978e49312b3abe26c24845b672be5055fd4a4a2cab9e2aa9d1acf79aa63
+  metadata.gz: 4ac832ee93155b950d27dde3d7295fae2b817a8b2c55c881ec07f73e97736c59
+  data.tar.gz: 28251067c648230eed4435b9df7cf75bfc1c552162f37928a8ee6c5b485d6c3b
 SHA512:
-  metadata.gz: c6a9e8caa8f2424c8628153535deda5f1d8623253801ae329884ee51f10c5f0e5237d4c6e2548faa40d14c0e0e3dd982c050e6f67521a8b2a56e39a1bbab40aa
-  data.tar.gz: db1b34ff9023343b4f9580b1a72c54765cf7d446f59f96bea22ec9230825ce42cd42f0ac26ec168cd3b721e75e6cf5c2964a05afea1895a778d8f27932f61d65
+  metadata.gz: d7b9f1419af3bb823ff58de80dca6a933bf25a756e27ff35f76c5ad7948ad1aab1c1f63f8eaf20bc9fc05bb0b2ed4932fd5ff1e33bcd7a8cda061bee2c54e4ac
+  data.tar.gz: f410653e0e1646f36eeaf90746bfae17d8c344abdf44eb92df2a992897d24843d16e370ec54f6441bedfe779981d17e4b9c5316f73115adc77e0639cb8dba53f

data/lib/httpx/altsvc.rb

@@ -42,7 +42,23 @@ module HTTPX
 
       origin = request.origin
       host = request.uri.host
-      parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
+
+      altsvc = response.headers["alt-svc"]
+
+      # https://tools.ietf.org/html/rfc7838#section-3
+      # A field value containing the special value "clear" indicates that the
+      # origin requests all alternatives for that origin to be invalidated
+      # (including those specified in the same response, in case of an
+      # invalid reply containing both "clear" and alternative services).
+      if altsvc == "clear"
+        @altsvc_mutex.synchronize do
+          @altsvcs[origin].clear
+        end
+
+        return
+      end
+
+      parse(altsvc) do |alt_origin, alt_params|
         alt_origin.host ||= host
         yield(alt_origin, origin, alt_params)
       end
@@ -73,7 +89,7 @@ module HTTPX
         alt_proto, alt_origin = alt_origin.split("=")
         alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
         if alt_origin.start_with?(":")
-          alt_origin = "dummy#{alt_origin}"
+          alt_origin = "#{alt_proto}://dummy#{alt_origin}"
           uri = URI.parse(alt_origin)
           uri.host = nil
           uri

data/lib/httpx/chainable.rb

@@ -3,9 +3,11 @@
 module HTTPX
   module Chainable
     %i[head get post put delete trace options connect patch].each do |meth|
-      define_method meth do |*uri, **options|
-        request(meth, uri, **options)
-      end
+      class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        def #{meth}(*uri, **options)
+          request(:#{meth}, uri, **options)
+        end
+      MOD
     end
 
     def request(verb, uri, **options)

data/lib/httpx/connection.rb

@@ -67,6 +67,10 @@ module HTTPX
       else
         transition(:idle)
       end
+
+      @inflight = 0
+      @keep_alive_timeout = options.timeout.keep_alive_timeout
+      @keep_alive_timer = nil
     end
 
     # this is a semi-private method, to be used by the resolver
@@ -84,6 +88,8 @@ module HTTPX
 
       return false if exhausted?
 
+      return false if @keep_alive_timer && @keep_alive_timer.fires_in.negative?
+
       (
         (
           @origins.include?(uri.origin) &&
@@ -101,6 +107,8 @@ module HTTPX
 
       return false if exhausted?
 
+      return false if @keep_alive_timer && @keep_alive_timer.fires_in.negative?
+
       !(@io.addresses & connection.addresses).empty? && @options == connection.options
     end
 
@@ -140,7 +148,10 @@ module HTTPX
 
     def purge_pending
       pendings = []
-      pendings << @parser.pending if @parser
+      if @parser
+        @inflight -= @parser.pending.size
+        pendings << @parser.pending
+      end
       pendings << @pending
       pendings.each do |pending|
         pending.reject! do |request|
@@ -168,22 +179,45 @@ module HTTPX
     end
 
     def interests
-      return :w if @state == :idle
+      # connecting
+      if connecting?
+        connect
 
-      :rw
+        return @io.interests if connecting?
+      end
+
+      # if the write buffer is full, we drain it
+      return :w if @write_buffer.full?
+
+      return @parser.interests if @parser
+
+      nil
     end
 
     def to_io
+      @io.to_io
+    end
+
+    def call
       case @state
-      when :idle
-        transition(:open)
+      when :closed
+        return
+      when :closing
+        consume
+        transition(:closed)
+        emit(:close)
+      when :open
+        consume
       end
-      @io.to_io
+      nil
     end
 
     def close
       @parser.close if @parser
-      transition(:closing)
+      return unless @keep_alive_timer
+
+      @keep_alive_timer.cancel
+      remove_instance_variable(:@keep_alive_timer)
     end
 
     def reset
@@ -195,26 +229,14 @@ module HTTPX
     def send(request)
       if @parser && !@write_buffer.full?
         request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
+        @inflight += 1
+        @keep_alive_timer.pause if @keep_alive_timer
         parser.send(request)
       else
         @pending << request
       end
     end
 
-    def call
-      case @state
-      when :closed
-        return
-      when :closing
-        dwrite
-        transition(:closed)
-        emit(:close)
-      when :open
-        consume
-      end
-      nil
-    end
-
     def timeout
       return @timeout if defined?(@timeout)
 
@@ -225,54 +247,94 @@ module HTTPX
 
     private
 
+    def connect
+      transition(:open)
+    end
+
     def exhausted?
       @parser && parser.exhausted?
     end
 
     def consume
       catch(:called) do
-        dread
-        dwrite
-        parser.consume
-      end
-    end
-
-    def dread(wsize = @window_size)
-      loop do
-        siz = @io.read(wsize, @read_buffer)
-        unless siz
-          ex = EOFError.new("descriptor closed")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
-        end
-        return if siz.zero?
-
-        log { "READ: #{siz} bytes..." }
-        parser << @read_buffer.to_s
-        return if @state == :closing || @state == :closed
-      end
-    end
-
-    def dwrite
-      loop do
-        return if @write_buffer.empty?
-
-        siz = @io.write(@write_buffer)
-        unless siz
-          ex = EOFError.new("descriptor closed")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
+        loop do
+          parser.consume
+
+          # we exit if there's no more data to process
+          if @pending.size.zero? && @inflight.zero?
+            log(level: 3) { "NO MORE REQUESTS..." }
+            return
+          end
+
+          @timeout = @current_timeout
+
+          read_drained = false
+          write_drained = nil
+
+          # dread
+          loop do
+            siz = @io.read(@window_size, @read_buffer)
+            unless siz
+              ex = EOFError.new("descriptor closed")
+              ex.set_backtrace(caller)
+              on_error(ex)
+              return
+            end
+
+            log { "READ: #{siz} bytes..." }
+
+            if siz.zero?
+              read_drained = @read_buffer.empty?
+              break
+            end
+
+            parser << @read_buffer.to_s
+
+            break if @state == :closing || @state == :closed
+
+            # for HTTP/2, we just want to write goaway frame
+          end unless @state == :closing
+
+          # dwrite
+          loop do
+            if @write_buffer.empty?
+              # we only mark as drained on the first loop
+              write_drained = write_drained.nil? && @inflight.positive?
+              break
+            end
+
+            siz = @io.write(@write_buffer)
+            unless siz
+              ex = EOFError.new("descriptor closed")
+              ex.set_backtrace(caller)
+              on_error(ex)
+              return
+            end
+            log { "WRITE: #{siz} bytes..." }
+
+            if siz.zero?
+              write_drained = !@write_buffer.empty?
+              break
+            end
+
+            break if @state == :closing || @state == :closed
+
+            write_drained = false
+          end
+
+          # return if socket is drained
+          if read_drained && write_drained
+            log(level: 3) { "WAITING FOR EVENTS..." }
+            return
+          end
         end
-        log { "WRITE: #{siz} bytes..." }
-        return if siz.zero?
-        return if @state == :closing || @state == :closed
       end
     end
 
     def send_pending
       while !@write_buffer.full? && (request = @pending.shift)
+        @inflight += 1
+        @keep_alive_timer.pause if @keep_alive_timer
         parser.send(request)
       end
     end
@@ -292,6 +354,7 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           emit(:altsvc, alt_origin, origin, alt_params)
         end
+        handle_response
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
@@ -307,14 +370,21 @@ module HTTPX
       parser.on(:origin) do |origin|
         @origins << origin
       end
-      parser.on(:close) do
+      parser.on(:close) do |force|
         transition(:closing)
+        if force
+          transition(:closed)
+          emit(:close)
+        end
       end
       parser.on(:reset) do
-        transition(:closing)
-        unless parser.empty?
+        if parser.empty?
+          reset
+        else
+          transition(:closing)
           transition(:closed)
           emit(:reset)
+          @parser.reset if @parser
           transition(:idle)
           transition(:open)
         end
@@ -335,6 +405,9 @@ module HTTPX
 
     def transition(nextstate)
       case nextstate
+      when :idle
+        @timeout = @current_timeout = @options.timeout.connect_timeout
+
       when :open
         return if @state == :closed
 
@@ -344,6 +417,8 @@ module HTTPX
         return unless @io.connected?
 
         send_pending
+
+        @timeout = @current_timeout = @options.timeout.operation_timeout
         emit(:open)
       when :closing
         return unless @state == :open
@@ -358,6 +433,11 @@ module HTTPX
 
         @io.close
         @read_buffer.clear
+        if @keep_alive_timer
+          @keep_alive_timer.cancel
+          remove_instance_variable(:@keep_alive_timer)
+        end
+
         remove_instance_variable(:@timeout) if defined?(@timeout)
       when :already_open
         nextstate = :open
@@ -379,21 +459,43 @@ module HTTPX
       emit(:close)
     end
 
-    def on_error(ex)
-      handle_error(ex)
-      reset
+    def handle_response
+      @inflight -= 1
+      return unless @inflight.zero?
+
+      if @keep_alive_timer
+        @keep_alive_timer.resume
+        @keep_alive_timer.reset
+      else
+        @keep_alive_timer = @timers.after(@keep_alive_timeout) do
+          unless @inflight.zero?
+            log { "(#{object_id})) keep alive timeout expired, closing..." }
+            reset
+          end
+        end
+      end
     end
 
-    def handle_error(error)
+    def on_error(error)
       if error.instance_of?(TimeoutError)
         if @timeout
           @timeout -= error.timeout
           return unless @timeout <= 0
         end
 
-        error = error.to_connection_error if connecting?
+        if @total_timeout && @total_timeout.fires_in.negative?
+          ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
+          ex.set_backtrace(error.backtrace)
+          error = ex
+        elsif connecting?
+          error = error.to_connection_error
+        end
       end
+      handle_error(error)
+      reset
+    end
 
+    def handle_error(error)
       parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
       while (request = @pending.shift)
         request.emit(:response, ErrorResponse.new(request, error, @options))
@@ -408,8 +510,8 @@ module HTTPX
       @total_timeout ||= @timers.after(total) do
         ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
         ex.set_backtrace(caller)
-        @parser.close if @parser
         on_error(ex)
+        @parser.close if @parser
       end
     end
   end

data/lib/httpx/connection/http1.rb

@@ -23,6 +23,19 @@ module HTTPX
       @requests = []
     end
 
+    def interests
+      # this means we're processing incoming response already
+      return :r if @request
+
+      return if @requests.empty?
+
+      request = @requests.first
+
+      return :w if request.interests == :w || !@buffer.empty?
+
+      :r
+    end
+
     def reset
       @max_requests = @options.max_requests || MAX_REQUESTS
       @parser.reset!
@@ -30,7 +43,7 @@ module HTTPX
 
     def close
       reset
-      emit(:close)
+      emit(:close, true)
     end
 
     def exhausted?
@@ -53,16 +66,18 @@ module HTTPX
         return
       end
 
-      unless @requests.include?(request)
-        @requests << request
-        @pipelining = true if @requests.size > 1
-      end
+      return if @requests.include?(request)
 
-      handle(request)
+      @requests << request
+      @pipelining = true if @requests.size > 1
     end
 
     def consume
-      @requests.each do |request|
+      requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
+      @requests.each_with_index do |request, idx|
+        break if idx >= requests_limit
+        next if request.state == :done
+
         handle(request)
       end
     end
@@ -121,7 +136,7 @@ module HTTPX
 
     def dispatch
       if @request.expects?
-        reset
+        @parser.reset!
         return handle(@request)
       end
 
@@ -136,10 +151,10 @@ module HTTPX
         throw(:called)
       end
 
-      reset
+      @parser.reset!
       @max_requests -= 1
-      send(@pending.shift) unless @pending.empty?
       manage_connection(response)
+      send(@pending.shift) unless @pending.empty?
     end
 
     def handle_error(ex)
@@ -149,6 +164,9 @@ module HTTPX
         @requests.each do |request|
           emit(:error, request, ex)
         end
+        @pending.each do |request|
+          emit(:error, request, ex)
+        end
       end
     end