httpx 0.21.0 → 1.2.1

data/lib/httpx/connection/http1.rb

@@ -7,15 +7,17 @@ module HTTPX
     include Callbacks
     include Loggable
 
-    MAX_REQUESTS = 100
+    MAX_REQUESTS = 200
     CRLF = "\r\n"
 
     attr_reader :pending, :requests
 
+    attr_accessor :max_concurrent_requests
+
     def initialize(buffer, options)
       @options = Options.new(options)
       @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
-      @max_requests = @options.max_requests || MAX_REQUESTS
+      @max_requests = @options.max_requests
       @parser = Parser::HTTP1.new(self)
       @buffer = buffer
       @version = [1, 1]
@@ -47,6 +49,7 @@ module HTTPX
       @max_requests = @options.max_requests || MAX_REQUESTS
       @parser.reset!
       @handshake_completed = false
+      @pending.concat(@requests) unless @requests.empty?
     end
 
     def close
@@ -106,6 +109,7 @@ module HTTPX
 
     def on_headers(h)
       @request = @requests.first
+
       return if @request.response
 
       log(level: 2) { "headers received" }
@@ -132,33 +136,42 @@ module HTTPX
     end
 
     def on_data(chunk)
-      return unless @request
+      request = @request
+
+      return unless request
 
       log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
       log(level: 2, color: :green) { "-> #{chunk.inspect}" }
-      response = @request.response
+      response = request.response
 
       response << chunk
+    rescue StandardError => e
+      error_response = ErrorResponse.new(request, e, request.options)
+      request.response = error_response
+      dispatch
     end
 
     def on_complete
-      return unless @request
+      request = @request
+
+      return unless request
 
       log(level: 2) { "parsing complete" }
       dispatch
     end
 
     def dispatch
-      if @request.expects?
+      request = @request
+
+      if request.expects?
         @parser.reset!
-        return handle(@request)
+        return handle(request)
       end
 
-      request = @request
       @request = nil
       @requests.shift
       response = request.response
-      response.finish!
+      response.finish! unless response.is_a?(ErrorResponse)
       emit(:response, request, response)
 
       if @parser.upgrade?
@@ -168,9 +181,20 @@ module HTTPX
 
       @parser.reset!
       @max_requests -= 1
-      manage_connection(response)
+      if response.is_a?(ErrorResponse)
+        disable
+      else
+        manage_connection(request, response)
+      end
 
-      send(@pending.shift) unless @pending.empty?
+      if exhausted?
+        @pending.concat(@requests)
+        @requests.clear
+
+        emit(:exhausted)
+      else
+        send(@pending.shift) unless @pending.empty?
+      end
     end
 
     def handle_error(ex)
@@ -197,13 +221,14 @@ module HTTPX
     end
 
     def ping
+      reset
       emit(:reset)
       emit(:exhausted)
     end
 
     private
 
-    def manage_connection(response)
+    def manage_connection(request, response)
       connection = response.headers["connection"]
       case connection
       when /keep-alive/i
@@ -233,7 +258,7 @@ module HTTPX
         disable
       when nil
         # In HTTP/1.1, it's keep alive by default
-        return if response.version == "1.1"
+        return if response.version == "1.1" && request.headers["connection"] != "close"
 
         disable
       end
@@ -241,6 +266,7 @@ module HTTPX
 
     def disable
       disable_pipelining
+      reset
       emit(:reset)
       throw(:called)
     end
@@ -271,29 +297,31 @@ module HTTPX
         request.body.chunk!
       end
 
-      connection = request.headers["connection"]
+      extra_headers = {}
 
-      connection ||= if request.options.persistent
-        # when in a persistent connection, the request can't be at
-        # the edge of a renegotiation
-        if @requests.index(request) + 1 < @max_requests
-          "keep-alive"
-        else
-          "close"
-        end
-      else
-        # when it's not a persistent connection, it sets "Connection: close" always
-        # on the last request of the possible batch (either allowed max requests,
-        # or if smaller, the size of the batch itself)
-        requests_limit = [@max_requests, @requests.size].min
-        if request == @requests[requests_limit - 1]
-          "close"
+      unless request.headers.key?("connection")
+        connection_value = if request.persistent?
+          # when in a persistent connection, the request can't be at
+          # the edge of a renegotiation
+          if @requests.index(request) + 1 < @max_requests
+            "keep-alive"
+          else
+            "close"
+          end
         else
-          "keep-alive"
+          # when it's not a persistent connection, it sets "Connection: close" always
+          # on the last request of the possible batch (either allowed max requests,
+          # or if smaller, the size of the batch itself)
+          requests_limit = [@max_requests, @requests.size].min
+          if request == @requests[requests_limit - 1]
+            "close"
+          else
+            "keep-alive"
+          end
         end
-      end
 
-      extra_headers = { "connection" => connection }
+        extra_headers["connection"] = connection_value
+      end
       extra_headers["host"] = request.authority unless request.headers.key?("host")
       extra_headers
     end
@@ -312,7 +340,7 @@ module HTTPX
     end
 
     def join_headline(request)
-      "#{request.verb.to_s.upcase} #{request.path} HTTP/#{@version.join(".")}"
+      "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
     end
 
     def join_headers(request)
@@ -349,12 +377,10 @@ module HTTPX
     end
 
     def join_headers2(headers)
-      buffer = "".b
       headers.each do |field, value|
-        buffer << "#{capitalized(field)}: #{value}" << CRLF
+        buffer = "#{capitalized(field)}: #{value}#{CRLF}"
         log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
         @buffer << buffer
-        buffer.clear
       end
     end
 
@@ -367,5 +393,4 @@ module HTTPX
       UPCASED[field] || field.split("-").map(&:capitalize).join("-")
     end
   end
-  Connection.register "http/1.1", Connection::HTTP1
 end

data/lib/httpx/connection/http2.rb

@@ -35,7 +35,7 @@ module HTTPX
       @handshake_completed = false
       @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
       @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
-      @max_requests = @options.max_requests || 0
+      @max_requests = @options.max_requests
       init_connection
     end
 
@@ -55,7 +55,7 @@ module HTTPX
         return :w
       end
 
-      unless (@connection.state == :connected && @handshake_completed)
+      unless @connection.state == :connected && @handshake_completed
         return @buffer.empty? ? :r : :rw
       end
 
@@ -73,8 +73,11 @@ module HTTPX
     end
 
     def close
-      @connection.goaway unless @connection.state == :closed
-      emit(:close)
+      unless @connection.state == :closed
+        @connection.goaway
+        emit(:timeout, @options.timeout[:close_handshake_timeout])
+      end
+      emit(:close, true)
     end
 
     def empty?
@@ -82,9 +85,7 @@ module HTTPX
     end
 
     def exhausted?
-      return false if @max_requests.zero? && @connection.active_stream_count.zero?
-
-      @connection.active_stream_count >= @max_requests
+      !@max_requests.positive?
     end
 
     def <<(data)
@@ -92,13 +93,9 @@ module HTTPX
     end
 
     def can_buffer_more_requests?
-      if @handshake_completed
+      (@handshake_completed || !@wait_for_handshake) &&
         @streams.size < @max_concurrent_requests &&
-          @streams.size < @max_requests
-      else
-        !@wait_for_handshake &&
-          @streams.size < @max_concurrent_requests
-      end
+        @streams.size < @max_requests
     end
 
     def send(request)
@@ -116,7 +113,6 @@ module HTTPX
       true
     rescue HTTP2Next::Error::StreamLimitExceeded
       @pending.unshift(request)
-      emit(:exhausted)
     end
 
     def consume
@@ -154,6 +150,7 @@ module HTTPX
 
     def send_pending
       while (request = @pending.shift)
+        # TODO: this request should go back to top of stack
         break unless send(request)
       end
     end
@@ -172,7 +169,6 @@ module HTTPX
 
     def init_connection
       @connection = HTTP2Next::Client.new(@settings)
-      @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
       @connection.on(:frame, &method(:on_frame))
       @connection.on(:frame_sent, &method(:on_frame_sent))
       @connection.on(:frame_received, &method(:on_frame_received))
@@ -208,7 +204,7 @@ module HTTPX
     def set_protocol_headers(request)
       {
         ":scheme" => request.scheme,
-        ":method" => request.verb.to_s.upcase,
+        ":method" => request.verb,
         ":path" => request.path,
         ":authority" => request.authority,
       }
@@ -314,10 +310,11 @@ module HTTPX
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
         response = ErrorResponse.new(request, ex, request.options)
+        request.response = response
         emit(:response, request, response)
       else
         response = request.response
-        if response && response.status == 421
+        if response && response.is_a?(Response) && response.status == 421
           ex = MisdirectedRequestError.new(response)
           ex.set_backtrace(caller)
           emit(:error, request, ex)
@@ -339,14 +336,7 @@ module HTTPX
     def on_settings(*)
       @handshake_completed = true
       emit(:current_timeout)
-
-      if @max_requests.zero?
-        @max_requests = @connection.remote_settings[:settings_max_concurrent_streams]
-
-        @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
-      end
-
-      @max_concurrent_requests = [@max_concurrent_requests, @max_requests].min
+      @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
       send_pending
     end
 
@@ -365,7 +355,7 @@ module HTTPX
         ex.set_backtrace(caller)
         handle_error(ex)
       end
-      return unless is_connection_closed && @streams.size.zero?
+      return unless is_connection_closed && @streams.empty?
 
       emit(:close, is_connection_closed)
     end
@@ -412,5 +402,4 @@ module HTTPX
       end
     end
   end
-  Connection.register "h2", Connection::HTTP2
 end