httpx 0.6.7 → 0.9.0

data/lib/httpx/connection/http1.rb

@@ -7,14 +7,15 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    MAX_REQUESTS = 100
     CRLF = "\r\n"
 
     attr_reader :pending
 
     def initialize(buffer, options)
       @options = Options.new(options)
-      @max_concurrent_requests = @options.max_concurrent_requests
-      @max_requests = Float::INFINITY
+      @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
+      @max_requests = @options.max_requests || MAX_REQUESTS
       @parser = Parser::HTTP1.new(self)
       @buffer = buffer
       @version = [1, 1]
@@ -22,13 +23,31 @@ module HTTPX
       @requests = []
     end
 
+    def interests
+      # this means we're processing incoming response already
+      return :r if @request
+
+      return if @requests.empty?
+
+      request = @requests.first
+
+      return :w if request.interests == :w || !@buffer.empty?
+
+      :r
+    end
+
     def reset
+      @max_requests = @options.max_requests || MAX_REQUESTS
       @parser.reset!
     end
 
     def close
       reset
-      emit(:close)
+      emit(:close, true)
+    end
+
+    def exhausted?
+      !@max_requests.positive?
     end
 
     def empty?
@@ -42,20 +61,24 @@ module HTTPX
     end
 
     def send(request)
-      if @max_requests.positive? &&
-         @requests.size >= @max_concurrent_requests
+      unless @max_requests.positive?
         @pending << request
         return
       end
-      unless @requests.include?(request)
-        @requests << request
-        @pipelining = true if @requests.size > 1
-      end
-      handle(request)
+
+      return if @requests.include?(request)
+
+      @requests << request
+      @pipelining = true if @requests.size > 1
     end
 
     def consume
-      @requests.each do |request|
+      requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
+      @requests.each_with_index do |request, idx|
+        break if idx >= requests_limit
+        next if request.state == :done
+
+        request.headers["connection"] ||= request.options.persistent || idx < requests_limit - 1 ? "keep-alive" : "close"
         handle(request)
       end
     end
@@ -114,7 +137,7 @@ module HTTPX
 
     def dispatch
       if @request.expects?
-        reset
+        @parser.reset!
         return handle(@request)
       end
 
@@ -129,10 +152,10 @@ module HTTPX
         throw(:called)
       end
 
-      reset
+      @parser.reset!
       @max_requests -= 1
-      send(@pending.shift) unless @pending.empty?
       manage_connection(response)
+      send(@pending.shift) unless @pending.empty?
     end
 
     def handle_error(ex)
@@ -142,9 +165,17 @@ module HTTPX
         @requests.each do |request|
           emit(:error, request, ex)
         end
+        @pending.each do |request|
+          emit(:error, request, ex)
+        end
       end
     end
 
+    def ping
+      emit(:reset)
+      emit(:exhausted)
+    end
+
     private
 
     def manage_connection(response)
@@ -163,13 +194,11 @@ module HTTPX
           emit(:timeout, keep_alive_timeout)
         end
       when /close/i
-        @max_requests = Float::INFINITY
         disable
       when nil
         # In HTTP/1.1, it's keep alive by default
         return if response.version == "1.1"
 
-        @max_requests = Float::INFINITY
         disable
       end
     end
@@ -183,7 +212,14 @@ module HTTPX
     def disable_pipelining
       return if @requests.empty?
 
-      @requests.each { |r| r.transition(:idle) }
+      @requests.each do |r|
+        r.transition(:idle)
+
+        # when we disable pipelining, we still want to try keep-alive.
+        # only when keep-alive with one request fails, do we fallback to
+        # connection: close.
+        r.headers["connection"] = "close" if @max_concurrent_requests == 1
+      end
       # server doesn't handle pipelining, and probably
       # doesn't support keep-alive. Fallback to send only
       # 1 keep alive request.
@@ -193,7 +229,7 @@ module HTTPX
 
     def set_request_headers(request)
       request.headers["host"] ||= request.authority
-      request.headers["connection"] ||= "keep-alive"
+      request.headers["connection"] ||= request.options.persistent ? "keep-alive" : "close"
       if !request.headers.key?("content-length") &&
          request.body.bytesize == Float::INFINITY
         request.chunk!

data/lib/httpx/connection/http2.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "securerandom"
 require "io/wait"
 require "http/2/next"
 
@@ -8,6 +9,8 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    MAX_CONCURRENT_REQUESTS = HTTP2Next::DEFAULT_MAX_CONCURRENT_STREAMS
+
     Error = Class.new(Error) do
       def initialize(id, code)
         super("stream #{id} closed with error: #{code}")
@@ -18,30 +21,63 @@ module HTTPX
 
     def initialize(buffer, options)
       @options = Options.new(options)
-      @max_concurrent_requests = @options.max_concurrent_requests
+      @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
+      @max_requests = @options.max_requests || 0
       @pending = []
       @streams = {}
       @drains  = {}
+      @pings = []
       @buffer = buffer
       @handshake_completed = false
       init_connection
     end
 
-    def close
-      @connection.goaway
+    def interests
+      # waiting for WINDOW_UPDATE frames
+      return :r if @buffer.full?
+
+      return :w if @connection.state == :closed
+
+      unless (@connection.state == :connected && @handshake_completed)
+        return @buffer.empty? ? :r : :rw
+      end
+
+      return :w unless @pending.empty?
+
+      return :w if @streams.each_key.any? { |r| r.interests == :w }
+
+      return :r if @buffer.empty?
+
+      :rw
+    end
+
+    def reset
+      init_connection
+    end
+
+    def close(*args)
+      @connection.goaway(*args) unless @connection.state == :closed
+      emit(:close)
     end
 
     def empty?
       @connection.state == :closed || @streams.empty?
     end
 
+    def exhausted?
+      return false if @max_requests.zero? && @connection.active_stream_count.zero?
+
+      @connection.active_stream_count >= @max_requests
+    end
+
     def <<(data)
       @connection << data
     end
 
-    def send(request, **)
+    def send(request)
       if !@handshake_completed ||
-         @streams.size >= @max_concurrent_requests
+         @streams.size >= @max_concurrent_requests ||
+         @streams.size >= @max_requests
         @pending << request
         return
       end
@@ -49,13 +85,19 @@ module HTTPX
         stream = @connection.new_stream
         handle_stream(stream, request)
         @streams[request] = stream
+        @max_requests -= 1
       end
       handle(request, stream)
       true
+    rescue HTTP2Next::Error::StreamLimitExceeded
+      @pending.unshift(request)
+      emit(:exhausted)
     end
 
     def consume
       @streams.each do |request, stream|
+        next if request.state == :done
+
         handle(request, stream)
       end
     end
@@ -69,6 +111,13 @@ module HTTPX
       end
     end
 
+    def ping
+      ping = SecureRandom.gen_random(8)
+      @connection.ping(ping)
+    ensure
+      @pings << ping
+    end
+
     private
 
     def send_pending
@@ -95,6 +144,7 @@ module HTTPX
 
     def init_connection
       @connection = HTTP2Next::Client.new(@options.http2_settings)
+      @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
       @connection.on(:frame, &method(:on_frame))
       @connection.on(:frame_sent, &method(:on_frame_sent))
       @connection.on(:frame_received, &method(:on_frame_received))
@@ -102,6 +152,7 @@ module HTTPX
       @connection.on(:promise, &method(:on_promise))
       @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
       @connection.on(:settings_ack, &method(:on_settings))
+      @connection.on(:ack, &method(:on_pong))
       @connection.on(:goaway, &method(:on_close))
       #
       # Some servers initiate HTTP/2 negotiation right away, some don't.
@@ -115,7 +166,7 @@ module HTTPX
     def handle_stream(stream, request)
       stream.on(:close, &method(:on_stream_close).curry[stream, request])
       stream.on(:half_close) do
-        log(level: 2, label: "#{stream.id}: ") { "waiting for response..." }
+        log(level: 2) { "#{stream.id}: waiting for response..." }
       end
       stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
       stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
@@ -130,8 +181,8 @@ module HTTPX
       headers[":path"]      = headline_uri(request)
       headers[":authority"] = request.authority
       headers = headers.merge(request.headers)
-      log(level: 1, label: "#{stream.id}: ", color: :yellow) do
-        headers.map { |k, v| "-> HEADER: #{k}: #{v}" }.join("\n")
+      log(level: 1, color: :yellow) do
+        headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
       stream.headers(headers, end_stream: request.empty?)
     end
@@ -142,8 +193,8 @@ module HTTPX
       chunk = @drains.delete(request) || request.drain_body
       while chunk
         next_chunk = request.drain_body
-        log(level: 1, label: "#{stream.id}: ", color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
-        log(level: 2, label: "#{stream.id}: ", color: :green) { "-> #{chunk.inspect}" }
+        log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
+        log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
         stream.data(chunk, end_stream: !next_chunk)
         if next_chunk && @buffer.full?
           @drains[request] = next_chunk
@@ -158,25 +209,25 @@ module HTTPX
     ######
 
     def on_stream_headers(stream, request, h)
-      log(label: "#{stream.id}:", color: :yellow) do
-        h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n")
+      log(color: :yellow) do
+        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
       end
       _, status = h.shift
       headers = request.options.headers_class.new(h)
       response = request.options.response_class.new(request, status, "2.0", headers)
       request.response = response
       @streams[request] = stream
+
+      handle(request, stream) if request.expects?
     end
 
     def on_stream_data(stream, request, data)
-      log(level: 1, label: "#{stream.id}: ", color: :green) { "<- DATA: #{data.bytesize} bytes..." }
-      log(level: 2, label: "#{stream.id}: ", color: :green) { "<- #{data.inspect}" }
+      log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
+      log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
       request.response << data
     end
 
     def on_stream_close(stream, request, error)
-      return handle(request, stream) if request.expects?
-
       if error && error != :no_error
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
@@ -191,10 +242,14 @@ module HTTPX
           emit(:response, request, response)
         end
       end
-      log(level: 2, label: "#{stream.id}: ") { "closing stream" }
+      log(level: 2) { "#{stream.id}: closing stream" }
 
       @streams.delete(request)
       send(@pending.shift) unless @pending.empty?
+      return unless @streams.empty? && exhausted?
+
+      close
+      emit(:exhausted) unless @pending.empty?
     end
 
     def on_frame(bytes)
@@ -203,8 +258,14 @@ module HTTPX
 
     def on_settings(*)
       @handshake_completed = true
-      @max_concurrent_requests = [@max_concurrent_requests,
-                                  @connection.remote_settings[:settings_max_concurrent_streams]].min
+
+      if @max_requests.zero?
+        @max_requests = @connection.remote_settings[:settings_max_concurrent_streams]
+
+        @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
+      end
+
+      @max_concurrent_requests = [@max_concurrent_requests, @max_requests].min
       send_pending
     end
 
@@ -222,32 +283,26 @@ module HTTPX
     end
 
     def on_frame_sent(frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "frame was sent!" }
-      log(level: 2, label: "#{frame[:stream]}: ", color: :blue) do
-        case frame[:type]
-        when :data
-          frame.merge(payload: frame[:payload].bytesize).inspect
-        else
-          frame.inspect
-        end
+      log(level: 2) { "#{frame[:stream]}: frame was sent!" }
+      log(level: 2, color: :blue) do
+        payload = frame
+        payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
+        "#{frame[:stream]}: #{payload}"
       end
     end
 
     def on_frame_received(frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "frame was received!" }
-      log(level: 2, label: "#{frame[:stream]}: ", color: :magenta) do
-        case frame[:type]
-        when :data
-          frame.merge(payload: frame[:payload].bytesize).inspect
-        else
-          frame.inspect
-        end
+      log(level: 2) { "#{frame[:stream]}: frame was received!" }
+      log(level: 2, color: :magenta) do
+        payload = frame
+        payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
+        "#{frame[:stream]}: #{payload}"
       end
     end
 
     def on_altsvc(origin, frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "altsvc frame was received" }
-      log(level: 2, label: "#{frame[:stream]}: ") { frame.inspect }
+      log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
+      log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
       alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
       params = { "ma" => frame[:max_age] }
       emit(:altsvc, origin, alt_origin, origin, params)
@@ -261,6 +316,14 @@ module HTTPX
       emit(:origin, origin)
     end
 
+    def on_pong(ping)
+      if !@pings.delete(ping)
+        close(:protocol_error, "ping payload did not match")
+      else
+        emit(:pong)
+      end
+    end
+
     def respond_to_missing?(meth, *args)
       @connection.respond_to?(meth, *args) || super
     end

data/lib/httpx/extensions.rb

@@ -11,7 +11,7 @@ module HTTPX
     #
     # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
     #
-    module CurryMethods # :nodoc:
+    module CurryMethods
       # Backport for the Method#curry method, which is part of ruby core since 2.2 .
       #
       def curry(*args)
@@ -81,4 +81,4 @@ module HTTPX
       end
     end
   end
-end
+end