httpx 0.11.3 → 0.14.0

data/lib/httpx/connection/http1.rb

@@ -10,7 +10,7 @@ module HTTPX
     MAX_REQUESTS = 100
     CRLF = "\r\n"
 
-    attr_reader :pending
+    attr_reader :pending, :requests
 
     def initialize(buffer, options)
       @options = Options.new(options)
@@ -74,9 +74,10 @@ module HTTPX
     end
 
     def consume
-      requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
+      requests_limit = [@max_requests, @requests.size].min
+      concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
       @requests.each_with_index do |request, idx|
-        break if idx >= requests_limit
+        break if idx >= concurrent_requests_limit
         next if request.state == :done
 
         request.headers["connection"] ||= request.options.persistent || idx < requests_limit - 1 ? "keep-alive" : "close"
@@ -115,7 +116,7 @@ module HTTPX
       response = @request.response
       log(level: 2) { "trailer headers received" }
 
-      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
+      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
       response.merge_headers(h)
     end
 
@@ -161,13 +162,13 @@ module HTTPX
     end
 
     def handle_error(ex)
-      if ex.is_a?(EOFError) && @request && @request.response &&
+      if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
          !@request.response.headers.key?("content-length") &&
          !@request.response.headers.key?("transfer-encoding")
         # if the response does not contain a content-length header, the server closing the
         # connnection is the indicator of response consumed.
         # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
-        on_complete
+        catch(:called) { on_complete }
         return
       end
 
@@ -234,6 +235,8 @@ module HTTPX
 
     def disable_pipelining
       return if @requests.empty?
+      # do not disable pipelining if already set to 1 request at a time
+      return if @max_concurrent_requests == 1
 
       @requests.each do |r|
         r.transition(:idle)
@@ -250,7 +253,7 @@ module HTTPX
       @pipelining = false
     end
 
-    def set_request_headers(request)
+    def set_protocol_headers(request)
       request.headers["host"] ||= request.authority
       request.headers["connection"] ||= request.options.persistent ? "keep-alive" : "close"
       if !request.headers.key?("content-length") &&
@@ -264,28 +267,23 @@ module HTTPX
     end
 
     def handle(request)
-      set_request_headers(request)
       catch(:buffer_full) do
         request.transition(:headers)
         join_headers(request) if request.state == :headers
         request.transition(:body)
         join_body(request) if request.state == :body
+        request.transition(:trailers)
+        # HTTP/1.1 trailers should only work for chunked encoding
+        join_trailers(request) if request.body.chunked? && request.state == :trailers
         request.transition(:done)
       end
     end
 
     def join_headers(request)
-      buffer = +""
-      buffer << "#{request.verb.to_s.upcase} #{headline_uri(request)} HTTP/#{@version.join(".")}" << CRLF
-      log(color: :yellow) { "<- HEADLINE: #{buffer.chomp.inspect}" }
-      @buffer << buffer
-      buffer.clear
-      request.headers.each do |field, value|
-        buffer << "#{capitalized(field)}: #{value}" << CRLF
-        log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
-        @buffer << buffer
-        buffer.clear
-      end
+      @buffer << "#{request.verb.to_s.upcase} #{headline_uri(request)} HTTP/#{@version.join(".")}" << CRLF
+      log(color: :yellow) { "<- HEADLINE: #{@buffer.to_s.chomp.inspect}" }
+      set_protocol_headers(request)
+      join_headers2(request.headers)
       log { "<- " }
       @buffer << CRLF
     end
@@ -299,6 +297,26 @@ module HTTPX
         @buffer << chunk
         throw(:buffer_full, request) if @buffer.full?
       end
+
+      raise request.drain_error if request.drain_error
+    end
+
+    def join_trailers(request)
+      return unless request.trailers? && request.callbacks_for?(:trailers)
+
+      join_headers2(request.trailers)
+      log { "<- " }
+      @buffer << CRLF
+    end
+
+    def join_headers2(headers)
+      buffer = "".b
+      headers.each do |field, value|
+        buffer << "#{capitalized(field)}: #{value}" << CRLF
+        log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
+        @buffer << buffer
+        buffer.clear
+      end
     end
 
     UPCASED = {

data/lib/httpx/connection/http2.rb

@@ -21,14 +21,16 @@ module HTTPX
 
     def initialize(buffer, options)
       @options = Options.new(options)
-      @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
-      @max_requests = @options.max_requests || 0
+      @settings = @options.http2_settings
       @pending = []
       @streams = {}
       @drains  = {}
       @pings = []
       @buffer = buffer
       @handshake_completed = false
+      @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
+      @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
+      @max_requests = @options.max_requests || 0
       init_connection
     end
 
@@ -36,17 +38,25 @@ module HTTPX
       # waiting for WINDOW_UPDATE frames
       return :r if @buffer.full?
 
-      return :w if @connection.state == :closed
+      if @connection.state == :closed
+        return unless @handshake_completed
+
+        return :w
+      end
 
       unless (@connection.state == :connected && @handshake_completed)
         return @buffer.empty? ? :r : :rw
       end
 
-      return :w unless @pending.empty?
+      return :w if !@pending.empty? && can_buffer_more_requests?
 
       return :w if @streams.each_key.any? { |r| r.interests == :w }
 
-      return :r if @buffer.empty?
+      if @buffer.empty?
+        return if @streams.empty? && @pings.empty?
+
+        return :r
+      end
 
       :rw
     end
@@ -70,10 +80,18 @@ module HTTPX
       @connection << data
     end
 
+    def can_buffer_more_requests?
+      if @handshake_completed
+        @streams.size < @max_concurrent_requests &&
+          @streams.size < @max_requests
+      else
+        !@wait_for_handshake &&
+          @streams.size < @max_concurrent_requests
+      end
+    end
+
     def send(request)
-      if !@handshake_completed ||
-         @streams.size >= @max_concurrent_requests ||
-         @streams.size >= @max_requests
+      unless can_buffer_more_requests?
         @pending << request
         return
       end
@@ -126,20 +144,20 @@ module HTTPX
       request.path
     end
 
-    def set_request_headers(request); end
-
     def handle(request, stream)
       catch(:buffer_full) do
         request.transition(:headers)
         join_headers(stream, request) if request.state == :headers
         request.transition(:body)
         join_body(stream, request) if request.state == :body
+        request.transition(:trailers)
+        join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
         request.transition(:done)
       end
     end
 
     def init_connection
-      @connection = HTTP2Next::Client.new(@options.http2_settings)
+      @connection = HTTP2Next::Client.new(@settings)
       @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
       @connection.on(:frame, &method(:on_frame))
       @connection.on(:frame_sent, &method(:on_frame_sent))
@@ -163,6 +181,7 @@ module HTTPX
     public :reset
 
     def handle_stream(stream, request)
+      request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
       stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
       stream.on(:half_close) do
         log(level: 2) { "#{stream.id}: waiting for response..." }
@@ -172,18 +191,31 @@ module HTTPX
       stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
     end
 
+    def set_protocol_headers(request)
+      request.headers[":scheme"]    = request.scheme
+      request.headers[":method"]    = request.verb.to_s.upcase
+      request.headers[":path"]      = headline_uri(request)
+      request.headers[":authority"] = request.authority
+    end
+
     def join_headers(stream, request)
-      set_request_headers(request)
-      headers = {}
-      headers[":scheme"]    = request.scheme
-      headers[":method"]    = request.verb.to_s.upcase
-      headers[":path"]      = headline_uri(request)
-      headers[":authority"] = request.authority
-      headers = headers.merge(request.headers)
+      set_protocol_headers(request)
+      log(level: 1, color: :yellow) do
+        request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+      end
+      stream.headers(request.headers.each, end_stream: request.empty?)
+    end
+
+    def join_trailers(stream, request)
+      unless request.trailers?
+        stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
+        return
+      end
+
       log(level: 1, color: :yellow) do
-        headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+        request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
-      stream.headers(headers, end_stream: request.empty?)
+      stream.headers(request.trailers.each, end_stream: true)
     end
 
     def join_body(stream, request)
@@ -194,13 +226,15 @@ module HTTPX
         next_chunk = request.drain_body
         log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
         log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
-        stream.data(chunk, end_stream: !next_chunk)
-        if next_chunk && @buffer.full?
+        stream.data(chunk, end_stream: !(next_chunk || request.trailers? || request.callbacks_for?(:trailers)))
+        if next_chunk && (@buffer.full? || request.body.unbounded_body?)
           @drains[request] = next_chunk
           throw(:buffer_full)
         end
         chunk = next_chunk
       end
+
+      on_stream_refuse(stream, request, request.drain_error) if request.drain_error
     end
 
     ######
@@ -208,6 +242,11 @@ module HTTPX
     ######
 
     def on_stream_headers(stream, request, h)
+      if request.response && request.response.version == "2.0"
+        on_stream_trailers(stream, request, h)
+        return
+      end
+
       log(color: :yellow) do
         h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
       end
@@ -220,17 +259,34 @@ module HTTPX
       handle(request, stream) if request.expects?
     end
 
+    def on_stream_trailers(stream, request, h)
+      log(color: :yellow) do
+        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
+      end
+      request.response.merge_headers(h)
+    end
+
     def on_stream_data(stream, request, data)
       log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
       log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
       request.response << data
     end
 
+    def on_stream_refuse(stream, request, error)
+      stream.close
+      on_stream_close(stream, request, error)
+    end
+
     def on_stream_close(stream, request, error)
+      log(level: 2) { "#{stream.id}: closing stream" }
+      @drains.delete(request)
+      @streams.delete(request)
+
       if error && error != :no_error
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
-        emit(:error, request, ex)
+        response = ErrorResponse.new(request, ex, request.options)
+        emit(:response, request, response)
       else
         response = request.response
         if response.status == 421
@@ -241,9 +297,6 @@ module HTTPX
           emit(:response, request, response)
         end
       end
-      log(level: 2) { "#{stream.id}: closing stream" }
-
-      @streams.delete(request)
       send(@pending.shift) unless @pending.empty?
       return unless @streams.empty? && exhausted?
 
@@ -328,11 +381,9 @@ module HTTPX
     end
 
     def method_missing(meth, *args, &blk)
-      if @connection.respond_to?(meth)
-        @connection.__send__(meth, *args, &blk)
-      else
-        super
-      end
+      return super unless @connection.respond_to?(meth)
+
+      @connection.__send__(meth, *args, &blk)
     end
   end
   Connection.register "h2", Connection::HTTP2

data/lib/httpx/headers.rb

@@ -112,7 +112,7 @@ module HTTPX
     end
 
     def ==(other)
-      to_hash == Headers.new(other).to_hash
+      other == to_hash
     end
 
     # the headers store in Hash format

data/lib/httpx/io.rb

@@ -2,16 +2,29 @@
 
 require "socket"
 require "httpx/io/tcp"
-require "httpx/io/ssl"
 require "httpx/io/unix"
 require "httpx/io/udp"
 
 module HTTPX
   module IO
     extend Registry
-    register "tcp", TCP
-    register "ssl", SSL
     register "udp", UDP
     register "unix", HTTPX::UNIX
+    register "tcp", TCP
+
+    if RUBY_ENGINE == "jruby"
+      begin
+        require "httpx/io/tls"
+        register "ssl", TLS
+      rescue LoadError
+        # :nocov:
+        require "httpx/io/ssl"
+        register "ssl", SSL
+        # :nocov:
+      end
+    else
+      require "httpx/io/ssl"
+      register "ssl", SSL
+    end
   end
 end

data/lib/httpx/io/ssl.rb

@@ -3,6 +3,8 @@
 require "openssl"
 
 module HTTPX
+  TLSError = OpenSSL::SSL::SSLError
+
   class SSL < TCP
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
       { alpn_protocols: %w[h2 http/1.1] }
@@ -11,19 +13,14 @@ module HTTPX
     end
 
     def initialize(_, _, options)
+      super
       @ctx = OpenSSL::SSL::SSLContext.new
       ctx_options = TLS_OPTIONS.merge(options.ssl)
-      @tls_hostname = ctx_options.delete(:hostname)
+      @sni_hostname = ctx_options.delete(:hostname) || @hostname
       @ctx.set_params(ctx_options) unless ctx_options.empty?
-      super
-      @tls_hostname ||= @hostname
       @state = :negotiated if @keep_open
     end
 
-    def interests
-      @interests || super
-    end
-
     def protocol
       @io.alpn_protocol || super
     rescue StandardError
@@ -50,28 +47,30 @@ module HTTPX
 
     def connect
       super
-      if @keep_open
-        @state = :negotiated
-        return
-      end
       return if @state == :negotiated ||
                 @state != :connected
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @tls_hostname
+        @io.hostname = @sni_hostname
         @io.sync_close = true
       end
-      @io.connect_nonblock
-      @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
-      transition(:negotiated)
-    rescue ::IO::WaitReadable
-      @interests = :r
-    rescue ::IO::WaitWritable
-      @interests = :w
+      try_ssl_connect
     end
 
     if RUBY_VERSION < "2.3"
+      # :nocov:
+      def try_ssl_connect
+        @io.connect_nonblock
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue ::IO::WaitWritable
+        @interests = :w
+      end
+
       def read(_, buffer)
         super
       rescue ::IO::WaitWritable
@@ -84,7 +83,23 @@ module HTTPX
       rescue ::IO::WaitReadable
         0
       end
+      # :nocov:
     else
+      def try_ssl_connect
+        case @io.connect_nonblock(exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      end
+
+      # :nocov:
       if OpenSSL::VERSION < "2.0.6"
         def read(size, buffer)
           @io.read_nonblock(size, buffer)
@@ -97,11 +112,7 @@ module HTTPX
           nil
         end
       end
-    end
-
-    def inspect
-      id = @io.closed? ? "closed" : @io.to_io.fileno
-      "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+      # :nocov:
     end
 
     private