httpx 0.12.0 → 0.14.1

data/lib/httpx/connection/http2.rb

@@ -21,14 +21,16 @@ module HTTPX
 
     def initialize(buffer, options)
       @options = Options.new(options)
-      @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
-      @max_requests = @options.max_requests || 0
+      @settings = @options.http2_settings
       @pending = []
       @streams = {}
       @drains  = {}
       @pings = []
       @buffer = buffer
       @handshake_completed = false
+      @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
+      @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
+      @max_requests = @options.max_requests || 0
       init_connection
     end
 
@@ -36,7 +38,11 @@ module HTTPX
       # waiting for WINDOW_UPDATE frames
       return :r if @buffer.full?
 
-      return :w if @connection.state == :closed
+      if @connection.state == :closed
+        return unless @handshake_completed
+
+        return :w
+      end
 
       unless (@connection.state == :connected && @handshake_completed)
         return @buffer.empty? ? :r : :rw
@@ -75,9 +81,13 @@ module HTTPX
     end
 
     def can_buffer_more_requests?
-      @handshake_completed &&
+      if @handshake_completed
         @streams.size < @max_concurrent_requests &&
-        @streams.size < @max_requests
+          @streams.size < @max_requests
+      else
+        !@wait_for_handshake &&
+          @streams.size < @max_concurrent_requests
+      end
     end
 
     def send(request)
@@ -91,7 +101,6 @@ module HTTPX
         @streams[request] = stream
         @max_requests -= 1
       end
-      request.once(:headers, &method(:set_protocol_headers))
       handle(request, stream)
       true
     rescue HTTP2Next::Error::StreamLimitExceeded
@@ -141,12 +150,14 @@ module HTTPX
         join_headers(stream, request) if request.state == :headers
         request.transition(:body)
         join_body(stream, request) if request.state == :body
+        request.transition(:trailers)
+        join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
         request.transition(:done)
       end
     end
 
     def init_connection
-      @connection = HTTP2Next::Client.new(@options.http2_settings)
+      @connection = HTTP2Next::Client.new(@settings)
       @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
       @connection.on(:frame, &method(:on_frame))
       @connection.on(:frame_sent, &method(:on_frame_sent))
@@ -170,6 +181,7 @@ module HTTPX
     public :reset
 
     def handle_stream(stream, request)
+      request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
       stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
       stream.on(:half_close) do
         log(level: 2) { "#{stream.id}: waiting for response..." }
@@ -180,17 +192,32 @@ module HTTPX
     end
 
     def set_protocol_headers(request)
-      request.headers[":scheme"]    = request.scheme
-      request.headers[":method"]    = request.verb.to_s.upcase
-      request.headers[":path"]      = headline_uri(request)
-      request.headers[":authority"] = request.authority
+      {
+        ":scheme" => request.scheme,
+        ":method" => request.verb.to_s.upcase,
+        ":path" => headline_uri(request),
+        ":authority" => request.authority,
+      }
     end
 
     def join_headers(stream, request)
+      extra_headers = set_protocol_headers(request)
+      log(level: 1, color: :yellow) do
+        request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+      end
+      stream.headers(request.headers.each(extra_headers), end_stream: request.empty?)
+    end
+
+    def join_trailers(stream, request)
+      unless request.trailers?
+        stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
+        return
+      end
+
       log(level: 1, color: :yellow) do
-        request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+        request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
-      stream.headers(request.headers.each, end_stream: request.empty?)
+      stream.headers(request.trailers.each, end_stream: true)
     end
 
     def join_body(stream, request)
@@ -201,13 +228,15 @@ module HTTPX
         next_chunk = request.drain_body
         log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
         log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
-        stream.data(chunk, end_stream: !next_chunk)
-        if next_chunk && @buffer.full?
+        stream.data(chunk, end_stream: !(next_chunk || request.trailers? || request.callbacks_for?(:trailers)))
+        if next_chunk && (@buffer.full? || request.body.unbounded_body?)
           @drains[request] = next_chunk
           throw(:buffer_full)
         end
         chunk = next_chunk
       end
+
+      on_stream_refuse(stream, request, request.drain_error) if request.drain_error
     end
 
     ######
@@ -215,6 +244,11 @@ module HTTPX
     ######
 
     def on_stream_headers(stream, request, h)
+      if request.response && request.response.version == "2.0"
+        on_stream_trailers(stream, request, h)
+        return
+      end
+
       log(color: :yellow) do
         h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
       end
@@ -227,12 +261,24 @@ module HTTPX
       handle(request, stream) if request.expects?
     end
 
+    def on_stream_trailers(stream, request, h)
+      log(color: :yellow) do
+        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
+      end
+      request.response.merge_headers(h)
+    end
+
     def on_stream_data(stream, request, data)
       log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
       log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
       request.response << data
     end
 
+    def on_stream_refuse(stream, request, error)
+      stream.close
+      on_stream_close(stream, request, error)
+    end
+
     def on_stream_close(stream, request, error)
       log(level: 2) { "#{stream.id}: closing stream" }
       @drains.delete(request)

data/lib/httpx/headers.rb

@@ -103,16 +103,20 @@ module HTTPX
     # returns the enumerable headers store in pairs of header field + the values in
     # the comma-separated string format
     #
-    def each
-      return enum_for(__method__) { @headers.size } unless block_given?
+    def each(extra_headers = nil)
+      return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
 
       @headers.each do |field, value|
         yield(field, value.join(", ")) unless value.empty?
       end
+
+      extra_headers.each do |field, value|
+        yield(field, value) unless value.empty?
+      end if extra_headers
     end
 
     def ==(other)
-      to_hash == Headers.new(other).to_hash
+      other == to_hash
     end
 
     # the headers store in Hash format

data/lib/httpx/io/ssl.rb

@@ -47,10 +47,6 @@ module HTTPX
 
     def connect
       super
-      if @keep_open
-        @state = :negotiated
-        return
-      end
       return if @state == :negotiated ||
                 @state != :connected
 
@@ -59,17 +55,22 @@ module HTTPX
         @io.hostname = @sni_hostname
         @io.sync_close = true
       end
-      @io.connect_nonblock
-      @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
-      transition(:negotiated)
-      @interests = :w
-    rescue ::IO::WaitReadable
-      @interests = :r
-    rescue ::IO::WaitWritable
-      @interests = :w
+      try_ssl_connect
     end
 
     if RUBY_VERSION < "2.3"
+      # :nocov:
+      def try_ssl_connect
+        @io.connect_nonblock
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue ::IO::WaitWritable
+        @interests = :w
+      end
+
       def read(_, buffer)
         super
       rescue ::IO::WaitWritable
@@ -82,7 +83,23 @@ module HTTPX
       rescue ::IO::WaitReadable
         0
       end
+      # :nocov:
     else
+      def try_ssl_connect
+        case @io.connect_nonblock(exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      end
+
+      # :nocov:
       if OpenSSL::VERSION < "2.0.6"
         def read(size, buffer)
           @io.read_nonblock(size, buffer)
@@ -95,11 +112,7 @@ module HTTPX
           nil
         end
       end
-    end
-
-    def inspect
-      id = @io.closed? ? "closed" : @io.to_io.fileno
-      "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+      # :nocov:
     end
 
     private

data/lib/httpx/io/tcp.rb

@@ -7,6 +7,8 @@ module HTTPX
   class TCP
     include Loggable
 
+    using URIExtensions
+
     attr_reader :ip, :port, :addresses, :state, :interests
 
     alias_method :host, :ip
@@ -14,7 +16,6 @@ module HTTPX
     def initialize(origin, addresses, options)
       @state = :idle
       @hostname = origin.host
-      @addresses = addresses
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
@@ -26,17 +27,17 @@ module HTTPX
               else
                 @options.io
         end
+        raise Error, "Given IO objects do not match the request authority" unless @io
+
         _, _, _, @ip = @io.addr
         @addresses ||= [@ip]
         @ip_index = @addresses.size - 1
-        unless @io.nil?
-          @keep_open = true
-          @state = :connected
-        end
+        @keep_open = true
+        @state = :connected
       else
-        @ip_index = @addresses.size - 1
-        @ip = @addresses[@ip_index]
+        @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
       end
+      @ip_index = @addresses.size - 1
       @io ||= build_socket
     end
 
@@ -51,36 +52,40 @@ module HTTPX
     def connect
       return unless closed?
 
-      begin
-        if @io.closed?
-          transition(:idle)
-          @io = build_socket
-        end
-        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
-      rescue Errno::EISCONN
+      if @io.closed?
+        transition(:idle)
+        @io = build_socket
       end
-      @interests = :w
-
-      transition(:connected)
+      try_connect
     rescue Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
 
       @ip_index -= 1
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index <= 0
+      raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
 
       @ip_index -= 1
       retry
-    rescue Errno::EINPROGRESS,
-           Errno::EALREADY
-      @interests = :w
-    rescue ::IO::WaitReadable
-      @interests = :r
     end
 
     if RUBY_VERSION < "2.3"
       # :nocov:
+      def try_connect
+        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
+      rescue ::IO::WaitWritable, Errno::EALREADY
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue Errno::EISCONN
+        transition(:connected)
+        @interests = :w
+      else
+        transition(:connected)
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         @io.read_nonblock(size, buffer)
         log { "READ: #{buffer.bytesize} bytes..." }
@@ -104,6 +109,22 @@ module HTTPX
       end
       # :nocov:
     else
+      def try_connect
+        case @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        transition(:connected)
+        @interests = :w
+      rescue Errno::EALREADY
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         ret = @io.read_nonblock(size, buffer, exception: false)
         if ret == :wait_readable
@@ -148,14 +169,14 @@ module HTTPX
 
     # :nocov:
     def inspect
-      id = @io.closed? ? "closed" : @io.fileno
-      "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
+      "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
     end
     # :nocov:
 
     private
 
     def build_socket
+      @ip = @addresses[@ip_index]
       Socket.new(@ip.family, :STREAM, 0)
     end
 
@@ -178,9 +199,9 @@ module HTTPX
     def log_transition_state(nextstate)
       case nextstate
       when :connected
-        "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
+        "Connected to #{host} (##{@io.fileno})"
       else
-        "#{@ip}:#{@port} #{@state} -> #{nextstate}"
+        "#{host} #{@state} -> #{nextstate}"
       end
     end
   end

data/lib/httpx/io/udp.rb

@@ -39,16 +39,20 @@ module HTTPX
       end
     end
 
-    def write(buffer)
-      siz = @io.send(buffer.to_s, 0, @host, @port)
-      log { "WRITE: #{siz} bytes..." }
-      buffer.shift!(siz)
-      siz
-    end
-
     # :nocov:
     if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
        RUBY_VERSION < "2.3"
+      def write(buffer)
+        siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s))
+        log { "WRITE: #{siz} bytes..." }
+        buffer.shift!(siz)
+        siz
+      rescue ::IO::WaitWritable
+        0
+      rescue EOFError
+        nil
+      end
+
       def read(size, buffer)
         data, _ = @io.recvfrom_nonblock(size)
         buffer.replace(data)
@@ -59,6 +63,18 @@ module HTTPX
       rescue IOError
       end
     else
+
+      def write(buffer)
+        siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
+        return 0 if siz == :wait_writable
+        return if siz.nil?
+
+        log { "WRITE: #{siz} bytes..." }
+
+        buffer.shift!(siz)
+        siz
+      end
+
       def read(size, buffer)
         ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
         return 0 if ret == :wait_readable
@@ -68,6 +84,14 @@ module HTTPX
       rescue IOError
       end
     end
+
+    # In JRuby, sendmsg_nonblock is not implemented
+    def write(buffer)
+      siz = @io.send(buffer.to_s, 0, @host, @port)
+      log { "WRITE: #{siz} bytes..." }
+      buffer.shift!(siz)
+      siz
+    end if RUBY_ENGINE == "jruby"
     # :nocov:
   end
 end