httpx 0.11.0 → 0.13.0

data/lib/httpx/connection/http2.rb

@@ -42,11 +42,15 @@ module HTTPX
         return @buffer.empty? ? :r : :rw
       end
 
-      return :w unless @pending.empty?
+      return :w if !@pending.empty? && can_buffer_more_requests?
 
       return :w if @streams.each_key.any? { |r| r.interests == :w }
 
-      return :r if @buffer.empty?
+      if @buffer.empty?
+        return if @streams.empty? && @pings.empty?
+
+        return :r
+      end
 
       :rw
     end
@@ -70,10 +74,14 @@ module HTTPX
       @connection << data
     end
 
+    def can_buffer_more_requests?
+      @handshake_completed &&
+        @streams.size < @max_concurrent_requests &&
+        @streams.size < @max_requests
+    end
+
     def send(request)
-      if !@handshake_completed ||
-         @streams.size >= @max_concurrent_requests ||
-         @streams.size >= @max_requests
+      unless can_buffer_more_requests?
         @pending << request
         return
       end
@@ -126,8 +134,6 @@ module HTTPX
       request.path
     end
 
-    def set_request_headers(request); end
-
     def handle(request, stream)
       catch(:buffer_full) do
         request.transition(:headers)
@@ -172,18 +178,19 @@ module HTTPX
       stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
     end
 
+    def set_protocol_headers(request)
+      request.headers[":scheme"]    = request.scheme
+      request.headers[":method"]    = request.verb.to_s.upcase
+      request.headers[":path"]      = headline_uri(request)
+      request.headers[":authority"] = request.authority
+    end
+
     def join_headers(stream, request)
-      set_request_headers(request)
-      headers = {}
-      headers[":scheme"]    = request.scheme
-      headers[":method"]    = request.verb.to_s.upcase
-      headers[":path"]      = headline_uri(request)
-      headers[":authority"] = request.authority
-      headers = headers.merge(request.headers)
+      set_protocol_headers(request)
       log(level: 1, color: :yellow) do
-        headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+        request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
-      stream.headers(headers, end_stream: request.empty?)
+      stream.headers(request.headers.each, end_stream: request.empty?)
     end
 
     def join_body(stream, request)
@@ -227,10 +234,15 @@ module HTTPX
     end
 
     def on_stream_close(stream, request, error)
+      log(level: 2) { "#{stream.id}: closing stream" }
+      @drains.delete(request)
+      @streams.delete(request)
+
       if error && error != :no_error
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
-        emit(:error, request, ex)
+        response = ErrorResponse.new(request, ex, request.options)
+        emit(:response, request, response)
       else
         response = request.response
         if response.status == 421
@@ -241,9 +253,6 @@ module HTTPX
           emit(:response, request, response)
         end
       end
-      log(level: 2) { "#{stream.id}: closing stream" }
-
-      @streams.delete(request)
       send(@pending.shift) unless @pending.empty?
       return unless @streams.empty? && exhausted?
 
@@ -328,11 +337,9 @@ module HTTPX
     end
 
     def method_missing(meth, *args, &blk)
-      if @connection.respond_to?(meth)
-        @connection.__send__(meth, *args, &blk)
-      else
-        super
-      end
+      return super unless @connection.respond_to?(meth)
+
+      @connection.__send__(meth, *args, &blk)
     end
   end
   Connection.register "h2", Connection::HTTP2

data/lib/httpx/io.rb

@@ -2,16 +2,29 @@
 
 require "socket"
 require "httpx/io/tcp"
-require "httpx/io/ssl"
 require "httpx/io/unix"
 require "httpx/io/udp"
 
 module HTTPX
   module IO
     extend Registry
-    register "tcp", TCP
-    register "ssl", SSL
     register "udp", UDP
     register "unix", HTTPX::UNIX
+    register "tcp", TCP
+
+    if RUBY_ENGINE == "jruby"
+      begin
+        require "httpx/io/tls"
+        register "ssl", TLS
+      rescue LoadError
+        # :nocov:
+        require "httpx/io/ssl"
+        register "ssl", SSL
+        # :nocov:
+      end
+    else
+      require "httpx/io/ssl"
+      register "ssl", SSL
+    end
   end
 end

data/lib/httpx/io/ssl.rb

@@ -3,6 +3,8 @@
 require "openssl"
 
 module HTTPX
+  TLSError = OpenSSL::SSL::SSLError
+
   class SSL < TCP
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
       { alpn_protocols: %w[h2 http/1.1] }
@@ -11,19 +13,14 @@ module HTTPX
     end
 
     def initialize(_, _, options)
+      super
       @ctx = OpenSSL::SSL::SSLContext.new
       ctx_options = TLS_OPTIONS.merge(options.ssl)
-      @tls_hostname = ctx_options.delete(:hostname)
+      @sni_hostname = ctx_options.delete(:hostname) || @hostname
       @ctx.set_params(ctx_options) unless ctx_options.empty?
-      super
-      @tls_hostname ||= @hostname
       @state = :negotiated if @keep_open
     end
 
-    def interests
-      @interests || super
-    end
-
     def protocol
       @io.alpn_protocol || super
     rescue StandardError
@@ -50,28 +47,30 @@ module HTTPX
 
     def connect
       super
-      if @keep_open
-        @state = :negotiated
-        return
-      end
       return if @state == :negotiated ||
                 @state != :connected
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @tls_hostname
+        @io.hostname = @sni_hostname
         @io.sync_close = true
       end
-      @io.connect_nonblock
-      @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
-      transition(:negotiated)
-    rescue ::IO::WaitReadable
-      @interests = :r
-    rescue ::IO::WaitWritable
-      @interests = :w
+      try_ssl_connect
     end
 
     if RUBY_VERSION < "2.3"
+      # :nocov:
+      def try_ssl_connect
+        @io.connect_nonblock
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue ::IO::WaitWritable
+        @interests = :w
+      end
+
       def read(_, buffer)
         super
       rescue ::IO::WaitWritable
@@ -84,7 +83,23 @@ module HTTPX
       rescue ::IO::WaitReadable
         0
       end
+      # :nocov:
     else
+      def try_ssl_connect
+        case @io.connect_nonblock(exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      end
+
+      # :nocov:
       if OpenSSL::VERSION < "2.0.6"
         def read(size, buffer)
           @io.read_nonblock(size, buffer)
@@ -97,11 +112,7 @@ module HTTPX
           nil
         end
       end
-    end
-
-    def inspect
-      id = @io.closed? ? "closed" : @io.to_io.fileno
-      "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+      # :nocov:
     end
 
     private

data/lib/httpx/io/tcp.rb

@@ -7,17 +7,19 @@ module HTTPX
   class TCP
     include Loggable
 
-    attr_reader :ip, :port, :addresses, :state
+    using URIExtensions
+
+    attr_reader :ip, :port, :addresses, :state, :interests
 
     alias_method :host, :ip
 
     def initialize(origin, addresses, options)
       @state = :idle
       @hostname = origin.host
-      @addresses = addresses
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
+      @interests = :w
       if @options.io
         @io = case @options.io
               when Hash
@@ -25,24 +27,20 @@ module HTTPX
               else
                 @options.io
         end
+        raise Error, "Given IO objects do not match the request authority" unless @io
+
         _, _, _, @ip = @io.addr
         @addresses ||= [@ip]
         @ip_index = @addresses.size - 1
-        unless @io.nil?
-          @keep_open = true
-          @state = :connected
-        end
+        @keep_open = true
+        @state = :connected
       else
-        @ip_index = @addresses.size - 1
-        @ip = @addresses[@ip_index]
+        @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
       end
+      @ip_index = @addresses.size - 1
       @io ||= build_socket
     end
 
-    def interests
-      :w
-    end
-
     def to_io
       @io.to_io
     end
@@ -54,32 +52,40 @@ module HTTPX
     def connect
       return unless closed?
 
-      begin
-        if @io.closed?
-          transition(:idle)
-          @io = build_socket
-        end
-        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
-      rescue Errno::EISCONN
+      if @io.closed?
+        transition(:idle)
+        @io = build_socket
       end
-      transition(:connected)
+      try_connect
     rescue Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
 
       @ip_index -= 1
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeoutError, e.message if @ip_index <= 0
+      raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index <= 0
 
       @ip_index -= 1
       retry
-    rescue Errno::EINPROGRESS,
-           Errno::EALREADY,
-           ::IO::WaitReadable
     end
 
     if RUBY_VERSION < "2.3"
       # :nocov:
+      def try_connect
+        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
+      rescue ::IO::WaitWritable, Errno::EALREADY
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue Errno::EISCONN
+        transition(:connected)
+        @interests = :w
+      else
+        transition(:connected)
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         @io.read_nonblock(size, buffer)
         log { "READ: #{buffer.bytesize} bytes..." }
@@ -103,6 +109,20 @@ module HTTPX
       end
       # :nocov:
     else
+      def try_connect
+        case @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        transition(:connected)
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         ret = @io.read_nonblock(size, buffer, exception: false)
         if ret == :wait_readable
@@ -147,14 +167,14 @@ module HTTPX
 
     # :nocov:
     def inspect
-      id = @io.closed? ? "closed" : @io.fileno
-      "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
+      "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
     end
     # :nocov:
 
     private
 
     def build_socket
+      @ip = @addresses[@ip_index]
       Socket.new(@ip.family, :STREAM, 0)
     end
 
@@ -177,9 +197,9 @@ module HTTPX
     def log_transition_state(nextstate)
       case nextstate
       when :connected
-        "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
+        "Connected to #{host} (##{@io.fileno})"
       else
-        "#{@ip}:#{@port} #{@state} -> #{nextstate}"
+        "#{host} #{@state} -> #{nextstate}"
       end
     end
   end

data/lib/httpx/io/tls.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module HTTPX
+  class TLS < TCP
+    Error = Class.new(StandardError)
+
+    def initialize(_, _, options)
+      super
+      @encrypted = Buffer.new(Connection::BUFFER_SIZE)
+      @decrypted = "".b
+      tls_options = convert_tls_options(options.ssl)
+      @sni_hostname = tls_options[:hostname]
+      @ctx = TLS::Box.new(false, self, tls_options)
+      @state = :negotiated if @keep_open
+    end
+
+    def interests
+      @interests || super
+    end
+
+    def protocol
+      @protocol || super
+    end
+
+    def connected?
+      @state == :negotiated
+    end
+
+    def connect
+      super
+      if @keep_open
+        @state = :negotiated
+        return
+      end
+      return if @state == :negotiated ||
+                @state != :connected
+
+      super
+      @ctx.start
+      @interests = :r
+      read(@options.window_size, @decrypted)
+    end
+
+    # :nocov:
+    def inspect
+      id = @io.closed? ? "closed" : @io
+      "#<TLS(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+    end
+    # :nocov:
+
+    alias_method :transport_close, :close
+    def close
+      transport_close
+      @ctx.cleanup
+    end
+
+    def read(*, buffer)
+      ret = super
+      return ret if !ret || ret.zero?
+
+      @ctx.decrypt(buffer.to_s.dup)
+      buffer.replace(@decrypted)
+      @decrypted.clear
+      buffer.bytesize
+    end
+
+    alias_method :unencrypted_write, :write
+    def write(buffer)
+      @ctx.encrypt(buffer.to_s.dup)
+      buffer.clear
+      do_write
+    end
+
+    # TLS callback.
+    #
+    # buffers the encrypted +data+
+    def transmit_cb(data)
+      log { "TLS encrypted: #{data.bytesize} bytes" }
+      log(level: 2) { data.inspect }
+      @encrypted << data
+      do_write
+    end
+
+    # TLS callback.
+    #
+    # buffers the decrypted +data+
+    def dispatch_cb(data)
+      log { "TLS decrypted: #{data.bytesize} bytes" }
+      log(level: 2) { data.inspect }
+
+      @decrypted << data
+    end
+
+    # TLS callback.
+    #
+    # signals TLS invalid status / shutdown.
+    def close_cb(msg = nil)
+      log { "TLS Error: #{msg}, closing" }
+      raise Error, "certificate verify failed (#{msg})"
+    end
+
+    # TLS callback.
+    #
+    # alpn protocol negotiation (+protocol+).
+    #
+    def alpn_protocol_cb(protocol)
+      @protocol = protocol
+      log { "TLS ALPN protocol negotiated: #{@protocol}" }
+    end
+
+    # TLS callback.
+    #
+    # handshake finished.
+    #
+    def handshake_cb
+      log { "TLS handshake completed" }
+      transition(:negotiated)
+    end
+
+    # TLS callback.
+    #
+    # passed the peer +cert+ to be verified.
+    #
+    def verify_cb(cert)
+      raise Error, "Peer verification enabled, but no certificate received." if cert.nil?
+
+      log { "TLS verifying #{cert}" }
+      @peer_cert = OpenSSL::X509::Certificate.new(cert)
+
+      # by default one doesn't verify client certificates in the server
+      verify_hostname(@sni_hostname)
+    end
+
+    # copied from:
+    # https://github.com/ruby/ruby/blob/8cbf2dae5aadfa5d6241b0df2bf44d55db46704f/ext/openssl/lib/openssl/ssl.rb#L395-L409
+    #
+    def verify_hostname(host)
+      return false unless @ctx.verify_peer && @peer_cert
+
+      OpenSSL::SSL.verify_certificate_identity(@peer_cert, host)
+    end
+
+    private
+
+    def do_write
+      nwritten = 0
+      until @encrypted.empty?
+        siz = unencrypted_write(@encrypted)
+        break unless !siz || siz.zero?
+
+        nwritten += siz
+      end
+      nwritten
+    end
+
+    def convert_tls_options(ssl_options)
+      options = {}
+      options[:verify_peer] = !ssl_options.key?(:verify_mode) || ssl_options[:verify_mode] != OpenSSL::SSL::VERIFY_NONE
+      options[:version] = ssl_options[:ssl_version] if ssl_options.key?(:ssl_version)
+
+      if ssl_options.key?(:key)
+        private_key = ssl_options[:key]
+        private_key = private_key.to_pem if private_key.respond_to?(:to_pem)
+        options[:private_key] = private_key
+      end
+
+      if ssl_options.key?(:ca_path) || ssl_options.key?(:ca_file)
+        ca_path = ssl_options[:ca_path] || ssl_options[:ca_file].path
+        options[:cert_chain] = ca_path
+      end
+
+      options[:ciphers] = ssl_options[:ciphers] if ssl_options.key?(:ciphers)
+      options[:protocols] = ssl_options.fetch(:alpn_protocols, %w[h2 http/1.1])
+      options[:hostname] = ssl_options.fetch(:hostname, @hostname)
+      options
+    end
+
+    def transition(nextstate)
+      case nextstate
+      when :negotiated
+        return unless @state == :connected
+      when :closed
+        return unless @state == :negotiated ||
+                      @state == :connected
+      end
+      do_transition(nextstate)
+    end
+
+    def log_transition_state(nextstate)
+      return super unless nextstate == :negotiated
+
+      server_cert = @peer_cert
+
+      "#{super}\n\n" \
+        "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
+        "ALPN, server accepted to use #{protocol}\n" +
+        (if server_cert
+           "Server certificate:\n" \
+                 " subject: #{server_cert.subject}\n" \
+                 " start date: #{server_cert.not_before}\n" \
+                 " expire date: #{server_cert.not_after}\n" \
+                 " issuer: #{server_cert.issuer}\n" \
+                 " SSL certificate verify ok."
+         else
+           "SSL certificate verify failed."
+         end
+        )
+    end
+  end
+
+  TLSError = TLS::Error
+end
+
+require "httpx/io/tls/ffi"
+require "httpx/io/tls/context"
+require "httpx/io/tls/box"