httpx 0.10.2 → 0.12.0

data/lib/httpx/domain_name.rb

@@ -139,10 +139,8 @@ module HTTPX
       elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == DOT
         # The other is higher
         -1
-      elsif othername.end_with?(@hostname) && othername[-@hostname.size - 1, 1] == DOT
-        # The other is lower
-        1
       else
+        # The other is lower
         1
       end
     end

data/lib/httpx/errors.rb

@@ -24,6 +24,8 @@ module HTTPX
 
   ConnectTimeoutError = Class.new(TimeoutError)
 
+  ResolveTimeoutError = Class.new(TimeoutError)
+
   ResolveError = Class.new(Error)
 
   NativeResolveError = Class.new(ResolveError) do

data/lib/httpx/headers.rb

@@ -119,6 +119,7 @@ module HTTPX
     def to_hash
       Hash[to_a]
     end
+    alias_method :to_h, :to_hash
 
     # the headers store in array of pairs format
     def to_a

data/lib/httpx/io.rb

@@ -2,16 +2,29 @@
 
 require "socket"
 require "httpx/io/tcp"
-require "httpx/io/ssl"
 require "httpx/io/unix"
 require "httpx/io/udp"
 
 module HTTPX
   module IO
     extend Registry
-    register "tcp", TCP
-    register "ssl", SSL
     register "udp", UDP
     register "unix", HTTPX::UNIX
+    register "tcp", TCP
+
+    if RUBY_ENGINE == "jruby"
+      begin
+        require "httpx/io/tls"
+        register "ssl", TLS
+      rescue LoadError
+        # :nocov:
+        require "httpx/io/ssl"
+        register "ssl", SSL
+        # :nocov:
+      end
+    else
+      require "httpx/io/ssl"
+      register "ssl", SSL
+    end
   end
 end

data/lib/httpx/io/ssl.rb

@@ -3,27 +3,24 @@
 require "openssl"
 
 module HTTPX
+  TLSError = OpenSSL::SSL::SSLError
+
   class SSL < TCP
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
       { alpn_protocols: %w[h2 http/1.1] }
     else
-      # :nocov:
       {}
-      # :nocov:
     end
 
     def initialize(_, _, options)
+      super
       @ctx = OpenSSL::SSL::SSLContext.new
       ctx_options = TLS_OPTIONS.merge(options.ssl)
+      @sni_hostname = ctx_options.delete(:hostname) || @hostname
       @ctx.set_params(ctx_options) unless ctx_options.empty?
-      super
       @state = :negotiated if @keep_open
     end
 
-    def interests
-      @interests || super
-    end
-
     def protocol
       @io.alpn_protocol || super
     rescue StandardError
@@ -59,19 +56,19 @@ module HTTPX
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @hostname
+        @io.hostname = @sni_hostname
         @io.sync_close = true
       end
       @io.connect_nonblock
-      @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+      @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
       transition(:negotiated)
+      @interests = :w
     rescue ::IO::WaitReadable
       @interests = :r
     rescue ::IO::WaitWritable
       @interests = :w
     end
 
-    # :nocov:
     if RUBY_VERSION < "2.3"
       def read(_, buffer)
         super
@@ -99,14 +96,11 @@ module HTTPX
         end
       end
     end
-    # :nocov:
 
-    # :nocov:
     def inspect
       id = @io.closed? ? "closed" : @io.to_io.fileno
       "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
     end
-    # :nocov:
 
     private
 

data/lib/httpx/io/tcp.rb

@@ -7,7 +7,7 @@ module HTTPX
   class TCP
     include Loggable
 
-    attr_reader :ip, :port, :addresses, :state
+    attr_reader :ip, :port, :addresses, :state, :interests
 
     alias_method :host, :ip
 
@@ -18,6 +18,7 @@ module HTTPX
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
+      @interests = :w
       if @options.io
         @io = case @options.io
               when Hash
@@ -39,10 +40,6 @@ module HTTPX
       @io ||= build_socket
     end
 
-    def interests
-      :w
-    end
-
     def to_io
       @io.to_io
     end
@@ -62,6 +59,8 @@ module HTTPX
         @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
       rescue Errno::EISCONN
       end
+      @interests = :w
+
       transition(:connected)
     rescue Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
@@ -69,13 +68,15 @@ module HTTPX
       @ip_index -= 1
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeoutError, e.message if @ip_index <= 0
+      raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index <= 0
 
       @ip_index -= 1
       retry
     rescue Errno::EINPROGRESS,
-           Errno::EALREADY,
-           ::IO::WaitReadable
+           Errno::EALREADY
+      @interests = :w
+    rescue ::IO::WaitReadable
+      @interests = :r
     end
 
     if RUBY_VERSION < "2.3"

data/lib/httpx/io/tls.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module HTTPX
+  class TLS < TCP
+    Error = Class.new(StandardError)
+
+    def initialize(_, _, options)
+      super
+      @encrypted = Buffer.new(Connection::BUFFER_SIZE)
+      @decrypted = "".b
+      tls_options = convert_tls_options(options.ssl)
+      @sni_hostname = tls_options[:hostname]
+      @ctx = TLS::Box.new(false, self, tls_options)
+      @state = :negotiated if @keep_open
+    end
+
+    def interests
+      @interests || super
+    end
+
+    def protocol
+      @protocol || super
+    end
+
+    def connected?
+      @state == :negotiated
+    end
+
+    def connect
+      super
+      if @keep_open
+        @state = :negotiated
+        return
+      end
+      return if @state == :negotiated ||
+                @state != :connected
+
+      super
+      @ctx.start
+      @interests = :r
+      read(@options.window_size, @decrypted)
+    end
+
+    # :nocov:
+    def inspect
+      id = @io.closed? ? "closed" : @io
+      "#<TLS(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+    end
+    # :nocov:
+
+    alias_method :transport_close, :close
+    def close
+      transport_close
+      @ctx.cleanup
+    end
+
+    def read(*, buffer)
+      ret = super
+      return ret if !ret || ret.zero?
+
+      @ctx.decrypt(buffer.to_s.dup)
+      buffer.replace(@decrypted)
+      @decrypted.clear
+      buffer.bytesize
+    end
+
+    alias_method :unencrypted_write, :write
+    def write(buffer)
+      @ctx.encrypt(buffer.to_s.dup)
+      buffer.clear
+      do_write
+    end
+
+    # TLS callback.
+    #
+    # buffers the encrypted +data+
+    def transmit_cb(data)
+      log { "TLS encrypted: #{data.bytesize} bytes" }
+      log(level: 2) { data.inspect }
+      @encrypted << data
+      do_write
+    end
+
+    # TLS callback.
+    #
+    # buffers the decrypted +data+
+    def dispatch_cb(data)
+      log { "TLS decrypted: #{data.bytesize} bytes" }
+      log(level: 2) { data.inspect }
+
+      @decrypted << data
+    end
+
+    # TLS callback.
+    #
+    # signals TLS invalid status / shutdown.
+    def close_cb(msg = nil)
+      log { "TLS Error: #{msg}, closing" }
+      raise Error, "certificate verify failed (#{msg})"
+    end
+
+    # TLS callback.
+    #
+    # alpn protocol negotiation (+protocol+).
+    #
+    def alpn_protocol_cb(protocol)
+      @protocol = protocol
+      log { "TLS ALPN protocol negotiated: #{@protocol}" }
+    end
+
+    # TLS callback.
+    #
+    # handshake finished.
+    #
+    def handshake_cb
+      log { "TLS handshake completed" }
+      transition(:negotiated)
+    end
+
+    # TLS callback.
+    #
+    # passed the peer +cert+ to be verified.
+    #
+    def verify_cb(cert)
+      raise Error, "Peer verification enabled, but no certificate received." if cert.nil?
+
+      log { "TLS verifying #{cert}" }
+      @peer_cert = OpenSSL::X509::Certificate.new(cert)
+
+      # by default one doesn't verify client certificates in the server
+      verify_hostname(@sni_hostname)
+    end
+
+    # copied from:
+    # https://github.com/ruby/ruby/blob/8cbf2dae5aadfa5d6241b0df2bf44d55db46704f/ext/openssl/lib/openssl/ssl.rb#L395-L409
+    #
+    def verify_hostname(host)
+      return false unless @ctx.verify_peer && @peer_cert
+
+      OpenSSL::SSL.verify_certificate_identity(@peer_cert, host)
+    end
+
+    private
+
+    def do_write
+      nwritten = 0
+      until @encrypted.empty?
+        siz = unencrypted_write(@encrypted)
+        break unless !siz || siz.zero?
+
+        nwritten += siz
+      end
+      nwritten
+    end
+
+    def convert_tls_options(ssl_options)
+      options = {}
+      options[:verify_peer] = !ssl_options.key?(:verify_mode) || ssl_options[:verify_mode] != OpenSSL::SSL::VERIFY_NONE
+      options[:version] = ssl_options[:ssl_version] if ssl_options.key?(:ssl_version)
+
+      if ssl_options.key?(:key)
+        private_key = ssl_options[:key]
+        private_key = private_key.to_pem if private_key.respond_to?(:to_pem)
+        options[:private_key] = private_key
+      end
+
+      if ssl_options.key?(:ca_path) || ssl_options.key?(:ca_file)
+        ca_path = ssl_options[:ca_path] || ssl_options[:ca_file].path
+        options[:cert_chain] = ca_path
+      end
+
+      options[:ciphers] = ssl_options[:ciphers] if ssl_options.key?(:ciphers)
+      options[:protocols] = ssl_options.fetch(:alpn_protocols, %w[h2 http/1.1])
+      options[:hostname] = ssl_options.fetch(:hostname, @hostname)
+      options
+    end
+
+    def transition(nextstate)
+      case nextstate
+      when :negotiated
+        return unless @state == :connected
+      when :closed
+        return unless @state == :negotiated ||
+                      @state == :connected
+      end
+      do_transition(nextstate)
+    end
+
+    def log_transition_state(nextstate)
+      return super unless nextstate == :negotiated
+
+      server_cert = @peer_cert
+
+      "#{super}\n\n" \
+        "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
+        "ALPN, server accepted to use #{protocol}\n" +
+        (if server_cert
+           "Server certificate:\n" \
+                 " subject: #{server_cert.subject}\n" \
+                 " start date: #{server_cert.not_before}\n" \
+                 " expire date: #{server_cert.not_after}\n" \
+                 " issuer: #{server_cert.issuer}\n" \
+                 " SSL certificate verify ok."
+         else
+           "SSL certificate verify failed."
+         end
+        )
+    end
+  end
+
+  TLSError = TLS::Error
+end
+
+require "httpx/io/tls/ffi"
+require "httpx/io/tls/context"
+require "httpx/io/tls/box"

data/lib/httpx/io/tls/box.rb

@@ -0,0 +1,365 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2004-2013 Cotag Media
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is furnished
+# to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+class HTTPX::TLS
+  class Box
+    InstanceLookup = ::Concurrent::Map.new
+
+    READ_BUFFER = 2048
+    SSL_VERIFY_PEER = 0x01
+    SSL_VERIFY_CLIENT_ONCE = 0x04
+
+    VerifyCB = FFI::Function.new(:int, %i[int pointer]) do |preverify_ok, x509_store|
+      x509 = SSL.X509_STORE_CTX_get_current_cert(x509_store)
+      ssl = SSL.X509_STORE_CTX_get_ex_data(x509_store, SSL.SSL_get_ex_data_X509_STORE_CTX_idx)
+
+      bio_out = SSL.BIO_new(SSL.BIO_s_mem)
+      ret = SSL.PEM_write_bio_X509(bio_out, x509)
+      if ret
+        len = SSL.BIO_pending(bio_out)
+        buffer = FFI::MemoryPointer.new(:char, len, false)
+        size = SSL.BIO_read(bio_out, buffer, len)
+
+        # THis is the callback into the ruby class
+        cert = buffer.read_string(size)
+        SSL.BIO_free(bio_out)
+        # InstanceLookup[ssl.address].verify(cert) || preverify_ok.zero? ? 1 : 0
+        depth = SSL.X509_STORE_CTX_get_error_depth(ssl)
+        box = InstanceLookup[ssl.address]
+        if preverify_ok == 1
+
+          hostname_verify = box.verify(cert)
+          if hostname_verify
+            1
+          else
+            # SSL.X509_STORE_CTX_set_error(x509_store, SSL::X509_V_ERR_HOSTNAME_MISMATCH)
+            0
+          end
+        else
+          1
+        end
+      else
+        SSL.BIO_free(bio_out)
+        SSL.X509_STORE_CTX_set_error(x509_store, SSL::X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT)
+        0
+      end
+    end
+
+    attr_reader :is_server, :context, :handshake_completed, :hosts, :ssl_version, :cipher, :verify_peer
+
+    def initialize(is_server, transport, options = {})
+      @ready = true
+
+      @handshake_completed = false
+      @handshake_signaled = false
+      @alpn_negotiated = false
+      @transport = transport
+
+      @read_buffer = FFI::MemoryPointer.new(:char, READ_BUFFER, false)
+
+      @is_server = is_server
+      @context = Context.new(is_server, options)
+
+      @bioRead = SSL.BIO_new(SSL.BIO_s_mem)
+      @bioWrite = SSL.BIO_new(SSL.BIO_s_mem)
+      @ssl = SSL.SSL_new(@context.ssl_ctx)
+      SSL.SSL_set_bio(@ssl, @bioRead, @bioWrite)
+
+      @write_queue = []
+
+      InstanceLookup[@ssl.address] = self
+
+      @verify_peer = options[:verify_peer]
+      
+      if @verify_peer
+        SSL.SSL_set_verify(@ssl, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, VerifyCB)
+      end
+
+      # Add Server Name Indication (SNI) for client connections
+      if (hostname = options[:hostname])
+        if is_server
+          @hosts = ::Concurrent::Map.new
+          @hosts[hostname.to_s] = @context
+          @context.add_server_name_indication
+        else
+          SSL.SSL_set_tlsext_host_name(@ssl, hostname)
+        end
+      end
+
+      SSL.SSL_connect(@ssl) unless is_server
+    end
+
+    def add_host(hostname:, **options)
+      raise Error, "Server Name Indication (SNI) not configured for default host" unless @hosts
+      raise Error, "only valid for server mode context" unless @is_server
+
+      context = Context.new(true, options)
+      @hosts[hostname.to_s] = context
+      context.add_server_name_indication
+      nil
+    end
+
+    # Careful with this.
+    # If you remove all the hosts you'll end up with a segfault
+    def remove_host(hostname)
+      raise Error, "Server Name Indication (SNI) not configured for default host" unless @hosts
+      raise Error, "only valid for server mode context" unless @is_server
+
+      context = @hosts[hostname.to_s]
+      if context
+        @hosts.delete(hostname.to_s)
+        context.cleanup
+      end
+      nil
+    end
+
+    def get_peer_cert
+      return "" unless @ready
+
+      SSL.SSL_get_peer_certificate(@ssl)
+    end
+
+    def start
+      return unless @ready
+
+      dispatch_cipher_text
+    end
+
+    def encrypt(data)
+      return unless @ready
+
+      wrote = put_plain_text data
+      if wrote < 0
+        @transport.close_cb
+      else
+        dispatch_cipher_text
+      end
+    end
+
+    SSL_ERROR_WANT_READ = 2
+    SSL_ERROR_SSL = 1
+    def decrypt(data)
+      return unless @ready
+
+      put_cipher_text data
+
+      unless SSL.is_init_finished(@ssl)
+        resp = @is_server ? SSL.SSL_accept(@ssl) : SSL.SSL_connect(@ssl)
+
+        if resp < 0
+          err_code = SSL.SSL_get_error(@ssl, resp)
+          if err_code != SSL_ERROR_WANT_READ
+            if err_code == SSL_ERROR_SSL
+              verify_msg = SSL.X509_verify_cert_error_string(SSL.SSL_get_verify_result(@ssl))
+              @transport.close_cb(verify_msg)
+            end
+            return
+          end
+        end
+
+        @handshake_completed = true
+        @ssl_version = SSL.get_version(@ssl)
+        @cipher = SSL.get_current_cipher(@ssl)
+        signal_handshake unless @handshake_signaled
+      end
+
+      loop do
+        size = get_plain_text(@read_buffer, READ_BUFFER)
+        if size > 0
+          @transport.dispatch_cb @read_buffer.read_string(size)
+        else
+          break
+        end
+      end
+
+      dispatch_cipher_text
+    end
+
+    def signal_handshake
+      @handshake_signaled = true
+
+      # Check protocol support here
+      if @context.alpn_set
+        proto = alpn_negotiated_protocol
+
+        if proto == :failed
+          if @alpn_negotiated
+            # We should shutdown if this is the case
+            # TODO: send back proper error message
+            @transport.close_cb
+            return
+          end
+        end
+        @transport.alpn_protocol_cb(proto)
+      end
+
+      @transport.handshake_cb
+    end
+
+    def alpn_negotiated!
+      @alpn_negotiated = true
+    end
+
+    SSL_RECEIVED_SHUTDOWN = 2
+    def cleanup
+      return unless @ready
+
+      @ready = false
+
+      InstanceLookup.delete @ssl.address
+
+      if (SSL.SSL_get_shutdown(@ssl) & SSL_RECEIVED_SHUTDOWN) != 0
+        SSL.SSL_shutdown @ssl
+      else
+        SSL.SSL_clear @ssl
+      end
+
+      SSL.SSL_free @ssl
+
+      if @hosts
+        @hosts.each_value(&:cleanup)
+        @hosts = nil
+      else
+        @context.cleanup
+      end
+    end
+
+    # Called from class level callback function
+    def verify(cert)
+      @transport.verify_cb(cert)
+    end
+
+    def close(msg)
+      @transport.close_cb(msg)
+    end
+
+    private
+
+    def alpn_negotiated_protocol
+      return nil unless @context.alpn_set
+
+      proto = FFI::MemoryPointer.new(:pointer, 1, true)
+      len = FFI::MemoryPointer.new(:uint, 1, true)
+      SSL.SSL_get0_alpn_selected(@ssl, proto, len)
+
+      resp = proto.get_pointer(0)
+
+      return :failed if resp.address == 0
+
+      length = len.get_uint(0)
+      resp.read_string(length)
+    end
+
+    def get_plain_text(buffer, ready)
+      # Read the buffered clear text
+      size = SSL.SSL_read(@ssl, buffer, ready)
+      if size >= 0
+        size
+      else
+        SSL.SSL_get_error(@ssl, size) == SSL_ERROR_WANT_READ ? 0 : -1
+      end
+    end
+
+    def pending_data(bio)
+      SSL.BIO_pending(bio)
+    end
+
+    def get_cipher_text(buffer, length)
+      SSL.BIO_read(@bioWrite, buffer, length)
+    end
+
+    def put_cipher_text(data)
+      len = data.bytesize
+      wrote = SSL.BIO_write(@bioRead, data, len)
+      wrote == len
+    end
+
+    SSL_ERROR_WANT_WRITE = 3
+    def put_plain_text(data)
+      @write_queue.push(data) if data
+      return 0 unless SSL.is_init_finished(@ssl)
+
+      fatal = false
+      did_work = false
+
+      until @write_queue.empty?
+        data = @write_queue.pop
+        len = data.bytesize
+
+        wrote = SSL.SSL_write(@ssl, data, len)
+
+        if wrote > 0
+          did_work = true
+        else
+          err_code = SSL.SSL_get_error(@ssl, wrote)
+          if (err_code != SSL_ERROR_WANT_READ) && (err_code != SSL_ERROR_WANT_WRITE)
+            fatal = true
+          else
+            # Not fatal - add back to the queue
+            @write_queue.unshift data
+          end
+
+          break
+        end
+      end
+
+      if did_work
+        1
+      elsif fatal
+        -1
+      else
+        0
+      end
+    end
+
+    CIPHER_DISPATCH_FAILED = "Cipher text dispatch failed"
+    def dispatch_cipher_text
+      loop do
+        did_work = false
+
+        # Get all the encrypted data and transmit it
+        pending = pending_data(@bioWrite)
+        if pending > 0
+          buffer = FFI::MemoryPointer.new(:char, pending, false)
+
+          resp = get_cipher_text(buffer, pending)
+          raise Error, CIPHER_DISPATCH_FAILED unless resp > 0
+
+          @transport.transmit_cb(buffer.read_string(resp))
+          did_work = true
+        end
+
+        # Send any queued out going data
+        unless @write_queue.empty?
+          resp = put_plain_text nil
+          if resp > 0
+            did_work = true
+          elsif resp < 0
+            @transport.close_cb
+          end
+        end
+        break unless did_work
+      end
+    end
+  end
+end