httpx 0.20.0 → 1.3.1

data/lib/httpx/domain_name.rb

@@ -51,8 +51,6 @@ module HTTPX
     # non-canonical domain.
     attr_reader :domain
 
-    DOT = "." # :nodoc:
-
     class << self
       def new(domain)
         return domain if domain.is_a?(self)
@@ -63,8 +61,12 @@ module HTTPX
       # Normalizes a _domain_ using the Punycode algorithm as necessary.
       # The result will be a downcased, ASCII-only string.
       def normalize(domain)
-        domain = domain.ascii_only? ? domain : domain.chomp(DOT).unicode_normalize(:nfc)
-        Punycode.encode_hostname(domain).downcase
+        unless domain.ascii_only?
+          domain = domain.chomp(".").unicode_normalize(:nfc)
+          domain = Punycode.encode_hostname(domain)
+        end
+
+        domain.downcase
       end
     end
 
@@ -73,7 +75,7 @@ module HTTPX
     def initialize(hostname)
       hostname = String(hostname)
 
-      raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(DOT)
+      raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
 
       begin
         @ipaddr = IPAddr.new(hostname)
@@ -84,7 +86,7 @@ module HTTPX
       end
 
       @hostname = DomainName.normalize(hostname)
-      tld = if (last_dot = @hostname.rindex(DOT))
+      tld = if (last_dot = @hostname.rindex("."))
         @hostname[(last_dot + 1)..-1]
       else
         @hostname
@@ -94,7 +96,7 @@ module HTTPX
       @domain = if last_dot
         # fallback - accept cookies down to second level
         # cf. http://www.dkim-reputation.org/regdom-libs/
-        if (penultimate_dot = @hostname.rindex(DOT, last_dot - 1))
+        if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
           @hostname[(penultimate_dot + 1)..-1]
         else
           @hostname
@@ -126,17 +128,12 @@ module HTTPX
       @domain && self <= domain && domain <= @domain
     end
 
-    # def ==(other)
-    #   other = DomainName.new(other)
-    #   other.hostname == @hostname
-    # end
-
     def <=>(other)
       other = DomainName.new(other)
       othername = other.hostname
       if othername == @hostname
         0
-      elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == DOT
+      elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
         # The other is higher
         -1
       else

data/lib/httpx/errors.rb

@@ -1,18 +1,27 @@
 # frozen_string_literal: true
 
 module HTTPX
+  # the default exception class for exceptions raised by HTTPX.
   class Error < StandardError; end
 
   class UnsupportedSchemeError < Error; end
 
+  class ConnectionError < Error; end
+
+  # Error raised when there was a timeout. Its subclasses allow for finer-grained
+  # control of which timeout happened.
   class TimeoutError < Error
+    # The timeout value which caused this error to be raised.
     attr_reader :timeout
 
+    # initializes the timeout exception with the +timeout+ causing the error, and the
+    # error +message+ for it.
     def initialize(timeout, message)
       @timeout = timeout
       super(message)
     end
 
+    # clones this error into a HTTPX::ConnectionTimeoutError.
     def to_connection_error
       ex = ConnectTimeoutError.new(@timeout, message)
       ex.set_backtrace(backtrace)
@@ -20,19 +29,52 @@ module HTTPX
     end
   end
 
-  class TotalTimeoutError < TimeoutError; end
-
+  # Error raised when there was a timeout establishing the connection to a server.
+  # This may be raised due to timeouts during TCP and TLS (when applicable) connection
+  # establishment.
   class ConnectTimeoutError < TimeoutError; end
 
+  # Error raised when there was a timeout while sending a request, or receiving a response
+  # from the server.
+  class RequestTimeoutError < TimeoutError
+    # The HTTPX::Request request object this exception refers to.
+    attr_reader :request
+
+    # initializes the exception with the +request+ and +response+ it refers to, and the
+    # +timeout+ causing the error, and the
+    def initialize(request, response, timeout)
+      @request = request
+      @response = response
+      super(timeout, "Timed out after #{timeout} seconds")
+    end
+
+    def marshal_dump
+      [message]
+    end
+  end
+
+  # Error raised when there was a timeout while receiving a response from the server.
+  class ReadTimeoutError < RequestTimeoutError; end
+
+  # Error raised when there was a timeout while sending a request from the server.
+  class WriteTimeoutError < RequestTimeoutError; end
+
+  # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
   class SettingsTimeoutError < TimeoutError; end
 
+  # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
+  # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 
+  # Error raised when there was an error while resolving a domain to an IP
+  # using a HTTPX::Resolver::Native resolver.
   class NativeResolveError < ResolveError
     attr_reader :connection, :host
 
+    # initializes the exception with the +connection+ it refers to, the +host+ domain
+    # which failed to resolve, and the error +message+.
     def initialize(connection, host, message = "Can't resolve #{host}")
       @connection = connection
       @host = host
@@ -40,18 +82,26 @@ module HTTPX
     end
   end
 
+  # The exception class for HTTP responses with 4xx or 5xx status.
   class HTTPError < Error
+    # The HTTPX::Response response object this exception refers to.
     attr_reader :response
 
+    # Creates the instance and assigns the HTTPX::Response +response+.
     def initialize(response)
       @response = response
       super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
     end
 
+    # The HTTP response status.
+    #
+    #   error.status #=> 404
     def status
       @response.status
     end
   end
 
+  # error raised when a request was sent a server which can't reproduce a response, and
+  # has therefore returned an HTTP response using the 421 status code.
   class MisdirectedRequestError < HTTPError; end
 end

data/lib/httpx/extensions.rb

@@ -3,132 +3,40 @@
 require "uri"
 
 module HTTPX
-  unless Method.method_defined?(:curry)
-
-    # Backport
-    #
-    # Ruby 2.1 and lower implement curry only for Procs.
-    #
-    # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
-    #
-    module CurryMethods
-      # Backport for the Method#curry method, which is part of ruby core since 2.2 .
-      #
-      def curry(*args)
-        to_proc.curry(*args)
-      end
-    end
-    Method.__send__(:include, CurryMethods)
-  end
-
-  unless String.method_defined?(:+@)
-    # Backport for +"", to initialize unfrozen strings from the string literal.
-    #
-    module LiteralStringExtensions
-      def +@
-        frozen? ? dup : self
-      end
-    end
-    String.__send__(:include, LiteralStringExtensions)
-  end
-
-  unless Numeric.method_defined?(:positive?)
-    # Ruby 2.3 Backport (Numeric#positive?)
-    #
-    module PosMethods
-      def positive?
-        self > 0
-      end
-    end
-    Numeric.__send__(:include, PosMethods)
-  end
-
-  unless Numeric.method_defined?(:negative?)
-    # Ruby 2.3 Backport (Numeric#negative?)
-    #
-    module NegMethods
-      def negative?
-        self < 0
-      end
-    end
-    Numeric.__send__(:include, NegMethods)
-  end
-
-  module StringExtensions
-    refine String do
-      def delete_suffix!(suffix)
-        suffix = Backports.coerce_to_str(suffix)
-        chomp! if frozen?
-        len = suffix.length
-        if len > 0 && index(suffix, -len)
-          self[-len..-1] = ''
-          self
-        else
-          nil
-        end
-      end unless String.method_defined?(:delete_suffix!)
-    end
-  end
-
-  module HashExtensions
-    refine Hash do
-      def compact
-        h = {}
-        each do |key, value|
-          h[key] = value unless value == nil
-        end
-        h
-      end unless Hash.method_defined?(:compact)
-    end
-  end
-
   module ArrayExtensions
-    refine Array do
-
-      def filter_map
-        return to_enum(:filter_map) unless block_given?
-
-        each_with_object([]) do |item, res|
-          processed = yield(item)
-          res << processed if processed
+    module FilterMap
+      refine Array do
+        # Ruby 2.7 backport
+        def filter_map
+          return to_enum(:filter_map) unless block_given?
+
+          each_with_object([]) do |item, res|
+            processed = yield(item)
+            res << processed if processed
+          end
         end
       end unless Array.method_defined?(:filter_map)
-
-      def sum(accumulator = 0, &block)
-        values = block_given? ? map(&block) : self
-        values.inject(accumulator, :+)
-      end unless Array.method_defined?(:sum)
     end
-  end
-
-  module IOExtensions
-    refine IO do
-      # provides a fallback for rubies where IO#wait isn't implemented,
-      # but IO#wait_readable and IO#wait_writable are.
-      def wait(timeout = nil, _mode = :read_write)
-        r, w = IO.select([self], [self], nil, timeout)
-
-        return unless r || w
-
-        self
-      end unless IO.method_defined?(:wait) && IO.instance_method(:wait).arity == 2
-    end
-  end
 
-  module RegexpExtensions
-    # If you wonder why this is there: the oauth feature uses a refinement to enhance the
-    # Regexp class locally with #match? , but this is never tested, because ActiveSupport
-    # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
-    # :nocov:
-    refine(Regexp) do
-      def match?(*args)
-        !match(*args).nil?
-      end
+    module Intersect
+      refine Array do
+        # Ruby 3.1 backport
+        def intersect?(arr)
+          if size < arr.size
+            smaller = self
+          else
+            smaller, arr = arr, self
+          end
+          (arr & smaller).size > 0
+        end
+      end unless Array.method_defined?(:intersect?)
     end
   end
 
   module URIExtensions
+    # uri 0.11 backport, ships with ruby 3.1
     refine URI::Generic do
+
       def non_ascii_hostname
         @non_ascii_hostname
       end
@@ -146,21 +54,6 @@ module HTTPX
       def origin
         "#{scheme}://#{authority}"
       end unless URI::HTTP.method_defined?(:origin)
-
-      def altsvc_match?(uri)
-        uri = URI.parse(uri)
-
-        origin == uri.origin || begin
-          case scheme
-          when "h2"
-            (uri.scheme == "https" || uri.scheme == "h2") &&
-            host == uri.host &&
-            (port || default_port) == (uri.port || uri.default_port)
-          else
-            false
-          end
-        end
-      end
     end
   end
 end

data/lib/httpx/io/ssl.rb

@@ -4,26 +4,48 @@ require "openssl"
 
 module HTTPX
   TLSError = OpenSSL::SSL::SSLError
-  IPRegex = Regexp.union(Resolv::IPv4::Regex, Resolv::IPv6::Regex)
 
   class SSL < TCP
-    using RegexpExtensions unless Regexp.method_defined?(:match?)
+    # rubocop:disable Style/MutableConstant
+    TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
+    # https://github.com/jruby/jruby-openssl/issues/284
+    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
+    # rubocop:enable Style/MutableConstant
+    TLS_OPTIONS.freeze
 
-    TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
-      { alpn_protocols: %w[h2 http/1.1].freeze }.freeze
-    else
-      {}.freeze
-    end
+    attr_writer :ssl_session
 
     def initialize(_, _, options)
       super
-      @ctx = OpenSSL::SSL::SSLContext.new
+
       ctx_options = TLS_OPTIONS.merge(options.ssl)
       @sni_hostname = ctx_options.delete(:hostname) || @hostname
-      @ctx.set_params(ctx_options) unless ctx_options.empty?
-      @state = :negotiated if @keep_open
 
-      @hostname_is_ip = IPRegex.match?(@sni_hostname)
+      if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
+        # externally initiated ssl socket
+        @ctx = @io.context
+        @state = :negotiated
+      else
+        @ctx = OpenSSL::SSL::SSLContext.new
+        @ctx.set_params(ctx_options) unless ctx_options.empty?
+        unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
+          @ctx.session_cache_mode =
+            OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
+        end
+
+        yield(self) if block_given?
+      end
+
+      @verify_hostname = @ctx.verify_hostname
+    end
+
+    if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
+      def session_new_cb(&pr)
+        @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
+      end
+    else
+      # session_new_cb not implemented under JRuby
+      def session_new_cb; end
     end
 
     def protocol
@@ -32,6 +54,20 @@ module HTTPX
       super
     end
 
+    if RUBY_ENGINE == "jruby"
+      # in jruby, alpn_protocol may return ""
+      # https://github.com/jruby/jruby-openssl/issues/287
+      def protocol
+        proto = @io.alpn_protocol
+
+        return super if proto.nil? || proto.empty?
+
+        proto
+      rescue StandardError
+        super
+      end
+    end
+
     def can_verify_peer?
       @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
     end
@@ -43,85 +79,54 @@ module HTTPX
       OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
     end
 
-    def close
-      super
-      # allow reconnections
-      # connect only works if initial @io is a socket
-      @io = @io.io if @io.respond_to?(:io)
-    end
-
     def connected?
       @state == :negotiated
     end
 
+    def expired?
+      super || ssl_session_expired?
+    end
+
+    def ssl_session_expired?
+      @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
+    end
+
     def connect
       super
       return if @state == :negotiated ||
                 @state != :connected
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+        if (hostname_is_ip = (@ip == @sni_hostname))
+          # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
+          @sni_hostname = @ip.to_string
+          # IP addresses in SNI is not valid per RFC 6066, section 3.
+          @ctx.verify_hostname = false
+        end
+
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @sni_hostname unless @hostname_is_ip
+
+        @io.hostname = @sni_hostname unless hostname_is_ip
+        @io.session = @ssl_session unless ssl_session_expired?
         @io.sync_close = true
       end
       try_ssl_connect
     end
 
-    if RUBY_VERSION < "2.3"
-      # :nocov:
-      def try_ssl_connect
-        @io.connect_nonblock
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
-        transition(:negotiated)
-        @interests = :w
-      rescue ::IO::WaitReadable
+    def try_ssl_connect
+      ret = @io.connect_nonblock(exception: false)
+      log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
+      case ret
+      when :wait_readable
         @interests = :r
-      rescue ::IO::WaitWritable
+        return
+      when :wait_writable
         @interests = :w
+        return
       end
-
-      def read(_, buffer)
-        super
-      rescue ::IO::WaitWritable
-        buffer.clear
-        0
-      end
-
-      def write(*)
-        super
-      rescue ::IO::WaitReadable
-        0
-      end
-      # :nocov:
-    else
-      def try_ssl_connect
-        case @io.connect_nonblock(exception: false)
-        when :wait_readable
-          @interests = :r
-          return
-        when :wait_writable
-          @interests = :w
-          return
-        end
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
-        transition(:negotiated)
-        @interests = :w
-      end
-
-      # :nocov:
-      if OpenSSL::VERSION < "2.0.6"
-        def read(size, buffer)
-          @io.read_nonblock(size, buffer)
-          buffer.bytesize
-        rescue ::IO::WaitReadable,
-               ::IO::WaitWritable
-          buffer.clear
-          0
-        rescue EOFError
-          nil
-        end
-      end
-      # :nocov:
+      @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
+      transition(:negotiated)
+      @interests = :w
     end
 
     private
@@ -130,6 +135,7 @@ module HTTPX
       case nextstate
       when :negotiated
         return unless @state == :connected
+
       when :closed
         return unless @state == :negotiated ||
                       @state == :connected
@@ -145,12 +151,12 @@ module HTTPX
       "#{super}\n\n" \
         "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
-        "Server certificate:\n" \
-        " subject: #{server_cert.subject}\n" \
-        " start date: #{server_cert.not_before}\n" \
-        " expire date: #{server_cert.not_after}\n" \
-        " issuer: #{server_cert.issuer}\n" \
-        " SSL certificate verify ok."
+        "Server certificate:\n " \
+        "subject: #{server_cert.subject}\n " \
+        "start date: #{server_cert.not_before}\n " \
+        "expire date: #{server_cert.not_after}\n " \
+        "issuer: #{server_cert.issuer}\n " \
+        "SSL certificate verify ok."
     end
   end
 end