httpx-patched 1.6.2.1

data/lib/httpx/domain_name.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+#
+# domain_name.rb - Domain Name manipulation library for Ruby
+#
+# Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+require "ipaddr"
+
+module HTTPX
+  # Represents a domain name ready for extracting its registered domain
+  # and TLD.
+  class DomainName
+    include Comparable
+
+    # The full host name normalized, ASCII-ized and downcased using the
+    # Unicode NFC rules and the Punycode algorithm.  If initialized with
+    # an IP address, the string representation of the IP address
+    # suitable for opening a connection to.
+    attr_reader :hostname
+
+    # The Unicode representation of the #hostname property.
+    #
+    # :attr_reader: hostname_idn
+
+    # The least "universally original" domain part of this domain name.
+    # For example, "example.co.uk" for "www.sub.example.co.uk".  This
+    # may be nil if the hostname does not have one, like when it is an
+    # IP address, an effective TLD or higher itself, or of a
+    # non-canonical domain.
+    attr_reader :domain
+
+    class << self
+      def new(domain)
+        return domain if domain.is_a?(self)
+
+        super(domain)
+      end
+
+      # Normalizes a _domain_ using the Punycode algorithm as necessary.
+      # The result will be a downcased, ASCII-only string.
+      def normalize(domain)
+        unless domain.ascii_only?
+          domain = domain.chomp(".").unicode_normalize(:nfc)
+          domain = Punycode.encode_hostname(domain)
+        end
+
+        domain.downcase
+      end
+    end
+
+    # Parses _hostname_ into a DomainName object.  An IP address is also
+    # accepted.  An IPv6 address may be enclosed in square brackets.
+    def initialize(hostname)
+      hostname = String(hostname)
+
+      raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
+
+      begin
+        @ipaddr = IPAddr.new(hostname)
+        @hostname = @ipaddr.to_s
+        return
+      rescue IPAddr::Error
+        nil
+      end
+
+      @hostname = DomainName.normalize(hostname)
+      tld = if (last_dot = @hostname.rindex("."))
+        @hostname[(last_dot + 1)..-1]
+      else
+        @hostname
+      end
+
+      # unknown/local TLD
+      @domain = if last_dot
+        # fallback - accept cookies down to second level
+        # cf. http://www.dkim-reputation.org/regdom-libs/
+        if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
+          @hostname[(penultimate_dot + 1)..-1]
+        else
+          @hostname
+        end
+      else
+        # no domain part - must be a local hostname
+        tld
+      end
+    end
+
+    # Checks if the server represented by this domain is qualified to
+    # send and receive cookies with a domain attribute value of
+    # _domain_.  A true value given as the second argument represents
+    # cookies without a domain attribute value, in which case only
+    # hostname equality is checked.
+    def cookie_domain?(domain, host_only = false)
+      # RFC 6265 #5.3
+      # When the user agent "receives a cookie":
+      return self == @domain if host_only
+
+      domain = DomainName.new(domain)
+
+      # RFC 6265 #5.1.3
+      # Do not perform subdomain matching against IP addresses.
+      @hostname == domain.hostname if @ipaddr
+
+      # RFC 6265 #4.1.1
+      # Domain-value must be a subdomain.
+      @domain && self <= domain && domain <= @domain
+    end
+
+    def <=>(other)
+      other = DomainName.new(other)
+      othername = other.hostname
+      if othername == @hostname
+        0
+      elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
+        # The other is higher
+        -1
+      else
+        # The other is lower
+        1
+      end
+    end
+  end
+end

data/lib/httpx/errors.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module HTTPX
+  # the default exception class for exceptions raised by HTTPX.
+  class Error < StandardError; end
+
+  class UnsupportedSchemeError < Error; end
+
+  class ConnectionError < Error; end
+
+  # Error raised when there was a timeout. Its subclasses allow for finer-grained
+  # control of which timeout happened.
+  class TimeoutError < Error
+    # The timeout value which caused this error to be raised.
+    attr_reader :timeout
+
+    # initializes the timeout exception with the +timeout+ causing the error, and the
+    # error +message+ for it.
+    def initialize(timeout, message)
+      @timeout = timeout
+      super(message)
+    end
+
+    # clones this error into a HTTPX::ConnectionTimeoutError.
+    def to_connection_error
+      ex = ConnectTimeoutError.new(@timeout, message)
+      ex.set_backtrace(backtrace)
+      ex
+    end
+  end
+
+  # Raise when it can't acquire a connection from the pool.
+  class PoolTimeoutError < TimeoutError; end
+
+  # Error raised when there was a timeout establishing the connection to a server.
+  # This may be raised due to timeouts during TCP and TLS (when applicable) connection
+  # establishment.
+  class ConnectTimeoutError < TimeoutError; end
+
+  # Error raised when there was a timeout while sending a request, or receiving a response
+  # from the server.
+  class RequestTimeoutError < TimeoutError
+    # The HTTPX::Request request object this exception refers to.
+    attr_reader :request
+
+    # initializes the exception with the +request+ and +response+ it refers to, and the
+    # +timeout+ causing the error, and the
+    def initialize(request, response, timeout)
+      @request = request
+      @response = response
+      super(timeout, "Timed out after #{timeout} seconds")
+    end
+
+    def marshal_dump
+      [message]
+    end
+  end
+
+  # Error raised when there was a timeout while receiving a response from the server.
+  class ReadTimeoutError < RequestTimeoutError; end
+
+  # Error raised when there was a timeout while sending a request from the server.
+  class WriteTimeoutError < RequestTimeoutError; end
+
+  # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
+  class SettingsTimeoutError < TimeoutError; end
+
+  # Error raised when there was a timeout while resolving a domain to an IP.
+  class ResolveTimeoutError < TimeoutError; end
+
+  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  class OperationTimeoutError < TimeoutError; end
+
+  # Error raised when there was an error while resolving a domain to an IP.
+  class ResolveError < Error; end
+
+  # Error raised when there was an error while resolving a domain to an IP
+  # using a HTTPX::Resolver::Native resolver.
+  class NativeResolveError < ResolveError
+    attr_reader :host
+
+    attr_accessor :connection
+
+    # initializes the exception with the +connection+ it refers to, the +host+ domain
+    # which failed to resolve, and the error +message+.
+    def initialize(connection, host, message = "Can't resolve #{host}")
+      @connection = connection
+      @host = host
+      super(message)
+    end
+  end
+
+  # The exception class for HTTP responses with 4xx or 5xx status.
+  class HTTPError < Error
+    # The HTTPX::Response response object this exception refers to.
+    attr_reader :response
+
+    # Creates the instance and assigns the HTTPX::Response +response+.
+    def initialize(response)
+      @response = response
+      super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
+    end
+
+    # The HTTP response status.
+    #
+    #   error.status #=> 404
+    def status
+      @response.status
+    end
+  end
+end

data/lib/httpx/extensions.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module HTTPX
+  module ArrayExtensions
+    module FilterMap
+      refine Array do
+        # Ruby 2.7 backport
+        def filter_map
+          return to_enum(:filter_map) unless block_given?
+
+          each_with_object([]) do |item, res|
+            processed = yield(item)
+            res << processed if processed
+          end
+        end
+      end unless Array.method_defined?(:filter_map)
+    end
+
+    module Intersect
+      refine Array do
+        # Ruby 3.1 backport
+        def intersect?(arr)
+          if size < arr.size
+            smaller = self
+          else
+            smaller, arr = arr, self
+          end
+          (arr & smaller).size > 0
+        end
+      end unless Array.method_defined?(:intersect?)
+    end
+  end
+
+  module URIExtensions
+    # uri 0.11 backport, ships with ruby 3.1
+    refine URI::Generic do
+
+      def non_ascii_hostname
+        @non_ascii_hostname
+      end
+
+      def non_ascii_hostname=(hostname)
+        @non_ascii_hostname = hostname
+      end
+
+      def authority
+        return host if port == default_port
+
+        "#{host}:#{port}"
+      end unless URI::HTTP.method_defined?(:authority)
+
+      def origin
+        "#{scheme}://#{authority}"
+      end unless URI::HTTP.method_defined?(:origin)
+    end
+  end
+end

data/lib/httpx/headers.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class Headers
+    class << self
+      def new(headers = nil)
+        return headers if headers.is_a?(self)
+
+        super
+      end
+    end
+
+    def initialize(headers = nil)
+      if headers.nil? || headers.empty?
+        @headers = headers.to_h
+        return
+      end
+
+      @headers = {}
+
+      headers.each do |field, value|
+        field = downcased(field)
+
+        value = array_value(value)
+
+        current = @headers[field]
+
+        if current.nil?
+          @headers[field] = value
+        else
+          current.concat(value)
+        end
+      end
+    end
+
+    # cloned initialization
+    def initialize_clone(orig, **kwargs)
+      super
+      @headers = orig.instance_variable_get(:@headers).clone(**kwargs)
+    end
+
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      @headers = orig.instance_variable_get(:@headers).dup
+    end
+
+    # freezes the headers hash
+    def freeze
+      @headers.freeze
+      super
+    end
+
+    # merges headers with another header-quack.
+    # the merge rule is, if the header already exists,
+    # ignore what the +other+ headers has. Otherwise, set
+    #
+    def merge(other)
+      headers = dup
+      other.each do |field, value|
+        headers[downcased(field)] = value
+      end
+      headers
+    end
+
+    # returns the comma-separated values of the header field
+    # identified by +field+, or nil otherwise.
+    #
+    def [](field)
+      a = @headers[downcased(field)] || return
+      a.join(", ")
+    end
+
+    # sets +value+ (if not nil) as single value for the +field+ header.
+    #
+    def []=(field, value)
+      return unless value
+
+      @headers[downcased(field)] = array_value(value)
+    end
+
+    # deletes all values associated with +field+ header.
+    #
+    def delete(field)
+      canonical = downcased(field)
+      @headers.delete(canonical) if @headers.key?(canonical)
+    end
+
+    # adds additional +value+ to the existing, for header +field+.
+    #
+    def add(field, value)
+      (@headers[downcased(field)] ||= []) << String(value)
+    end
+
+    # helper to be used when adding an header field as a value to another field
+    #
+    #     h2_headers.add_header("vary", "accept-encoding")
+    #     h2_headers["vary"] #=> "accept-encoding"
+    #     h1_headers.add_header("vary", "accept-encoding")
+    #     h1_headers["vary"] #=> "Accept-Encoding"
+    #
+    alias_method :add_header, :add
+
+    # returns the enumerable headers store in pairs of header field + the values in
+    # the comma-separated string format
+    #
+    def each(extra_headers = nil)
+      return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
+
+      @headers.each do |field, value|
+        yield(field, value.join(", ")) unless value.empty?
+      end
+
+      extra_headers.each do |field, value|
+        yield(field, value) unless value.empty?
+      end if extra_headers
+    end
+
+    def ==(other)
+      other == to_hash
+    end
+
+    def empty?
+      @headers.empty?
+    end
+
+    # the headers store in Hash format
+    def to_hash
+      Hash[to_a]
+    end
+    alias_method :to_h, :to_hash
+
+    # the headers store in array of pairs format
+    def to_a
+      Array(each)
+    end
+
+    # headers as string
+    def to_s
+      @headers.to_s
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "#{to_hash.inspect}>"
+    end
+    # :nocov:
+
+    # this is internal API and doesn't abide to other public API
+    # guarantees, like downcasing strings.
+    # Please do not use this outside of core!
+    #
+    def key?(downcased_key)
+      @headers.key?(downcased_key)
+    end
+
+    # returns the values for the +field+ header in array format.
+    # This method is more internal, and for this reason doesn't try
+    # to "correct" the user input, i.e. it doesn't downcase the key.
+    #
+    def get(field)
+      @headers[field] || EMPTY
+    end
+
+    private
+
+    def array_value(value)
+      Array(value)
+    end
+
+    def downcased(field)
+      String(field).downcase
+    end
+  end
+end

data/lib/httpx/io/ssl.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module HTTPX
+  TLSError = OpenSSL::SSL::SSLError
+
+  class SSL < TCP
+    # rubocop:disable Style/MutableConstant
+    TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
+    # https://github.com/jruby/jruby-openssl/issues/284
+    # TODO: remove when dropping support for jruby-openssl < 0.15.4
+    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
+    # rubocop:enable Style/MutableConstant
+    TLS_OPTIONS.freeze
+
+    attr_writer :ssl_session
+
+    def initialize(_, _, options)
+      super
+
+      @ssl_session = nil
+      ctx_options = TLS_OPTIONS.merge(options.ssl)
+      @sni_hostname = ctx_options.delete(:hostname) || @hostname
+
+      if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
+        # externally initiated ssl socket
+        @ctx = @io.context
+        @state = :negotiated
+      else
+        @ctx = OpenSSL::SSL::SSLContext.new
+        @ctx.set_params(ctx_options) unless ctx_options.empty?
+        unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
+          @ctx.session_cache_mode =
+            OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
+        end
+
+        yield(self) if block_given?
+      end
+
+      @verify_hostname = @ctx.verify_hostname
+    end
+
+    if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
+      def session_new_cb(&pr)
+        @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
+      end
+    else
+      # session_new_cb not implemented under JRuby
+      def session_new_cb; end
+    end
+
+    def protocol
+      @io.alpn_protocol || super
+    rescue StandardError
+      super
+    end
+
+    if RUBY_ENGINE == "jruby"
+      # in jruby, alpn_protocol may return ""
+      # https://github.com/jruby/jruby-openssl/issues/287
+      def protocol
+        proto = @io.alpn_protocol
+
+        return super if proto.nil? || proto.empty?
+
+        proto
+      rescue StandardError
+        super
+      end
+    end
+
+    def can_verify_peer?
+      @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
+    end
+
+    def verify_hostname(host)
+      return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
+      return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
+
+      OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
+    end
+
+    def connected?
+      @state == :negotiated
+    end
+
+    def ssl_session_expired?
+      @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
+    end
+
+    def connect
+      return if @state == :negotiated
+
+      unless @state == :connected
+        super
+        return unless @state == :connected
+      end
+
+      unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+        if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
+          # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
+          @sni_hostname = @ip.to_string
+          # IP addresses in SNI is not valid per RFC 6066, section 3.
+          @ctx.verify_hostname = false
+        end
+
+        @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
+
+        @io.hostname = @sni_hostname unless hostname_is_ip
+        @io.session = @ssl_session unless ssl_session_expired?
+        @io.sync_close = true
+      end
+      try_ssl_connect
+    end
+
+    def try_ssl_connect
+      ret = @io.connect_nonblock(exception: false)
+      log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
+      case ret
+      when :wait_readable
+        @interests = :r
+        return
+      when :wait_writable
+        @interests = :w
+        return
+      end
+      @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
+      transition(:negotiated)
+      @interests = :w
+    end
+
+    private
+
+    def transition(nextstate)
+      case nextstate
+      when :negotiated
+        return unless @state == :connected
+
+      when :closed
+        return unless @state == :negotiated ||
+                      @state == :connected
+      end
+      do_transition(nextstate)
+    end
+
+    def log_transition_state(nextstate)
+      return super unless nextstate == :negotiated
+
+      server_cert = @io.peer_cert
+
+      "#{super}\n\n" \
+        "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
+        "ALPN, server accepted to use #{protocol}\n" \
+        "Server certificate:\n " \
+        "subject: #{server_cert.subject}\n " \
+        "start date: #{server_cert.not_before}\n " \
+        "expire date: #{server_cert.not_after}\n " \
+        "issuer: #{server_cert.issuer}\n " \
+        "SSL certificate verify ok."
+    end
+  end
+end