httpx 0.21.0 → 0.22.4

data/lib/httpx/plugins/rate_limiter.rb

@@ -9,7 +9,7 @@ module HTTPX
     # * when the server is unavailable (503);
     # * when a 3xx request comes with a "retry-after" value
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    # https://gitlab.com/os85/httpx/wikis/RateLimiter
     #
     module RateLimiter
       class << self
@@ -23,6 +23,8 @@ module HTTPX
         end
 
         def retry_on_rate_limited_response(response)
+          return false unless response.is_a?(Response)
+
           status = response.status
 
           RATE_LIMIT_CODES.include?(status)

data/lib/httpx/plugins/response_cache.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin adds support for retrying requests when certain errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
     module ResponseCache
       CACHEABLE_VERBS = %i[get head].freeze

data/lib/httpx/plugins/retries.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin adds support for retrying requests when certain errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Retries
+    # https://gitlab.com/os85/httpx/wikis/Retries
     #
     module Retries
       MAX_RETRIES = 3
@@ -23,9 +23,10 @@ module HTTPX
         Parser::Error,
         TLSError,
         TimeoutError,
+        ConnectionError,
         Connection::HTTP2::GoawayError,
       ].freeze
-      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
       if ENV.key?("HTTPX_NO_JITTER")
         def self.extra_options(options)

data/lib/httpx/plugins/stream.rb

@@ -95,8 +95,6 @@ module HTTPX
     #
     module Stream
       module InstanceMethods
-        private
-
         def request(*args, stream: false, **options)
           return super(*args, **options) unless stream
 

data/lib/httpx/plugins/upgrade.rb

@@ -35,7 +35,9 @@ module HTTPX
           response = super
 
           if response
-            return response unless response.respond_to?(:headers) && response.headers.key?("upgrade")
+            return response unless response.is_a?(Response)
+
+            return response unless response.headers.key?("upgrade")
 
             upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 

data/lib/httpx/plugins/webdav.rb

@@ -32,6 +32,8 @@ module HTTPX
                 "</D:lockinfo>"
           response = request(:lock, path, headers: headers, xml: xml)
 
+          return response unless response.is_a?(Response)
+
           return response unless blk && response.status == 200
 
           lock_token = response.headers["lock-token"]

data/lib/httpx/pool.rb

@@ -72,7 +72,7 @@ module HTTPX
     end
 
     def init_connection(connection, _options)
-      resolve_connection(connection)
+      resolve_connection(connection) unless connection.family
       connection.timers = @timers
       connection.on(:open) do
         @connected_connections += 1
@@ -116,19 +116,57 @@ module HTTPX
         #    resolve a name (not the same as name being an IP, yet)
         # 2. when the connection is initialized with an external already open IO.
         #
+        connection.once(:connect_error, &connection.method(:handle_error))
         on_resolver_connection(connection)
         return
       end
 
       find_resolver_for(connection) do |resolver|
-        resolver << connection
+        resolver << try_clone_connection(connection, resolver.family)
         next if resolver.empty?
 
         select_connection(resolver)
       end
     end
 
+    def try_clone_connection(connection, family)
+      connection.family ||= family
+
+      return connection if connection.family == family
+
+      new_connection = connection.class.new(connection.type, connection.origin, connection.options)
+      new_connection.family = family
+
+      connection.once(:tcp_open) { new_connection.force_reset }
+      connection.once(:connect_error) do |err|
+        if new_connection.connecting?
+          new_connection.merge(connection)
+        else
+          connection.handle_error(err)
+        end
+      end
+
+      new_connection.once(:tcp_open) do |new_conn|
+        if new_conn != connection
+          new_conn.merge(connection)
+          connection.force_reset
+        end
+      end
+      new_connection.once(:connect_error) do |err|
+        if connection.connecting?
+          # main connection has the requests
+          connection.merge(new_connection)
+        else
+          new_connection.handle_error(err)
+        end
+      end
+
+      init_connection(new_connection, connection.options)
+      new_connection
+    end
+
     def on_resolver_connection(connection)
+      @connections << connection unless @connections.include?(connection)
       found_connection = @connections.find do |ch|
         ch != connection && ch.mergeable?(connection)
       end
@@ -186,6 +224,7 @@ module HTTPX
     def coalesce_connections(conn1, conn2)
       return register_connection(conn2) unless conn1.coalescable?(conn2)
 
+      conn2.emit(:tcp_open, conn1)
       conn1.merge(conn2)
       @connections.delete(conn2)
     end

data/lib/httpx/request.rb

@@ -86,7 +86,7 @@ module HTTPX
     def response=(response)
       return unless response
 
-      if response.is_a?(Response) && response.status == 100
+      if response.is_a?(Response) && response.status == 100 && @headers.key?("expect")
         @informational_status = response.status
         return
       end

data/lib/httpx/resolver/https.rb

@@ -69,7 +69,8 @@ module HTTPX
         @building_connection = true
         connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
         @pool.init_connection(connection, @options)
-        emit_addresses(connection, @family, @uri_addresses)
+        # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
+        emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
         @building_connection = false
         connection
       end
@@ -135,9 +136,22 @@ module HTTPX
         emit_resolve_error(connection, connection.origin.host, e)
         return
       end
-      if answers.nil? || answers.empty?
+
+      if answers.nil?
+        # Indicates no such domain was found.
+
         host = @requests.delete(request)
         connection = @queries.delete(host)
+
+        emit_resolve_error(connection) unless @queries.value?(connection)
+      elsif answers.empty?
+        # no address found, eliminate candidates
+        host = @requests.delete(request)
+        connection = @queries.delete(host)
+
+        # eliminate other candidates
+        @queries.delete_if { |_, conn| connection == conn }
+
         emit_resolve_error(connection)
         return
 

data/lib/httpx/resolver/native.rb

@@ -13,27 +13,15 @@ module HTTPX
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
+      }
     else
       {
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
-    end
-
-    # nameservers for ipv6 are misconfigured in certain systems;
-    # this can use an unexpected endless loop
-    # https://gitlab.com/honeyryderchuck/httpx/issues/56
-    DEFAULTS[:nameserver].select! do |nameserver|
-      begin
-        IPAddr.new(nameserver)
-        true
-      rescue IPAddr::InvalidAddressError
-        false
-      end
-    end if DEFAULTS[:nameserver]
+      }
+    end.freeze
 
     DNS_PORT = 53
 
@@ -152,10 +140,21 @@ module HTTPX
       host = connection.origin.host
       timeout = (@timeouts[host][0] -= loop_time)
 
-      return unless timeout.negative?
+      return unless timeout <= 0
 
       @timeouts[host].shift
-      if @timeouts[host].empty?
+
+      if !@timeouts[host].empty?
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
+      elsif @ns_index + 1 < @nameserver.size
+        # try on the next nameserver
+        @ns_index += 1
+        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        transition(:idle)
+        resolve(connection)
+      else
+
         @timeouts.delete(host)
         @queries.delete(h)
 
@@ -165,9 +164,6 @@ module HTTPX
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
         raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
-      else
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-        resolve(connection)
       end
     end
 
@@ -205,7 +201,8 @@ module HTTPX
         raise ex
       end
 
-      if addresses.nil? || addresses.empty?
+      if addresses.nil?
+        # Indicates no such domain was found.
         hostname, connection = @queries.first
         @queries.delete(hostname)
         @timeouts.delete(hostname)
@@ -214,6 +211,14 @@ module HTTPX
           @connections.delete(connection)
           raise NativeResolveError.new(connection, connection.origin.host)
         end
+      elsif addresses.empty?
+        # no address found, eliminate candidates
+        _, connection = @queries.first
+        candidates = @queries.select { |_, conn| connection == conn }.keys
+        @queries.delete_if { |hs, _| candidates.include?(hs) }
+        @timeouts.delete_if { |hs, _| candidates.include?(hs) }
+        @connections.delete(connection)
+        raise NativeResolveError.new(connection, connection.origin.host)
       else
         address = addresses.first
         name = address["name"]
@@ -299,11 +304,8 @@ module HTTPX
 
       ip, port = @nameserver[@ns_index]
       port ||= DNS_PORT
-      uri = URI::Generic.build(scheme: "udp", port: port)
-      uri.hostname = ip
-      type = IO.registry(uri.scheme)
-      log { "resolver: server: #{uri}..." }
-      @io = type.new(uri, [IPAddr.new(ip)], @options)
+      log { "resolver: server: #{ip}:#{port}..." }
+      @io = UDP.new(ip, port, @options)
     end
 
     def transition(nextstate)

data/lib/httpx/resolver/resolver.rb

@@ -54,7 +54,7 @@ module HTTPX
       # double emission check
       return if connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
+      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
          connection.options.ip_families.size > 1 &&
@@ -63,20 +63,21 @@ module HTTPX
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
           # double emission check
-          unless connection.addresses && addresses.intersect?(connection.addresses)
-
-            connection.addresses = addresses
-            emit(:resolve, connection)
-          end
+          emit_resolved_connection(connection, addresses) unless connection.addresses && addresses.intersect?(connection.addresses)
         end
       else
-        connection.addresses = addresses
-        emit(:resolve, connection)
+        emit_resolved_connection(connection, addresses)
       end
     end
 
     private
 
+    def emit_resolved_connection(connection, addresses)
+      connection.addresses = addresses
+
+      emit(:resolve, connection)
+    end
+
     def early_resolve(connection, hostname: connection.origin.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 

data/lib/httpx/resolver.rb

@@ -108,7 +108,15 @@ module HTTPX
 
     def decode_dns_answer(payload)
       message = Resolv::DNS::Message.decode(payload)
+
+      # no domain was found
+      return if message.rcode == Resolv::DNS::RCode::NXDomain
+
       addresses = []
+
+      # TODO: raise an "other dns OtherResolvError" type of error
+      return addresses if message.rcode != Resolv::DNS::RCode::NoError
+
       message.each_answer do |question, _, value|
         case value
         when Resolv::DNS::Resource::IN::CNAME

data/lib/httpx/response.rb

@@ -128,6 +128,8 @@ module HTTPX
     end
 
     class Body
+      attr_reader :encoding
+
       def initialize(response, options)
         @response = response
         @headers = response.headers
@@ -179,6 +181,12 @@ module HTTPX
         end
       end
 
+      def filename
+        return unless @headers.key?("content-disposition")
+
+        Utils.get_filename(@headers["content-disposition"])
+      end
+
       def to_s
         case @buffer
         when StringIO

data/lib/httpx/transcoder/xml.rb

@@ -38,7 +38,6 @@ module HTTPX::Transcoder
     begin
       require "nokogiri"
 
-      # rubocop:disable Lint/DuplicateMethods
       def decode(response)
         content_type = response.content_type.mime_type
 
@@ -51,7 +50,6 @@ module HTTPX::Transcoder
         raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
       end
     end
-    # rubocop:enable Lint/DuplicateMethods
   end
   register "xml", Xml
 end

data/lib/httpx/transcoder.rb

@@ -73,7 +73,7 @@ module HTTPX
     end
 
     def params_hash_has_key?(hash, key)
-      return false if /\[\]/.match?(key)
+      return false if key.include?("[]")
 
       key.split(/[\[\]]+/).inject(hash) do |h, part|
         next h if part == ""

data/lib/httpx/utils.rb

@@ -3,6 +3,12 @@
 module HTTPX
   module Utils
     using URIExtensions
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
+    FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
 
     module_function
 
@@ -25,6 +31,30 @@ module HTTPX
       time - Time.now
     end
 
+    def get_filename(header, _prefix_regex = nil)
+      filename = nil
+      case header
+      when FILENAME_REGEX
+        filename = Regexp.last_match(1)
+        filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+      when FILENAME_EXTENSION_REGEX
+        filename = Regexp.last_match(1)
+        encoding, _, filename = filename.split("'", 3)
+      end
+
+      return unless filename
+
+      filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+      filename.scrub!
+
+      filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+      filename.force_encoding ::Encoding.find(encoding) if encoding
+
+      filename
+    end
+
     if RUBY_VERSION < "2.3"
 
       def to_uri(uri)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.21.0"
+  VERSION = "0.22.4"
 end

data/lib/httpx.rb

@@ -67,3 +67,9 @@ end
 
 require "httpx/session"
 require "httpx/session_extensions"
+
+# load integrations when possible
+
+require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog)
+require "httpx/adapters/sentry" if defined?(Sentry)
+require "httpx/adapters/webmock" if defined?(WebMock)

data/sig/connection.rbs

@@ -21,6 +21,7 @@ module HTTPX
 
     BUFFER_SIZE: Integer
 
+    attr_reader type: io_type
     attr_reader origin: URI::Generic
     attr_reader origins: Array[String]
     attr_reader state: Symbol
@@ -28,7 +29,8 @@ module HTTPX
     attr_reader options: Options
     attr_writer timers: Timers
 
-    @type: io_type
+    attr_accessor family: Integer?
+
     @window_size: Integer
     @read_buffer: Buffer
     @write_buffer: Buffer

data/sig/plugins/multipart.rbs

@@ -68,8 +68,6 @@ module HTTPX
         def initialize: (Response response) -> void
 
         def parse: () -> void
-
-        def get_filename: (String head) -> String?
       end
 
       class FilePart #  < SimpleDelegator

data/sig/pool.rbs

@@ -22,7 +22,9 @@ module HTTPX
 
     private
 
-    def initialize: () -> untyped
+    def initialize: () -> void
+
+    def try_clone_connection: (Connection connection, Integer? family) -> Connection
 
     def resolve_connection: (Connection) -> void
 

data/sig/resolver/https.rbs

@@ -6,7 +6,8 @@ module HTTPX
       DEFAULTS: Hash[Symbol, untyped]
       FAMILY_TYPES: Hash[singleton(Resolv::DNS::Resource), String]
 
-      @family: ip_family
+      attr_reader family: ip_family
+
       @options: Options
       @requests: Hash[Request, String]
       @connections: Array[Connection]
@@ -33,7 +34,7 @@ module HTTPX
 
       def build_request: (String hostname) -> Request
 
-      def decode_response_body: (Response) -> Array[dns_result]
+      def decode_response_body: (Response) -> Array[dns_result]?
     end
   end
 end

data/sig/resolver/native.rbs

@@ -7,7 +7,8 @@ module HTTPX
       DEFAULTS: Hash[Symbol, untyped]
       DNS_PORT: Integer
 
-      @family: ip_family
+      attr_reader family: ip_family
+
       @options: Options
       @ns_index: Integer
       @nameserver: Array[String]?

data/sig/resolver/resolver.rbs

@@ -6,7 +6,7 @@ module HTTPX
 
       RECORD_TYPES: Hash[Integer, singleton(Resolv::DNS::Resource)]
 
-      attr_reader family: ip_family
+      attr_reader family: ip_family?
 
       @record_type: singleton(Resolv::DNS::Resource)
       @options: Options
@@ -24,6 +24,8 @@ module HTTPX
 
       private
 
+      def emit_resolved_connection: (Connection connection, Array[IPAddr] addresses) -> void
+
       def initialize: (ip_family? family, options options) -> void
 
       def early_resolve: (Connection connection, ?hostname: String) -> void

data/sig/resolver.rbs

@@ -30,6 +30,6 @@ module HTTPX
 
     def self?.encode_dns_query: (String hostname, ?type: dns_resource) -> String
 
-    def self?.decode_dns_answer: (String) -> Array[dns_result]
+    def self?.decode_dns_answer: (String) -> Array[dns_result]?
   end
 end

data/sig/response.rbs

@@ -48,13 +48,14 @@ module HTTPX
       include _ToS
       include _ToStr
 
+      attr_reader encoding: String
+
       @response: Response
       @headers: Headers
       @options: Options
       @state: :idle | :memory | :buffer | :closed
       @threshold_size: Integer
       @window_size: Integer
-      @encoding: String
       @length: Integer
       @buffer: StringIO | Tempfile | nil
 
@@ -63,6 +64,8 @@ module HTTPX
       def each: () { (String) -> void } -> void
               | () -> Enumerable[String]
 
+      def filename: () -> String?
+
       def bytesize: () -> (Integer | Float)
       def empty?: () -> bool
       def copy_to: (String | File | _Writer destination) -> void

data/sig/utils.rbs

@@ -9,5 +9,7 @@ module HTTPX
     def self?.elapsed_time: (Integer | Float monotonic_time) -> Float
 
     def self?.to_uri: (generic_uri uri) -> URI::Generic
+
+    def self?.get_filename: (String header) -> String?
   end
 end