httpx 0.18.0 → 0.19.3

data/lib/httpx/connection.rb

@@ -44,7 +44,7 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :origin, :state, :pending, :options
+    attr_reader :io, :origin, :origins, :state, :pending, :options
 
     attr_writer :timers
 
@@ -78,7 +78,11 @@ module HTTPX
     # this is a semi-private method, to be used by the resolver
     # to initiate the io object.
     def addresses=(addrs)
-      @io ||= IO.registry(@type).new(@origin, addrs, @options) # rubocop:disable Naming/MemoizedInstanceVariableName
+      if @io
+        @io.add_addresses(addrs)
+      else
+        @io = IO.registry(@type).new(@origin, addrs, @options)
+      end
     end
 
     def addresses
@@ -117,7 +121,8 @@ module HTTPX
     def coalescable?(connection)
       if @io.protocol == "h2" &&
          @origin.scheme == "https" &&
-         connection.origin.scheme == "https"
+         connection.origin.scheme == "https" &&
+         @io.can_verify_peer?
         @io.verify_hostname(connection.origin.host)
       else
         @origin == connection.origin
@@ -241,7 +246,7 @@ module HTTPX
         if elapsed_time.negative?
           ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
           ex.set_backtrace(caller)
-          on_error(@total_timeout)
+          on_error(ex)
           return
         end
 
@@ -463,6 +468,7 @@ module HTTPX
           transition(:closing)
           transition(:closed)
           emit(:reset)
+
           @parser.reset if @parser
           transition(:idle)
           transition(:open)
@@ -487,6 +493,18 @@ module HTTPX
     end
 
     def transition(nextstate)
+      handle_transition(nextstate)
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           TLSError => e
+      # connect errors, exit gracefully
+      handle_error(e)
+      @state = :closed
+      emit(:close)
+    end
+
+    def handle_transition(nextstate)
       case nextstate
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
@@ -523,14 +541,6 @@ module HTTPX
         emit(:activate)
       end
       @state = nextstate
-    rescue Errno::ECONNREFUSED,
-           Errno::EADDRNOTAVAIL,
-           Errno::EHOSTUNREACH,
-           TLSError => e
-      # connect errors, exit gracefully
-      handle_error(e)
-      @state = :closed
-      emit(:close)
     end
 
     def purge_after_closed
@@ -548,6 +558,10 @@ module HTTPX
           ex.set_backtrace(error.backtrace)
           error = ex
         else
+          # inactive connections do not contribute to the select loop, therefore
+          # they should fail due to such errors.
+          return if @state == :inactive
+
           if @timeout
             @timeout -= error.timeout
             return unless @timeout <= 0

data/lib/httpx/extensions.rb

@@ -54,6 +54,22 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module StringExtensions
+    refine String do
+      def delete_suffix!(suffix)
+        suffix = Backports.coerce_to_str(suffix)
+        chomp! if frozen?
+        len = suffix.length
+        if len > 0 && index(suffix, -len)
+          self[-len..-1] = ''
+          self
+        else
+          nil
+        end
+      end unless String.method_defined?(:delete_suffix!)
+    end
+  end
+
   module HashExtensions
     refine Hash do
       def compact

data/lib/httpx/headers.rb

@@ -2,8 +2,6 @@
 
 module HTTPX
   class Headers
-    EMPTY = [].freeze
-
     class << self
       def new(headers = nil)
         return headers if headers.is_a?(self)

data/lib/httpx/io/ssl.rb

@@ -27,6 +27,10 @@ module HTTPX
       super
     end
 
+    def can_verify_peer?
+      @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
+    end
+
     def verify_hostname(host)
       return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
       return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?

data/lib/httpx/io/tcp.rb

@@ -15,6 +15,7 @@ module HTTPX
 
     def initialize(origin, addresses, options)
       @state = :idle
+      @addresses = []
       @hostname = origin.host
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
@@ -30,15 +31,29 @@ module HTTPX
         raise Error, "Given IO objects do not match the request authority" unless @io
 
         _, _, _, @ip = @io.addr
-        @addresses ||= [@ip]
-        @ip_index = @addresses.size - 1
+        @addresses << @ip
         @keep_open = true
         @state = :connected
       else
-        @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+        add_addresses(addresses)
       end
       @ip_index = @addresses.size - 1
-      @io ||= build_socket
+      # @io ||= build_socket
+    end
+
+    def add_addresses(addrs)
+      return if addrs.empty?
+
+      addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+
+      ip_index = @ip_index || (@addresses.size - 1)
+      if addrs.first.ipv6?
+        # should be the next in line
+        @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
+      else
+        @addresses.unshift(*addrs)
+        @ip_index += addrs.size if @ip_index
+      end
     end
 
     def to_io
@@ -52,20 +67,26 @@ module HTTPX
     def connect
       return unless closed?
 
-      if @io.closed?
+      if !@io || @io.closed?
         transition(:idle)
         @io = build_socket
       end
       try_connect
-    rescue Errno::EHOSTUNREACH => e
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
 
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
       @ip_index -= 1
+      @io = build_socket
       retry
     rescue Errno::ETIMEDOUT => e
       raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
 
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
       @ip_index -= 1
+      @io = build_socket
       retry
     end
 

data/lib/httpx/io/udp.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "socket"
 require "ipaddr"
 
 module HTTPX

data/lib/httpx/options.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "socket"
+
 module HTTPX
   class Options
     WINDOW_SIZE = 1 << 14 # 16K
@@ -9,6 +11,18 @@ module HTTPX
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
 
+    # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
+    ip_address_families = begin
+      list = Socket.ip_address_list
+      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+        [Socket::AF_INET6, Socket::AF_INET]
+      else
+        [Socket::AF_INET]
+      end
+    rescue NotImplementedError
+      [Socket::AF_INET]
+    end
+
     DEFAULT_OPTIONS = {
       :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
@@ -37,6 +51,7 @@ module HTTPX
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
       :resolver_options => { cache: true },
+      :ip_families => ip_address_families,
     }.freeze
 
     begin
@@ -110,20 +125,18 @@ module HTTPX
     end
 
     def initialize(options = {})
-      defaults = DEFAULT_OPTIONS.merge(options)
-      defaults.each do |k, v|
-        next if v.nil?
-
-        begin
-          value = __send__(:"option_#{k}", v)
-          instance_variable_set(:"@#{k}", value)
-        rescue NoMethodError
-          raise Error, "unknown option: #{k}"
-        end
-      end
+      __initialize__(options)
       freeze
     end
 
+    def freeze
+      super
+      @origin.freeze
+      @timeout.freeze
+      @headers.freeze
+      @addresses.freeze
+    end
+
     def option_origin(value)
       URI(value)
     end
@@ -174,6 +187,10 @@ module HTTPX
       Array(value)
     end
 
+    def option_ip_families(value)
+      Array(value)
+    end
+
     %i[
       params form json body ssl http2_settings
       request_class response_class headers_class request_body_class
@@ -249,5 +266,21 @@ module HTTPX
         end
       end
     end
+
+    private
+
+    def __initialize__(options = {})
+      defaults = DEFAULT_OPTIONS.merge(options)
+      defaults.each do |k, v|
+        next if v.nil?
+
+        begin
+          value = __send__(:"option_#{k}", v)
+          instance_variable_set(:"@#{k}", value)
+        rescue NoMethodError
+          raise Error, "unknown option: #{k}"
+        end
+      end
+    end
   end
 end

data/lib/httpx/plugins/cookies.rb

@@ -18,12 +18,6 @@ module HTTPX
         require "httpx/plugins/cookies/set_cookie_parser"
       end
 
-      module OptionsMethods
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
-      end
-
       module InstanceMethods
         extend Forwardable
 
@@ -77,7 +71,7 @@ module HTTPX
       end
 
       module OptionsMethods
-        def initialize(*)
+        def __initialize__(*)
           super
 
           return unless @headers.key?("cookie")
@@ -89,6 +83,10 @@ module HTTPX
             end
           end
         end
+
+        def option_cookies(value)
+          value.is_a?(Jar) ? value : Jar.new(value)
+        end
       end
     end
     register_plugin :cookies, Cookies

data/lib/httpx/plugins/internal_telemetry.rb

@@ -81,7 +81,7 @@ module HTTPX
           super
         end
 
-        def transition(nextstate)
+        def handle_transition(nextstate)
           state = @state
           super
           meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -8,11 +8,25 @@ module HTTPX
       DEFAULT_MIMETYPE = "application/octet-stream"
 
       # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
-      if defined?(MIME::Types)
+      if defined?(FileMagic)
+        MAGIC_NUMBER = 256 * 1024
 
-        def call(_file, filename)
-          mime = MIME::Types.of(filename).first
-          mime.content_type if mime
+        def call(file, _)
+          return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
+
+          mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
+            filemagic.buffer(file.read(MAGIC_NUMBER))
+          end
+
+          file.rewind
+
+          mime
+        end
+      elsif defined?(Marcel)
+        def call(file, filename)
+          return nil if file.eof? # marcel returns "application/octet-stream" for empty files
+
+          Marcel::MimeType.for(file, name: filename)
         end
 
       elsif defined?(MimeMagic)

data/lib/httpx/plugins/proxy/http.rb

@@ -13,7 +13,7 @@ module HTTPX
 
           private
 
-          def transition(nextstate)
+          def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "http"
 
             case nextstate
@@ -23,7 +23,8 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser = ConnectProxyParser.new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+              @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+              @parser.extend(ProxyParser)
               @parser.once(:response, &method(:__http_on_connect))
               @parser.on(:close) { transition(:closing) }
               __http_proxy_connect
@@ -36,7 +37,7 @@ module HTTPX
                 @parser.close
                 @parser = nil
               when :idle
-                @parser = ProxyParser.new(@write_buffer, @options)
+                @parser.callbacks.clear
                 set_parser_callbacks(@parser)
               end
             end
@@ -54,7 +55,7 @@ module HTTPX
               @inflight += 1
               parser.send(connect_request)
             else
-              transition(:connected)
+              handle_transition(:connected)
             end
           end
 
@@ -76,9 +77,11 @@ module HTTPX
           end
         end
 
-        class ProxyParser < Connection::HTTP1
-          def headline_uri(request)
-            request.uri.to_s
+        module ProxyParser
+          def join_headline(request)
+            return super if request.verb == :connect
+
+            "#{request.verb.to_s.upcase} #{request.uri} HTTP/#{@version.join(".")}"
           end
 
           def set_protocol_headers(request)
@@ -91,22 +94,6 @@ module HTTPX
           end
         end
 
-        class ConnectProxyParser < ProxyParser
-          attr_reader :pending
-
-          def headline_uri(request)
-            return super unless request.verb == :connect
-
-            tunnel = request.path
-            log { "establishing HTTP proxy tunnel to #{tunnel}" }
-            tunnel
-          end
-
-          def empty?
-            @requests.reject { |r| r.verb == :connect }.empty? || @requests.all? { |request| !request.response.nil? }
-          end
-        end
-
         class ConnectRequest < Request
           def initialize(uri, _options)
             super(:connect, uri, {})

data/lib/httpx/plugins/proxy/socks4.rb

@@ -27,7 +27,7 @@ module HTTPX
 
           private
 
-          def transition(nextstate)
+          def handle_transition(nextstate)
             return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
 
             case nextstate

data/lib/httpx/plugins/proxy/socks5.rb

@@ -46,7 +46,7 @@ module HTTPX
 
           private
 
-          def transition(nextstate)
+          def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
 
             case nextstate

data/lib/httpx/plugins/proxy.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "resolv"
-require "ipaddr"
-require "forwardable"
-
 module HTTPX
   class HTTPProxyError < Error; end
 
@@ -85,7 +81,7 @@ module HTTPX
             end
             uris
           end
-          options.proxy.merge(uri: @_proxy_uris.first) unless @_proxy_uris.empty?
+          { uri: @_proxy_uris.first } unless @_proxy_uris.empty?
         end
 
         def find_connection(request, connections, options)
@@ -109,12 +105,15 @@ module HTTPX
           return super unless proxy
 
           connection = options.connection_class.new("tcp", uri, options)
-          pool.init_connection(connection, options)
-          connection
+          catch(:coalesced) do
+            pool.init_connection(connection, options)
+            connection
+          end
         end
 
         def fetch_response(request, connections, options)
           response = super
+
           if response.is_a?(ErrorResponse) &&
              __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
@@ -138,10 +137,20 @@ module HTTPX
           error = response.error
           case error
           when NativeResolveError
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            origin = error.connection.origin
+
             # failed resolving proxy domain
-            error.connection.origin.to_s == @_proxy_uris.first
+            origin.host == proxy_uri.host && origin.port == proxy_uri.port
           when ResolveError
-            error.message.end_with?(@_proxy_uris.first)
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            error.message.end_with?(proxy_uri.to_s)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true
@@ -160,7 +169,9 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          @origin = URI(@options.proxy.uri.origin)
+          proxy_uri = URI(@options.proxy.uri)
+          @origin.host = proxy_uri.host
+          @origin.port = proxy_uri.port
         end
 
         def match?(uri, options)
@@ -169,11 +180,20 @@ module HTTPX
           super && @options.proxy == options.proxy
         end
 
-        # should not coalesce connections here, as the IP is the IP of the proxy
-        def coalescable?(*)
+        def coalescable?(connection)
           return super unless @options.proxy
 
-          false
+          if @io.protocol == "h2" &&
+             @origin.scheme == "https" &&
+             connection.origin.scheme == "https" &&
+             @io.can_verify_peer?
+            # in proxied connections, .origin is the proxy ; Given names
+            # are stored in .origins, this is what is used.
+            origin = URI(connection.origins.first)
+            @io.verify_hostname(origin.host)
+          else
+            @origin == connection.origin
+          end
         end
 
         def send(request)
@@ -222,13 +242,13 @@ module HTTPX
           end
         end
 
-        def transition(nextstate)
+        def handle_transition(nextstate)
           return super unless @options.proxy
 
           case nextstate
           when :closing
             # this is a hack so that we can use the super method
-            # and it'll thing that the current state is open
+            # and it'll think that the current state is open
             @state = :open if @state == :connecting
           end
           super

data/lib/httpx/plugins/retries.rb

@@ -12,16 +12,19 @@ module HTTPX
       # TODO: pass max_retries in a configure/load block
 
       IDEMPOTENT_METHODS = %i[get options head put delete].freeze
-      RETRYABLE_ERRORS = [IOError,
-                          EOFError,
-                          Errno::ECONNRESET,
-                          Errno::ECONNABORTED,
-                          Errno::EPIPE,
-                          TLSError,
-                          TimeoutError,
-                          Parser::Error,
-                          Errno::EINVAL,
-                          Errno::ETIMEDOUT].freeze
+      RETRYABLE_ERRORS = [
+        IOError,
+        EOFError,
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        Errno::EINVAL,
+        Errno::ETIMEDOUT,
+        Parser::Error,
+        TLSError,
+        TimeoutError,
+        Connection::HTTP2::GoawayError,
+      ].freeze
       DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
 
       if ENV.key?("HTTPX_NO_JITTER")
@@ -38,7 +41,7 @@ module HTTPX
         def option_retry_after(value)
           # return early if callable
           unless value.respond_to?(:call)
-            value = Integer(value)
+            value = Float(value)
             raise TypeError, ":retry_after must be positive" unless value.positive?
           end
 
@@ -93,8 +96,8 @@ module HTTPX
                # rubocop:enable Style/MultilineTernaryOperator
              )
             response.close if response.respond_to?(:close)
-            request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
+            request.retries -= 1
             request.transition(:idle)
 
             retry_after = options.retry_after