httpx 0.18.7 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab9b9577b0e266b140e21030307b3f2436af346a13f1e0931af05020ba42f7e6
-  data.tar.gz: c5d043448ef41b0532c08d1622540d80c68511ef59cffcc810a6f8bd92f348d5
+  metadata.gz: 1d2af1a18388c76620d8a5b28e0f1d2dd90a4d2d0ee2c514be85ea7ba1b05c45
+  data.tar.gz: ca44331c601c46d51813cd5ccab8b258e94cf517913e26b78716885234e6cf86
 SHA512:
-  metadata.gz: 8b4ec58f2ab031d23f28a86bc47b40d0f0d79eea9aea64590b0cc3d07269738513de1364608e0c0282a6956d816b78b900520dbad4daab1f73522b8b7f73d138
-  data.tar.gz: f2d2c297b530bfb3567ebcd1678bad1d729187dd75c9107839c925def1a708c4648fe80df614a01af02bce309594e256e274e042f3c6b8347182aebff1df9935
+  metadata.gz: eac8b6a85fdd44dda2bcaf000cc7dc5dfa19b075165b52fa709dd8b575e9b4a919293e724242d2449794b74e28cf34c00687c6f99e23baee563c45626429b3c7
+  data.tar.gz: 564f2e8f6d580070eaffa3cda012f0efb032948ce4851b96f947cd5a004d1e8fde9496f29031b341709ccc44040ea3473304697e5d5c7c659b7c2cd286702b78

data/doc/release_notes/0_19_0.md

@@ -0,0 +1,39 @@
+# 0.19.0
+
+## Features
+
+### Happy Eyeballs v2
+
+When the system supports dual-stack networking, `httpx` implements the Happy Eyeballs v2 algorithm (RFC 8305) to resolve hostnames to both IPv6 and IPv4 addresses while privileging IPv6 connectivity. This is implemented by `httpx` both for the `:native` as well as the `:https` (DoH) resolver (which do not perform address sorting, thereby being "DNS-based load-balancing" friendly), and "outsourced" to `getaddrinfo` when using the `:system` resolver.
+
+IPv6 connectivity will also be privileged for `/etc/hosts` local DNS (i.e. `localhost` connections will connec to `::1`).
+
+A new option, `:ip_families`, will also be available (`[Socket::AF_INET6, Socket::AF_INET]` in dual-stack systems). If you'd like to i.e. force IPv4 connectivity, you can do use it (`client = HTTPX.with(ip_families: [Socket::AF_INET])`).
+
+## Improvements
+
+### DNS: :system resolver uses getaddrinfo (instead of the resolver lib)
+
+The `:system` resolver switched to using the `getaddinfo` system function to perform DNS requests. Not only is this call **not** blocking the session event loop anymore (unlike pre-0.19.0 `:system` resolver), it adds a lot of functionality that the stdlib `resolv` library just doesn't support at the moment (such as SRV records).
+
+### HTTP/2 proxy support
+
+The `:proxy` plugin handles "prior-knowledge" HTTP/2 proxies.
+
+```ruby
+HTTPX.plugin(:proxy, fallback_protocol: "h2").with_proxy(uri: "http://http2-proxy:3128").get(...
+```
+
+Connection coalescing has also been enabled for proxied connections (also `CONNECT`-tunneled connections).
+
+### curl-to-httpx
+
+widget in [project website](https://honeyryderchuck.gitlab.io/httpx/) to turn curl commands into the equivalent `httpx` code.
+
+## Bugfixes
+
+* faraday adapter now supports passing session options.
+* proxy: several fixes which enabled env-var (`HTTP(S)_PROXY`) defined proxy support.
+* proxy: fixed graceful recovery from proxy tcp connect errors.
+* several fixes around CNAMEs timeouts with the native resolver.
+* https resolver is now closed when wrapping session closes (it was left open).

data/lib/httpx/adapters/faraday.rb

@@ -81,6 +81,9 @@ module Faraday
 
           def response=(response)
             super
+
+            return if response.is_a?(::HTTPX::ErrorResponse)
+
             response.body.on_data = @response_on_data
           end
         end
@@ -136,7 +139,7 @@ module Faraday
 
           def on_response(&blk)
             if blk
-              @on_response = lambda do |response|
+              @on_response = ->(response) do
                 blk.call(response)
               end
               self
@@ -200,9 +203,9 @@ module Faraday
         end
       end
 
-      def initialize(app)
+      def initialize(app, options = {})
         super(app)
-        @session = Session.new
+        @session = Session.new(options)
       end
 
       def call(env)
@@ -210,6 +213,7 @@ module Faraday
         if parallel?(env)
           handler = env[:parallel_manager].enqueue(env)
           handler.on_response do |response|
+            response.raise_for_status
             save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
               response_headers.merge!(response.headers)
             end

data/lib/httpx/connection.rb

@@ -44,7 +44,7 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :origin, :origins, :state, :pending, :options
+    attr_reader :io, :origin, :origins, :state, :pending, :options
 
     attr_writer :timers
 
@@ -78,7 +78,11 @@ module HTTPX
     # this is a semi-private method, to be used by the resolver
     # to initiate the io object.
     def addresses=(addrs)
-      @io ||= IO.registry(@type).new(@origin, addrs, @options) # rubocop:disable Naming/MemoizedInstanceVariableName
+      if @io
+        @io.add_addresses(addrs)
+      else
+        @io = IO.registry(@type).new(@origin, addrs, @options)
+      end
     end
 
     def addresses
@@ -490,14 +494,14 @@ module HTTPX
 
     def transition(nextstate)
       handle_transition(nextstate)
-   rescue Errno::ECONNREFUSED,
-          Errno::EADDRNOTAVAIL,
-          Errno::EHOSTUNREACH,
-          TLSError => e
-     # connect errors, exit gracefully
-     handle_error(e)
-     @state = :closed
-     emit(:close)
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           TLSError => e
+      # connect errors, exit gracefully
+      handle_error(e)
+      @state = :closed
+      emit(:close)
     end
 
     def handle_transition(nextstate)

data/lib/httpx/extensions.rb

@@ -54,6 +54,22 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module StringExtensions
+    refine String do
+      def delete_suffix!(suffix)
+        suffix = Backports.coerce_to_str(suffix)
+        chomp! if frozen?
+        len = suffix.length
+        if len > 0 && index(suffix, -len)
+          self[-len..-1] = ''
+          self
+        else
+          nil
+        end
+      end unless String.method_defined?(:delete_suffix!)
+    end
+  end
+
   module HashExtensions
     refine Hash do
       def compact

data/lib/httpx/headers.rb

@@ -2,8 +2,6 @@
 
 module HTTPX
   class Headers
-    EMPTY = [].freeze
-
     class << self
       def new(headers = nil)
         return headers if headers.is_a?(self)

data/lib/httpx/io/tcp.rb

@@ -15,6 +15,7 @@ module HTTPX
 
     def initialize(origin, addresses, options)
       @state = :idle
+      @addresses = []
       @hostname = origin.host
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
@@ -30,15 +31,29 @@ module HTTPX
         raise Error, "Given IO objects do not match the request authority" unless @io
 
         _, _, _, @ip = @io.addr
-        @addresses ||= [@ip]
-        @ip_index = @addresses.size - 1
+        @addresses << @ip
         @keep_open = true
         @state = :connected
       else
-        @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+        add_addresses(addresses)
       end
       @ip_index = @addresses.size - 1
-      @io ||= build_socket
+      # @io ||= build_socket
+    end
+
+    def add_addresses(addrs)
+      return if addrs.empty?
+
+      addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+
+      ip_index = @ip_index || (@addresses.size - 1)
+      if addrs.first.ipv6?
+        # should be the next in line
+        @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
+      else
+        @addresses.unshift(*addrs)
+        @ip_index += addrs.size if @ip_index
+      end
     end
 
     def to_io
@@ -52,7 +67,7 @@ module HTTPX
     def connect
       return unless closed?
 
-      if @io.closed?
+      if !@io || @io.closed?
         transition(:idle)
         @io = build_socket
       end
@@ -62,12 +77,16 @@ module HTTPX
            Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
 
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
       @ip_index -= 1
+      @io = build_socket
       retry
     rescue Errno::ETIMEDOUT => e
       raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
 
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
       @ip_index -= 1
+      @io = build_socket
       retry
     end
 

data/lib/httpx/options.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "socket"
+
 module HTTPX
   class Options
     WINDOW_SIZE = 1 << 14 # 16K
@@ -9,6 +11,18 @@ module HTTPX
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
 
+    # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
+    ip_address_families = begin
+      list = Socket.ip_address_list
+      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+        [Socket::AF_INET6, Socket::AF_INET]
+      else
+        [Socket::AF_INET]
+      end
+    rescue NotImplementedError
+      [Socket::AF_INET]
+    end
+
     DEFAULT_OPTIONS = {
       :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
@@ -37,6 +51,7 @@ module HTTPX
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
       :resolver_options => { cache: true },
+      :ip_families => ip_address_families,
     }.freeze
 
     begin
@@ -110,20 +125,18 @@ module HTTPX
     end
 
     def initialize(options = {})
-      defaults = DEFAULT_OPTIONS.merge(options)
-      defaults.each do |k, v|
-        next if v.nil?
-
-        begin
-          value = __send__(:"option_#{k}", v)
-          instance_variable_set(:"@#{k}", value)
-        rescue NoMethodError
-          raise Error, "unknown option: #{k}"
-        end
-      end
+      __initialize__(options)
       freeze
     end
 
+    def freeze
+      super
+      @origin.freeze
+      @timeout.freeze
+      @headers.freeze
+      @addresses.freeze
+    end
+
     def option_origin(value)
       URI(value)
     end
@@ -174,6 +187,10 @@ module HTTPX
       Array(value)
     end
 
+    def option_ip_families(value)
+      Array(value)
+    end
+
     %i[
       params form json body ssl http2_settings
       request_class response_class headers_class request_body_class
@@ -249,5 +266,21 @@ module HTTPX
         end
       end
     end
+
+    private
+
+    def __initialize__(options = {})
+      defaults = DEFAULT_OPTIONS.merge(options)
+      defaults.each do |k, v|
+        next if v.nil?
+
+        begin
+          value = __send__(:"option_#{k}", v)
+          instance_variable_set(:"@#{k}", value)
+        rescue NoMethodError
+          raise Error, "unknown option: #{k}"
+        end
+      end
+    end
   end
 end

data/lib/httpx/plugins/cookies.rb

@@ -18,12 +18,6 @@ module HTTPX
         require "httpx/plugins/cookies/set_cookie_parser"
       end
 
-      module OptionsMethods
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
-      end
-
       module InstanceMethods
         extend Forwardable
 
@@ -77,7 +71,7 @@ module HTTPX
       end
 
       module OptionsMethods
-        def initialize(*)
+        def __initialize__(*)
           super
 
           return unless @headers.key?("cookie")
@@ -89,6 +83,10 @@ module HTTPX
             end
           end
         end
+
+        def option_cookies(value)
+          value.is_a?(Jar) ? value : Jar.new(value)
+        end
       end
     end
     register_plugin :cookies, Cookies

data/lib/httpx/plugins/proxy.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "resolv"
-require "ipaddr"
-require "forwardable"
-
 module HTTPX
   class HTTPProxyError < Error; end
 
@@ -117,6 +113,7 @@ module HTTPX
 
         def fetch_response(request, connections, options)
           response = super
+
           if response.is_a?(ErrorResponse) &&
              __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
@@ -251,7 +248,7 @@ module HTTPX
           case nextstate
           when :closing
             # this is a hack so that we can use the super method
-            # and it'll thing that the current state is open
+            # and it'll think that the current state is open
             @state = :open if @state == :connecting
           end
           super

data/lib/httpx/plugins/retries.rb

@@ -96,8 +96,8 @@ module HTTPX
                # rubocop:enable Style/MultilineTernaryOperator
              )
             response.close if response.respond_to?(:close)
-            request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
+            request.retries -= 1
             request.transition(:idle)
 
             retry_after = options.retry_after

data/lib/httpx/pool.rb

@@ -14,7 +14,6 @@ module HTTPX
 
     def initialize
       @resolvers = {}
-      @_resolver_ios = {}
       @timers = Timers.new
       @selector = Selector.new
       @connections = []
@@ -56,9 +55,20 @@ module HTTPX
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
       next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
+
+      # close resolvers
+      outstanding_connections = @connections
+      resolver_connections = @resolvers.each_value.flat_map(&:connections).compact
+      outstanding_connections -= resolver_connections
+
+      return unless outstanding_connections.empty?
+
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
-      end if @connections.empty?
+      end
+      # for https resolver
+      resolver_connections.each(&:close)
+      next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
     end
 
     def init_connection(connection, _options)
@@ -107,11 +117,12 @@ module HTTPX
         return
       end
 
-      resolver = find_resolver_for(connection)
-      resolver << connection
-      return if resolver.empty?
+      find_resolver_for(connection) do |resolver|
+        resolver << connection
+        next if resolver.empty?
 
-      @_resolver_ios[resolver] ||= select_connection(resolver)
+        select_connection(resolver)
+      end
     end
 
     def on_resolver_connection(connection)
@@ -138,12 +149,11 @@ module HTTPX
 
     def on_resolver_close(resolver)
       resolver_type = resolver.class
-      return unless @resolvers[resolver_type] == resolver
+      return if resolver.closed?
 
       @resolvers.delete(resolver_type)
 
       deselect_connection(resolver)
-      @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
 
@@ -174,12 +184,10 @@ module HTTPX
     end
 
     def coalesce_connections(conn1, conn2)
-      if conn1.coalescable?(conn2)
-        conn1.merge(conn2)
-        @connections.delete(conn2)
-      else
-        register_connection(conn2)
-      end
+      return register_connection(conn2) unless conn1.coalescable?(conn2)
+
+      conn1.merge(conn2)
+      @connections.delete(conn2)
     end
 
     def next_timeout
@@ -196,13 +204,25 @@ module HTTPX
       resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
 
       @resolvers[resolver_type] ||= begin
-        resolver = resolver_type.new(connection_options)
-        resolver.pool = self if resolver.respond_to?(:pool=)
-        resolver.on(:resolve, &method(:on_resolver_connection))
-        resolver.on(:error, &method(:on_resolver_error))
-        resolver.on(:close) { on_resolver_close(resolver) }
-        resolver
+        resolver_manager = if resolver_type.multi?
+          Resolver::Multi.new(resolver_type, connection_options)
+        else
+          resolver_type.new(connection_options)
+        end
+        resolver_manager.on(:resolve, &method(:on_resolver_connection))
+        resolver_manager.on(:error, &method(:on_resolver_error))
+        resolver_manager.on(:close, &method(:on_resolver_close))
+        resolver_manager
+      end
+
+      manager = @resolvers[resolver_type]
+
+      (manager.is_a?(Resolver::Multi) && manager.early_resolve(connection)) || manager.resolvers.each do |resolver|
+        resolver.pool = self
+        yield resolver
       end
+
+      manager
     end
   end
 end