httpx 0.20.1 → 0.20.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d7bcf0e6c97e369caf28c209d551d4f9667745b6a6a530ae2efd328c611e0e5
-  data.tar.gz: 822c0437afcefb35e7d4319f91e71c5b4c81720bd0d7a7763db1d1f8bfbc4b1a
+  metadata.gz: 3ae7fc4057762abf47253c487544c4e3cd07e31326adec158235f1b53130a8db
+  data.tar.gz: 0a09ef99c59c1b3fd1e516066b656b78d888ffd4eed1160c6c9d96f001669410
 SHA512:
-  metadata.gz: 747e7d2ca2a4aee92f1774473e9b38565fec3879ee6b6d0ab3c4eeb1313740c6bbd3648c8c6f21b40903f4bf9bc842647b531968debcda747652afb483577b11
-  data.tar.gz: 88d3bafe38437bebb42960d980dddc85e9e12a917c492203744ded3f93c79736feedf839604cd1a4b080a59c64c96c26c545808cc29d19bcdaa93fa55154bbef
+  metadata.gz: 61c786041e8e27c55d45c30e5f8558b1423470254c5be819c2a9daaa61b8bb049e7e892e0a2673799d26339435fd237fbe772bcb02a0373969e7f382eaa76186
+  data.tar.gz: f24c113213513d847aa693242902d334e1052deaacdfd944def81d91ea9545113ad8a6a973c5b97ec141fc6474f5513b2a3228c9ba644f8560c29806c10c2886

data/doc/release_notes/0_20_2.md

@@ -0,0 +1,7 @@
+# 0.20.2
+
+## Bugfixes
+
+* fix for selector timeout errors closing all connections and ignoring resolvers.
+
+Timeout errors on select were being propagated to all pooled connections, although not all of them were being selected on, and not all of them having timed out. plus, resolver timeouts were doing the same, making connections fail with connection timeout error, rather than resolve timeout error. A patch was implemented, where the selector now yields an error to the selected connections, rather than plain raising exception.

data/doc/release_notes/0_20_3.md

@@ -0,0 +1,6 @@
+# 0.20.3
+
+## Bugfixes
+
+* DoH resolver wasn't working for non-absolute (the large majority) of domains since v0.19.
+* Allowing a single IP string to be passed to the resolver option `:nameserver` (just like the `resolv` library does), besides the already supported list of IPs.

data/doc/release_notes/0_24_4.md

@@ -0,0 +1,17 @@
+# 0.20.4
+
+## Improvements
+
+The `:response_cache` plugin is now more compliant with how the RFC 2616 defines which behaviour caches shall have:
+
+* it caches only responses with one of the following status codes: 200, 203, 300, 301, 410.
+* it discards cached responses which become stale.
+* it supports "cache-control" header directives to decided when to cache, to store, what the response "age" is.
+* it can cache more than one response for the same request, provided that the request presents different header values for the headers declared in the "vary" response header (previously, it was only caching the first response, and discarding the remainder).
+
+
+
+## Bugfixes
+
+* fixed DNS resolution bug which caused a loop when a failed connection attempt would cause a new DNS request to be triggered for the same domain, filling up and giving preference to the very IP which failed the attempt.
+* response_cache: request verb is now taken into account, not causing HEAD/GET confusion for the same URL.

data/lib/httpx/connection.rb

@@ -276,6 +276,12 @@ module HTTPX
       @state == :open || @state == :inactive
     end
 
+    def raise_timeout_error(interval)
+      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
+
     private
 
     def connect

data/lib/httpx/extensions.rb

@@ -83,22 +83,41 @@ module HTTPX
   end
 
   module ArrayExtensions
-    refine Array do
+    module FilterMap
+      refine Array do
 
-      def filter_map
-        return to_enum(:filter_map) unless block_given?
+        def filter_map
+          return to_enum(:filter_map) unless block_given?
 
-        each_with_object([]) do |item, res|
-          processed = yield(item)
-          res << processed if processed
+          each_with_object([]) do |item, res|
+            processed = yield(item)
+            res << processed if processed
+          end
         end
       end unless Array.method_defined?(:filter_map)
+    end
 
-      def sum(accumulator = 0, &block)
-        values = block_given? ? map(&block) : self
-        values.inject(accumulator, :+)
+    module Sum
+      refine Array do
+        def sum(accumulator = 0, &block)
+          values = block_given? ? map(&block) : self
+          values.inject(accumulator, :+)
+        end
       end unless Array.method_defined?(:sum)
     end
+
+    module Intersect
+      refine Array do
+        def intersect?(arr)
+          if size < arr.size
+            smaller = self
+          else
+            smaller, arr = arr, self
+          end
+          (array & smaller).size > 0
+        end
+      end unless Array.method_defined?(:intersect?)
+    end
   end
 
   module IOExtensions

data/lib/httpx/io/ssl.rb

@@ -145,12 +145,12 @@ module HTTPX
       "#{super}\n\n" \
         "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
-        "Server certificate:\n" \
-        " subject: #{server_cert.subject}\n" \
-        " start date: #{server_cert.not_before}\n" \
-        " expire date: #{server_cert.not_after}\n" \
-        " issuer: #{server_cert.issuer}\n" \
-        " SSL certificate verify ok."
+        "Server certificate:\n " \
+        "subject: #{server_cert.subject}\n " \
+        "start date: #{server_cert.not_before}\n " \
+        "expire date: #{server_cert.not_after}\n " \
+        "issuer: #{server_cert.issuer}\n " \
+        "SSL certificate verify ok."
     end
   end
 end

data/lib/httpx/plugins/compression.rb

@@ -72,7 +72,7 @@ module HTTPX
       end
 
       module ResponseBodyMethods
-        using ArrayExtensions
+        using ArrayExtensions::FilterMap
 
         attr_reader :encodings
 

data/lib/httpx/plugins/response_cache/store.rb

@@ -13,35 +13,38 @@ module HTTPX::Plugins
         @store = {}
       end
 
-      def lookup(uri)
-        @store[uri]
+      def lookup(request)
+        responses = @store[request.response_cache_key]
+
+        return unless responses
+
+        response = responses.find(&method(:match_by_vary?).curry(2)[request])
+
+        return unless response && response.fresh?
+
+        response
       end
 
-      def cached?(uri)
-        @store.key?(uri)
+      def cached?(request)
+        lookup(request)
       end
 
-      def cache(uri, response)
-        @store[uri] = response
+      def cache(request, response)
+        return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
+
+        responses = (@store[request.response_cache_key] ||= [])
+
+        responses.reject!(&method(:match_by_vary?).curry(2)[request])
+
+        responses << response
       end
 
       def prepare(request)
-        cached_response = @store[request.uri]
+        cached_response = lookup(request)
 
         return unless cached_response
 
-        original_request = cached_response.instance_variable_get(:@request)
-
-        if (vary = cached_response.headers["vary"])
-          if vary == "*"
-            return unless request.headers.same_headers?(original_request.headers)
-          else
-            return unless vary.split(/ *, */).all? do |cache_field|
-              cache_field.downcase!
-              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
-            end
-          end
-        end
+        return unless match_by_vary?(request, cached_response)
 
         if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
           request.headers.add("if-modified-since", last_modified)
@@ -51,6 +54,23 @@ module HTTPX::Plugins
           request.headers.add("if-none-match", etag)
         end
       end
+
+      private
+
+      def match_by_vary?(request, response)
+        vary = response.vary
+
+        return true unless vary
+
+        original_request = response.instance_variable_get(:@request)
+
+        return request.headers.same_headers?(original_request.headers) if vary == %w[*]
+
+        vary.all? do |cache_field|
+          cache_field.downcase!
+          !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+        end
+      end
     end
   end
 end

data/lib/httpx/plugins/response_cache.rb

@@ -9,7 +9,9 @@ module HTTPX
     #
     module ResponseCache
       CACHEABLE_VERBS = %i[get head].freeze
+      CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
       private_constant :CACHEABLE_VERBS
+      private_constant :CACHEABLE_STATUS_CODES
 
       class << self
         def load_dependencies(*)
@@ -17,14 +19,28 @@ module HTTPX
         end
 
         def cacheable_request?(request)
-          CACHEABLE_VERBS.include?(request.verb)
+          CACHEABLE_VERBS.include?(request.verb) &&
+            (
+              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
+            )
         end
 
         def cacheable_response?(response)
           response.is_a?(Response) &&
-            # partial responses shall not be cached, only full ones.
+            (
+              response.cache_control.nil? ||
+              # TODO: !response.cache_control.include?("private") && is shared cache
+              !response.cache_control.include?("no-store")
+            ) &&
+            CACHEABLE_STATUS_CODES.include?(response.status) &&
+            # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
+            # 410 MAY be stored by a cache and used in reply to a subsequent
+            # request, subject to the expiration mechanism, unless a cache-control
+            # directive prohibits caching. However, a cache that does not support
+            # the Range and Content-Range headers MUST NOT cache 206 (Partial
+            # Content) responses.
             response.status != 206 && (
-            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+            response.headers.key?("etag") || response.headers.key?("last-modified-at") || response.fresh?
           )
         end
 
@@ -52,7 +68,7 @@ module HTTPX
 
         def build_request(*)
           request = super
-          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
 
           @options.response_cache_store.prepare(request)
 
@@ -62,25 +78,99 @@ module HTTPX
         def fetch_response(request, *)
           response = super
 
-          if response && ResponseCache.cached_response?(response)
+          return unless response
+
+          if ResponseCache.cached_response?(response)
             log { "returning cached response for #{request.uri}" }
-            cached_response = @options.response_cache_store.lookup(request.uri)
+            cached_response = @options.response_cache_store.lookup(request)
 
             response.copy_from_cached(cached_response)
-          end
 
-          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+          else
+            @options.response_cache_store.cache(request, response)
+          end
 
           response
         end
       end
 
+      module RequestMethods
+        def response_cache_key
+          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}#{@uri}")
+        end
+      end
+
       module ResponseMethods
         def copy_from_cached(other)
           @body = other.body
 
           @body.__send__(:rewind)
         end
+
+        # A response is fresh if its age has not yet exceeded its freshness lifetime.
+        def fresh?
+          if cache_control
+            return false if cache_control.include?("no-cache")
+
+            # check age: max-age
+            max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
+
+            max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
+
+            max_age = max_age[/age=(\d+)/, 1] if max_age
+
+            max_age = max_age.to_i if max_age
+
+            return max_age > age if max_age
+          end
+
+          # check age: expires
+          if @headers.key?("expires")
+            begin
+              expires = Time.httpdate(@headers["expires"])
+            rescue ArgumentError
+              return true
+            end
+
+            return (expires - Time.now).to_i.positive?
+          end
+
+          true
+        end
+
+        def cache_control
+          return @cache_control if defined?(@cache_control)
+
+          @cache_control = begin
+            return unless @headers.key?("cache-control")
+
+            @headers["cache-control"].split(/ *, */)
+          end
+        end
+
+        def vary
+          return @vary if defined?(@vary)
+
+          @vary = begin
+            return unless @headers.key?("vary")
+
+            @headers["vary"].split(/ *, */)
+          end
+        end
+
+        private
+
+        def age
+          return @headers["age"].to_i if @headers.key?("age")
+
+          (Time.now - date).to_i
+        end
+
+        def date
+          @date ||= Time.httpdate(@headers["date"])
+        rescue NoMethodError, ArgumentError
+          Time.now.httpdate
+        end
       end
     end
     register_plugin :response_cache, ResponseCache

data/lib/httpx/pool.rb

@@ -7,7 +7,7 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
     extend Forwardable
 
     def_delegator :@timers, :after

data/lib/httpx/resolver/https.rb

@@ -39,6 +39,7 @@ module HTTPX
       @uri_addresses = nil
       @resolver = Resolv::DNS.new
       @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
+      @resolver.lazy_initialize
     end
 
     def <<(connection)

data/lib/httpx/resolver/multi.rb

@@ -6,7 +6,7 @@ require "resolv"
 module HTTPX
   class Resolver::Multi
     include Callbacks
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
 
     attr_reader :resolvers
 

data/lib/httpx/resolver/native.rb

@@ -45,7 +45,7 @@ module HTTPX
       super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
-      @nameserver = @resolver_options[:nameserver]
+      @nameserver = Array(@resolver_options[:nameserver]) if @resolver_options[:nameserver]
       @ndots = @resolver_options[:ndots]
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
@@ -118,6 +118,10 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
+    def raise_timeout_error(interval)
+      do_retry(interval)
+    end
+
     private
 
     def calculate_interests
@@ -134,10 +138,10 @@ module HTTPX
       dwrite if calculate_interests == :w
     end
 
-    def do_retry
+    def do_retry(loop_time = nil)
       return if @queries.empty? || !@start_timeout
 
-      loop_time = Utils.elapsed_time(@start_timeout)
+      loop_time ||= Utils.elapsed_time(@start_timeout)
 
       query = @queries.first
 

data/lib/httpx/resolver/resolver.rb

@@ -8,6 +8,8 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    using ArrayExtensions::Intersect
+
     RECORD_TYPES = {
       Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
       Socket::AF_INET => Resolv::DNS::Resource::IN::A,
@@ -48,6 +50,10 @@ module HTTPX
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
       end
+
+      # double emission check
+      return if connection.addresses && !addresses.intersect?(connection.addresses)
+
       log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
@@ -56,8 +62,12 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          connection.addresses = addresses
-          emit(:resolve, connection)
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
+
+            connection.addresses = addresses
+            emit(:resolve, connection)
+          end
         end
       else
         connection.addresses = addresses

data/lib/httpx/response.rb

@@ -56,7 +56,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<Response:#{object_id} "\
+      "#<Response:#{object_id} " \
         "HTTP/#{version} " \
         "@status=#{@status} " \
         "@headers=#{@headers} " \
@@ -310,9 +310,12 @@ module HTTPX
 
   class ErrorResponse
     include Loggable
+    extend Forwardable
 
     attr_reader :request, :error
 
+    def_delegator :@request, :uri
+
     def initialize(request, error, options)
       @request = request
       @error = error

data/lib/httpx/selector.rb

@@ -74,7 +74,10 @@ class HTTPX::Selector
 
       readers, writers = IO.select(r, w, nil, interval)
 
-      raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") if readers.nil? && writers.nil? && interval
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.raise_timeout_error(interval) }
+        return
+      end
     rescue IOError, SystemCallError
       @selectables.reject!(&:closed?)
       retry
@@ -108,7 +111,11 @@ class HTTPX::Selector
              when nil then return
     end
 
-    raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") unless result || interval.nil?
+    unless result || interval.nil?
+      io.raise_timeout_error(interval)
+      return
+    end
+    # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
 
     yield io
   rescue IOError, SystemCallError

data/lib/httpx/transcoder/body.rb

@@ -9,7 +9,7 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
-      using HTTPX::ArrayExtensions
+      using HTTPX::ArrayExtensions::Sum
       extend Forwardable
 
       def_delegator :@raw, :to_s

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.20.1"
+  VERSION = "0.20.4"
 end

data/sig/connection.rbs

@@ -72,6 +72,9 @@ module HTTPX
     def timeout: () -> Numeric?
 
     def deactivate: () -> void
+
+    def raise_timeout_error: (Numeric interval) -> void
+
     private
 
     def initialize: (String, URI::Generic, options) -> untyped

data/sig/plugins/response_cache.rbs

@@ -8,15 +8,19 @@ module HTTPX
       def self?.cached_response?: (response response) -> bool
 
       class Store
-        @store: Hash[URI::Generic, Response]
+        @store: Hash[String, Array[Response]]
 
-        def lookup: (URI::Generic uri) -> Response?
+        def lookup: (Request request) -> Response?
 
-        def cached?: (URI::Generic uri) -> bool
+        def cached?: (Request request) -> boolish
 
-        def cache: (URI::Generic uri, Response response) -> void
+        def cache: (Request request, Response response) -> void
 
         def prepare: (Request request) -> void
+
+        private
+
+        def match_by_vary?: (Request request, Response response) -> bool
       end
 
       module InstanceMethods
@@ -25,8 +29,24 @@ module HTTPX
         def clear_response_cache: () -> void
       end
 
+      module RequestMethods
+        def response_cache_key: () -> String
+      end
+
       module ResponseMethods
         def copy_from_cached: (Response other) -> void
+
+        def fresh?: () -> bool
+
+        def cache_control: () -> Array[String]?
+
+        def vary: () -> Array[String]?
+
+        private
+
+        def age: () -> Integer
+
+        def date: () -> Time
       end
     end
 

data/sig/resolver/native.rbs

@@ -27,6 +27,8 @@ module HTTPX
 
       def timeout: () -> Numeric?
 
+      def raise_timeout_error: (Numeric interval) -> void
+
       private
 
       def initialize: (ip_family family, options options) -> void
@@ -35,7 +37,7 @@ module HTTPX
 
       def consume: () -> void
 
-      def do_retry: () -> void
+      def do_retry: (?Numeric loop_time) -> void
 
       def dread: (Integer) -> void
                | () -> void

data/sig/response.rbs

@@ -96,6 +96,7 @@ module HTTPX
   class ErrorResponse
     include _Response
     include Loggable
+    extend Forwardable
 
     @options: Options
     @error: Exception
@@ -104,6 +105,8 @@ module HTTPX
 
     def status: () -> (Integer | _ToS)
 
+    def uri: () -> URI::Generic
+
     private
 
     def initialize: (Request, Exception, options) -> untyped

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.20.4
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-21 00:00:00.000000000 Z
+date: 2022-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -82,6 +82,9 @@ extra_rdoc_files:
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_20_0.md
 - doc/release_notes/0_20_1.md
+- doc/release_notes/0_20_2.md
+- doc/release_notes/0_20_3.md
+- doc/release_notes/0_24_4.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
 - doc/release_notes/0_3_0.md
@@ -156,6 +159,9 @@ files:
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_20_0.md
 - doc/release_notes/0_20_1.md
+- doc/release_notes/0_20_2.md
+- doc/release_notes/0_20_3.md
+- doc/release_notes/0_24_4.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
 - doc/release_notes/0_3_0.md
@@ -360,7 +366,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: HTTPX, to the future, and beyond