httpx 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6b2befec2e4b0093acd45f7ef9ef448ad41516510710616aded46934f1e3981
-  data.tar.gz: a9858adfacbdc27e1097b98b958f7dd1614f1207c74ec4290a54268df6270f69
+  metadata.gz: 372f89cdf05727a32d23467503299ae77831cb5d23356467dc0c7f3a22bbfd0c
+  data.tar.gz: bbb3df9079bf4f5a449f37f34dd6b1d3208a978e8a8eaa7fdd69ea9d1829ac6c
 SHA512:
-  metadata.gz: 86276d59efaf3a15efe0a27fbd59bff2d005bb3bab83ee9917599854bf46eba1bfbb016b9df55172c41799d1fe82195e4bb7d82c008c1996814ec2e393be71b3
-  data.tar.gz: bd113e65cf5700f231992bb485f6c59511f114372d53078ea53c019a654e449e5012b2a1a19f889d79cee1eecd24345d3e03f3f9fa50aa6c69060138327e1d09
+  metadata.gz: 38211af1e56fbc823ad0780198579ff11d4fa4b5b64677af95e78361f8b274d0dc43b124697358f1a0b17c2d45e4cdd620f97ca983af7fa60ce59515e9b51785
+  data.tar.gz: 2c9d48f3ad7a499046ab94ac3ee1831f1623ce44990de0f08320ac6e5ce3767ca938381cd7b96c3799477f201a811aa5c446d62f88bd58c6e1e0afd84c53af75

data/doc/release_notes/0_18_0.md

@@ -0,0 +1,69 @@
+# 0.18.0
+
+## Features
+
+### Response Cache
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Response-Cache
+
+The `:response_cache` plugin handles transparent usage of HTTP caching and conditional requests to improve performance and bandwidth usage.
+
+```ruby
+client = HTTPX.plugin(:response_cache)
+r1 = client.get("https://nghttp2.org/httpbin/cache")
+r2 = client.get("https://nghttp2.org/httpbin/cache")
+
+r1.status #=> 200
+r2.status #=> 304
+r1.body == r2.body #=> true
+```
+
+### jitter on "retry-after"
+
+On the `:retries` plugin, jitter calculation is now applied to the value in seconds defined by user after which a request should be retried (i.e. if `:retry_after` option is set to `2`, the retry interval may be `1.5422312` seconds, for example). This is important to avoid cases of synchronized "thundering herd", where server rejects requests, but they all get retried at the same time because the retry interval is exactly the same.
+
+You can override the jitter calculation function by using the [:retry_jitter](https://gitlab.com/honeyryderchuck/httpx/-/wikis/Retries#retry_jitter) option:
+
+```ruby
+HTTPX.plugin(:retries, retry_after: 2, retry_jitter: ->(interval) { interval + rand }) # interval is 3
+```
+
+You can opt out of this by setting `HTTPX_NO_JITTER=1` environment variable.
+
+### Response#error
+
+`HTTPX::Response#error` was added, to match `HTTPX::Response#error`. It returns an exception for 4xx/5xx responses (`HTTPX::HTTPError`), `nil` otherwise. It allows for end users to write such code:
+
+```ruby
+if (response = HTTPX.get(uri)).error
+  # success
+else
+  #error
+end
+```
+
+## Improvements
+
+* `webmock` adapter: added support for "stub_http_request#to_timeout" (https://gitlab.com/honeyryderchuck/httpx/-/merge_requests/165).
+
+## timers not a dependency
+
+The functionality provided by the `timers` gem was replaced by a simpler custom implementation. Although powerful, its complexity was somewhat unnecessary for `httpx`'s simpler event loop, where user-defined timeouts are usually the same for a given batch of requests. The removal of `timers` reduces the number of dependencies to 1, which is `http-2-next` and is maintained by me.
+
+## AWS plugins
+
+* `aws_sdk_authentication` plugin: removed implementation relying on  `aws-sdk-s3`, replacing it with an `aws-sdk-core` relying implementation, which only uses credentials strategies and region discovery (the whole point of this SDK is to use a minimal subset of AWS SDK).
+
+
+## Bugfixes
+
+* Fixed Error class declaration on response decoders when mime type is invalid (https://gitlab.com/honeyryderchuck/httpx/-/merge_requests/166).
+* `ErrorResponse#to_s` now removes ANSI escape sequences from error backtraces.
+* Persistent connections were kept around both in the pool and in the selector; the first is necessary, but the second caused busy loop scenarios all over; they are now removed when no requests are being handled.
+* Connections which failed connection handshake were removed from the pool, but not from the selector list, causing busy loop scenarios in a few cases; this has been fixed.
+* Fixed issue where HTTP/2 streams were being closed twice (and signaling it also twice), messing connection accounting in the pool.
+* DoH resolver was always subscribed to the default thread "connection pool", which broke scenarios were session was patched to use its custom pool; it now ensures that it subscribes to the pool it was created in.
+* `:aws_sigv4` plugin: removed require of `aws-sdk-s3`, left there by mistake (the whole point of the plugin is to run without the AWS SDK).
+## Chore
+
+* `HTTPX::ErrorResponse#status` is now deprecated.

data/lib/httpx/adapters/datadog.rb

@@ -64,7 +64,7 @@ module Datadog
           def finish(response)
             return unless @span
 
-            if response.respond_to?(:error)
+            if response.is_a?(::HTTPX::ErrorResponse)
               @span.set_error(response.error)
             else
               @span.set_tag(Datadog::Ext::HTTP::STATUS_CODE, response.status.to_s)

data/lib/httpx/adapters/faraday.rb

@@ -22,6 +22,8 @@ module Faraday
       # :nocov:
 
       module RequestMixin
+        using ::HTTPX::HashExtensions
+
         private
 
         def build_request(env)
@@ -38,7 +40,7 @@ module Faraday
           timeout_options = {
             connect_timeout: env.request.open_timeout,
             operation_timeout: env.request.timeout,
-          }.reject { |_, v| v.nil? }
+          }.compact
 
           options = {
             ssl: {},
@@ -101,7 +103,7 @@ module Faraday
           end
 
           def on_response(&blk)
-            if block_given?
+            if blk
               @on_response = lambda do |response|
                 blk.call(response)
               end
@@ -112,7 +114,7 @@ module Faraday
           end
 
           def on_complete(&blk)
-            if block_given?
+            if blk
               @on_complete = blk
               self
             else

data/lib/httpx/adapters/webmock.rb

@@ -86,7 +86,9 @@ module WebMock
         end
 
         def _build_from_webmock_response(request, webmock_response)
-          return ErrorResponse.new(request, webmock_response.exception, request.options) if webmock_response.exception
+          return _build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
+
+          return _build_error_response(request, webmock_response.exception) if webmock_response.exception
 
           response = request.options.response_class.new(request,
                                                         webmock_response.status[0],
@@ -95,6 +97,10 @@ module WebMock
           response << webmock_response.body.dup
           response
         end
+
+        def _build_error_response(request, exception)
+          HTTPX::ErrorResponse.new(request, exception, request.options)
+        end
       end
     end
 

data/lib/httpx/altsvc.rb

@@ -10,14 +10,14 @@ module HTTPX
     module_function
 
     def cached_altsvc(origin)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       @altsvc_mutex.synchronize do
         lookup(origin, now)
       end
     end
 
     def cached_altsvc_set(origin, entry)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       @altsvc_mutex.synchronize do
         return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
 

data/lib/httpx/chainable.rb

@@ -4,9 +4,9 @@ module HTTPX
   module Chainable
     %i[head get post put delete trace options connect patch].each do |meth|
       class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-        def #{meth}(*uri, **options)
-          request(:#{meth}, uri, **options)
-        end
+        def #{meth}(*uri, **options)        # def get(*uri, **options)
+          request(:#{meth}, uri, **options) #   request(:get, uri, **options)
+        end                                 # end
       MOD
     end
 

data/lib/httpx/connection/http1.rb

@@ -283,10 +283,10 @@ module HTTPX
         # on the last request of the possible batch (either allowed max requests,
         # or if smaller, the size of the batch itself)
         requests_limit = [@max_requests, @requests.size].min
-        if request != @requests[requests_limit - 1]
-          "keep-alive"
-        else
+        if request == @requests[requests_limit - 1]
           "close"
+        else
+          "keep-alive"
         end
       end
 

data/lib/httpx/connection/http2.rb

@@ -291,11 +291,13 @@ module HTTPX
     end
 
     def on_stream_refuse(stream, request, error)
-      stream.close
       on_stream_close(stream, request, error)
+      stream.close
     end
 
     def on_stream_close(stream, request, error)
+      return if error == :stream_closed && !@streams.key?(request)
+
       log(level: 2) { "#{stream.id}: closing stream" }
       @drains.delete(request)
       @streams.delete(request)
@@ -388,10 +390,10 @@ module HTTPX
     end
 
     def on_pong(ping)
-      if !@pings.delete(ping.to_s)
-        close(:protocol_error, "ping payload did not match")
-      else
+      if @pings.delete(ping.to_s)
         emit(:pong)
+      else
+        close(:protocol_error, "ping payload did not match")
       end
     end
   end

data/lib/httpx/connection.rb

@@ -70,7 +70,7 @@ module HTTPX
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-      @keep_alive_timer = nil
+      @total_timeout = @options.timeout[:total_timeout]
 
       self.addresses = @options.addresses if @options.addresses
     end
@@ -200,11 +200,9 @@ module HTTPX
     end
 
     def close
-      @parser.close if @parser
-      return unless @keep_alive_timer
+      transition(:active) if @state == :inactive
 
-      @keep_alive_timer.cancel
-      remove_instance_variable(:@keep_alive_timer)
+      @parser.close if @parser
     end
 
     def reset
@@ -216,26 +214,40 @@ module HTTPX
     def send(request)
       if @parser && !@write_buffer.full?
         request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-        if @keep_alive_timer
+
+        if @response_received_at && @keep_alive_timeout &&
+           Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
           # is the possibility that the connection might have extended the keep alive timeout.
           # for such cases, we want to ping for availability before deciding to shovel requests.
-          if @keep_alive_timer.fires_in.negative?
-            @pending << request
-            parser.ping
-            return
-          end
-
-          @keep_alive_timer.pause
+          @pending << request
+          parser.ping
+          transition(:active) if @state == :inactive
+          return
         end
-        @inflight += 1
-        parser.send(request)
+
+        send_request_to_parser(request)
       else
         @pending << request
       end
     end
 
     def timeout
+      if @total_timeout
+        return @total_timeout unless @connected_at
+
+        elapsed_time = @total_timeout - Utils.elapsed_time(@connected_at)
+
+        if elapsed_time.negative?
+          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
+          ex.set_backtrace(caller)
+          on_error(@total_timeout)
+          return
+        end
+
+        return elapsed_time
+      end
+
       return @timeout if defined?(@timeout)
 
       return @options.timeout[:connect_timeout] if @state == :idle
@@ -243,6 +255,14 @@ module HTTPX
       @options.timeout[:operation_timeout]
     end
 
+    def deactivate
+      transition(:inactive)
+    end
+
+    def open?
+      @state == :open || @state == :inactive
+    end
+
     private
 
     def connect
@@ -379,9 +399,7 @@ module HTTPX
 
     def send_pending
       while !@write_buffer.full? && (request = @pending.shift)
-        @inflight += 1
-        @keep_alive_timer.pause if @keep_alive_timer
-        parser.send(request)
+        send_request_to_parser(request)
       end
     end
 
@@ -389,6 +407,15 @@ module HTTPX
       @parser ||= build_parser
     end
 
+    def send_request_to_parser(request)
+      @inflight += 1
+      parser.send(request)
+
+      return unless @state == :inactive
+
+      transition(:active)
+    end
+
     def build_parser(protocol = @io.protocol)
       parser = registry(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
@@ -400,7 +427,8 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           emit(:altsvc, alt_origin, origin, alt_params)
         end
-        handle_response
+        @response_received_at = Utils.now
+        @inflight -= 1
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
@@ -420,7 +448,7 @@ module HTTPX
       end
       parser.on(:close) do |force|
         transition(:closing)
-        if force
+        if force || @state == :idle
           transition(:closed)
           emit(:close)
         end
@@ -466,15 +494,17 @@ module HTTPX
       when :open
         return if @state == :closed
 
-        total_timeout
-
         @io.connect
         return unless @io.connected?
 
+        @connected_at = Utils.now
+
         send_pending
 
         @timeout = @current_timeout = parser.timeout
         emit(:open)
+      when :inactive
+        return unless @state == :open
       when :closing
         return unless @state == :open
 
@@ -482,15 +512,15 @@ module HTTPX
         return unless @state == :closing
         return unless @write_buffer.empty?
 
-        if @total_timeout
-          @total_timeout.cancel
-          remove_instance_variable(:@total_timeout)
-        end
-
         purge_after_closed
       when :already_open
         nextstate = :open
         send_pending
+      when :active
+        return unless @state == :inactive
+
+        nextstate = :open
+        emit(:activate)
       end
       @state = nextstate
     rescue Errno::ECONNREFUSED,
@@ -506,44 +536,24 @@ module HTTPX
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
-      if @keep_alive_timer
-        @keep_alive_timer.cancel
-        remove_instance_variable(:@keep_alive_timer)
-      end
-
       remove_instance_variable(:@timeout) if defined?(@timeout)
     end
 
-    def handle_response
-      @inflight -= 1
-      return unless @inflight.zero?
-
-      if @keep_alive_timer
-        @keep_alive_timer.resume
-        @keep_alive_timer.reset
-      else
-        @keep_alive_timer = @timers.after(@keep_alive_timeout) do
-          unless @inflight.zero?
-            log { "(#{@origin}): keep alive timeout expired" }
-            parser.ping
-          end
-        end
-      end
-    end
-
     def on_error(error)
       if error.instance_of?(TimeoutError)
-        if @timeout
-          @timeout -= error.timeout
-          return unless @timeout <= 0
-        end
 
-        if @total_timeout && @total_timeout.fires_in.negative?
-          ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
+        if @total_timeout && @connected_at &&
+           Utils.elapsed_time(@connected_at) > @total_timeout
+          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
           ex.set_backtrace(error.backtrace)
           error = ex
-        elsif connecting?
-          error = error.to_connection_error
+        else
+          if @timeout
+            @timeout -= error.timeout
+            return unless @timeout <= 0
+          end
+
+          error = error.to_connection_error if connecting?
         end
       end
       handle_error(error)
@@ -558,18 +568,5 @@ module HTTPX
         request.emit(:response, response)
       end
     end
-
-    def total_timeout
-      total = @options.timeout[:total_timeout]
-
-      return unless total
-
-      @total_timeout ||= @timers.after(total) do
-        ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
-        ex.set_backtrace(caller)
-        on_error(ex)
-        @parser.close if @parser
-      end
-    end
   end
 end

data/lib/httpx/domain_name.rb

@@ -123,7 +123,7 @@ module HTTPX
 
       # RFC 6265 #4.1.1
       # Domain-value must be a subdomain.
-      @domain && self <= domain && domain <= @domain ? true : false
+      @domain && self <= domain && domain <= @domain
     end
 
     # def ==(other)

data/lib/httpx/extensions.rb

@@ -54,6 +54,51 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module HashExtensions
+    refine Hash do
+      def compact
+        h = {}
+        each do |key, value|
+          h[key] = value unless value == nil
+        end
+        h
+      end unless Hash.method_defined?(:compact)
+    end
+  end
+
+  module ArrayExtensions
+    refine Array do
+
+      def filter_map
+        return to_enum(:filter_map) unless block_given?
+
+        each_with_object([]) do |item, res|
+          processed = yield(item)
+          res << processed if processed
+        end
+      end unless Array.method_defined?(:filter_map)
+
+      def sum(accumulator = 0, &block)
+        values = block_given? ? map(&block) : self
+        values.inject(accumulator, :+)
+      end unless Array.method_defined?(:sum)
+    end
+  end
+
+  module IOExtensions
+    refine IO do
+      # provides a fallback for rubies where IO#wait isn't implemented,
+      # but IO#wait_readable and IO#wait_writable are.
+      def wait(timeout = nil, _mode = :read_write)
+        r, w = IO.select([self], [self], nil, timeout)
+
+        return unless r || w
+
+        self
+      end unless IO.method_defined?(:wait) && IO.instance_method(:wait).arity == 2
+    end
+  end
+
   module RegexpExtensions
     # If you wonder why this is there: the oauth feature uses a refinement to enhance the
     # Regexp class locally with #match? , but this is never tested, because ActiveSupport
@@ -77,13 +122,14 @@ module HTTPX
       end
 
       def authority
-        port_string = port == default_port ? nil : ":#{port}"
-        "#{host}#{port_string}"
-      end
+        return host if port == default_port
+
+        "#{host}:#{port}"
+      end unless URI::HTTP.method_defined?(:authority)
 
       def origin
         "#{scheme}://#{authority}"
-      end
+      end unless URI::HTTP.method_defined?(:origin)
 
       def altsvc_match?(uri)
         uri = URI.parse(uri)

data/lib/httpx/io/ssl.rb

@@ -134,7 +134,7 @@ module HTTPX
       server_cert = @io.peer_cert
 
       "#{super}\n\n" \
-      "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
+        "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
         "Server certificate:\n" \
         " subject: #{server_cert.subject}\n" \

data/lib/httpx/io/tls.rb

@@ -194,15 +194,15 @@ module HTTPX
       server_cert = @peer_cert
 
       "#{super}\n\n" \
-        "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
-        "ALPN, server accepted to use #{protocol}\n" +
+      "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
+      "ALPN, server accepted to use #{protocol}\n" +
         (if server_cert
            "Server certificate:\n" \
-                 " subject: #{server_cert.subject}\n" \
-                 " start date: #{server_cert.not_before}\n" \
-                 " expire date: #{server_cert.not_after}\n" \
-                 " issuer: #{server_cert.issuer}\n" \
-                 " SSL certificate verify ok."
+             " subject: #{server_cert.subject}\n" \
+             " start date: #{server_cert.not_before}\n" \
+             " expire date: #{server_cert.not_after}\n" \
+             " issuer: #{server_cert.issuer}\n" \
+             " SSL certificate verify ok."
          else
            "SSL certificate verify failed."
          end

data/lib/httpx/loggable.rb

@@ -24,15 +24,13 @@ module HTTPX
       debug_stream << message
     end
 
-    if !Exception.instance_methods.include?(:full_message)
+    if Exception.instance_methods.include?(:full_message)
 
       def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        message = +"#{ex.message} (#{ex.class})"
-        message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
-        log(level: level, color: color) { message }
+        log(level: level, color: color) { ex.full_message }
       end
 
     else
@@ -41,7 +39,9 @@ module HTTPX
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        log(level: level, color: color) { ex.full_message }
+        message = +"#{ex.message} (#{ex.class})"
+        message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
+        log(level: level, color: color) { message }
       end
 
     end

data/lib/httpx/options.rb

@@ -81,9 +81,9 @@ module HTTPX
       end
 
       def def_option(optname, *args, &block)
-        if args.size.zero? && !block_given?
+        if args.size.zero? && !block
           class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(v); v; end
+            def option_#{optname}(v); v; end # def option_smth(v); v; end
           OUT
           return
         end
@@ -93,15 +93,15 @@ module HTTPX
 
       def deprecated_def_option(optname, layout = nil, &interpreter)
         warn "DEPRECATION WARNING: using `def_option(#{optname})` for setting options is deprecated. " \
-          "Define module OptionsMethods and `def option_#{optname}(val)` instead."
+             "Define module OptionsMethods and `def option_#{optname}(val)` instead."
 
         if layout
           class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(value)
-              #{layout}
-            end
+            def option_#{optname}(value)  # def option_origin(v)
+              #{layout}                   #   URI(v)
+            end                           # end
           OUT
-        elsif block_given?
+        elsif interpreter
           define_method(:"option_#{optname}") do |value|
             instance_exec(value, &interpreter)
           end