httpx 0.20.5 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47766fc5b4fb9cb70e7b0feb8333656b987a25aea262c645c36de53e7cc3f464
-  data.tar.gz: 2f49417d21841b803d9f44b76c199785a54b7983c7712cf8a39b4837900e892b
+  metadata.gz: c62cb1d027ee7e62770de459b00411614372578117ea56354df0d9114cf4397b
+  data.tar.gz: 18275725fc8adeac596f02f83f034dfcfe24a7e4c96ec34bc11400e8ca0e9567
 SHA512:
-  metadata.gz: f74fd61a16e738fbe668312e14d7c658af062f4f3dbcf65360b33892e27f31a524e7b675136de5e7693e327ba9a388c74c995e43b4e49a4593ec63fa385d07df
-  data.tar.gz: b04ae701b05663bd8c94d74416fd6a124d3a700e5860065d373f452383cc6f48a5569e8696dfac5a7f398824c163b3ddb62553a56c465022dc35ac34e627a479
+  metadata.gz: 840bf59a4cbbb45d26a836ccd4060007470d380314b4aedea58e49cb434b050a8252f7db3823daf9c2dfdb5d54075215e4ee1216c80852afbb7abfd57e1dfea2
+  data.tar.gz: 35a666b92236240e8ccd09b1dc29a2ccf83061222b0511d35eee6fb7f3794dc737ca86ac9ca7d87ba01e03d8e648b07e6e52290b02b25674bd783b7f9d2cfbce

data/doc/release_notes/0_13_0.md

@@ -34,7 +34,7 @@ HTTPX.get("http://example.com", addresses: %w[172.5.3.1 172.5.3.2]))
 You should also use it to connect to HTTP servers bound to a UNIX socket, in which case you'll have to provide a path:
 
 ```ruby
-HTTPX.get("http://example.com", addresses: %w[/path/to/usocket]))
+HTTPX.get("http://example.com", transport: "unix", addresses: %w[/path/to/usocket]))
 ```
 
 The `:transport_options` are therefore deprecated, and will be moved in a major version.

data/doc/release_notes/0_21_0.md

@@ -0,0 +1,94 @@
+# 0.21.0
+
+## Features
+
+### `:write_timeout`, `:read_timeout` and `:request_timeout`
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Timeouts
+
+The following timeouts are now supported:
+
+* `:write_timeout`: total time (in seconds) to write a request to the server;
+* `:read_timeout`: total time (in seconds) to read aa response from the server;
+* `:request_timeout`: tracks both of the above (time to write the request and read a response);
+
+```ruby
+HTTPX.with(timeout: { request_timeout: 60}).get(...
+```
+
+Just like `:connect_timeout`, the new timeouts are deadline-oriented, rather than op-oriented, meaning that they do not reset on each socket operation (as most ruby HTTP clients do).
+
+None of them has a default value, in order not to break integrations, but that'll change in a future v1, where they'll become the default timeouts.
+
+### Circuit Breaker plugin
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Circuit-Breaker
+
+The `:circuit_breaker` plugin wraps around errors happening when performing HTTP requests, and support options for setting maximum number of attempts before circuit opens (`:circuit_breaker_max_attempts`), period after which attempts should be reset (`:circuit_breaker_reset_attempts_in`), timespan until circuit half-opens (`circuit_breaker_break_in`), respective half-open drip rate (`:circuit_breaker_half_open_drip_rate`), and a callback to do your own check on whether a response has failed, in case you want HTTP level errors to be marked as failed attempts (`:circuit_breaker_break_on`).
+
+```ruby
+http = HTTPX.plugin(:circuit_breaker)
+# that's it!
+http.get(...
+```
+
+### WebDAV plugin
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/WebDav
+
+The `:webdav` introduces some "convenience" methods to perform common WebDAV operations.
+
+```ruby
+webdav = HTTPX.plugin(:webdav, origin: "http://webdav-server")
+              .plugin(:digest_authentication).digest_auth("user", "pass")
+
+res = webdav.put("/file.html", body: "this is the file body")
+res = webdav.copy("/file.html", "/newdir/copy.html")
+# ...
+```
+
+### XML transcoder, `:xml` option and `response.xml`
+
+A new transcoder was added fot the XML mime type, which requires `"nokogiri"` to be installed; it can both serialize Nokogiri nodes in a request, and parse response content into nokogiri nodes:
+
+```ruby
+response = HTTPX.post("https://xml-server.com", xml: Nokogiri::XML("<xml ..."))
+response.xml #=> #(Document:0x16e4 { name = "document", children = ...
+```
+
+## Improvements
+
+### `:proxy` plugin: `:no_proxy` option
+
+Support was added, in the `:proxy` plugin, to declare domains, either via regexp patterns, or strings, for which requests should bypass the proxy.
+
+```ruby
+http = HTTPX.plugin(:proxy).with_proxy(
+    uri: "http://10.10.0.1:51432",
+    no_proxy: ["gitlab.local", /*.google.com/]
+)
+http.get("https://duckduckgo.com/?q=httpx") #=> proxied
+http.get("https://google.com/?q=httpx") #=> not proxied
+http.get("https://gitlab.com") #=> proxied
+http.get("https://gitlab.local") #=> not proxied
+```
+
+### OOTB support for other JSON libraries
+
+If one of `multi_json`, `oj` or `yajl` is available, all `httpx` operations doing JSON parsing or dumping will use it (the `json` standard library will be used otherwise).
+
+```ruby
+require "oj"
+require "httpx"
+
+response = HTTPX.post("https://somedomain.json", json: { "foo" => "bar" }) # will use "oj"
+puts response.json # will use "oj"
+```
+
+## Bugfixes
+
+* `:expect` plugin: `:expect_timeout` can accept floats (not just integers).
+
+## Chore
+
+* DoH `:https` resolver: support was removed for the "application/dns-json" mime-type (it was only supported in practice by the Google DoH resolver, which has since added support for the standardized "application/dns-message").

data/lib/httpx/connection/http1.rb

@@ -118,7 +118,7 @@ module HTTPX
       log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
 
       @request.response = response
-      on_complete if response.complete?
+      on_complete if response.finished?
     end
 
     def on_trailers(h)
@@ -158,6 +158,7 @@ module HTTPX
       @request = nil
       @requests.shift
       response = request.response
+      response.finish!
       emit(:response, request, response)
 
       if @parser.upgrade?

data/lib/httpx/connection.rb

@@ -34,6 +34,7 @@ module HTTPX
     include Callbacks
 
     using URIExtensions
+    using NumericExtensions
 
     require "httpx/connection/http2"
     require "httpx/connection/http1"
@@ -233,6 +234,7 @@ module HTTPX
           # when pushing a request into an existing connection, we have to check whether there
           # is the possibility that the connection might have extended the keep alive timeout.
           # for such cases, we want to ping for availability before deciding to shovel requests.
+          log(level: 3) { "keep alive timeout expired, pinging connection..." }
           @pending << request
           parser.ping
           transition(:active) if @state == :inactive
@@ -430,6 +432,8 @@ module HTTPX
       @inflight += 1
       parser.send(request)
 
+      set_request_timeouts(request)
+
       return unless @state == :inactive
 
       transition(:active)
@@ -573,7 +577,7 @@ module HTTPX
           error = ex
         else
           # inactive connections do not contribute to the select loop, therefore
-          # they should fail due to such errors.
+          # they should not fail due to such errors.
           return if @state == :inactive
 
           if @timeout
@@ -591,10 +595,45 @@ module HTTPX
     def handle_error(error)
       parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
       while (request = @pending.shift)
-        response = ErrorResponse.new(request, error, @options)
+        response = ErrorResponse.new(request, error, request.options)
         request.response = response
         request.emit(:response, response)
       end
     end
+
+    def set_request_timeouts(request)
+      write_timeout = request.write_timeout
+      request.once(:headers) do
+        @timers.after(write_timeout) { write_timeout_callback(request, write_timeout) }
+      end unless write_timeout.nil? || write_timeout.infinite?
+
+      read_timeout = request.read_timeout
+      request.once(:done) do
+        @timers.after(read_timeout) { read_timeout_callback(request, read_timeout) }
+      end unless read_timeout.nil? || read_timeout.infinite?
+
+      request_timeout = request.request_timeout
+      request.once(:headers) do
+        @timers.after(request_timeout) { read_timeout_callback(request, request_timeout, RequestTimeoutError) }
+      end unless request_timeout.nil? || request_timeout.infinite?
+    end
+
+    def write_timeout_callback(request, write_timeout)
+      return if request.state == :done
+
+      @write_buffer.clear
+      error = WriteTimeoutError.new(request, nil, write_timeout)
+      on_error(error)
+    end
+
+    def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
+      response = request.response
+
+      return if response && response.finished?
+
+      @write_buffer.clear
+      error = error_type.new(request, request.response, read_timeout)
+      on_error(error)
+    end
   end
 end

data/lib/httpx/errors.rb

@@ -24,6 +24,24 @@ module HTTPX
 
   class ConnectTimeoutError < TimeoutError; end
 
+  class RequestTimeoutError < TimeoutError
+    attr_reader :request
+
+    def initialize(request, response, timeout)
+      @request = request
+      @response = response
+      super(timeout, "Timed out after #{timeout} seconds")
+    end
+
+    def marshal_dump
+      [message]
+    end
+  end
+
+  class ReadTimeoutError < RequestTimeoutError; end
+
+  class WriteTimeoutError < RequestTimeoutError; end
+
   class SettingsTimeoutError < TimeoutError; end
 
   class ResolveTimeoutError < TimeoutError; end

data/lib/httpx/extensions.rb

@@ -54,6 +54,14 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module NumericExtensions
+    refine Numeric do
+      def infinite?
+        self == Float::INFINITY
+      end unless Numeric.method_defined?(:infinite?)
+    end
+  end
+
   module StringExtensions
     refine String do
       def delete_suffix!(suffix)
@@ -135,10 +143,6 @@ module HTTPX
   end
 
   module RegexpExtensions
-    # If you wonder why this is there: the oauth feature uses a refinement to enhance the
-    # Regexp class locally with #match? , but this is never tested, because ActiveSupport
-    # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
-    # :nocov:
     refine(Regexp) do
       def match?(*args)
         !match(*args).nil?

data/lib/httpx/io/unix.rb

@@ -33,7 +33,7 @@ module HTTPX
       else
         if @options.transport_options
           # :nocov:
-          warn ":#{__method__} is deprecated, use :addresses instead"
+          warn ":transport_options is deprecated, use :addresses instead"
           @path = @options.transport_options[:path]
           # :nocov:
         else

data/lib/httpx/options.rb

@@ -10,6 +10,7 @@ module HTTPX
     OPERATION_TIMEOUT = 60
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
+    READ_TIMEOUT = WRITE_TIMEOUT = REQUEST_TIMEOUT = Float::INFINITY
 
     # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
     ip_address_families = begin
@@ -34,6 +35,9 @@ module HTTPX
         settings_timeout: SETTINGS_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
+        read_timeout: READ_TIMEOUT,
+        write_timeout: WRITE_TIMEOUT,
+        request_timeout: REQUEST_TIMEOUT,
       },
       :headers => {},
       :window_size => WINDOW_SIZE,
@@ -197,7 +201,7 @@ module HTTPX
     end
 
     %i[
-      params form json body ssl http2_settings
+      params form json xml body ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
       io fallback_protocol debug debug_level transport_options resolver_class resolver_options
@@ -206,7 +210,7 @@ module HTTPX
       def_option(method_name)
     end
 
-    REQUEST_IVARS = %i[@params @form @json @body].freeze
+    REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
     private_constant :REQUEST_IVARS
 
     def ==(other)
@@ -263,7 +267,7 @@ module HTTPX
         instance_variables.each do |ivar|
           value = other.instance_variable_get(ivar)
           value = case value
-                  when Symbol, Fixnum, TrueClass, FalseClass # rubocop:disable Lint/UnifiedInteger
+                  when Symbol, Numeric, TrueClass, FalseClass
                     value
                   else
                     value.dup

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::CircuitBreaker
+    #
+    # A circuit is assigned to a given absoolute url or origin.
+    #
+    # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
+    # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
+    # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
+    # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
+    #
+    class Circuit
+      def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
+        @max_attempts = max_attempts
+        @reset_attempts_in = reset_attempts_in
+        @break_in = break_in
+        @circuit_breaker_half_open_drip_rate = 1 - circuit_breaker_half_open_drip_rate
+        @attempts = 0
+        @state = :closed
+      end
+
+      def respond
+        try_close
+
+        case @state
+        when :closed
+          nil
+        when :half_open
+          # return nothing or smth based on ratio
+          return if Random::DEFAULT.rand >= @circuit_breaker_half_open_drip_rate
+
+          @response
+        when :open
+
+          @response
+        end
+      end
+
+      def try_open(response)
+        return unless @state == :closed
+
+        now = Utils.now
+
+        if @attempts.positive?
+          @attempts = 0 if now - @attempted_at > @reset_attempts_in
+        else
+          @attempted_at = now
+        end
+
+        @attempts += 1
+
+        return unless @attempts >= @max_attempts
+
+        @state = :open
+        @opened_at = now
+        @response = response
+      end
+
+      def try_close
+        case @state
+        when :closed
+          nil
+        when :half_open
+          # reset!
+          @attempts = 0
+          @opened_at = @attempted_at = @response = nil
+          @state = :closed
+
+        when :open
+          @state = :half_open if Utils.elapsed_time(@opened_at) > @break_in
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit_store.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins::CircuitBreaker
+  using HTTPX::URIExtensions
+
+  class CircuitStore
+    def initialize(options)
+      @circuits = Hash.new do |h, k|
+        h[k] = Circuit.new(
+          options.circuit_breaker_max_attempts,
+          options.circuit_breaker_reset_attempts_in,
+          options.circuit_breaker_break_in,
+          options.circuit_breaker_half_open_drip_rate
+        )
+      end
+    end
+
+    def try_open(uri, response)
+      circuit = get_circuit_for_uri(uri)
+
+      circuit.try_open(response)
+    end
+
+    # if circuit is open, it'll respond with the stored response.
+    # if not, nil.
+    def try_respond(request)
+      circuit = get_circuit_for_uri(request.uri)
+
+      circuit.respond
+    end
+
+    private
+
+    def get_circuit_for_uri(uri)
+      uri = URI(uri)
+
+      if @circuits.key?(uri.origin)
+        @circuits[uri.origin]
+      else
+        @circuits[uri.to_s]
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements a circuit breaker around connection errors.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/Circuit-Breaker
+    #
+    module CircuitBreaker
+      using URIExtensions
+
+      def self.load_dependencies(*)
+        require_relative "circuit_breaker/circuit"
+        require_relative "circuit_breaker/circuit_store"
+      end
+
+      def self.extra_options(options)
+        options.merge(circuit_breaker_max_attempts: 3, circuit_breaker_reset_attempts_in: 60, circuit_breaker_break_in: 60,
+                      circuit_breaker_half_open_drip_rate: 1)
+      end
+
+      module InstanceMethods
+        def initialize(*)
+          super
+          @circuit_store = CircuitStore.new(@options)
+        end
+
+        def initialize_dup(orig)
+          super
+          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
+        end
+
+        def send_requests(*requests)
+          # @type var short_circuit_responses: Array[response]
+          short_circuit_responses = []
+
+          # run all requests through the circuit breaker, see if the circuit is
+          # open for any of them.
+          real_requests = requests.each_with_object([]) do |req, real_reqs|
+            short_circuit_response = @circuit_store.try_respond(req)
+            if short_circuit_response.nil?
+              real_reqs << req
+              next
+            end
+            short_circuit_responses[requests.index(req)] = short_circuit_response
+          end
+
+          # run requests for the remainder
+          unless real_requests.empty?
+            responses = super(*real_requests)
+
+            real_requests.each_with_index do |request, idx|
+              short_circuit_responses[requests.index(request)] = responses[idx]
+            end
+          end
+
+          short_circuit_responses
+        end
+
+        def on_response(request, response)
+          if response.is_a?(ErrorResponse)
+            case response.error
+            when RequestTimeoutError
+              @circuit_store.try_open(request.uri, response)
+            else
+              @circuit_store.try_open(request.origin, response)
+            end
+          elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
+            @circuit_store.try_open(request.uri, response)
+          end
+
+          super
+        end
+      end
+
+      module OptionsMethods
+        def option_circuit_breaker_max_attempts(value)
+          attempts = Integer(value)
+          raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
+
+          attempts
+        end
+
+        def option_circuit_breaker_reset_attempts_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_break_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_half_open_drip_rate(value)
+          ratio = Float(value)
+          raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
+
+          ratio
+        end
+
+        def option_circuit_breaker_break_on(value)
+          raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+    end
+    register_plugin :circuit_breaker, CircuitBreaker
+  end
+end

data/lib/httpx/plugins/cookies.rb

@@ -42,7 +42,7 @@ module HTTPX
 
         private
 
-        def on_response(reuest, response)
+        def on_response(_request, response)
           if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
 
             log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH

data/lib/httpx/plugins/expect.rb

@@ -22,7 +22,7 @@ module HTTPX
 
       module OptionsMethods
         def option_expect_timeout(value)
-          seconds = Integer(value)
+          seconds = Float(value)
           raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
 
           seconds

data/lib/httpx/plugins/multipart/decoder.rb

@@ -61,7 +61,7 @@ module HTTPX::Plugins
         @state = :idle
       end
 
-      def call(response, _)
+      def call(response, *)
         response.body.each do |chunk|
           @buffer << chunk
 

data/lib/httpx/plugins/proxy.rb

@@ -105,6 +105,10 @@ module HTTPX
           end
           return if @_proxy_uris.empty?
 
+          proxy = options.proxy
+
+          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+
           proxy_opts = { uri: @_proxy_uris.first }
           proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
           proxy_opts
@@ -117,7 +121,9 @@ module HTTPX
           next_proxy = proxy_uris(uri, options)
           raise Error, "Failed to connect to proxy" unless next_proxy
 
-          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
+          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+
+          proxy_options = options.merge(proxy: proxy)
           connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection

data/lib/httpx/plugins/retries.rb

@@ -67,7 +67,7 @@ module HTTPX
         end
 
         def option_retry_on(value)
-          raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+          raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
 
           value
         end