httpx 0.20.5 → 0.21.1

data/lib/httpx/plugins/circuit_breaker.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements a circuit breaker around connection errors.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/Circuit-Breaker
+    #
+    module CircuitBreaker
+      using URIExtensions
+
+      def self.load_dependencies(*)
+        require_relative "circuit_breaker/circuit"
+        require_relative "circuit_breaker/circuit_store"
+      end
+
+      def self.extra_options(options)
+        options.merge(circuit_breaker_max_attempts: 3, circuit_breaker_reset_attempts_in: 60, circuit_breaker_break_in: 60,
+                      circuit_breaker_half_open_drip_rate: 1)
+      end
+
+      module InstanceMethods
+        def initialize(*)
+          super
+          @circuit_store = CircuitStore.new(@options)
+        end
+
+        def initialize_dup(orig)
+          super
+          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
+        end
+
+        def send_requests(*requests)
+          # @type var short_circuit_responses: Array[response]
+          short_circuit_responses = []
+
+          # run all requests through the circuit breaker, see if the circuit is
+          # open for any of them.
+          real_requests = requests.each_with_object([]) do |req, real_reqs|
+            short_circuit_response = @circuit_store.try_respond(req)
+            if short_circuit_response.nil?
+              real_reqs << req
+              next
+            end
+            short_circuit_responses[requests.index(req)] = short_circuit_response
+          end
+
+          # run requests for the remainder
+          unless real_requests.empty?
+            responses = super(*real_requests)
+
+            real_requests.each_with_index do |request, idx|
+              short_circuit_responses[requests.index(request)] = responses[idx]
+            end
+          end
+
+          short_circuit_responses
+        end
+
+        def on_response(request, response)
+          if response.is_a?(ErrorResponse)
+            case response.error
+            when RequestTimeoutError
+              @circuit_store.try_open(request.uri, response)
+            else
+              @circuit_store.try_open(request.origin, response)
+            end
+          elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
+            @circuit_store.try_open(request.uri, response)
+          end
+
+          super
+        end
+      end
+
+      module OptionsMethods
+        def option_circuit_breaker_max_attempts(value)
+          attempts = Integer(value)
+          raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
+
+          attempts
+        end
+
+        def option_circuit_breaker_reset_attempts_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_break_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_half_open_drip_rate(value)
+          ratio = Float(value)
+          raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
+
+          ratio
+        end
+
+        def option_circuit_breaker_break_on(value)
+          raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+    end
+    register_plugin :circuit_breaker, CircuitBreaker
+  end
+end

data/lib/httpx/plugins/cookies.rb

@@ -42,7 +42,7 @@ module HTTPX
 
         private
 
-        def on_response(reuest, response)
+        def on_response(_request, response)
           if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
 
             log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH

data/lib/httpx/plugins/expect.rb

@@ -22,7 +22,7 @@ module HTTPX
 
       module OptionsMethods
         def option_expect_timeout(value)
-          seconds = Integer(value)
+          seconds = Float(value)
           raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
 
           seconds

data/lib/httpx/plugins/multipart/decoder.rb

@@ -61,7 +61,7 @@ module HTTPX::Plugins
         @state = :idle
       end
 
-      def call(response, _)
+      def call(response, *)
         response.body.each do |chunk|
           @buffer << chunk
 

data/lib/httpx/plugins/proxy.rb

@@ -105,6 +105,10 @@ module HTTPX
           end
           return if @_proxy_uris.empty?
 
+          proxy = options.proxy
+
+          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+
           proxy_opts = { uri: @_proxy_uris.first }
           proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
           proxy_opts
@@ -117,7 +121,9 @@ module HTTPX
           next_proxy = proxy_uris(uri, options)
           raise Error, "Failed to connect to proxy" unless next_proxy
 
-          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
+          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+
+          proxy_options = options.merge(proxy: proxy)
           connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection

data/lib/httpx/plugins/retries.rb

@@ -67,7 +67,7 @@ module HTTPX
         end
 
         def option_retry_on(value)
-          raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+          raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
 
           value
         end

data/lib/httpx/plugins/webdav.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for performing WEBDAV requests.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/WEBDAV
+    #
+    module WebDav
+      module InstanceMethods
+        def copy(src, dest)
+          request(:copy, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def move(src, dest)
+          request(:move, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def lock(path, timeout: nil, &blk)
+          headers = {}
+          headers["timeout"] = if timeout && timeout.positive?
+            "Second-#{timeout}"
+          else
+            "Infinite, Second-4100000000"
+          end
+          xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
+                "<D:lockinfo xmlns:D=\"DAV:\">" \
+                "<D:lockscope><D:exclusive/></D:lockscope>" \
+                "<D:locktype><D:write/></D:locktype>" \
+                "<D:owner>null</D:owner>" \
+                "</D:lockinfo>"
+          response = request(:lock, path, headers: headers, xml: xml)
+
+          return response unless blk && response.status == 200
+
+          lock_token = response.headers["lock-token"]
+
+          begin
+            blk.call(response)
+          ensure
+            unlock(path, lock_token)
+          end
+        end
+
+        def unlock(path, lock_token)
+          request(:unlock, path, headers: { "lock-token" => lock_token })
+        end
+
+        def mkcol(dir)
+          request(:mkcol, dir)
+        end
+
+        def propfind(path, xml = nil)
+          body = case xml
+                 when :acl
+                   '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
+                   "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
+                 when nil
+                   '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
+                 else
+                   xml
+          end
+
+          request(:propfind, path, headers: { "depth" => "1" }, xml: body)
+        end
+
+        def proppatch(path, xml)
+          body = "<?xml version=\"1.0\"?>" \
+                 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
+          request(:proppatch, path, xml: body)
+        end
+        # %i[ orderpatch acl report search]
+      end
+    end
+    register_plugin(:webdav, WebDav)
+  end
+end

data/lib/httpx/request.rb

@@ -9,29 +9,6 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
-    METHODS = [
-      # RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
-      :options, :get, :head, :post, :put, :delete, :trace, :connect,
-
-      # RFC 2518: HTTP Extensions for Distributed Authoring -- WEBDAV
-      :propfind, :proppatch, :mkcol, :copy, :move, :lock, :unlock,
-
-      # RFC 3648: WebDAV Ordered Collections Protocol
-      :orderpatch,
-
-      # RFC 3744: WebDAV Access Control Protocol
-      :acl,
-
-      # RFC 6352: vCard Extensions to WebDAV -- CardDAV
-      :report,
-
-      # RFC 5789: PATCH Method for HTTP
-      :patch,
-
-      # draft-reschke-webdav-search: WebDAV Search
-      :search
-    ].freeze
-
     USER_AGENT = "httpx.rb/#{VERSION}"
 
     attr_reader :verb, :uri, :headers, :body, :state, :options, :response
@@ -54,8 +31,6 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
-      raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
-
       @headers = @options.headers_class.new(@options.headers)
       @headers["user-agent"] ||= USER_AGENT
       @headers["accept"]     ||= "*/*"
@@ -64,6 +39,18 @@ module HTTPX
       @state = :idle
     end
 
+    def read_timeout
+      @options.timeout[:read_timeout]
+    end
+
+    def write_timeout
+      @options.timeout[:write_timeout]
+    end
+
+    def request_timeout
+      @options.timeout[:request_timeout]
+    end
+
     def trailers?
       defined?(@trailers)
     end
@@ -108,6 +95,7 @@ module HTTPX
 
     def path
       path = uri.path.dup
+      path =  +"" if path.nil?
       path << "/" if path.empty?
       path << "?#{query}" unless query.empty?
       path
@@ -174,6 +162,8 @@ module HTTPX
           Transcoder.registry("form").encode(options.form)
         elsif options.json
           Transcoder.registry("json").encode(options.json)
+        elsif options.xml
+          Transcoder.registry("xml").encode(options.xml)
         end
         return if @body.nil?
 

data/lib/httpx/resolver/https.rb

@@ -102,7 +102,7 @@ module HTTPX
         @requests[request] = hostname
         resolver_connection.send(request)
         @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
+      rescue ResolveError, Resolv::DNS::EncodeError => e
         @queries.delete(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
@@ -129,7 +129,7 @@ module HTTPX
     def parse(request, response)
       begin
         answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError, JSON::JSONError => e
+      rescue Resolv::DNS::DecodeError => e
         host, connection = @queries.first
         @queries.delete(host)
         emit_resolve_error(connection, connection.origin.host, e)
@@ -203,11 +203,6 @@ module HTTPX
 
     def decode_response_body(response)
       case response.headers["content-type"]
-      when "application/dns-json",
-           "application/json",
-           %r{^application/x-javascript} # because google...
-        payload = JSON.parse(response.to_s)
-        payload["Answer"]
       when "application/dns-udpwireformat",
            "application/dns-message"
         Resolver.decode_dns_answer(response.to_s)

data/lib/httpx/resolver/native.rb

@@ -13,14 +13,14 @@ module HTTPX
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
+      }
     else
       {
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
+      }
     end
 
     # nameservers for ipv6 are misconfigured in certain systems;
@@ -35,6 +35,8 @@ module HTTPX
       end
     end if DEFAULTS[:nameserver]
 
+    DEFAULTS.freeze
+
     DNS_PORT = 53
 
     def_delegator :@connections, :empty?
@@ -77,7 +79,8 @@ module HTTPX
       nil
     rescue Errno::EHOSTUNREACH => e
       @ns_index += 1
-      if @ns_index < @nameserver.size
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
         log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
       else
@@ -151,10 +154,21 @@ module HTTPX
       host = connection.origin.host
       timeout = (@timeouts[host][0] -= loop_time)
 
-      return unless timeout.negative?
+      return unless timeout <= 0
 
       @timeouts[host].shift
-      if @timeouts[host].empty?
+
+      if !@timeouts[host].empty?
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
+      elsif @ns_index + 1 < @nameserver.size
+        # try on the next nameserver
+        @ns_index += 1
+        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        transition(:idle)
+        resolve(connection)
+      else
+
         @timeouts.delete(host)
         @queries.delete(h)
 
@@ -164,9 +178,6 @@ module HTTPX
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
         raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
-      else
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-        resolve(connection)
       end
     end
 

data/lib/httpx/response.rb

@@ -31,6 +31,7 @@ module HTTPX
       @status = Integer(status)
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
+      @finished = complete?
     end
 
     def merge_headers(h)
@@ -41,15 +42,24 @@ module HTTPX
       @body.write(data)
     end
 
+    def content_type
+      @content_type ||= ContentType.new(@headers["content-type"])
+    end
+
+    def finished?
+      @finished
+    end
+
+    def finish!
+      @finished = true
+      @headers.freeze
+    end
+
     def bodyless?
       @request.verb == :head ||
         no_data?
     end
 
-    def content_type
-      @content_type ||= ContentType.new(@headers["content-type"])
-    end
-
     def complete?
       bodyless? || (@request.verb == :connect && @status == 200)
     end
@@ -76,17 +86,21 @@ module HTTPX
       raise err
     end
 
-    def json(options = nil)
-      decode("json", options)
+    def json(*args)
+      decode("json", *args)
     end
 
     def form
       decode("form")
     end
 
+    def xml
+      decode("xml")
+    end
+
     private
 
-    def decode(format, options = nil)
+    def decode(format, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
       transcoder = Transcoder.registry(format)
 
@@ -96,13 +110,13 @@ module HTTPX
 
       raise Error, "no decoder available for \"#{format}\"" unless decoder
 
-      decoder.call(self, options)
+      decoder.call(self, *args)
     rescue Registry::Error
       raise Error, "no decoder available for \"#{format}\""
     end
 
     def no_data?
-      @status < 200 ||
+      @status < 200 || # informational response
         @status == 204 ||
         @status == 205 ||
         @status == 304 || begin
@@ -339,6 +353,10 @@ module HTTPX
       end
     end
 
+    def finished?
+      true
+    end
+
     def raise_for_status
       raise @error
     end

data/lib/httpx/timers.rb

@@ -37,9 +37,12 @@ module HTTPX
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
       @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
+
+      @next_interval_at = nil if @intervals.empty?
     end
 
     def cancel
+      @next_interval_at = nil
       @intervals.clear
     end
 

data/lib/httpx/transcoder/form.rb

@@ -36,7 +36,7 @@ module HTTPX::Transcoder
     module Decoder
       module_function
 
-      def call(response, _)
+      def call(response, *)
         URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
           HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
         end

data/lib/httpx/transcoder/json.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "json"
 
 module HTTPX::Transcoder
   module JSON
@@ -19,7 +18,7 @@ module HTTPX::Transcoder
       def_delegator :@raw, :bytesize
 
       def initialize(json)
-        @raw = ::JSON.dump(json)
+        @raw = JSON.json_dump(json)
         @charset = @raw.encoding.name.downcase
       end
 
@@ -37,8 +36,25 @@ module HTTPX::Transcoder
 
       raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
 
-      ::JSON.method(:parse)
+      method(:json_load)
     end
+
+    # rubocop:disable Style/SingleLineMethods
+    if defined?(MultiJson)
+      def json_load(*args); MultiJson.load(*args); end
+      def json_dump(*args); MultiJson.dump(*args); end
+    elsif defined?(Oj)
+      def json_load(response, *args); Oj.load(response.to_s, *args); end
+      def json_dump(*args); Oj.dump(*args); end
+    elsif defined?(Yajl)
+      def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
+      def json_dump(*args); Yajl::Encoder.encode(*args); end
+    else
+      require "json"
+      def json_load(*args); ::JSON.parse(*args); end
+      def json_dump(*args); ::JSON.dump(*args); end
+    end
+    # rubocop:enable Style/SingleLineMethods
   end
   register "json", JSON
 end

data/lib/httpx/transcoder/xml.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "forwardable"
+require "uri"
+
+module HTTPX::Transcoder
+  module Xml
+    using HTTPX::RegexpExtensions
+
+    module_function
+
+    MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
+
+    class Encoder
+      def initialize(xml)
+        @raw = xml
+      end
+
+      def content_type
+        charset = @raw.respond_to?(:encoding) ? @raw.encoding.to_s.downcase : "utf-8"
+        "application/xml; charset=#{charset}"
+      end
+
+      def bytesize
+        @raw.to_s.bytesize
+      end
+
+      def to_s
+        @raw.to_s
+      end
+    end
+
+    def encode(xml)
+      Encoder.new(xml)
+    end
+
+    begin
+      require "nokogiri"
+
+      # rubocop:disable Lint/DuplicateMethods
+      def decode(response)
+        content_type = response.content_type.mime_type
+
+        raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
+
+        Nokogiri::XML.method(:parse)
+      end
+    rescue LoadError
+      def decode(_response)
+        raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
+      end
+    end
+    # rubocop:enable Lint/DuplicateMethods
+  end
+  register "xml", Xml
+end

data/lib/httpx/transcoder.rb

@@ -90,4 +90,5 @@ end
 require "httpx/transcoder/body"
 require "httpx/transcoder/form"
 require "httpx/transcoder/json"
+require "httpx/transcoder/xml"
 require "httpx/transcoder/chunker"

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.20.5"
+  VERSION = "0.21.1"
 end

data/sig/buffer.rbs

@@ -15,7 +15,7 @@ module HTTPX
     # delegated
     def <<: (string data) -> String
     def empty?: () -> bool
-    def bytesize: () -> Integer
+    def bytesize: () -> (Integer | Float)
     def clear: () -> void
     def replace: (string) -> void
 

data/sig/chainable.rbs

@@ -33,6 +33,7 @@ module HTTPX
               | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session
               | (:grpc, ?options) -> Plugins::grpcSession
               | (:response_cache, ?options) -> Plugins::sessionResponseCache
+              | (:circuit_breaker, ?options) -> Plugins::sessionCircuitBreaker
               | (Symbol | Module, ?options) { (Class) -> void } -> Session
               | (Symbol | Module, ?options) -> Session