httpx 0.20.3 → 0.21.0

data/lib/httpx/plugins/proxy.rb

@@ -105,6 +105,10 @@ module HTTPX
           end
           return if @_proxy_uris.empty?
 
+          proxy = options.proxy
+
+          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+
           proxy_opts = { uri: @_proxy_uris.first }
           proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
           proxy_opts
@@ -117,7 +121,9 @@ module HTTPX
           next_proxy = proxy_uris(uri, options)
           raise Error, "Failed to connect to proxy" unless next_proxy
 
-          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
+          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+
+          proxy_options = options.merge(proxy: proxy)
           connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection

data/lib/httpx/plugins/response_cache/store.rb

@@ -13,35 +13,38 @@ module HTTPX::Plugins
         @store = {}
       end
 
-      def lookup(uri)
-        @store[uri]
+      def lookup(request)
+        responses = @store[request.response_cache_key]
+
+        return unless responses
+
+        response = responses.find(&method(:match_by_vary?).curry(2)[request])
+
+        return unless response && response.fresh?
+
+        response
       end
 
-      def cached?(uri)
-        @store.key?(uri)
+      def cached?(request)
+        lookup(request)
       end
 
-      def cache(uri, response)
-        @store[uri] = response
+      def cache(request, response)
+        return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
+
+        responses = (@store[request.response_cache_key] ||= [])
+
+        responses.reject!(&method(:match_by_vary?).curry(2)[request])
+
+        responses << response
       end
 
       def prepare(request)
-        cached_response = @store[request.uri]
+        cached_response = lookup(request)
 
         return unless cached_response
 
-        original_request = cached_response.instance_variable_get(:@request)
-
-        if (vary = cached_response.headers["vary"])
-          if vary == "*"
-            return unless request.headers.same_headers?(original_request.headers)
-          else
-            return unless vary.split(/ *, */).all? do |cache_field|
-              cache_field.downcase!
-              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
-            end
-          end
-        end
+        return unless match_by_vary?(request, cached_response)
 
         if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
           request.headers.add("if-modified-since", last_modified)
@@ -51,6 +54,23 @@ module HTTPX::Plugins
           request.headers.add("if-none-match", etag)
         end
       end
+
+      private
+
+      def match_by_vary?(request, response)
+        vary = response.vary
+
+        return true unless vary
+
+        original_request = response.instance_variable_get(:@request)
+
+        return request.headers.same_headers?(original_request.headers) if vary == %w[*]
+
+        vary.all? do |cache_field|
+          cache_field.downcase!
+          !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+        end
+      end
     end
   end
 end

data/lib/httpx/plugins/response_cache.rb

@@ -9,7 +9,9 @@ module HTTPX
     #
     module ResponseCache
       CACHEABLE_VERBS = %i[get head].freeze
+      CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
       private_constant :CACHEABLE_VERBS
+      private_constant :CACHEABLE_STATUS_CODES
 
       class << self
         def load_dependencies(*)
@@ -17,14 +19,28 @@ module HTTPX
         end
 
         def cacheable_request?(request)
-          CACHEABLE_VERBS.include?(request.verb)
+          CACHEABLE_VERBS.include?(request.verb) &&
+            (
+              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
+            )
         end
 
         def cacheable_response?(response)
           response.is_a?(Response) &&
-            # partial responses shall not be cached, only full ones.
+            (
+              response.cache_control.nil? ||
+              # TODO: !response.cache_control.include?("private") && is shared cache
+              !response.cache_control.include?("no-store")
+            ) &&
+            CACHEABLE_STATUS_CODES.include?(response.status) &&
+            # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
+            # 410 MAY be stored by a cache and used in reply to a subsequent
+            # request, subject to the expiration mechanism, unless a cache-control
+            # directive prohibits caching. However, a cache that does not support
+            # the Range and Content-Range headers MUST NOT cache 206 (Partial
+            # Content) responses.
             response.status != 206 && (
-            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+            response.headers.key?("etag") || response.headers.key?("last-modified-at") || response.fresh?
           )
         end
 
@@ -52,7 +68,7 @@ module HTTPX
 
         def build_request(*)
           request = super
-          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
 
           @options.response_cache_store.prepare(request)
 
@@ -62,25 +78,99 @@ module HTTPX
         def fetch_response(request, *)
           response = super
 
-          if response && ResponseCache.cached_response?(response)
+          return unless response
+
+          if ResponseCache.cached_response?(response)
             log { "returning cached response for #{request.uri}" }
-            cached_response = @options.response_cache_store.lookup(request.uri)
+            cached_response = @options.response_cache_store.lookup(request)
 
             response.copy_from_cached(cached_response)
-          end
 
-          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+          else
+            @options.response_cache_store.cache(request, response)
+          end
 
           response
         end
       end
 
+      module RequestMethods
+        def response_cache_key
+          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}#{@uri}")
+        end
+      end
+
       module ResponseMethods
         def copy_from_cached(other)
           @body = other.body
 
           @body.__send__(:rewind)
         end
+
+        # A response is fresh if its age has not yet exceeded its freshness lifetime.
+        def fresh?
+          if cache_control
+            return false if cache_control.include?("no-cache")
+
+            # check age: max-age
+            max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
+
+            max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
+
+            max_age = max_age[/age=(\d+)/, 1] if max_age
+
+            max_age = max_age.to_i if max_age
+
+            return max_age > age if max_age
+          end
+
+          # check age: expires
+          if @headers.key?("expires")
+            begin
+              expires = Time.httpdate(@headers["expires"])
+            rescue ArgumentError
+              return true
+            end
+
+            return (expires - Time.now).to_i.positive?
+          end
+
+          true
+        end
+
+        def cache_control
+          return @cache_control if defined?(@cache_control)
+
+          @cache_control = begin
+            return unless @headers.key?("cache-control")
+
+            @headers["cache-control"].split(/ *, */)
+          end
+        end
+
+        def vary
+          return @vary if defined?(@vary)
+
+          @vary = begin
+            return unless @headers.key?("vary")
+
+            @headers["vary"].split(/ *, */)
+          end
+        end
+
+        private
+
+        def age
+          return @headers["age"].to_i if @headers.key?("age")
+
+          (Time.now - date).to_i
+        end
+
+        def date
+          @date ||= Time.httpdate(@headers["date"])
+        rescue NoMethodError, ArgumentError
+          Time.now.httpdate
+        end
       end
     end
     register_plugin :response_cache, ResponseCache

data/lib/httpx/plugins/retries.rb

@@ -67,7 +67,7 @@ module HTTPX
         end
 
         def option_retry_on(value)
-          raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+          raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
 
           value
         end

data/lib/httpx/plugins/webdav.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for performing WEBDAV requests.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/WEBDAV
+    #
+    module WebDav
+      module InstanceMethods
+        def copy(src, dest)
+          request(:copy, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def move(src, dest)
+          request(:move, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def lock(path, timeout: nil, &blk)
+          headers = {}
+          headers["timeout"] = if timeout && timeout.positive?
+            "Second-#{timeout}"
+          else
+            "Infinite, Second-4100000000"
+          end
+          xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
+                "<D:lockinfo xmlns:D=\"DAV:\">" \
+                "<D:lockscope><D:exclusive/></D:lockscope>" \
+                "<D:locktype><D:write/></D:locktype>" \
+                "<D:owner>null</D:owner>" \
+                "</D:lockinfo>"
+          response = request(:lock, path, headers: headers, xml: xml)
+
+          return response unless blk && response.status == 200
+
+          lock_token = response.headers["lock-token"]
+
+          begin
+            blk.call(response)
+          ensure
+            unlock(path, lock_token)
+          end
+        end
+
+        def unlock(path, lock_token)
+          request(:unlock, path, headers: { "lock-token" => lock_token })
+        end
+
+        def mkcol(dir)
+          request(:mkcol, dir)
+        end
+
+        def propfind(path, xml = nil)
+          body = case xml
+                 when :acl
+                   '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
+                   "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
+                 when nil
+                   '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
+                 else
+                   xml
+          end
+
+          request(:propfind, path, headers: { "depth" => "1" }, xml: body)
+        end
+
+        def proppatch(path, xml)
+          body = "<?xml version=\"1.0\"?>" \
+                 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
+          request(:proppatch, path, xml: body)
+        end
+        # %i[ orderpatch acl report search]
+      end
+    end
+    register_plugin(:webdav, WebDav)
+  end
+end

data/lib/httpx/pool.rb

@@ -7,7 +7,7 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
     extend Forwardable
 
     def_delegator :@timers, :after

data/lib/httpx/request.rb

@@ -9,29 +9,6 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
-    METHODS = [
-      # RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
-      :options, :get, :head, :post, :put, :delete, :trace, :connect,
-
-      # RFC 2518: HTTP Extensions for Distributed Authoring -- WEBDAV
-      :propfind, :proppatch, :mkcol, :copy, :move, :lock, :unlock,
-
-      # RFC 3648: WebDAV Ordered Collections Protocol
-      :orderpatch,
-
-      # RFC 3744: WebDAV Access Control Protocol
-      :acl,
-
-      # RFC 6352: vCard Extensions to WebDAV -- CardDAV
-      :report,
-
-      # RFC 5789: PATCH Method for HTTP
-      :patch,
-
-      # draft-reschke-webdav-search: WebDAV Search
-      :search
-    ].freeze
-
     USER_AGENT = "httpx.rb/#{VERSION}"
 
     attr_reader :verb, :uri, :headers, :body, :state, :options, :response
@@ -54,8 +31,6 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
-      raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
-
       @headers = @options.headers_class.new(@options.headers)
       @headers["user-agent"] ||= USER_AGENT
       @headers["accept"]     ||= "*/*"
@@ -64,6 +39,18 @@ module HTTPX
       @state = :idle
     end
 
+    def read_timeout
+      @options.timeout[:read_timeout]
+    end
+
+    def write_timeout
+      @options.timeout[:write_timeout]
+    end
+
+    def request_timeout
+      @options.timeout[:request_timeout]
+    end
+
     def trailers?
       defined?(@trailers)
     end
@@ -108,6 +95,7 @@ module HTTPX
 
     def path
       path = uri.path.dup
+      path =  +"" if path.nil?
       path << "/" if path.empty?
       path << "?#{query}" unless query.empty?
       path
@@ -174,6 +162,8 @@ module HTTPX
           Transcoder.registry("form").encode(options.form)
         elsif options.json
           Transcoder.registry("json").encode(options.json)
+        elsif options.xml
+          Transcoder.registry("xml").encode(options.xml)
         end
         return if @body.nil?
 

data/lib/httpx/resolver/https.rb

@@ -102,7 +102,7 @@ module HTTPX
         @requests[request] = hostname
         resolver_connection.send(request)
         @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
+      rescue ResolveError, Resolv::DNS::EncodeError => e
         @queries.delete(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
@@ -129,7 +129,7 @@ module HTTPX
     def parse(request, response)
       begin
         answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError, JSON::JSONError => e
+      rescue Resolv::DNS::DecodeError => e
         host, connection = @queries.first
         @queries.delete(host)
         emit_resolve_error(connection, connection.origin.host, e)
@@ -203,11 +203,6 @@ module HTTPX
 
     def decode_response_body(response)
       case response.headers["content-type"]
-      when "application/dns-json",
-           "application/json",
-           %r{^application/x-javascript} # because google...
-        payload = JSON.parse(response.to_s)
-        payload["Answer"]
       when "application/dns-udpwireformat",
            "application/dns-message"
         Resolver.decode_dns_answer(response.to_s)

data/lib/httpx/resolver/multi.rb

@@ -6,7 +6,7 @@ require "resolv"
 module HTTPX
   class Resolver::Multi
     include Callbacks
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
 
     attr_reader :resolvers
 

data/lib/httpx/resolver/native.rb

@@ -77,7 +77,8 @@ module HTTPX
       nil
     rescue Errno::EHOSTUNREACH => e
       @ns_index += 1
-      if @ns_index < @nameserver.size
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
         log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
       else

data/lib/httpx/resolver/resolver.rb

@@ -8,6 +8,8 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    using ArrayExtensions::Intersect
+
     RECORD_TYPES = {
       Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
       Socket::AF_INET => Resolv::DNS::Resource::IN::A,
@@ -48,6 +50,10 @@ module HTTPX
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
       end
+
+      # double emission check
+      return if connection.addresses && !addresses.intersect?(connection.addresses)
+
       log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
@@ -56,8 +62,12 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          connection.addresses = addresses
-          emit(:resolve, connection)
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
+
+            connection.addresses = addresses
+            emit(:resolve, connection)
+          end
         end
       else
         connection.addresses = addresses

data/lib/httpx/response.rb

@@ -31,6 +31,7 @@ module HTTPX
       @status = Integer(status)
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
+      @finished = complete?
     end
 
     def merge_headers(h)
@@ -41,15 +42,24 @@ module HTTPX
       @body.write(data)
     end
 
+    def content_type
+      @content_type ||= ContentType.new(@headers["content-type"])
+    end
+
+    def finished?
+      @finished
+    end
+
+    def finish!
+      @finished = true
+      @headers.freeze
+    end
+
     def bodyless?
       @request.verb == :head ||
         no_data?
     end
 
-    def content_type
-      @content_type ||= ContentType.new(@headers["content-type"])
-    end
-
     def complete?
       bodyless? || (@request.verb == :connect && @status == 200)
     end
@@ -76,17 +86,21 @@ module HTTPX
       raise err
     end
 
-    def json(options = nil)
-      decode("json", options)
+    def json(*args)
+      decode("json", *args)
     end
 
     def form
       decode("form")
     end
 
+    def xml
+      decode("xml")
+    end
+
     private
 
-    def decode(format, options = nil)
+    def decode(format, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
       transcoder = Transcoder.registry(format)
 
@@ -96,13 +110,13 @@ module HTTPX
 
       raise Error, "no decoder available for \"#{format}\"" unless decoder
 
-      decoder.call(self, options)
+      decoder.call(self, *args)
     rescue Registry::Error
       raise Error, "no decoder available for \"#{format}\""
     end
 
     def no_data?
-      @status < 200 ||
+      @status < 200 || # informational response
         @status == 204 ||
         @status == 205 ||
         @status == 304 || begin
@@ -310,9 +324,12 @@ module HTTPX
 
   class ErrorResponse
     include Loggable
+    extend Forwardable
 
     attr_reader :request, :error
 
+    def_delegator :@request, :uri
+
     def initialize(request, error, options)
       @request = request
       @error = error
@@ -336,6 +353,10 @@ module HTTPX
       end
     end
 
+    def finished?
+      true
+    end
+
     def raise_for_status
       raise @error
     end

data/lib/httpx/timers.rb

@@ -37,9 +37,12 @@ module HTTPX
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
       @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
+
+      @next_interval_at = nil if @intervals.empty?
     end
 
     def cancel
+      @next_interval_at = nil
       @intervals.clear
     end
 

data/lib/httpx/transcoder/body.rb

@@ -9,7 +9,7 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
-      using HTTPX::ArrayExtensions
+      using HTTPX::ArrayExtensions::Sum
       extend Forwardable
 
       def_delegator :@raw, :to_s

data/lib/httpx/transcoder/form.rb

@@ -36,7 +36,7 @@ module HTTPX::Transcoder
     module Decoder
       module_function
 
-      def call(response, _)
+      def call(response, *)
         URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
           HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
         end