httpx 1.4.4 → 1.5.0

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -1,39 +1,139 @@
 # frozen_string_literal: true
 
 require "pathname"
-require_relative "store"
 
 module HTTPX::Plugins
   module ResponseCache
-    class FileStore < Store
+    # Implementation of a file system based cache store.
+    #
+    # It stores cached responses in a file under a directory pointed by the +dir+
+    # variable (defaults to the default temp directory from the OS), in a custom
+    # format (similar but different from HTTP/1.1 request/response framing).
+    class FileStore
+      CRLF = HTTPX::Connection::HTTP1::CRLF
+
+      attr_reader :dir
+
       def initialize(dir = Dir.tmpdir)
-        @dir = Pathname.new(dir)
+        @dir = Pathname.new(dir).join("httpx-response-cache")
+
+        FileUtils.mkdir_p(@dir)
       end
 
       def clear
-        # delete all files
+        FileUtils.rm_rf(@dir)
       end
 
-      def cached?(request)
-        file_path = @dir.join(request.response_cache_key)
+      def get(request)
+        path = file_path(request)
+
+        return unless File.exist?(path)
+
+        File.open(path, mode: File::RDONLY | File::BINARY) do |f|
+          f.flock(File::Constants::LOCK_SH)
 
-        exist?(file_path)
+          read_from_file(request, f)
+        end
       end
 
-      private
+      def set(request, response)
+        path = file_path(request)
+
+        file_exists = File.exist?(path)
+
+        mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
+
+        File.open(path, mode: mode | File::BINARY) do |f|
+          f.flock(File::Constants::LOCK_EX)
+
+          if file_exists
+            cached_response = read_from_file(request, f)
+
+            if cached_response
+              next if cached_response == request.cached_response
+
+              cached_response.close
+
+              f.truncate(0)
+
+              f.rewind
+            end
+          end
+          # cache the request headers
+          f << request.verb << CRLF
+          f << request.uri << CRLF
+
+          request.headers.each do |field, value|
+            f << field << ":" << value << CRLF
+          end
+
+          f << CRLF
+
+          # cache the response
+          f << response.status << CRLF
+          f << response.version << CRLF
 
-      def _get(request)
-        return unless cached?(request)
+          response.headers.each do |field, value|
+            f << field << ":" << value << CRLF
+          end
 
-        File.open(@dir.join(request.response_cache_key))
+          f << CRLF
+
+          response.body.rewind
+
+          ::IO.copy_stream(response.body, f)
+        end
+      end
+
+      private
+
+      def file_path(request)
+        @dir.join(request.response_cache_key)
       end
 
-      def _set(request, response)
-        file_path = @dir.join(request.response_cache_key)
+      def read_from_file(request, f)
+        # if it's an empty file
+        return if f.eof?
+
+        # read request data
+        verb = f.readline.delete_suffix!(CRLF)
+        uri = f.readline.delete_suffix!(CRLF)
+
+        request_headers = {}
+        while (line = f.readline) != CRLF
+          line.delete_suffix!(CRLF)
+          sep_index = line.index(":")
+
+          field = line.byteslice(0..(sep_index - 1))
+          value = line.byteslice((sep_index + 1)..-1)
+
+          request_headers[field] = value
+        end
+
+        status = f.readline.delete_suffix!(CRLF)
+        version = f.readline.delete_suffix!(CRLF)
+
+        response_headers = {}
+        while (line = f.readline) != CRLF
+          line.delete_suffix!(CRLF)
+          sep_index = line.index(":")
+
+          field = line.byteslice(0..(sep_index - 1))
+          value = line.byteslice((sep_index + 1)..-1)
+
+          response_headers[field] = value
+        end
+
+        original_request = request.options.request_class.new(verb, uri, request.options)
+        original_request.merge_headers(request_headers)
+
+        response = request.options.response_class.new(request, status, version, response_headers)
+        response.original_request = original_request
+        response.finish!
 
-        response.copy_to(file_path)
+        ::IO.copy_stream(f, response.body)
 
-        response.body.rewind
+        response
       end
     end
   end

data/lib/httpx/plugins/response_cache/store.rb

@@ -2,6 +2,7 @@
 
 module HTTPX::Plugins
   module ResponseCache
+    # Implementation of a thread-safe in-memory cache store.
     class Store
       def initialize
         @store = {}
@@ -12,80 +13,19 @@ module HTTPX::Plugins
         @store_mutex.synchronize { @store.clear }
       end
 
-      def lookup(request)
-        responses = _get(request)
-
-        return unless responses
-
-        responses.find(&method(:match_by_vary?).curry(2)[request])
-      end
-
-      def cached?(request)
-        lookup(request)
-      end
-
-      def cache(request, response)
-        return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
-
-        _set(request, response)
-      end
-
-      def prepare(request)
-        cached_response = lookup(request)
-
-        return unless cached_response
-
-        return unless match_by_vary?(request, cached_response)
-
-        if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
-          request.headers.add("if-modified-since", last_modified)
-        end
-
-        if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
-          request.headers.add("if-none-match", etag)
-        end
-      end
-
-      private
-
-      def match_by_vary?(request, response)
-        vary = response.vary
-
-        return true unless vary
-
-        original_request = response.instance_variable_get(:@request)
-
-        return request.headers.same_headers?(original_request.headers) if vary == %w[*]
-
-        vary.all? do |cache_field|
-          cache_field.downcase!
-          !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
-        end
-      end
-
-      def _get(request)
+      def get(request)
         @store_mutex.synchronize do
-          responses = @store[request.response_cache_key]
-
-          return unless responses
-
-          responses.select! do |res|
-            !res.body.closed? && res.fresh?
-          end
-
-          responses
+          @store[request.response_cache_key]
         end
       end
 
-      def _set(request, response)
+      def set(request, response)
         @store_mutex.synchronize do
-          responses = (@store[request.response_cache_key] ||= [])
+          cached_response = @store[request.response_cache_key]
 
-          responses.reject! do |res|
-            res.body.closed? || !res.fresh? || match_by_vary?(request, res)
-          end
+          cached_response.close if cached_response
 
-          responses << response
+          @store[request.response_cache_key] = response
         end
       end
     end

data/lib/httpx/plugins/response_cache.rb

@@ -10,21 +10,18 @@ module HTTPX
     module ResponseCache
       CACHEABLE_VERBS = %w[GET HEAD].freeze
       CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
+      SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
       private_constant :CACHEABLE_VERBS
       private_constant :CACHEABLE_STATUS_CODES
 
       class << self
         def load_dependencies(*)
           require_relative "response_cache/store"
+          require_relative "response_cache/file_store"
         end
 
-        def cacheable_request?(request)
-          CACHEABLE_VERBS.include?(request.verb) &&
-            (
-              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
-            )
-        end
-
+        # whether the +response+ can be stored in the response cache.
+        # (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
         def cacheable_response?(response)
           response.is_a?(Response) &&
             (
@@ -39,82 +36,230 @@ module HTTPX
             # directive prohibits caching. However, a cache that does not support
             # the Range and Content-Range headers MUST NOT cache 206 (Partial
             # Content) responses.
-            response.status != 206 && (
-            response.headers.key?("etag") || response.headers.key?("last-modified") || response.fresh?
-          )
+            response.status != 206
         end
 
-        def cached_response?(response)
+        # whether the +response+
+        def not_modified?(response)
           response.is_a?(Response) && response.status == 304
         end
 
         def extra_options(options)
-          options.merge(response_cache_store: Store.new)
+          options.merge(
+            supported_vary_headers: SUPPORTED_VARY_HEADERS,
+            response_cache_store: :store,
+          )
         end
       end
 
+      # adds support for the following options:
+      #
+      # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
+      #                            (defaults to {SUPPORTED_VARY_HEADERS}).
+      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
+      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
+      #                          abides by the Cache Store Interface
+      #
+      # The Cache Store Interface requires implementation of the following methods:
+      #
+      # * +#get(request) -> response or nil+
+      # * +#set(request, response) -> void+
+      # * +#clear() -> void+)
+      #
       module OptionsMethods
         def option_response_cache_store(value)
-          raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
+          case value
+          when :store
+            Store.new
+          when :file_store
+            FileStore.new
+          else
+            value
+          end
+        end
 
-          value
+        def option_supported_vary_headers(value)
+          Array(value).sort
         end
       end
 
       module InstanceMethods
+        # wipes out all cached responses from the cache store.
         def clear_response_cache
           @options.response_cache_store.clear
         end
 
         def build_request(*)
           request = super
-          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
+          return request unless cacheable_request?(request)
 
-          @options.response_cache_store.prepare(request)
+          prepare_cache(request)
 
           request
         end
 
+        private
+
+        def send_request(request, *)
+          return request if request.response
+
+          super
+        end
+
         def fetch_response(request, *)
           response = super
 
           return unless response
 
-          if ResponseCache.cached_response?(response)
+          if ResponseCache.not_modified?(response)
             log { "returning cached response for #{request.uri}" }
-            cached_response = @options.response_cache_store.lookup(request)
-
-            response.copy_from_cached(cached_response)
 
-          else
-            @options.response_cache_store.cache(request, response)
+            response.copy_from_cached!
+          elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
+            request.options.response_cache_store.set(request, response) unless response.cached?
           end
 
           response
         end
+
+        # will either assign a still-fresh cached response to +request+, or set up its HTTP
+        # cache invalidation headers in case it's not fresh anymore.
+        def prepare_cache(request)
+          cached_response = request.options.response_cache_store.get(request)
+
+          return unless cached_response && match_by_vary?(request, cached_response)
+
+          cached_response.body.rewind
+
+          if cached_response.fresh?
+            cached_response = cached_response.dup
+            cached_response.mark_as_cached!
+            request.response = cached_response
+            request.emit(:response, cached_response)
+            return
+          end
+
+          request.cached_response = cached_response
+
+          if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
+            request.headers.add("if-modified-since", last_modified)
+          end
+
+          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+            request.headers.add("if-none-match", etag)
+          end
+        end
+
+        def cacheable_request?(request)
+          request.cacheable_verb? &&
+            (
+              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
+            )
+        end
+
+        # whether the +response+ complies with the directives set by the +request+ "vary" header
+        # (true when none is available).
+        def match_by_vary?(request, response)
+          vary = response.vary
+
+          return true unless vary
+
+          original_request = response.original_request
+
+          if vary == %w[*]
+            request.options.supported_vary_headers.each do |field|
+              return false unless request.headers[field] == original_request.headers[field]
+            end
+
+            return true
+          end
+
+          vary.all? do |field|
+            !original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
+          end
+        end
       end
 
       module RequestMethods
+        # points to a previously cached Response corresponding to this request.
+        attr_accessor :cached_response
+
+        def initialize(*)
+          super
+          @cached_response = nil
+        end
+
+        def merge_headers(*)
+          super
+          @response_cache_key = nil
+        end
+
+        # returns whether this request is cacheable as per HTTP caching rules.
+        def cacheable_verb?
+          CACHEABLE_VERBS.include?(@verb)
+        end
+
+        # returns a unique cache key as a String identifying this request
         def response_cache_key
-          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
+          @response_cache_key ||= begin
+            keys = [@verb, @uri]
+
+            @options.supported_vary_headers.each do |field|
+              value = @headers[field]
+
+              keys << value if value
+            end
+            Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
+          end
         end
       end
 
       module ResponseMethods
-        def copy_from_cached(other)
+        attr_writer :original_request
+
+        def initialize(*)
+          super
+          @cached = false
+        end
+
+        # a copy of the request this response was originally cached from
+        def original_request
+          @original_request || @request
+        end
+
+        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
+        def cached?
+          @cached
+        end
+
+        # sets this Response as being duplicated from a previously cached response.
+        def mark_as_cached!
+          @cached = true
+        end
+
+        # eager-copies the response headers and body from {RequestMethods#cached_response}.
+        def copy_from_cached!
+          cached_response = @request.cached_response
+
+          return unless cached_response
+
           # 304 responses do not have content-type, which are needed for decoding.
-          @headers = @headers.class.new(other.headers.merge(@headers))
+          @headers = @headers.class.new(cached_response.headers.merge(@headers))
 
-          @body = other.body.dup
+          @body = cached_response.body.dup
 
           @body.rewind
         end
 
         # A response is fresh if its age has not yet exceeded its freshness lifetime.
+        # other (#cache_control} directives may influence the outcome, as per the rules
+        # from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
         def fresh?
           if cache_control
             return false if cache_control.include?("no-cache")
 
+            return true if cache_control.include?("immutable")
+
             # check age: max-age
             max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
 
@@ -132,15 +277,16 @@ module HTTPX
             begin
               expires = Time.httpdate(@headers["expires"])
             rescue ArgumentError
-              return true
+              return false
             end
 
             return (expires - Time.now).to_i.positive?
           end
 
-          true
+          false
         end
 
+        # returns the "cache-control" directives as an Array of String(s).
         def cache_control
           return @cache_control if defined?(@cache_control)
 
@@ -151,24 +297,28 @@ module HTTPX
           end
         end
 
+        # returns the "vary" header value as an Array of (String) headers.
         def vary
           return @vary if defined?(@vary)
 
           @vary = begin
             return unless @headers.key?("vary")
 
-            @headers["vary"].split(/ *, */)
+            @headers["vary"].split(/ *, */).map(&:downcase)
           end
         end
 
         private
 
+        # returns the value of the "age" header as an Integer (time since epoch).
+        # if no "age" of header exists, it returns the number of seconds since {#date}.
         def age
           return @headers["age"].to_i if @headers.key?("age")
 
           (Time.now - date).to_i
         end
 
+        # returns the value of the "date" header as a Time object
         def date
           @date ||= Time.httpdate(@headers["date"])
         rescue NoMethodError, ArgumentError

data/lib/httpx/plugins/retries.rb

@@ -17,7 +17,9 @@ module HTTPX
       # TODO: pass max_retries in a configure/load block
 
       IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
-      RETRYABLE_ERRORS = [
+
+      # subset of retryable errors which are safe to retry when reconnecting
+      RECONNECTABLE_ERRORS = [
         IOError,
         EOFError,
         Errno::ECONNRESET,
@@ -25,12 +27,15 @@ module HTTPX
         Errno::EPIPE,
         Errno::EINVAL,
         Errno::ETIMEDOUT,
-        Parser::Error,
-        TLSError,
-        TimeoutError,
         ConnectionError,
-        Connection::HTTP2::GoawayError,
+        TLSError,
+        Connection::HTTP2::Error,
       ].freeze
+
+      RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        Parser::Error,
+        TimeoutError,
+      ]).freeze
       DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
       if ENV.key?("HTTPX_NO_JITTER")
@@ -88,6 +93,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
         def max_retries(n)
           with(max_retries: n)
         end
@@ -99,16 +105,16 @@ module HTTPX
 
           if response &&
              request.retries.positive? &&
-             __repeatable_request?(request, options) &&
+             repeatable_request?(request, options) &&
              (
                (
-                 response.is_a?(ErrorResponse) && __retryable_error?(response.error)
+                 response.is_a?(ErrorResponse) && retryable_error?(response.error)
                ) ||
                (
                  options.retry_on && options.retry_on.call(response)
                )
              )
-            __try_partial_retry(request, response)
+            try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
             request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
             request.transition(:idle)
@@ -125,9 +131,10 @@ module HTTPX
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               selector.after(retry_after) do
-                if request.response
+                if (response = request.response)
+                  response.finish!
                   # request has terminated abruptly meanwhile
-                  request.emit(:response, request.response)
+                  request.emit(:response, response)
                 else
                   log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                   send_request(request, selector, options)
@@ -142,11 +149,13 @@ module HTTPX
           response
         end
 
-        def __repeatable_request?(request, options)
+        # returns whether +request+ can be retried.
+        def repeatable_request?(request, options)
           IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
         end
 
-        def __retryable_error?(ex)
+        # returns whether the +ex+ exception happend for a retriable request.
+        def retryable_error?(ex)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
@@ -155,11 +164,11 @@ module HTTPX
         end
 
         #
-        # Atttempt to set the request to perform a partial range request.
+        # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and
         # the last response contains some body payload.
         #
-        def __try_partial_retry(request, response)
+        def try_partial_retry(request, response)
           response = response.response if response.is_a?(ErrorResponse)
 
           return unless response
@@ -180,10 +189,13 @@ module HTTPX
       end
 
       module RequestMethods
+        # number of retries left.
         attr_accessor :retries
 
+        # a response partially received before.
         attr_writer :partial_response
 
+        # initializes the request instance, sets the number of retries for the request.
         def initialize(*args)
           super
           @retries = @options.max_retries