httpx-patched 1.6.2.1

data/lib/httpx/plugins/query.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for using the experimental QUERY HTTP method
+    #
+    # https://gitlab.com/os85/httpx/wikis/Query
+    module Query
+      def self.subplugins
+        {
+          retries: QueryRetries,
+        }
+      end
+
+      module InstanceMethods
+        def query(*uri, **options)
+          request("QUERY", uri, **options)
+        end
+      end
+
+      module QueryRetries
+        module InstanceMethods
+          private
+
+          def repeatable_request?(request, options)
+            super || request.verb == "QUERY"
+          end
+        end
+      end
+    end
+
+    register_plugin :query, Query
+  end
+end

data/lib/httpx/plugins/rate_limiter.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when the request:
+    #
+    # * is rate limited;
+    # * when the server is unavailable (503);
+    # * when a 3xx request comes with a "retry-after" value
+    #
+    # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
+    #
+    module RateLimiter
+      class << self
+        RATE_LIMIT_CODES = [429, 503].freeze
+
+        def configure(klass)
+          klass.plugin(:retries,
+                       retry_change_requests: true,
+                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_after: method(:retry_after_rate_limit))
+        end
+
+        def retry_on_rate_limited_response(response)
+          return false unless response.is_a?(Response)
+
+          status = response.status
+
+          RATE_LIMIT_CODES.include?(status)
+        end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def retry_after_rate_limit(_, response)
+          return unless response.is_a?(Response)
+
+          retry_after = response.headers["retry-after"]
+
+          return unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
+      end
+    end
+
+    register_plugin :rate_limiter, RateLimiter
+  end
+end

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module HTTPX::Plugins
+  module ResponseCache
+    # Implementation of a file system based cache store.
+    #
+    # It stores cached responses in a file under a directory pointed by the +dir+
+    # variable (defaults to the default temp directory from the OS), in a custom
+    # format (similar but different from HTTP/1.1 request/response framing).
+    class FileStore
+      CRLF = HTTPX::Connection::HTTP1::CRLF
+
+      attr_reader :dir
+
+      def initialize(dir = Dir.tmpdir)
+        @dir = Pathname.new(dir).join("httpx-response-cache")
+
+        FileUtils.mkdir_p(@dir)
+      end
+
+      def clear
+        FileUtils.rm_rf(@dir)
+      end
+
+      def get(request)
+        path = file_path(request)
+
+        return unless File.exist?(path)
+
+        File.open(path, mode: File::RDONLY | File::BINARY) do |f|
+          f.flock(File::Constants::LOCK_SH)
+
+          read_from_file(request, f)
+        end
+      end
+
+      def set(request, response)
+        path = file_path(request)
+
+        file_exists = File.exist?(path)
+
+        mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
+
+        File.open(path, mode: mode | File::BINARY) do |f|
+          f.flock(File::Constants::LOCK_EX)
+
+          if file_exists
+            cached_response = read_from_file(request, f)
+
+            if cached_response
+              next if cached_response == request.cached_response
+
+              cached_response.close
+
+              f.truncate(0)
+
+              f.rewind
+            end
+          end
+          # cache the request headers
+          f << request.verb << CRLF
+          f << request.uri << CRLF
+
+          request.headers.each do |field, value|
+            f << field << ":" << value << CRLF
+          end
+
+          f << CRLF
+
+          # cache the response
+          f << response.status << CRLF
+          f << response.version << CRLF
+
+          response.headers.each do |field, value|
+            f << field << ":" << value << CRLF
+          end
+
+          f << CRLF
+
+          response.body.rewind
+
+          IO.copy_stream(response.body, f)
+        end
+      end
+
+      private
+
+      def file_path(request)
+        @dir.join(request.response_cache_key)
+      end
+
+      def read_from_file(request, f)
+        # if it's an empty file
+        return if f.eof?
+
+        # read request data
+        verb = f.readline.delete_suffix!(CRLF)
+        uri = f.readline.delete_suffix!(CRLF)
+
+        request_headers = {}
+        while (line = f.readline) != CRLF
+          line.delete_suffix!(CRLF)
+          sep_index = line.index(":")
+
+          field = line.byteslice(0..(sep_index - 1))
+          value = line.byteslice((sep_index + 1)..-1)
+
+          request_headers[field] = value
+        end
+
+        status = f.readline.delete_suffix!(CRLF)
+        version = f.readline.delete_suffix!(CRLF)
+
+        response_headers = {}
+        while (line = f.readline) != CRLF
+          line.delete_suffix!(CRLF)
+          sep_index = line.index(":")
+
+          field = line.byteslice(0..(sep_index - 1))
+          value = line.byteslice((sep_index + 1)..-1)
+
+          response_headers[field] = value
+        end
+
+        original_request = request.options.request_class.new(verb, uri, request.options)
+        original_request.merge_headers(request_headers)
+
+        response = request.options.response_class.new(request, status, version, response_headers)
+        response.original_request = original_request
+        response.finish!
+
+        IO.copy_stream(f, response.body)
+
+        response
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache/store.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  module ResponseCache
+    # Implementation of a thread-safe in-memory cache store.
+    class Store
+      def initialize
+        @store = {}
+        @store_mutex = Thread::Mutex.new
+      end
+
+      def clear
+        @store_mutex.synchronize { @store.clear }
+      end
+
+      def get(request)
+        @store_mutex.synchronize do
+          @store[request.response_cache_key]
+        end
+      end
+
+      def set(request, response)
+        @store_mutex.synchronize do
+          cached_response = @store[request.response_cache_key]
+
+          cached_response.close if cached_response
+
+          @store[request.response_cache_key] = response
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache.rb

@@ -0,0 +1,333 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when certain errors happen.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Response-Cache
+    #
+    module ResponseCache
+      CACHEABLE_VERBS = %w[GET HEAD].freeze
+      CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
+      SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
+      private_constant :CACHEABLE_VERBS
+      private_constant :CACHEABLE_STATUS_CODES
+
+      class << self
+        def load_dependencies(*)
+          require_relative "response_cache/store"
+          require_relative "response_cache/file_store"
+        end
+
+        # whether the +response+ can be stored in the response cache.
+        # (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
+        def cacheable_response?(response)
+          response.is_a?(Response) &&
+            (
+              response.cache_control.nil? ||
+              # TODO: !response.cache_control.include?("private") && is shared cache
+              !response.cache_control.include?("no-store")
+            ) &&
+            CACHEABLE_STATUS_CODES.include?(response.status) &&
+            # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
+            # 410 MAY be stored by a cache and used in reply to a subsequent
+            # request, subject to the expiration mechanism, unless a cache-control
+            # directive prohibits caching. However, a cache that does not support
+            # the Range and Content-Range headers MUST NOT cache 206 (Partial
+            # Content) responses.
+            response.status != 206
+        end
+
+        # whether the +response+
+        def not_modified?(response)
+          response.is_a?(Response) && response.status == 304
+        end
+
+        def extra_options(options)
+          options.merge(
+            supported_vary_headers: SUPPORTED_VARY_HEADERS,
+            response_cache_store: :store,
+          )
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
+      #                            (defaults to {SUPPORTED_VARY_HEADERS}).
+      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
+      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
+      #                          abides by the Cache Store Interface
+      #
+      # The Cache Store Interface requires implementation of the following methods:
+      #
+      # * +#get(request) -> response or nil+
+      # * +#set(request, response) -> void+
+      # * +#clear() -> void+)
+      #
+      module OptionsMethods
+        private
+
+        def option_response_cache_store(value)
+          case value
+          when :store
+            Store.new
+          when :file_store
+            FileStore.new
+          else
+            value
+          end
+        end
+
+        def option_supported_vary_headers(value)
+          Array(value).sort
+        end
+      end
+
+      module InstanceMethods
+        # wipes out all cached responses from the cache store.
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless cacheable_request?(request)
+
+          prepare_cache(request)
+
+          request
+        end
+
+        private
+
+        def send_request(request, *)
+          return request if request.response
+
+          super
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          return unless response
+
+          if ResponseCache.not_modified?(response)
+            log { "returning cached response for #{request.uri}" }
+
+            response.copy_from_cached!
+          elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
+            request.options.response_cache_store.set(request, response) unless response.cached?
+          end
+
+          response
+        end
+
+        # will either assign a still-fresh cached response to +request+, or set up its HTTP
+        # cache invalidation headers in case it's not fresh anymore.
+        def prepare_cache(request)
+          cached_response = request.options.response_cache_store.get(request)
+
+          return unless cached_response && match_by_vary?(request, cached_response)
+
+          cached_response.body.rewind
+
+          if cached_response.fresh?
+            cached_response = cached_response.dup
+            cached_response.mark_as_cached!
+            request.response = cached_response
+            request.emit(:response, cached_response)
+            return
+          end
+
+          request.cached_response = cached_response
+
+          if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
+            request.headers.add("if-modified-since", last_modified)
+          end
+
+          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+            request.headers.add("if-none-match", etag)
+          end
+        end
+
+        def cacheable_request?(request)
+          request.cacheable_verb? &&
+            (
+              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
+            )
+        end
+
+        # whether the +response+ complies with the directives set by the +request+ "vary" header
+        # (true when none is available).
+        def match_by_vary?(request, response)
+          vary = response.vary
+
+          return true unless vary
+
+          original_request = response.original_request
+
+          if vary == %w[*]
+            request.options.supported_vary_headers.each do |field|
+              return false unless request.headers[field] == original_request.headers[field]
+            end
+
+            return true
+          end
+
+          vary.all? do |field|
+            !original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
+          end
+        end
+      end
+
+      module RequestMethods
+        # points to a previously cached Response corresponding to this request.
+        attr_accessor :cached_response
+
+        def initialize(*)
+          super
+          @cached_response = nil
+        end
+
+        def merge_headers(*)
+          super
+          @response_cache_key = nil
+        end
+
+        # returns whether this request is cacheable as per HTTP caching rules.
+        def cacheable_verb?
+          CACHEABLE_VERBS.include?(@verb)
+        end
+
+        # returns a unique cache key as a String identifying this request
+        def response_cache_key
+          @response_cache_key ||= begin
+            keys = [@verb, @uri]
+
+            @options.supported_vary_headers.each do |field|
+              value = @headers[field]
+
+              keys << value if value
+            end
+            Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
+          end
+        end
+      end
+
+      module ResponseMethods
+        attr_writer :original_request
+
+        def initialize(*)
+          super
+          @cached = false
+        end
+
+        # a copy of the request this response was originally cached from
+        def original_request
+          @original_request || @request
+        end
+
+        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
+        def cached?
+          @cached
+        end
+
+        # sets this Response as being duplicated from a previously cached response.
+        def mark_as_cached!
+          @cached = true
+        end
+
+        # eager-copies the response headers and body from {RequestMethods#cached_response}.
+        def copy_from_cached!
+          cached_response = @request.cached_response
+
+          return unless cached_response
+
+          # 304 responses do not have content-type, which are needed for decoding.
+          @headers = @headers.class.new(cached_response.headers.merge(@headers))
+
+          @body = cached_response.body.dup
+
+          @body.rewind
+        end
+
+        # A response is fresh if its age has not yet exceeded its freshness lifetime.
+        # other (#cache_control} directives may influence the outcome, as per the rules
+        # from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
+        def fresh?
+          if cache_control
+            return false if cache_control.include?("no-cache")
+
+            return true if cache_control.include?("immutable")
+
+            # check age: max-age
+            max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
+
+            max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
+
+            max_age = max_age[/age=(\d+)/, 1] if max_age
+
+            max_age = max_age.to_i if max_age
+
+            return max_age > age if max_age
+          end
+
+          # check age: expires
+          if @headers.key?("expires")
+            begin
+              expires = Time.httpdate(@headers["expires"])
+            rescue ArgumentError
+              return false
+            end
+
+            return (expires - Time.now).to_i.positive?
+          end
+
+          false
+        end
+
+        # returns the "cache-control" directives as an Array of String(s).
+        def cache_control
+          return @cache_control if defined?(@cache_control)
+
+          @cache_control = begin
+            return unless @headers.key?("cache-control")
+
+            @headers["cache-control"].split(/ *, */)
+          end
+        end
+
+        # returns the "vary" header value as an Array of (String) headers.
+        def vary
+          return @vary if defined?(@vary)
+
+          @vary = begin
+            return unless @headers.key?("vary")
+
+            @headers["vary"].split(/ *, */).map(&:downcase)
+          end
+        end
+
+        private
+
+        # returns the value of the "age" header as an Integer (time since epoch).
+        # if no "age" of header exists, it returns the number of seconds since {#date}.
+        def age
+          return @headers["age"].to_i if @headers.key?("age")
+
+          (Time.now - date).to_i
+        end
+
+        # returns the value of the "date" header as a Time object
+        def date
+          @date ||= Time.httpdate(@headers["date"])
+        rescue NoMethodError, ArgumentError
+          Time.now
+        end
+      end
+    end
+    register_plugin :response_cache, ResponseCache
+  end
+end