httpx 0.20.0 → 1.3.1

data/lib/httpx/plugins/proxy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class HTTPProxyError < Error; end
+  class HTTPProxyError < ConnectionError; end
 
   module Plugins
     #
@@ -12,12 +12,24 @@ module HTTPX
     # * Socks4/4a proxies
     # * Socks5 proxies
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Proxy
+    # https://gitlab.com/os85/httpx/wikis/Proxy
     #
     module Proxy
       Error = HTTPProxyError
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
+      class << self
+        def configure(klass)
+          klass.plugin(:"proxy/http")
+          klass.plugin(:"proxy/socks4")
+          klass.plugin(:"proxy/socks5")
+        end
+
+        def extra_options(options)
+          options.merge(supported_proxy_protocols: [])
+        end
+      end
+
       class Parameters
         attr_reader :uri, :username, :password, :scheme
 
@@ -41,7 +53,7 @@ module HTTPX
 
           auth_scheme = scheme.to_s.capitalize
 
-          require_relative "authentication/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+          require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
 
           @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
         end
@@ -77,48 +89,35 @@ module HTTPX
         end
       end
 
-      class << self
-        def configure(klass)
-          klass.plugin(:"proxy/http")
-          klass.plugin(:"proxy/socks4")
-          klass.plugin(:"proxy/socks5")
-        end
-      end
-
+      # adds support for the following options:
+      #
+      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
+      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
       module OptionsMethods
         def option_proxy(value)
           value.is_a?(Parameters) ? value : Hash[value]
         end
+
+        def option_supported_proxy_protocols(value)
+          raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
+
+          value.map(&:to_s)
+        end
       end
 
       module InstanceMethods
         private
 
-        def proxy_uris(uri, options)
-          @_proxy_uris ||= begin
-            uris = options.proxy ? Array(options.proxy[:uri]) : []
-            if uris.empty?
-              uri = URI(uri).find_proxy
-              uris << uri if uri
-            end
-            uris
-          end
-          return if @_proxy_uris.empty?
-
-          proxy_opts = { uri: @_proxy_uris.first }
-          proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
-          proxy_opts
-        end
-
         def find_connection(request, connections, options)
           return super unless options.respond_to?(:proxy)
 
-          uri = URI(request.uri)
-          next_proxy = proxy_uris(uri, options)
-          raise Error, "Failed to connect to proxy" unless next_proxy
+          uri = request.uri
+
+          proxy_options = proxy_options(uri, options)
+
+          return super(request, connections, proxy_options) unless proxy_options.proxy
 
-          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
-          connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
+          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection
             set_connection_callbacks(connection, connections, options)
@@ -126,40 +125,61 @@ module HTTPX
           connection
         end
 
-        def build_connection(uri, options)
-          proxy = options.proxy
-          return super unless proxy
+        def proxy_options(request_uri, options)
+          proxy_opts = if (next_proxy = request_uri.find_proxy)
+            { uri: next_proxy }
+          else
+            proxy = options.proxy
+
+            return options unless proxy
+
+            return options.merge(proxy: nil) unless proxy.key?(:uri)
+
+            @_proxy_uris ||= Array(proxy[:uri])
+
+            next_proxy = @_proxy_uris.first
+            raise Error, "Failed to connect to proxy" unless next_proxy
+
+            next_proxy = URI(next_proxy)
+
+            raise Error,
+                  "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
-          connection = options.connection_class.new("tcp", uri, options)
-          catch(:coalesced) do
-            pool.init_connection(connection, options)
-            connection
+            if proxy.key?(:no_proxy)
+
+              no_proxy = proxy[:no_proxy]
+              no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
+
+              return options.merge(proxy: nil) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
+                                                                              next_proxy.port, no_proxy)
+            end
+
+            proxy.merge(uri: next_proxy)
           end
+
+          proxy = Parameters.new(**proxy_opts)
+
+          options.merge(proxy: proxy)
         end
 
         def fetch_response(request, connections, options)
           response = super
 
-          if response.is_a?(ErrorResponse) &&
-             __proxy_error?(response) && !@_proxy_uris.empty?
+          if response.is_a?(ErrorResponse) && proxy_error?(request, response)
             @_proxy_uris.shift
+
+            # return last error response if no more proxies to try
+            return response if @_proxy_uris.empty?
+
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
-            connection = find_connection(request, connections, options)
-            connections << connection unless connections.include?(connection)
-            connection.send(request)
+            send_request(request, connections, options)
             return
           end
           response
         end
 
-        def build_altsvc_connection(_, _, _, _, _, options)
-          return if options.proxy
-
-          super
-        end
-
-        def __proxy_error?(response)
+        def proxy_error?(_request, response)
           error = response.error
           case error
           when NativeResolveError
@@ -216,13 +236,6 @@ module HTTPX
           end
         end
 
-        def send(request)
-          return super unless @options.proxy
-          return super unless connecting?
-
-          @pending << request
-        end
-
         def connecting?
           return super unless @options.proxy
 
@@ -244,13 +257,19 @@ module HTTPX
           return super unless @options.proxy
 
           @state = :open
-          transition(:closing)
-          transition(:closed)
+
+          super
           emit(:close)
         end
 
         private
 
+        def initialize_type(uri, options)
+          return super unless options.proxy
+
+          "tcp"
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -278,7 +297,7 @@ module HTTPX
     register_plugin :proxy, Proxy
   end
 
-  class ProxySSL < IO.registry["ssl"]
+  class ProxySSL < SSL
     def initialize(tcp, request_uri, options)
       @io = tcp.to_io
       super(request_uri, tcp.addresses, options)

data/lib/httpx/plugins/push_promise.rb

@@ -8,7 +8,7 @@ module HTTPX
     # In order to benefit from this, requests are sent one at a time, so that
     # no push responses are received after corresponding request has been sent.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Server-Push
+    # https://gitlab.com/os85/httpx/wikis/Server-Push
     #
     module PushPromise
       def self.extra_options(options)

data/lib/httpx/plugins/rate_limiter.rb

@@ -9,7 +9,7 @@ module HTTPX
     # * when the server is unavailable (503);
     # * when a 3xx request comes with a "retry-after" value
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
       class << self
@@ -23,6 +23,8 @@ module HTTPX
         end
 
         def retry_on_rate_limited_response(response)
+          return false unless response.is_a?(Response)
+
           status = response.status
 
           RATE_LIMIT_CODES.include?(status)
@@ -37,6 +39,8 @@ module HTTPX
         # the redirected request.
         #
         def retry_after_rate_limit(_, response)
+          return unless response.is_a?(Response)
+
           retry_after = response.headers["retry-after"]
 
           return unless retry_after

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "pathname"
+require_relative "store"
+
+module HTTPX::Plugins
+  module ResponseCache
+    class FileStore < Store
+      def initialize(dir = Dir.tmpdir)
+        @dir = Pathname.new(dir)
+      end
+
+      def clear
+        # delete all files
+      end
+
+      def cached?(request)
+        file_path = @dir.join(request.response_cache_key)
+
+        exist?(file_path)
+      end
+
+      private
+
+      def _get(request)
+        return unless cached?(request)
+
+        File.open(@dir.join(request.response_cache_key))
+      end
+
+      def _set(request, response)
+        file_path = @dir.join(request.response_cache_key)
+
+        response.copy_to(file_path)
+
+        response.body.rewind
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache/store.rb

@@ -1,47 +1,41 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module HTTPX::Plugins
   module ResponseCache
     class Store
-      extend Forwardable
-
-      def_delegator :@store, :clear
-
       def initialize
         @store = {}
+        @store_mutex = Thread::Mutex.new
+      end
+
+      def clear
+        @store_mutex.synchronize { @store.clear }
       end
 
-      def lookup(uri)
-        @store[uri]
+      def lookup(request)
+        responses = _get(request)
+
+        return unless responses
+
+        responses.find(&method(:match_by_vary?).curry(2)[request])
       end
 
-      def cached?(uri)
-        @store.key?(uri)
+      def cached?(request)
+        lookup(request)
       end
 
-      def cache(uri, response)
-        @store[uri] = response
+      def cache(request, response)
+        return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
+
+        _set(request, response)
       end
 
       def prepare(request)
-        cached_response = @store[request.uri]
+        cached_response = lookup(request)
 
         return unless cached_response
 
-        original_request = cached_response.instance_variable_get(:@request)
-
-        if (vary = cached_response.headers["vary"])
-          if vary == "*"
-            return unless request.headers.same_headers?(original_request.headers)
-          else
-            return unless vary.split(/ *, */).all? do |cache_field|
-              cache_field.downcase!
-              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
-            end
-          end
-        end
+        return unless match_by_vary?(request, cached_response)
 
         if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
           request.headers.add("if-modified-since", last_modified)
@@ -51,6 +45,49 @@ module HTTPX::Plugins
           request.headers.add("if-none-match", etag)
         end
       end
+
+      private
+
+      def match_by_vary?(request, response)
+        vary = response.vary
+
+        return true unless vary
+
+        original_request = response.instance_variable_get(:@request)
+
+        return request.headers.same_headers?(original_request.headers) if vary == %w[*]
+
+        vary.all? do |cache_field|
+          cache_field.downcase!
+          !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+        end
+      end
+
+      def _get(request)
+        @store_mutex.synchronize do
+          responses = @store[request.response_cache_key]
+
+          return unless responses
+
+          responses.select! do |res|
+            !res.body.closed? && res.fresh?
+          end
+
+          responses
+        end
+      end
+
+      def _set(request, response)
+        @store_mutex.synchronize do
+          responses = (@store[request.response_cache_key] ||= [])
+
+          responses.reject! do |res|
+            res.body.closed? || !res.fresh? || match_by_vary?(request, res)
+          end
+
+          responses << response
+        end
+      end
     end
   end
 end

data/lib/httpx/plugins/response_cache.rb

@@ -5,11 +5,13 @@ module HTTPX
     #
     # This plugin adds support for retrying requests when certain errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
     module ResponseCache
-      CACHEABLE_VERBS = %i[get head].freeze
+      CACHEABLE_VERBS = %w[GET HEAD].freeze
+      CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
       private_constant :CACHEABLE_VERBS
+      private_constant :CACHEABLE_STATUS_CODES
 
       class << self
         def load_dependencies(*)
@@ -17,14 +19,28 @@ module HTTPX
         end
 
         def cacheable_request?(request)
-          CACHEABLE_VERBS.include?(request.verb)
+          CACHEABLE_VERBS.include?(request.verb) &&
+            (
+              !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
+            )
         end
 
         def cacheable_response?(response)
           response.is_a?(Response) &&
-            # partial responses shall not be cached, only full ones.
+            (
+              response.cache_control.nil? ||
+              # TODO: !response.cache_control.include?("private") && is shared cache
+              !response.cache_control.include?("no-store")
+            ) &&
+            CACHEABLE_STATUS_CODES.include?(response.status) &&
+            # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
+            # 410 MAY be stored by a cache and used in reply to a subsequent
+            # request, subject to the expiration mechanism, unless a cache-control
+            # directive prohibits caching. However, a cache that does not support
+            # the Range and Content-Range headers MUST NOT cache 206 (Partial
+            # Content) responses.
             response.status != 206 && (
-            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+            response.headers.key?("etag") || response.headers.key?("last-modified") || response.fresh?
           )
         end
 
@@ -52,7 +68,7 @@ module HTTPX
 
         def build_request(*)
           request = super
-          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
 
           @options.response_cache_store.prepare(request)
 
@@ -62,24 +78,101 @@ module HTTPX
         def fetch_response(request, *)
           response = super
 
-          if response && ResponseCache.cached_response?(response)
+          return unless response
+
+          if ResponseCache.cached_response?(response)
             log { "returning cached response for #{request.uri}" }
-            cached_response = @options.response_cache_store.lookup(request.uri)
+            cached_response = @options.response_cache_store.lookup(request)
 
             response.copy_from_cached(cached_response)
-          end
 
-          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+          else
+            @options.response_cache_store.cache(request, response)
+          end
 
           response
         end
       end
 
+      module RequestMethods
+        def response_cache_key
+          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
+        end
+      end
+
       module ResponseMethods
         def copy_from_cached(other)
-          @body = other.body
+          # 304 responses do not have content-type, which are needed for decoding.
+          @headers = @headers.class.new(other.headers.merge(@headers))
+
+          @body = other.body.dup
+
+          @body.rewind
+        end
+
+        # A response is fresh if its age has not yet exceeded its freshness lifetime.
+        def fresh?
+          if cache_control
+            return false if cache_control.include?("no-cache")
+
+            # check age: max-age
+            max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
+
+            max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
+
+            max_age = max_age[/age=(\d+)/, 1] if max_age
+
+            max_age = max_age.to_i if max_age
+
+            return max_age > age if max_age
+          end
+
+          # check age: expires
+          if @headers.key?("expires")
+            begin
+              expires = Time.httpdate(@headers["expires"])
+            rescue ArgumentError
+              return true
+            end
+
+            return (expires - Time.now).to_i.positive?
+          end
+
+          true
+        end
+
+        def cache_control
+          return @cache_control if defined?(@cache_control)
+
+          @cache_control = begin
+            return unless @headers.key?("cache-control")
+
+            @headers["cache-control"].split(/ *, */)
+          end
+        end
+
+        def vary
+          return @vary if defined?(@vary)
+
+          @vary = begin
+            return unless @headers.key?("vary")
+
+            @headers["vary"].split(/ *, */)
+          end
+        end
+
+        private
+
+        def age
+          return @headers["age"].to_i if @headers.key?("age")
+
+          (Time.now - date).to_i
+        end
 
-          @body.__send__(:rewind)
+        def date
+          @date ||= Time.httpdate(@headers["date"])
+        rescue NoMethodError, ArgumentError
+          Time.now
         end
       end
     end