httpx 1.2.6 → 1.4.4

data/lib/httpx/plugins/proxy.rb

@@ -31,31 +31,53 @@ module HTTPX
       end
 
       class Parameters
-        attr_reader :uri, :username, :password, :scheme
+        attr_reader :uri, :username, :password, :scheme, :no_proxy
 
-        def initialize(uri:, scheme: nil, username: nil, password: nil, **extra)
-          @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
-          @username = username || @uri.user
-          @password = password || @uri.password
+        def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
+          @no_proxy = Array(no_proxy) if no_proxy
+          @uris = Array(uri)
+          uri = @uris.first
+
+          @username = username
+          @password = password
 
-          return unless @username && @password
+          @ns = 0
 
-          scheme ||= case @uri.scheme
-                     when "socks5"
-                       @uri.scheme
-                     when "http", "https"
-                       "basic"
-                     else
-                       return
+          if uri
+            @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
+            @username ||= @uri.user
+            @password ||= @uri.password
           end
 
           @scheme = scheme
 
-          auth_scheme = scheme.to_s.capitalize
+          return unless @uri && @username && @password
 
-          require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+          @authenticator = nil
+          @scheme ||= infer_default_auth_scheme(@uri)
+
+          return unless @scheme
 
-          @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
+          @authenticator = load_authenticator(@scheme, @username, @password, **extra)
+        end
+
+        def shift
+          # TODO: this operation must be synchronized
+          @ns += 1
+          @uri = @uris[@ns]
+
+          return unless @uri
+
+          @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
+
+          scheme = infer_default_auth_scheme(@uri)
+
+          return unless scheme != @scheme
+
+          @scheme = scheme
+          @username = username || @uri.user
+          @password = password || @uri.password
+          @authenticator = load_authenticator(scheme, @username, @password)
         end
 
         def can_authenticate?(*args)
@@ -87,11 +109,34 @@ module HTTPX
             super
           end
         end
+
+        private
+
+        def infer_default_auth_scheme(uri)
+          case uri.scheme
+          when "socks5"
+            uri.scheme
+          when "http", "https"
+            "basic"
+          end
+        end
+
+        def load_authenticator(scheme, username, password, **extra)
+          auth_scheme = scheme.to_s.capitalize
+
+          require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+
+          Authentication.const_get(auth_scheme).new(username, password, **extra)
+        end
       end
 
+      # adds support for the following options:
+      #
+      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
+      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
       module OptionsMethods
         def option_proxy(value)
-          value.is_a?(Parameters) ? value : Hash[value]
+          value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
         end
 
         def option_supported_proxy_protocols(value)
@@ -102,88 +147,68 @@ module HTTPX
       end
 
       module InstanceMethods
-        private
-
-        def find_connection(request, connections, options)
+        def find_connection(request_uri, selector, options)
           return super unless options.respond_to?(:proxy)
 
-          uri = URI(request.uri)
-
-          proxy_opts = if (next_proxy = uri.find_proxy)
-            { uri: next_proxy }
-          else
-            proxy = options.proxy
-
-            return super unless proxy
-
-            return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri)
-
-            @_proxy_uris ||= Array(proxy[:uri])
+          if (next_proxy = request_uri.find_proxy)
+            return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
+          end
 
-            next_proxy = @_proxy_uris.first
-            raise Error, "Failed to connect to proxy" unless next_proxy
+          proxy = options.proxy
 
-            next_proxy = URI(next_proxy)
+          return super unless proxy
 
-            raise Error,
-                  "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
+          next_proxy = proxy.uri
 
-            if proxy.key?(:no_proxy)
+          raise Error, "Failed to connect to proxy" unless next_proxy
 
-              no_proxy = proxy[:no_proxy]
-              no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
+          raise Error,
+                "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
-              return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host,
-                                                                                                           next_proxy.port, no_proxy)
-            end
+          if (no_proxy = proxy.no_proxy)
+            no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
-            proxy.merge(uri: next_proxy)
+            # TODO: setting proxy to nil leaks the connection object in the pool
+            return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
+                                                                                                          next_proxy.port, no_proxy)
           end
 
-          proxy = Parameters.new(**proxy_opts)
-
-          proxy_options = options.merge(proxy: proxy)
-          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
-          unless connections.nil? || connections.include?(connection)
-            connections << connection
-            set_connection_callbacks(connection, connections, options)
-          end
-          connection
+          super(request_uri, selector, options.merge(proxy: proxy))
         end
 
-        def fetch_response(request, connections, options)
+        private
+
+        def fetch_response(request, selector, options)
           response = super
 
-          if response.is_a?(ErrorResponse) && proxy_error?(request, response)
-            @_proxy_uris.shift
+          if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
+            options.proxy.shift
 
             # return last error response if no more proxies to try
-            return response if @_proxy_uris.empty?
+            return response if options.proxy.uri.nil?
 
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
-            send_request(request, connections, options)
+            send_request(request, selector, options)
             return
           end
           response
         end
 
-        def proxy_error?(_request, response)
+        def proxy_error?(_request, response, options)
+          return false unless options.proxy
+
           error = response.error
           case error
           when NativeResolveError
-            return false unless @_proxy_uris && !@_proxy_uris.empty?
-
-            proxy_uri = URI(@_proxy_uris.first)
+            proxy_uri = URI(options.proxy.uri)
 
-            origin = error.connection.origin
+            peer = error.connection.peer
 
             # failed resolving proxy domain
-            origin.host == proxy_uri.host && origin.port == proxy_uri.port
+            peer.host == proxy_uri.host && peer.port == proxy_uri.port
           when ResolveError
-            return false unless @_proxy_uris && !@_proxy_uris.empty?
-
-            proxy_uri = URI(@_proxy_uris.first)
+            proxy_uri = URI(options.proxy.uri)
 
             error.message.end_with?(proxy_uri.to_s)
           when *PROXY_ERRORS
@@ -204,25 +229,11 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          proxy_uri = URI(@options.proxy.uri)
-          @origin.host = proxy_uri.host
-          @origin.port = proxy_uri.port
+          @proxy_uri = URI(@options.proxy.uri)
         end
 
-        def coalescable?(connection)
-          return super unless @options.proxy
-
-          if @io.protocol == "h2" &&
-             @origin.scheme == "https" &&
-             connection.origin.scheme == "https" &&
-             @io.can_verify_peer?
-            # in proxied connections, .origin is the proxy ; Given names
-            # are stored in .origins, this is what is used.
-            origin = URI(connection.origins.first)
-            @io.verify_hostname(origin.host)
-          else
-            @origin == connection.origin
-          end
+        def peer
+          @proxy_uri || super
         end
 
         def connecting?
@@ -248,7 +259,7 @@ module HTTPX
           @state = :open
 
           super
-          emit(:close)
+          # emit(:close)
         end
 
         private

data/lib/httpx/plugins/retries.rb

@@ -3,7 +3,12 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds support for retrying requests when certain errors happen.
+    # This plugin adds support for retrying requests when errors happen.
+    #
+    # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
+    # after which it will return the last response, error or not. It will **not** raise an exception.
+    #
+    # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
     #
     # https://gitlab.com/os85/httpx/wikis/Retries
     #
@@ -38,6 +43,14 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
+      # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
+      # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
+      # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
+      #              (i.e. <tt>->(res) { ... }</tt>).
       module OptionsMethods
         def option_retry_after(value)
           # return early if callable
@@ -76,12 +89,12 @@ module HTTPX
 
       module InstanceMethods
         def max_retries(n)
-          with(max_retries: n.to_i)
+          with(max_retries: n)
         end
 
         private
 
-        def fetch_response(request, connections, options)
+        def fetch_response(request, selector, options)
           response = super
 
           if response &&
@@ -97,7 +110,7 @@ module HTTPX
              )
             __try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1
+            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
             request.transition(:idle)
 
             retry_after = options.retry_after
@@ -111,12 +124,17 @@ module HTTPX
 
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
-              pool.after(retry_after) do
-                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                send_request(request, connections, options)
+              selector.after(retry_after) do
+                if request.response
+                  # request has terminated abruptly meanwhile
+                  request.emit(:response, request.response)
+                else
+                  log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                  send_request(request, selector, options)
+                end
               end
             else
-              send_request(request, connections, options)
+              send_request(request, selector, options)
             end
 
             return
@@ -132,7 +150,7 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
-        def proxy_error?(request, response)
+        def proxy_error?(request, response, _)
           super && !request.retries.positive?
         end
 
@@ -149,7 +167,7 @@ module HTTPX
           unless response.headers.key?("accept-ranges") &&
                  response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
                  (original_body = response.body)
-            response.close if response.respond_to?(:close)
+            response.body.close
             return
           end
 

data/lib/httpx/plugins/ssrf_filter.rb

@@ -87,6 +87,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
       module OptionsMethods
         def option_allowed_schemes(value)
           Array(value)
@@ -100,7 +103,7 @@ module HTTPX
 
             error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
             error.set_backtrace(caller)
-            response = ErrorResponse.new(request, error, request.options)
+            response = ErrorResponse.new(request, error)
             request.emit(:response, response)
             response
           end

data/lib/httpx/plugins/stream.rb

@@ -4,27 +4,39 @@ module HTTPX
   class StreamResponse
     def initialize(request, session)
       @request = request
+      @options = @request.options
       @session = session
-      @response = nil
+      @response_enum = nil
+      @buffered_chunks = []
     end
 
     def each(&block)
       return enum_for(__method__) unless block
 
+      if (response_enum = @response_enum)
+        @response_enum = nil
+        # streaming already started, let's finish it
+
+        while (chunk = @buffered_chunks.shift)
+          block.call(chunk)
+        end
+
+        # consume enum til the end
+        begin
+          while (chunk = response_enum.next)
+            block.call(chunk)
+          end
+        rescue StopIteration
+          return
+        end
+      end
+
       @request.stream = self
 
       begin
         @on_chunk = block
 
-        if @request.response
-          # if we've already started collecting the payload, yield it first
-          # before proceeding.
-          body = @request.response.body
-
-          body.each do |chunk|
-            on_chunk(chunk)
-          end
-        end
+        response = @session.request(@request)
 
         response.raise_for_status
       ensure
@@ -64,27 +76,39 @@ module HTTPX
     # :nocov:
 
     def to_s
-      response.to_s
+      if @request.response
+        @request.response.to_s
+      else
+        @buffered_chunks.join
+      end
     end
 
     private
 
     def response
-      return @response if @response
-
       @request.response || begin
-        @response = @session.request(@request)
+        response_enum = each
+        while (chunk = response_enum.next)
+          @buffered_chunks << chunk
+          break if @request.response
+        end
+        @response_enum = response_enum
+        @request.response
       end
     end
 
-    def respond_to_missing?(meth, *args)
-      response.respond_to?(meth, *args) || super
+    def respond_to_missing?(meth, include_private)
+      if (response = @request.response)
+        response.respond_to_missing?(meth, include_private)
+      else
+        @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
+      end || super
     end
 
-    def method_missing(meth, *args, &block)
+    def method_missing(meth, *args, **kwargs, &block)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, &block)
+      response.__send__(meth, *args, **kwargs, &block)
     end
   end
 

data/lib/httpx/plugins/upgrade.rb

@@ -28,7 +28,7 @@ module HTTPX
       end
 
       module InstanceMethods
-        def fetch_response(request, connections, options)
+        def fetch_response(request, selector, options)
           response = super
 
           if response
@@ -45,7 +45,7 @@ module HTTPX
             return response unless protocol_handler
 
             log { "upgrading to #{upgrade_protocol}..." }
-            connection = find_connection(request, connections, options)
+            connection = find_connection(request.uri, selector, options)
 
             # do not upgrade already upgraded connections
             return if connection.upgrade_protocol == upgrade_protocol
@@ -60,14 +60,6 @@ module HTTPX
 
           response
         end
-
-        def close(*args)
-          return super if args.empty?
-
-          connections, = args
-
-          pool.close(connections.reject(&:hijacked))
-        end
       end
 
       module ConnectionMethods
@@ -75,6 +67,9 @@ module HTTPX
 
         def hijack_io
           @hijacked = true
+
+          # connection is taken away from selector and not given back to the pool.
+          @current_session.deselect_connection(self, @current_selector, true)
         end
       end
     end

data/lib/httpx/plugins/webdav.rb

@@ -8,6 +8,10 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/WebDav
     #
     module WebDav
+      def self.configure(klass)
+        klass.plugin(:xml)
+      end
+
       module InstanceMethods
         def copy(src, dest)
           request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
@@ -43,6 +47,8 @@ module HTTPX
           ensure
             unlock(path, lock_token)
           end
+
+          response
         end
 
         def unlock(path, lock_token)

data/lib/httpx/plugins/xml.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin supports request XML encoding/response decoding using the nokogiri gem.
+    #
+    # https://gitlab.com/os85/httpx/wikis/XML
+    #
+    module XML
+      MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
+      module Transcoder
+        module_function
+
+        class Encoder
+          def initialize(xml)
+            @raw = xml
+          end
+
+          def content_type
+            charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
+            "application/xml; charset=#{charset}"
+          end
+
+          def bytesize
+            @raw.to_s.bytesize
+          end
+
+          def to_s
+            @raw.to_s
+          end
+        end
+
+        def encode(xml)
+          Encoder.new(xml)
+        end
+
+        def decode(response)
+          content_type = response.content_type.mime_type
+
+          raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
+
+          Nokogiri::XML.method(:parse)
+        end
+      end
+
+      class << self
+        def load_dependencies(*)
+          require "nokogiri"
+        end
+      end
+
+      module ResponseMethods
+        # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
+        # "application/xml" (requires the "nokogiri" gem).
+        def xml
+          decode(Transcoder)
+        end
+      end
+
+      module RequestBodyClassMethods
+        #   ..., xml: Nokogiri::XML::Node #=> xml encoder
+        def initialize_body(params)
+          if (xml = params.delete(:xml))
+            # @type var xml: Nokogiri::XML::Node | String
+            return Transcoder.encode(xml)
+          end
+
+          super
+        end
+      end
+    end
+
+    register_plugin(:xml, XML)
+  end
+end