httpx 1.2.6 → 1.3.1

data/lib/httpx/plugins/follow_redirects.rb

@@ -4,12 +4,17 @@ module HTTPX
   InsecureRedirectError = Class.new(Error)
   module Plugins
     #
-    # This plugin adds support for following redirect (status 30X) responses.
+    # This plugin adds support for automatically following redirect (status 30X) responses.
     #
-    # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
-    # will return the last redirect response. It will **not** raise an exception.
+    # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
+    # after which it will return the last redirect response. It will **not** raise an exception.
     #
-    # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    #
+    # It doesn't propagate authorization related headers to requests redirecting to different origins
+    # (see *allow_auth_to_other_origins*) to override.
+    #
+    # It allows customization of when to redirect via the *redirect_on* callback option).
     #
     # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
     #
@@ -20,6 +25,14 @@ module HTTPX
 
       using URIExtensions
 
+      # adds support for the following options:
+      #
+      # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
+      # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
+      #                               (defaults to <tt>false</tt>).
+      # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
+      #                                 (defaults to <tt>false</tt>).
+      # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
       module OptionsMethods
         def option_max_redirects(value)
           num = Integer(value)
@@ -44,6 +57,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a session with the *max_redirects* option set to +n+
         def max_redirects(n)
           with(max_redirects: n.to_i)
         end
@@ -71,40 +85,40 @@ module HTTPX
           # build redirect request
           request_body = redirect_request.body
           redirect_method = "GET"
+          redirect_params = {}
 
           if response.status == 305 && options.respond_to?(:proxy)
             request_body.rewind
             # The requested resource MUST be accessed through the proxy given by
             # the Location field. The Location field gives the URI of the proxy.
-            retry_options = options.merge(headers: redirect_request.headers,
-                                          proxy: { uri: redirect_uri },
-                                          body: request_body,
-                                          max_redirects: max_redirects - 1)
+            redirect_options = options.merge(headers: redirect_request.headers,
+                                             proxy: { uri: redirect_uri },
+                                             max_redirects: max_redirects - 1)
+
+            redirect_params[:body] = request_body
             redirect_uri = redirect_request.uri
-            options = retry_options
+            options = redirect_options
           else
             redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
-
-            retry_opts = Hash[options].merge(max_redirects: max_redirects - 1)
+            redirect_opts = Hash[options]
+            redirect_params[:max_redirects] = max_redirects - 1
 
             unless request_body.empty?
               if response.status == 307
                 # The method and the body of the original request are reused to perform the redirected request.
                 redirect_method = redirect_request.verb
                 request_body.rewind
-                retry_opts[:body] = request_body
+                redirect_params[:body] = request_body
               else
                 # redirects are **ALWAYS** GET, so remove body-related headers
                 REQUEST_BODY_HEADERS.each do |h|
                   redirect_headers.delete(h)
                 end
-                retry_opts.delete(:body)
+                redirect_params[:body] = nil
               end
             end
 
-            retry_opts[:headers] = redirect_headers.to_h
-
-            retry_options = options.class.new(retry_opts)
+            options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
           end
 
           redirect_uri = Utils.to_uri(redirect_uri)
@@ -114,27 +128,35 @@ module HTTPX
              redirect_uri.scheme == "http"
             error = InsecureRedirectError.new(redirect_uri.to_s)
             error.set_backtrace(caller)
-            return ErrorResponse.new(request, error, options)
+            return ErrorResponse.new(request, error)
           end
 
-          retry_request = build_request(redirect_method, redirect_uri, retry_options)
+          retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
 
           request.redirect_request = retry_request
 
-          retry_after = response.headers["retry-after"]
+          redirect_after = response.headers["retry-after"]
 
-          if retry_after
+          if redirect_after
             # Servers send the "Retry-After" header field to indicate how long the
             # user agent ought to wait before making a follow-up request.
             # When sent with any 3xx (Redirection) response, Retry-After indicates
             # the minimum time that the user agent is asked to wait before issuing
             # the redirected request.
             #
-            retry_after = Utils.parse_retry_after(retry_after)
+            redirect_after = Utils.parse_retry_after(redirect_after)
 
-            log { "redirecting after #{retry_after} secs..." }
-            pool.after(retry_after) do
-              send_request(retry_request, connections, options)
+            log { "redirecting after #{redirect_after} secs..." }
+
+            deactivate_connection(request, connections, options)
+
+            pool.after(redirect_after) do
+              if request.response
+                # request has terminated abruptly meanwhile
+                retry_request.emit(:response, request.response)
+              else
+                send_request(retry_request, connections, options)
+              end
             end
           else
             send_request(retry_request, connections, options)
@@ -142,6 +164,7 @@ module HTTPX
           nil
         end
 
+        # :nodoc:
         def redirect_request_headers(original_uri, redirect_uri, headers, options)
           headers = headers.dup
 
@@ -149,14 +172,14 @@ module HTTPX
 
           return headers unless headers.key?("authorization")
 
-          unless original_uri.origin == redirect_uri.origin
-            headers = headers.dup
-            headers.delete("authorization")
-          end
+          return headers if original_uri.origin == redirect_uri.origin
+
+          headers.delete("authorization")
 
           headers
         end
 
+        # :nodoc:
         def __get_location_from_response(response)
           # @type var location_uri: http_uri
           location_uri = URI(response.headers["location"])
@@ -166,12 +189,15 @@ module HTTPX
       end
 
       module RequestMethods
+        # returns the top-most original HTTPX::Request from the redirect chain
         attr_accessor :root_request
 
+        # returns the follow-up redirect request, or itself
         def redirect_request
           @redirect_request || self
         end
 
+        # sets the follow-up redirect request
         def redirect_request=(req)
           @redirect_request = req
           req.root_request = @root_request || self
@@ -179,7 +205,7 @@ module HTTPX
         end
 
         def response
-          return super unless @redirect_request
+          return super unless @redirect_request && @response.nil?
 
           @redirect_request.response
         end
@@ -188,6 +214,16 @@ module HTTPX
           @options.max_redirects || MAX_REDIRECTS
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def set_request_request_timeout(request)
+          return unless request.root_request.nil?
+
+          super
+        end
+      end
     end
     register_plugin :follow_redirects, FollowRedirects
   end

data/lib/httpx/plugins/grpc.rb

@@ -110,10 +110,10 @@ module HTTPX
       end
 
       module RequestBodyMethods
-        def initialize(headers, _)
+        def initialize(*, **)
           super
 
-          if (compression = headers["grpc-encoding"])
+          if (compression = @headers["grpc-encoding"])
             deflater_body = self.class.initialize_deflater_body(@body, compression)
             @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
           else

data/lib/httpx/plugins/h2c.rb

@@ -39,7 +39,7 @@ module HTTPX
           upgrade_request.headers.add("connection", "upgrade")
           upgrade_request.headers.add("connection", "http2-settings")
           upgrade_request.headers["upgrade"] = "h2c"
-          upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
+          upgrade_request.headers["http2-settings"] = ::HTTP2::Client.settings_header(upgrade_request.options.http2_settings)
 
           super(upgrade_request, *remainder)
         end

data/lib/httpx/plugins/oauth.rb

@@ -155,7 +155,7 @@ module HTTPX
           with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
         end
 
-        def build_request(*, _)
+        def build_request(*)
           request = super
 
           return request if request.headers.key?("authorization")

data/lib/httpx/plugins/proxy/http.rb

@@ -32,9 +32,14 @@ module HTTPX
                !request.headers.key?("proxy-authorization") &&
                response.headers.key?("proxy-authenticate")
 
-              connection = find_connection(request, connections, options)
+              uri = request.uri
 
-              if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+              proxy_options = proxy_options(uri, options)
+              connection = connections.find do |conn|
+                conn.match?(uri, proxy_options)
+              end
+
+              if connection && connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
                 request.transition(:idle)
                 request.headers["proxy-authorization"] =
                   connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
@@ -163,8 +168,8 @@ module HTTPX
         end
 
         class ConnectRequest < Request
-          def initialize(uri, _options)
-            super("CONNECT", uri, {})
+          def initialize(uri, options)
+            super("CONNECT", uri, options)
             @headers.delete("accept")
           end
 

data/lib/httpx/plugins/proxy/socks4.rb

@@ -89,7 +89,7 @@ module HTTPX
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
+            @options = options
           end
 
           def close; end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -141,7 +141,7 @@ module HTTPX
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
+            @options = options
           end
 
           def close; end

data/lib/httpx/plugins/proxy.rb

@@ -89,6 +89,10 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
+      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
       module OptionsMethods
         def option_proxy(value)
           value.is_a?(Parameters) ? value : Hash[value]
@@ -107,16 +111,29 @@ module HTTPX
         def find_connection(request, connections, options)
           return super unless options.respond_to?(:proxy)
 
-          uri = URI(request.uri)
+          uri = request.uri
 
-          proxy_opts = if (next_proxy = uri.find_proxy)
+          proxy_options = proxy_options(uri, options)
+
+          return super(request, connections, proxy_options) unless proxy_options.proxy
+
+          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
+          unless connections.nil? || connections.include?(connection)
+            connections << connection
+            set_connection_callbacks(connection, connections, options)
+          end
+          connection
+        end
+
+        def proxy_options(request_uri, options)
+          proxy_opts = if (next_proxy = request_uri.find_proxy)
             { uri: next_proxy }
           else
             proxy = options.proxy
 
-            return super unless proxy
+            return options unless proxy
 
-            return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri)
+            return options.merge(proxy: nil) unless proxy.key?(:uri)
 
             @_proxy_uris ||= Array(proxy[:uri])
 
@@ -133,8 +150,8 @@ module HTTPX
               no_proxy = proxy[:no_proxy]
               no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
-              return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host,
-                                                                                                           next_proxy.port, no_proxy)
+              return options.merge(proxy: nil) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
+                                                                              next_proxy.port, no_proxy)
             end
 
             proxy.merge(uri: next_proxy)
@@ -142,13 +159,7 @@ module HTTPX
 
           proxy = Parameters.new(**proxy_opts)
 
-          proxy_options = options.merge(proxy: proxy)
-          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
-          unless connections.nil? || connections.include?(connection)
-            connections << connection
-            set_connection_callbacks(connection, connections, options)
-          end
-          connection
+          options.merge(proxy: proxy)
         end
 
         def fetch_response(request, connections, options)

data/lib/httpx/plugins/retries.rb

@@ -3,7 +3,12 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds support for retrying requests when certain errors happen.
+    # This plugin adds support for retrying requests when errors happen.
+    #
+    # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
+    # after which it will return the last response, error or not. It will **not** raise an exception.
+    #
+    # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
     #
     # https://gitlab.com/os85/httpx/wikis/Retries
     #
@@ -38,6 +43,14 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
+      # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
+      # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
+      # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
+      #              (i.e. <tt>->(res) { ... }</tt>).
       module OptionsMethods
         def option_retry_after(value)
           # return early if callable
@@ -76,7 +89,7 @@ module HTTPX
 
       module InstanceMethods
         def max_retries(n)
-          with(max_retries: n.to_i)
+          with(max_retries: n)
         end
 
         private
@@ -111,9 +124,17 @@ module HTTPX
 
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
+
+              deactivate_connection(request, connections, options)
+
               pool.after(retry_after) do
-                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                send_request(request, connections, options)
+                if request.response
+                  # request has terminated abruptly meanwhile
+                  request.emit(:response, request.response)
+                else
+                  log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                  send_request(request, connections, options)
+                end
               end
             else
               send_request(request, connections, options)

data/lib/httpx/plugins/ssrf_filter.rb

@@ -87,6 +87,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
       module OptionsMethods
         def option_allowed_schemes(value)
           Array(value)
@@ -100,7 +103,7 @@ module HTTPX
 
             error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
             error.set_backtrace(caller)
-            response = ErrorResponse.new(request, error, request.options)
+            response = ErrorResponse.new(request, error)
             request.emit(:response, response)
             response
           end

data/lib/httpx/pool/synch_pool.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "thread"
+
+module HTTPX
+  class SynchPool < Pool
+    def initialize(options)
+      super
+
+      @connections = ConnectionStore.new(options)
+    end
+
+    # TODO: #wrap
+    def find_or_new_connection(uri, options, &blk)
+      @connections.find_or_new(uri, options) do |new_conn|
+        catch(:coalesced) do
+          init_connection(new_conn, options)
+          blk.call(new_conn) if blk
+          new_conn
+        end
+      end
+      find_connection(uri, options) || new_connection(uri, options, &blk)
+    end
+
+    class ConnectionManager
+      include Enumerable
+
+      def initialize(limit = 3)
+        @connections = []
+        @used = 0
+        @limit = limit
+      end
+
+      def each(*args, &blk)
+        @connections.each(*args, &blk)
+      end
+
+      def find_or_new(uri, options, &blk)
+        raise "over limit" if @used >= @limit
+
+        @used += 1
+        conn = @connections.find do |connection|
+          connection.match?(uri, options)
+        end
+
+        if conn
+          @connections.delete(conn)
+        else
+          conn = options.connection_class.new(uri, options)
+          blk[conn]
+        end
+
+        conn
+      end
+    end
+
+    class ConnectionStore
+      include Enumerable
+
+      def initialize(options)
+        @connections = Hash.new { |hs, k| hs[k] ||= ConnectionManager.new }
+        @conn_mtx = Thread::Mutex.new
+        @conn_waiter = ConditionVariable.new
+        @timeout = Float(options.fetch(:pool_timeout, 5))
+      end
+
+      def each(&block)
+        return enum_for(__meth__) unless block
+
+        @conn_mtx.synchronize do
+          @connections.each_value do |conns|
+            conns.each(&block)
+          end
+        end
+      end
+
+      def find_or_new(uri, options, &blk)
+        @connections[uri.origin].find_or_new(uri, options, &blk)
+      end
+
+      # def <<(conn)
+      #   @conn_mtx.synchronize do
+      #     origin, conns = @connections.find { |_orig, _| conn.origins.include?(origin) }
+      #     (conns || @connections[conn.origin.to_s]) << conn
+      #   end
+      # end
+
+      def empty?
+        @conn_mtx.synchronize { super }
+      end
+    end
+  end
+end

data/lib/httpx/pool.rb

@@ -108,7 +108,7 @@ module HTTPX
       resolve_connection(connection) unless connection.family
     end
 
-    def deactivate(connections)
+    def deactivate(*connections)
       connections.each do |connection|
         connection.deactivate
         deselect_connection(connection) if connection.state == :inactive

data/lib/httpx/request/body.rb

@@ -4,30 +4,53 @@ module HTTPX
   # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
   class Request::Body < SimpleDelegator
     class << self
-      def new(_, options)
-        return options.body if options.body.is_a?(self)
+      def new(_, options, body: nil, **params)
+        if body.is_a?(self)
+          # request derives its options from body
+          body.options = options.merge(params)
+          return body
+        end
 
         super
       end
     end
 
-    # inits the instance with the request +headers+ and +options+, which contain the payload definition.
-    def initialize(headers, options)
-      @headers = headers
+    attr_accessor :options
 
-      # forego compression in the Range request case
-      if @headers.key?("range")
-        @headers.delete("accept-encoding")
-      else
-        @headers["accept-encoding"] ||= options.supported_compression_formats
+    # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
+    # it wraps the given body with the appropriate encoder on initialization.
+    #
+    #   ..., json: { foo: "bar" }) #=> json encoder
+    #   ..., form: { foo: "bar" }) #=> form urlencoded encoder
+    #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { body: "bla") }) #=> raw data encoder
+    def initialize(headers, options, body: nil, form: nil, json: nil, xml: nil, **params)
+      @headers = headers
+      @options = options.merge(params)
+
+      @body = if body
+        Transcoder::Body.encode(body)
+      elsif form
+        Transcoder::Form.encode(form)
+      elsif json
+        Transcoder::JSON.encode(json)
+      elsif xml
+        Transcoder::Xml.encode(xml)
       end
 
-      initialize_body(options)
+      if @body
+        if @options.compress_request_body && @headers.key?("content-encoding")
 
-      return if @body.nil?
+          @headers.get("content-encoding").each do |encoding|
+            @body = self.class.initialize_deflater_body(@body, encoding)
+          end
+        end
+
+        @headers["content-type"] ||= @body.content_type
+        @headers["content-length"] = @body.bytesize unless unbounded_body?
+      end
 
-      @headers["content-type"] ||= @body.content_type
-      @headers["content-length"] = @body.bytesize unless unbounded_body?
       super(@body)
     end
 
@@ -99,33 +122,6 @@ module HTTPX
     end
     # :nocov:
 
-    private
-
-    # wraps the given body with the appropriate encoder.
-    #
-    #   ..., json: { foo: "bar" }) #=> json encoder
-    #   ..., form: { foo: "bar" }) #=> form urlencoded encoder
-    #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
-    #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
-    #   ..., form: { body: "bla") }) #=> raw data encoder
-    def initialize_body(options)
-      @body = if options.body
-        Transcoder::Body.encode(options.body)
-      elsif options.form
-        Transcoder::Form.encode(options.form)
-      elsif options.json
-        Transcoder::JSON.encode(options.json)
-      elsif options.xml
-        Transcoder::Xml.encode(options.xml)
-      end
-
-      return unless @body && options.compress_request_body && @headers.key?("content-encoding")
-
-      @headers.get("content-encoding").each do |encoding|
-        @body = self.class.initialize_deflater_body(@body, encoding)
-      end
-    end
-
     class << self
       # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
       def initialize_deflater_body(body, encoding)