httpx 1.9.0 → 1.9.1

data/lib/httpx/plugins/content_digest.rb

@@ -81,15 +81,13 @@ module HTTPX
         end
 
         def write(chunk)
-          @content_digest_buffer.write(chunk) if @content_digest_buffer
+          @content_digest_buffer&.write(chunk)
           super
         end
 
         def close
-          if @content_digest_buffer
-            @content_digest_buffer.close
-            @content_digest_buffer = nil
-          end
+          @content_digest_buffer&.close
+          @content_digest_buffer = nil
           super
         end
       end

data/lib/httpx/plugins/expect.rb

@@ -104,9 +104,15 @@ module HTTPX
       end
 
       module ConnectionMethods
-        def send_request_to_parser(request)
+        private
+
+        def set_request_timeouts(request)
           super
 
+          set_request_expect_timeout(request)
+        end
+
+        def set_request_expect_timeout(request)
           return unless request.headers["expect"] == "100-continue"
 
           expect_timeout = request.options.expect_timeout
@@ -118,6 +124,7 @@ module HTTPX
             if request.state == :expect && !request.expects?
               Expect.no_expect_store.add(request.origin)
               request.headers.delete("expect")
+              request.transition(:body)
               consume
             end
           end

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -76,10 +76,10 @@ module HTTPX
           super
         end
 
-        def interests
-          return if connecting? && @pending.none?(&:current_context?)
-
-          super
+        def closed_for_select?
+          # @fiber-switch-guard
+          # a closed connection still holding requests will reconnect once the fiber closing its socket resumes.
+          super && @pending.empty?
         end
 
         def on_io_error(e)
@@ -88,17 +88,12 @@ module HTTPX
           # @fiber-switch-guard
           # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
           # socket may be used in. this check verifies that this is actually about this socket.
-          return unless to_io.closed?
-
-          if @state == :closing
-            # @fiber-switch-guard
-            # if the connection is reused across fibers, the socket may have been closed in the other fiber
-            # and switched here during the process, so continue what it was doing and transition to closed
-            # via #call.
+          unless to_io.closed? && !backlog?
             call
-          elsif !backlog?
-            super
+            return
           end
+
+          super
         end
 
         def on_connect_error(e)
@@ -107,24 +102,50 @@ module HTTPX
           # @fiber-switch-guard
           # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
           # socket may be used in. this check verifies that this is actually about this socket.
-          return unless to_io.closed? && !backlog?
+          unless to_io.closed? && !backlog?
+            call
+            return
+          end
 
           super
         end
 
         def pending_count
-          count = @pending.count(&:current_context?)
-          count += @parser.pending.count(&:current_context?) if @parser.respond_to?(:pending)
-          count
+          @pending.count(&:current_context?)
+        end
+
+        def interests
+          idling if (@state == :closed) && !no_more_requests?
+
+          super
         end
 
         private
 
+        def handle_transition(nextstate)
+          case nextstate
+          when :inactive
+            # @fiber-switch-guard
+            # connections may have requests (pending or inflight) from other fibers.
+            return if @state == :open && backlog?
+          end
+          super
+        end
+
+        def no_requests_to_handle?
+          pending_count.zero? && @parser&.no_requests_to_handle?
+        end
+
+        # @fiber-switch-guard
+        # pending requests from other fibers are still being waited for.
+        def clear_pending_requests
+          @pending.reject!(&:current_context?)
+        end
+
         # checks whether the connection has any pending request (which the connection itself may
         # have stored, or it may be somewhere in the parser).
         def backlog?
-          @pending.any? ||
-            (@parser && (@parser.pending.any? || @parser.requests.any?))
+          @pending.any? || @parser&.backlog? || @parser&.waiting_for_ping?
         end
       end
 
@@ -138,6 +159,14 @@ module HTTPX
 
           super
         end
+
+        def backlog?
+          !no_more_requests?
+        end
+
+        def no_requests_to_handle?
+          @requests.none?(&:current_context?)
+        end
       end
 
       module HTTP2Methods
@@ -162,6 +191,14 @@ module HTTPX
           super
         end
 
+        def backlog?
+          !no_more_requests?
+        end
+
+        def no_requests_to_handle?
+          @streams.each_key.none?(&:current_context?)
+        end
+
         private
 
         def on_close(_, error, _)

data/lib/httpx/plugins/h2c.rb

@@ -68,29 +68,30 @@ module HTTPX
         end
 
         def upgrade_to_h2c(request, response)
-          enqueue_pending_requests_from_parser(@parser)
+          @parser.reset_requests
 
-          @parser = request.options.h2c_class.new(@write_buffer, @options)
+          @parser = request.options.h2c_class.new(@write_buffer, @pending, @options)
           set_parser_callbacks(@parser)
-          @inflight += 1 # request is being completed below
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
         end
 
         private
 
-        def send_request_to_parser(request)
-          super
+        def send_request_to_parser(request, *)
+          super.tap do |is_inflight|
+            next unless is_inflight
 
-          return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
+            next unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
 
-          max_concurrent_requests = parser.max_concurrent_requests
+            max_concurrent_requests = parser.max_concurrent_requests
 
-          return if max_concurrent_requests == 1
+            next if max_concurrent_requests == 1
 
-          parser.max_concurrent_requests = 1
-          request.once(:response) do
-            parser.max_concurrent_requests = max_concurrent_requests
+            parser.max_concurrent_requests = 1
+            request.once(:response) do
+              parser.max_concurrent_requests = max_concurrent_requests
+            end
           end
         end
       end

data/lib/httpx/plugins/hedging.rb

@@ -270,10 +270,10 @@ module HTTPX
 
         private
 
-        def send_request_to_parser(request)
+        def send_request_to_parser(request, *)
           # if original request was in the queue, but has since been finished
           # in the hedge connection.
-          return if request.response&.finished?
+          return false if request.response&.finished?
 
           super
         end

data/lib/httpx/plugins/ntlm_auth.rb

@@ -41,9 +41,10 @@ module HTTPX
 
             if ntlm
               request.authorize(ntlm.negotiate)
-              probe_response = wrap { super(request).first }
+              responses = wrap { super(request) }
+              probe_response = responses.first
 
-              return probe_response unless probe_response.is_a?(Response)
+              return responses unless probe_response.is_a?(Response)
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
@@ -51,7 +52,7 @@ module HTTPX
                 request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else
-                probe_response
+                responses
               end
             else
               super(request)

data/lib/httpx/plugins/ntlm_v2_auth.rb

@@ -67,9 +67,10 @@ module HTTPX
 
             if ntlm
               request.authorize(ntlm.negotiate)
-              probe_response = wrap { super(request).first }
+              responses = wrap { super(request) }
+              probe_response = responses.first
 
-              return probe_response unless probe_response.is_a?(Response)
+              return responses unless probe_response.is_a?(Response)
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
@@ -78,7 +79,7 @@ module HTTPX
                                                     probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else
-                probe_response
+                responses
               end
             else
               super(request)

data/lib/httpx/plugins/oauth.rb

@@ -290,7 +290,7 @@ module HTTPX
 
         def reset_auth_header_value!
           super.tap do
-            @oauth_session.reset if @oauth_session
+            @oauth_session&.reset!
           end
         end
 

data/lib/httpx/plugins/proxy/http.rb

@@ -16,9 +16,22 @@ module HTTPX
         # :enable_proxy_tunnel :: when <tt>true</tt>, it forces the establishment of the CONNECT proxy
         #                         tunnel, in cases where it wouldn't otherwise normally establish it
         #                         (i.e. if the request is an http:// request).
+        # :connect_request_class :: class of the request object used for the CONNECT tunnel request.
         module OptionsMethods
+          def extend_with_plugin_classes(pl)
+            super
+
+            @connect_request_class = @request_class.dup
+            Options::SET_TEMPORARY_NAME[@connect_request_class, pl]
+            @connect_request_class.__send__(:include, ConnectRequestMethods)
+          end
+
           private
 
+          def option_connect_request_class(val)
+            val
+          end
+
           def option_proxy_headers(value)
             value = value.dup if value.frozen?
 
@@ -80,8 +93,6 @@ module HTTPX
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
               @parser.reset
-              @inflight -= @parser.pending.size
-              @parser.pending.clear
             end
 
             super
@@ -114,12 +125,12 @@ module HTTPX
               @tunnel_parser || begin
                 if @parser
                   # in case this happens, requests from @parser need to be reenqueued for delivery.
-                  enqueue_pending_requests_from_parser(@parser)
+                  @parser.reset_requests
                   @parser = nil
                 end
 
                 @tunnel_parser = @parser = parser =
-                                   parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                                   parser_type(@io.protocol).new(@write_buffer, @pending, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
@@ -127,11 +138,12 @@ module HTTPX
 
                   reset
                 end
+                parser.on(:error, &method(:on_parser_error))
                 parser.on(:reset) do
-                  if parser.pending.empty? && parser.empty?
+                  if parser.empty?
                     reset
                   else
-                    enqueue_pending_requests_from_parser(parser)
+                    parser.reset_requests
 
                     initial_state = @state
 
@@ -152,7 +164,7 @@ module HTTPX
                     if initial_state == :connecting
                       parser = tunnel_parser
                       parser.reset
-                      if @pending.first.is_a?(ConnectRequest)
+                      if @pending.first.is_a?(@options.connect_request_class)
                         connect_request = @pending.shift # this happened when reenqueing
                       end
                     end
@@ -160,10 +172,7 @@ module HTTPX
                     idling
 
                     @tunnel_parser = @parser = parser
-                    if connect_request
-                      @inflight += 1
-                      parser.send(connect_request)
-                    end
+                    parser.send(connect_request) if connect_request
                     transition(:connecting)
                   end
                 end
@@ -197,7 +206,7 @@ module HTTPX
               # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
               # and therefore, will share the connection.
               #
-              connect_request = ConnectRequest.new(req.uri, @options)
+              connect_request = @options.connect_request_class.new(req.uri, @options)
               send(connect_request)
             else
               handle_transition(:connected)
@@ -205,7 +214,6 @@ module HTTPX
           end
 
           def __http_on_connect(request, response)
-            @inflight -= 1
             if response.is_a?(Response) && response.status == 200
               req = @pending.first
               request_uri = req.uri
@@ -221,8 +229,7 @@ module HTTPX
               request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
               send(request)
             else
-              pending = @pending + @parser.pending
-              while (req = pending.shift)
+              while (req = @pending.shift)
                 response.finish!
                 req.response = response
                 req.emit_response(response)
@@ -252,7 +259,7 @@ module HTTPX
           end
         end
 
-        class ConnectRequest < Request
+        module ConnectRequestMethods
           def initialize(uri, options)
             super("CONNECT", uri, options)
             @headers = options.headers_class.new

data/lib/httpx/plugins/proxy/socks5.rb

@@ -42,10 +42,6 @@ module HTTPX
             end
           end
 
-          def connecting?
-            super || @state == :authenticating || @state == :negotiating
-          end
-
           def interests
             if @state == :connecting || @state == :authenticating || @state == :negotiating
               return @write_buffer.empty? ? :r : :w
@@ -56,6 +52,10 @@ module HTTPX
 
           private
 
+          def proxy_connecting?
+            super || @state == :authenticating || @state == :negotiating
+          end
+
           def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
 

data/lib/httpx/plugins/proxy.rb

@@ -11,7 +11,7 @@ module HTTPX
     #
     # * HTTP proxies
     # * HTTPS proxies
-    # * Socks4 /Socks4a proxies
+    # * Socks4 proxies (also supports Socks4a)
     # * Socks5 proxies
     #
     # https://gitlab.com/os85/httpx/wikis/Proxy
@@ -157,7 +157,7 @@ module HTTPX
       # :username :: the username to use when authenticating with the proxy
       # :password :: the password to use when authenticating with the proxy
       # :no_proxy :: A list of hostnames (or regex patterns) for which the proxy should be bypassed
-      #              (ex: <tt>["example.com", "localhost", /.*\.google\.com/]</tt>).
+      #              (ex: <tt>["example.com", "localhost", /.*\.google\.com/]</tt>). Can also receive ["*"] as a catch-all.
       #
       # if you systems setting environment variables like `HTTP_PROXY` or `NO_PROXY` though, you won't need to set
       # most of the above.
@@ -287,7 +287,7 @@ module HTTPX
         def connecting?
           return super unless @options.proxy
 
-          super || @state == :connecting || @state == :connected
+          super || proxy_connecting?
         end
 
         def call
@@ -309,15 +309,6 @@ module HTTPX
           raise e
         end
 
-        def reset
-          return super unless @options.proxy
-
-          @state = :open
-
-          super
-          # emit(:close)
-        end
-
         private
 
         def initialize_type(uri, options)
@@ -326,6 +317,10 @@ module HTTPX
           "tcp"
         end
 
+        def proxy_connecting?
+          @state == :connecting || @state == :connected
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -342,9 +337,7 @@ module HTTPX
 
           case nextstate
           when :closing
-            # this is a hack so that we can use the super method
-            # and it'll think that the current state is open
-            @state = :open if @state == :connecting
+            @parser = nil if proxy_connecting?
           end
           super
         end

data/lib/httpx/plugins/query.rb

@@ -14,6 +14,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # sends a QUERY request for the given +uri+, using +options+ for each.
         def query(*uri, **options)
           request("QUERY", uri, **options)
         end

data/lib/httpx/plugins/stream.rb

@@ -125,12 +125,12 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
-      STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
+      STREAM_REQUEST_OPTIONS = { timeout: { operation_timeout: 60 }.freeze }.freeze
 
       def self.extra_options(options)
         options.merge(
           stream: false,
-          timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
+          timeout: { operation_timeout: 60 },
           stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
         )
       end
@@ -140,14 +140,6 @@ module HTTPX
       # :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
       # :stream_response_class :: Class used to build the stream response object.
       module OptionsMethods
-        def option_stream(val)
-          val
-        end
-
-        def option_stream_response_class(value)
-          value
-        end
-
         def extend_with_plugin_classes(pl)
           return super unless defined?(pl::StreamResponseMethods)
 
@@ -157,6 +149,16 @@ module HTTPX
 
           super
         end
+
+        private
+
+        def option_stream(val)
+          val
+        end
+
+        def option_stream_response_class(value)
+          value
+        end
       end
 
       module InstanceMethods

data/lib/httpx/plugins/stream_bidi.rb

@@ -158,6 +158,10 @@ module HTTPX
           @closed ? :closed : :open
         end
 
+        def closed_for_select?
+          @closed
+        end
+
         # noop
         def log(**, &_); end