httpx 1.8.3 → 1.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5faeb9a08adff992ddd2661b4ccd5176cb2c49d17dbb6a0ea593e1a00c10f35b
-  data.tar.gz: 04f81429bbe03762c641531f0886c9b133f8fa04f8607943534ae36736f705e7
+  metadata.gz: daca70f040a0f0afcc413a6eb3b2e586ae56a5405477c0045cf226aaddf5b49c
+  data.tar.gz: 53910e19b4bf264a2d3cf4aed4298de57c4c6373f868e3d9e187140c15f2ed63
 SHA512:
-  metadata.gz: e750874b2e206ff8604e804fa526847cc40c5511608d818c3a4a7c3136b383f5ca7c8baffbc856bd080ca0b77becc9ebc10d88e7743b5a757c7dd00b49a7e57c
-  data.tar.gz: 0e6e2f5aaf85be60d2fc044cad8ad7e215ae9a7768c11681f0ea5a5bcb264f3f7b8fb3aaf5a21bb2c94a8b03445ac26e0972e637ebc3a30d4f204238d79dbb96
+  metadata.gz: 8b19e3219fb672bfbb45c4e5427c68b9f7ba715003f5b39796a2314bd8b991b58c5bc185767e303557159f04d61d84c19523f25b953b604a74753e55859defb8
+  data.tar.gz: 844bf656335bdbc5599003d6ee9724ff54b05478bfa9de37d5e251acff788154d4d7cea4b866b56c7d4c42d07b1afa8ece6982b02c5236d981843f092ee25ceb

data/doc/release_notes/1_8_4.md

@@ -0,0 +1,13 @@
+# 1.8.4
+
+## Improvements
+
+* send requests which were not sent, or were not processed by the server, when the server sends back a GOAWAY frame (which is deemed recoverable, like when there's no error), after forcing a reconnection (instead of bubbling up the GOAWAY error for all of them).
+
+## Bugfixes
+
+* `:retries` plugin: also retry requests which failed due to a RST_STREAM frame with REFUSED_STREAM error code.
+* `:retries` plugin: resend requests which were piped to a closed connection internally.
+* `:proxy` plugin: fix issue related with reconnecting inactive connections shut down by the peer, when used with the `:persistent` plugin, where the CONNECT request wasn't being sent to establish the tunnel.
+* fixed compatibility issue starting with the `json` gem 3.0.0, when calling `HTTPX::Response#json`.
+* avoid bookkepping issue in session, where the `on_response_arrived` on a request may be called more than once (which it can happen in the `:follow_redirects` plugin, where a request may have completed due to an error, while the request is waiting to be resent due to `retry-after`).

data/lib/httpx/callbacks.rb

@@ -14,9 +14,9 @@ module HTTPX
       end
     end
 
-    def emit(type, *args)
+    def emit(type, ...)
       log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
-      callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
+      callbacks(type).delete_if { |pr| :delete == pr.call(...) } # rubocop:disable Style/YodaCondition
     end
 
     def callbacks_for?(type)

data/lib/httpx/chainable.rb

@@ -13,8 +13,8 @@ module HTTPX
     end
 
     # delegates to the default session (see HTTPX::Session#request).
-    def request(*args, **options)
-      branch(default_options).request(*args, **options)
+    def request(...)
+      branch(default_options).request(...)
     end
 
     def accept(type)

data/lib/httpx/connection/http2.rb

@@ -23,9 +23,29 @@ module HTTPX
     end
 
     class GoawayError < Error
-      def initialize(code = :no_error)
+      UNRECOVERABLE_ERRORS = %i[settings_timeout inadequate_security].freeze
+
+      def initialize(code, last_stream_id)
+        @code = code
+        @last_stream_id = last_stream_id
         super(0, code)
       end
+
+      def last_stream_id
+        return Float::INFINITY if unrecoverable?
+
+        @last_stream_id
+      end
+
+      def unrecoverable?
+        UNRECOVERABLE_ERRORS.include?(@code)
+      end
+    end
+
+    class RstStreamError < Error
+    end
+
+    class RefusedStreamError < RstStreamError
     end
 
     attr_reader :streams, :pending
@@ -149,18 +169,43 @@ module HTTPX
     end
 
     def handle_error(ex, request = nil)
-      if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
-        @connection.goaway(:settings_timeout, "closing due to settings timeout")
-        emit(:close_handshake)
-        settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
-        settings_ex.set_backtrace(ex.backtrace)
-        ex = settings_ex
-      end
-      while (req, _ = @streams.shift)
+      last_stream_id = Float::INFINITY
+      case ex
+      when OperationTimeoutError
+        if !@handshake_completed && @connection.state != :closed
+          @connection.goaway(:settings_timeout, "closing due to settings timeout")
+          emit(:close_handshake)
+          settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
+          settings_ex.set_backtrace(ex.backtrace)
+          ex = settings_ex
+        end
+      when GoawayError
+        last_stream_id = ex.last_stream_id
+      end
+
+      inflight_unprocessed_requests = [] #: Array[Request]
+
+      while (req, stream = @streams.shift)
         next if request && request == req
 
+        if stream.id > last_stream_id
+          req.transition(:idle)
+          # unprocessed request
+          inflight_unprocessed_requests << req
+
+          next
+        end
+
         emit(:error, req, ex)
       end
+
+      if ex.is_a?(GoawayError) && !ex.unrecoverable?
+        # resend unprocessed requests on a different connection
+        @pending.unshift(*inflight_unprocessed_requests) if inflight_unprocessed_requests.any?
+        emit(:exhausted, ex) if @pending.any?
+        return
+      end
+
       while (req = @pending.shift)
         next if request && request == req
 
@@ -395,7 +440,14 @@ module HTTPX
         when :http_1_1_required
           emit(:error, request, error)
         else
-          ex = Error.new(stream.id, error)
+          ex =
+            case error
+            when :refused_stream
+              RefusedStreamError.new(stream.id, error)
+            else
+              RstStreamError.new(stream.id, error)
+            end
+
           ex.set_backtrace(caller)
           response = ErrorResponse.new(request, ex)
           request.response = response
@@ -432,7 +484,7 @@ module HTTPX
       send_pending
     end
 
-    def on_close(_last_frame, error, _payload)
+    def on_close(last_stream_id, error, _payload)
       is_connection_closed = @connection.closed?
       if error
         @buffer.clear if is_connection_closed
@@ -442,12 +494,11 @@ module HTTPX
             emit(:error, request, error)
           end
         else
-          ex = GoawayError.new(error)
+          ex = GoawayError.new(error, last_stream_id)
           ex.set_backtrace(caller)
 
           handle_error(ex)
           teardown
-
         end
       end
       return unless is_connection_closed && @streams.empty?

data/lib/httpx/connection.rb

@@ -51,7 +51,10 @@ module HTTPX
                                                   @callbacks = @ping_timer = @family =
                                                                  @io = @ssl_session = @timeout = @connected_at = @response_received_at = nil
 
-      @exhausted = @cloned = @main_sibling = false
+      @exhausted = @cloned = @main_sibling =
+                     # variable used to gate against a potential endless loop where the peer continuously closes the connection with
+                     # GOAWAY frames without ever processing a request.
+                     @previously_exhausted_with_error = false
 
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
@@ -711,6 +714,7 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           build_altsvc_connection(alt_origin, origin, alt_params)
         end
+        @previously_exhausted_with_error = false
         @response_received_at = Utils.now
         @no_more_requests_counter = 0
         @inflight -= 1
@@ -726,9 +730,19 @@ module HTTPX
       parser.on(:promise) do |request, stream|
         request.emit(:promise, parser, stream)
       end
-      parser.on(:exhausted) do
+      parser.on(:exhausted) do |error|
         enqueue_pending_requests_from_parser(parser)
 
+        if error
+          if @previously_exhausted_with_error
+            @previously_exhausted_with_error = false
+            on_error(error)
+            next
+          else
+            @previously_exhausted_with_error = true
+          end
+        end
+
         @exhausted = true
         parser.close
 

data/lib/httpx/plugins/cookies/cookie.rb

@@ -68,7 +68,7 @@ module HTTPX
       end
 
       class << self
-        def new(cookie, *args)
+        def new(cookie, *)
           case cookie
           when self
             cookie

data/lib/httpx/plugins/cookies.rb

@@ -46,8 +46,8 @@ module HTTPX
 
         # factory method to return a Jar to the user, which can then manipulate
         # externally to the session.
-        def make_jar(*args)
-          Jar.new(*args)
+        def make_jar(...)
+          Jar.new(...)
         end
 
         private

data/lib/httpx/plugins/follow_redirects.rb

@@ -75,9 +75,12 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
-          return response unless response.is_a?(Response)
-          return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
-          return response unless max_redirects.positive?
+          return response unless response.is_a?(Response) &&
+                                 # it's a redirect response with location
+                                 REDIRECT_STATUS.include?(response.status) &&
+                                 response.headers.key?("location") &&
+                                 # maximum number of redirects hasn't been reached yet
+                                 max_redirects.positive?
 
           redirect_uri = __get_location_from_response(response)
 
@@ -190,8 +193,7 @@ module HTTPX
 
         # :nodoc:
         def __get_location_from_response(response)
-          # @type var location_uri: http_uri
-          location_uri = URI(response.headers["location"])
+          location_uri = URI(response.headers["location"]) #: http_uri
           location_uri = response.uri.merge(location_uri) if location_uri.relative?
           location_uri
         end

data/lib/httpx/plugins/grpc/call.rb

@@ -48,12 +48,12 @@ module HTTPX
           end
         end
 
-        def respond_to_missing?(meth, *args, &blk)
-          grpc_response.respond_to?(meth, *args) || super
+        def respond_to_missing?(meth, ...)
+          grpc_response.respond_to?(meth, ...) || super
         end
 
-        def method_missing(meth, *args, &blk)
-          return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
+        def method_missing(meth, ...)
+          return grpc_response.__send__(meth, ...) if grpc_response.respond_to?(meth)
 
           super
         end

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -78,8 +78,8 @@ module HTTPX
         end
       end
 
-      def self.encode(*args, **kwargs)
-        Deflater.new(*args, **kwargs)
+      def self.encode(...)
+        Deflater.new(...)
       end
 
       def self.decode(response)

data/lib/httpx/plugins/proxy/http.rb

@@ -46,6 +46,11 @@ module HTTPX
         end
 
         module ConnectionMethods
+          def initialize(*)
+            @tunnel_parser = nil
+            super
+          end
+
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
@@ -59,6 +64,12 @@ module HTTPX
 
           private
 
+          def purge_after_closed
+            super
+
+            @tunnel_parser = nil
+          end
+
           def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "http"
 
@@ -69,8 +80,15 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser || begin
-                @parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+              @tunnel_parser || begin
+                if @parser
+                  # in case this happens, requests from @parser need to be reenqueued for delivery.
+                  enqueue_pending_requests_from_parser(@parser)
+                  @parser = nil
+                end
+
+                @tunnel_parser = @parser = parser =
+                                   parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
@@ -86,6 +104,9 @@ module HTTPX
 
                     initial_state = @state
 
+                    # because #reset will nillify @parser...
+                    tunnel_parser = @parser
+
                     reset
 
                     if @pending.empty?
@@ -98,8 +119,8 @@ module HTTPX
                     connect_request = parser = nil
 
                     if initial_state == :connecting
-                      parser = @parser
-                      @parser.reset
+                      parser = tunnel_parser
+                      parser.reset
                       if @pending.first.is_a?(ConnectRequest)
                         connect_request = @pending.shift # this happened when reenqueing
                       end
@@ -107,7 +128,7 @@ module HTTPX
 
                     idling
 
-                    @parser = parser
+                    @tunnel_parser = @parser = parser
                     if connect_request
                       @inflight += 1
                       parser.send(connect_request)
@@ -124,7 +145,7 @@ module HTTPX
               case @state
               when :connecting
                 parser = @parser
-                @parser = nil
+                @tunnel_parser = @parser = nil
                 parser.close
               when :idle
                 @parser.callbacks.clear

data/lib/httpx/plugins/proxy.rb

@@ -92,16 +92,16 @@ module HTTPX
           @authenticator = load_authenticator(scheme, @username, @password)
         end
 
-        def can_authenticate?(*args)
+        def can_authenticate?(...)
           return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
 
-          @authenticator.can_authenticate?(*args)
+          @authenticator.can_authenticate?(...)
         end
 
-        def authenticate(*args)
+        def authenticate(...)
           return unless @authenticator
 
-          @authenticator.authenticate(*args)
+          @authenticator.authenticate(...)
         end
 
         def ==(other)

data/lib/httpx/plugins/retries.rb

@@ -31,11 +31,21 @@ module HTTPX
         TLSError,
         Zlib::BufError,
         PingTimeoutError,
+        # HTTP/2 GOAWAY or RST_STREAM errors guaranteed to be retriable for all
+        # types of requests.
+        Connection::HTTP2::RefusedStreamError,
         Connection::HTTP2::GoawayError,
         Connection::HTTP2::PingError,
+
+        # happens if connections were somehow sent to a closed or closing
+        # HTTP2::Connection, request guaranteed to not have been processed.
+        ::HTTP2::Error::ConnectionClosed,
       ].freeze
 
       RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        # most RST_STREAM errors aren't safely retriable, as request may have
+        # been partially processed.
+        Connection::HTTP2::RstStreamError,
         Parser::Error,
         TimeoutError,
       ]).freeze
@@ -256,7 +266,7 @@ module HTTPX
         attr_writer :partial_response
 
         # initializes the request instance, sets the number of retries for the request.
-        def initialize(*args)
+        def initialize(*)
           super
           @retries = @options.max_retries
           @partial_response = nil

data/lib/httpx/plugins/server_sent_events.rb

@@ -40,7 +40,7 @@ module HTTPX
       end
 
       module InstanceMethods
-        def request(*args, **options)
+        def request(*, **options)
           options[:stream] = true if options[:event_stream]
 
           super

data/lib/httpx/plugins/stream.rb

@@ -111,10 +111,10 @@ module HTTPX
       end || super
     end
 
-    def method_missing(meth, *args, **kwargs, &block)
+    def method_missing(meth, ...)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, **kwargs, &block)
+      response.__send__(meth, ...)
     end
   end
 

data/lib/httpx/plugins/tracing.rb

@@ -36,9 +36,9 @@ module HTTPX::Plugins
       %i[start finish reset enabled?].each do |callback|
         class_eval(<<-OUT, __FILE__, __LINE__ + 1)
           # proxies ##{callback} calls to wrapper tracers.
-          def #{callback}(*args)                        # def start(*args)
-            @tracers.each { |t| t.#{callback}(*args) }  # @tracers.each { |t| t.start(*args) }
-          end                                           # end
+          def #{callback}(...)                        # def start(...)
+            @tracers.each { |t| t.#{callback}(...) }  # @tracers.each { |t| t.start(....) }
+          end                                         # end
         OUT
       end
     end

data/lib/httpx/request.rb

@@ -107,7 +107,7 @@ module HTTPX
       @connection = @response =
         @drainer = @peer_address = @callbacks =
                      @informational_status = @on_response_arrived = nil
-      @ping = @started = false
+      @ping = @started = @complete = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
@@ -121,8 +121,11 @@ module HTTPX
     end
 
     def complete!(response = @response)
+      return false if @complete
+
       emit(:complete, response)
       reset_timers(true)
+      @complete = true
     end
 
     # whether request has been buffered with a ping
@@ -296,7 +299,7 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
-        @ping = false
+        @ping = @complete = false
         @response = @drainer = nil
 
         # request may be sent to a different connection and will be

data/lib/httpx/resolver/multi.rb

@@ -35,8 +35,8 @@ module HTTPX
       @resolvers.each { |r| r.current_session = s }
     end
 
-    def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
+    def log(...)
+      @resolvers.each { |r| r.log(...) }
     end
 
     def closed?

data/lib/httpx/resolver/resolver.rb

@@ -56,9 +56,9 @@ module HTTPX
       call
     end
 
-    def force_close(*args)
+    def force_close(...)
       while (connection = @connections.shift)
-        connection.force_close(*args)
+        connection.force_close(...)
       end
     end
 

data/lib/httpx/response/body.rb

@@ -69,7 +69,7 @@ module HTTPX
     end
 
     # reads a chunk from the payload (implementation of the IO reader protocol).
-    def read(*args)
+    def read(...)
       return unless @buffer
 
       unless @reader
@@ -77,7 +77,7 @@ module HTTPX
         @reader = @buffer
       end
 
-      @reader.read(*args)
+      @reader.read(...)
     end
 
     # size of the decoded response payload. May differ from "content-length" header if

data/lib/httpx/response.rb

@@ -176,8 +176,8 @@ module HTTPX
     #
     #   response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
     #   response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
-    def json(*args)
-      decode(Transcoder::JSON, *args)
+    def json(...)
+      decode(Transcoder::JSON, ...)
     end
 
     # decodes the response payload into a ruby object **if** the payload is valid
@@ -200,7 +200,7 @@ module HTTPX
     #
     # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
     # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
-    def decode(transcoder, *args)
+    def decode(transcoder, ...)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
 
       decoder = transcoder.decode(self)
@@ -209,7 +209,7 @@ module HTTPX
 
       @body.rewind
 
-      decoder.call(self, *args)
+      decoder.call(self, ...)
     end
   end
 

data/lib/httpx/session.rb

@@ -355,7 +355,8 @@ module HTTPX
           # making the next loop cheaper (because we're dropping).
           next unless response
 
-          request.complete!(response)
+          next unless request.complete!(response)
+
           responses[idx] = response
           request.on_response_arrived = nil
           pending -= 1

data/lib/httpx/transcoder/chunker.rb

@@ -24,8 +24,8 @@ module HTTPX::Transcoder
         yield "0#{CRLF}"
       end
 
-      def respond_to_missing?(meth, *args)
-        @raw.respond_to?(meth, *args) || super
+      def respond_to_missing?(...)
+        @raw.respond_to?(...) || super
       end
     end
 

data/lib/httpx/transcoder/json.rb

@@ -53,18 +53,18 @@ module HTTPX::Transcoder
 
     # rubocop:disable-next Style/SingleLineMethods
     if defined?(MultiJson)
-      def json_load(*args); MultiJson.load(*args); end
-      def json_dump(*args); MultiJson.dump(*args); end
+      def json_load(...); MultiJson.load(...); end
+      def json_dump(...); MultiJson.dump(...); end
     elsif defined?(Oj)
-      def json_load(response, *args); Oj.load(response.to_s, *args); end
+      def json_load(response, ...); Oj.load(response.to_s, ...); end
       def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
     elsif defined?(Yajl)
-      def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
-      def json_dump(*args); Yajl::Encoder.encode(*args); end
+      def json_load(response, ...); Yajl::Parser.new(...).parse(response.to_s); end
+      def json_dump(...); Yajl::Encoder.encode(...); end
     else
       require "json"
-      def json_load(*args); ::JSON.parse(*args); end
-      def json_dump(*args); ::JSON.generate(*args); end
+      def json_load(...); ::JSON.parse(...); end
+      def json_dump(...); ::JSON.generate(...); end
     end
   end
 end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.8.3"
+  VERSION = "1.8.4"
 end

data/sig/connection/http2.rbs

@@ -116,11 +116,25 @@ module HTTPX
     end
 
     class GoawayError < Error
-      def initialize: (?Symbol code) -> void
+      UNRECOVERABLE_ERRORS: Array[Symbol]
+
+      @code: Symbol
+
+      def initialize: (Symbol code, Integer last_stream_id) -> void
+
+      def last_stream_id: () -> Numeric
+
+      def unrecoverable?: () -> bool
     end
 
     class PingError < Error
       def initialize: () -> void
     end
+
+    class RstStreamError < Error
+    end
+
+    class RefusedStreamError < Error
+    end
   end
 end

data/sig/connection.rbs

@@ -47,6 +47,7 @@ module HTTPX
     @response_received_at: Float
     @exhausted: bool
     @cloned: bool
+    @previously_exhausted_with_error: bool
     @coalesced_connection: instance?
     @altsvc_connection: instance?
     @sibling: instance?

data/sig/plugins/proxy/http.rbs

@@ -12,6 +12,8 @@ module HTTPX
         end
 
         module ConnectionMethods
+          @tunnel_parser: Connection::HTTP1 | Connection::HTTP2
+
           def __http_proxy_connect: (Connection::_Parser parser) ->  void
           def __http_on_connect: (top, Response) ->  void
         end

data/sig/request.rbs

@@ -32,10 +32,11 @@ module HTTPX
     @query: String?
     @drainer: Enumerator[String, void]?
     @started: bool
+    @complete: bool
 
     def initialize: (Symbol | String verb, generic_uri uri, Options options, ?request_params params) -> untyped
 
-    def complete!: (?response response) -> void
+    def complete!: (?response response) -> bool
 
     def started?: () -> bool
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 1.8.3
+  version: 1.8.4
 platform: ruby
 authors:
 - Tiago Cardoso
@@ -172,6 +172,7 @@ extra_rdoc_files:
 - doc/release_notes/1_8_1.md
 - doc/release_notes/1_8_2.md
 - doc/release_notes/1_8_3.md
+- doc/release_notes/1_8_4.md
 files:
 - LICENSE.txt
 - README.md
@@ -316,6 +317,7 @@ files:
 - doc/release_notes/1_8_1.md
 - doc/release_notes/1_8_2.md
 - doc/release_notes/1_8_3.md
+- doc/release_notes/1_8_4.md
 - lib/httpx.rb
 - lib/httpx/adapters/datadog.rb
 - lib/httpx/adapters/faraday.rb