grpc 1.47.0-x86_64-linux → 1.49.0.pre1-x86_64-linux

data/src/core/ext/transport/chttp2/transport/flow_control.h

@@ -22,20 +22,23 @@
 #include <grpc/support/port_platform.h>
 
 #include <stdint.h>
-#include <stdlib.h>
 
-#include <algorithm>
+#include <iosfwd>
+#include <string>
+
+#include "absl/functional/function_ref.h"
+#include "absl/status/status.h"
+#include "absl/types/optional.h"
+#include "absl/utility/utility.h"
+
+#include <grpc/support/log.h>
 
-#include "src/core/ext/transport/chttp2/transport/http2_settings.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/time.h"
-#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/transport/bdp_estimator.h"
 #include "src/core/lib/transport/pid_controller.h"
 
-struct grpc_chttp2_transport;
-struct grpc_chttp2_stream;
-
 extern grpc_core::TraceFlag grpc_flowctl_trace;
 
 namespace grpc {
@@ -48,18 +51,19 @@ namespace grpc_core {
 namespace chttp2 {
 
 static constexpr uint32_t kDefaultWindow = 65535;
+static constexpr uint32_t kDefaultFrameSize = 16384;
 static constexpr int64_t kMaxWindow = static_cast<int64_t>((1u << 31) - 1);
 // TODO(ncteisen): Tune this
 static constexpr uint32_t kFrameSize = 1024 * 1024;
 static constexpr const uint32_t kMinInitialWindowSize = 128;
 static constexpr const uint32_t kMaxInitialWindowSize = (1u << 30);
 // The maximum per-stream flow control window delta to advertise.
-static constexpr const uint32_t kMaxWindowDelta = (1u << 20);
+static constexpr const int64_t kMaxWindowDelta = (1u << 20);
 
 class TransportFlowControl;
 class StreamFlowControl;
 
-extern bool g_test_only_transport_flow_control_window_check;
+enum class StallEdge { kNoChange, kStalled, kUnstalled };
 
 // Encapsulates a collections of actions the transport needs to take with
 // regard to flow control. Each action comes with urgencies that tell the
@@ -109,7 +113,20 @@ class FlowControlAction {
   }
 
   static const char* UrgencyString(Urgency u);
-  void Trace(grpc_chttp2_transport* t) const;
+  std::string DebugString() const;
+
+  void AssertEmpty() { GPR_ASSERT(*this == FlowControlAction()); }
+
+  bool operator==(const FlowControlAction& other) const {
+    return send_stream_update_ == other.send_stream_update_ &&
+           send_transport_update_ == other.send_transport_update_ &&
+           send_initial_window_update_ == other.send_initial_window_update_ &&
+           send_max_frame_size_update_ == other.send_max_frame_size_update_ &&
+           (send_initial_window_update_ == Urgency::NO_ACTION_NEEDED ||
+            initial_window_size_ == other.initial_window_size_) &&
+           (send_max_frame_size_update_ == Urgency::NO_ACTION_NEEDED ||
+            max_frame_size_ == other.max_frame_size_);
+  }
 
  private:
   Urgency send_stream_update_ = Urgency::NO_ACTION_NEEDED;
@@ -120,120 +137,16 @@ class FlowControlAction {
   uint32_t max_frame_size_ = 0;
 };
 
-class FlowControlTrace {
- public:
-  FlowControlTrace(const char* reason, TransportFlowControl* tfc,
-                   StreamFlowControl* sfc) {
-    if (enabled_) Init(reason, tfc, sfc);
-  }
-
-  ~FlowControlTrace() {
-    if (enabled_) Finish();
-  }
-
- private:
-  void Init(const char* reason, TransportFlowControl* tfc,
-            StreamFlowControl* sfc);
-  void Finish();
-
-  const bool enabled_ = GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace);
-
-  TransportFlowControl* tfc_;
-  StreamFlowControl* sfc_;
-  const char* reason_;
-  int64_t remote_window_;
-  int64_t target_window_;
-  int64_t announced_window_;
-  int64_t remote_window_delta_;
-  int64_t local_window_delta_;
-  int64_t announced_window_delta_;
-};
-
-// Fat interface with all methods a flow control implementation needs to
-// support.
-class TransportFlowControlBase {
- public:
-  TransportFlowControlBase() {}
-  virtual ~TransportFlowControlBase() {}
-
-  // Is flow control enabled? This is needed in other codepaths like the checks
-  // in parsing and in writing.
-  virtual bool flow_control_enabled() const = 0;
-
-  // Called to check if the transport needs to send a WINDOW_UPDATE frame
-  virtual uint32_t MaybeSendUpdate(bool /* writing_anyway */) = 0;
-
-  // Using the protected members, returns and Action to be taken by the
-  // tranport.
-  virtual FlowControlAction MakeAction() = 0;
-
-  // Using the protected members, returns and Action to be taken by the
-  // tranport. Also checks for updates to our BDP estimate and acts
-  // accordingly.
-  virtual FlowControlAction PeriodicUpdate() = 0;
-
-  // Called to do bookkeeping when a stream owned by this transport sends
-  // data on the wire
-  virtual void StreamSentData(int64_t /* size */) = 0;
-
-  // Called to do bookkeeping when a stream owned by this transport receives
-  // data from the wire. Also does error checking for frame size.
-  virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0;
-
-  // Called to do bookkeeping when we receive a WINDOW_UPDATE frame.
-  virtual void RecvUpdate(uint32_t /* size */) = 0;
-
-  // Returns the BdpEstimator held by this object. Caller is responsible for
-  // checking for nullptr. TODO(ncteisen): consider fully encapsulating all
-  // bdp estimator actions inside TransportFlowControl
-  virtual BdpEstimator* bdp_estimator() { return nullptr; }
-
-  // Getters
-  int64_t remote_window() const { return remote_window_; }
-  virtual int64_t target_window() const { return target_initial_window_size_; }
-  int64_t announced_window() const { return announced_window_; }
-
-  // Used in certain benchmarks in which we don't want FlowControl to be a
-  // factor
-  virtual void TestOnlyForceHugeWindow() {}
-
- protected:
-  friend class grpc::testing::TrickledCHTTP2;
-  int64_t remote_window_ = kDefaultWindow;
-  int64_t target_initial_window_size_ = kDefaultWindow;
-  int64_t announced_window_ = kDefaultWindow;
-};
-
-// Implementation of flow control that does NOTHING. Always returns maximum
-// values, never initiates writes, and assumes that the remote peer is doing
-// the same. To be used to narrow down on flow control as the cause of negative
-// performance.
-class TransportFlowControlDisabled final : public TransportFlowControlBase {
- public:
-  // Maxes out all values
-  explicit TransportFlowControlDisabled(grpc_chttp2_transport* t);
-
-  bool flow_control_enabled() const override { return false; }
-
-  // Never do anything.
-  uint32_t MaybeSendUpdate(bool /* writing_anyway */) override { return 0; }
-  FlowControlAction MakeAction() override { return FlowControlAction(); }
-  FlowControlAction PeriodicUpdate() override { return FlowControlAction(); }
-  void StreamSentData(int64_t /* size */) override {}
-  grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override {
-    return GRPC_ERROR_NONE;
-  }
-  void RecvUpdate(uint32_t /* size */) override {}
-};
+std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency urgency);
+std::ostream& operator<<(std::ostream& out, const FlowControlAction& action);
 
 // Implementation of flow control that abides to HTTP/2 spec and attempts
 // to be as performant as possible.
-class TransportFlowControl final : public TransportFlowControlBase {
+class TransportFlowControl final {
  public:
-  TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe);
-  ~TransportFlowControl() override {}
-
-  bool flow_control_enabled() const override { return true; }
+  explicit TransportFlowControl(const char* name, bool enable_bdp_probe,
+                                MemoryOwner* memory_owner);
+  ~TransportFlowControl() {}
 
   bool bdp_probe() const { return enable_bdp_probe_; }
 
@@ -241,85 +154,113 @@ class TransportFlowControl final : public TransportFlowControlBase {
   // else returns zero; writing_anyway indicates if a write would happen
   // regardless of the send - if it is false and this function returns non-zero,
   // this announce will cause a write to occur
-  uint32_t MaybeSendUpdate(bool writing_anyway) override;
+  uint32_t MaybeSendUpdate(bool writing_anyway);
+
+  // Track an update to the incoming flow control counters - that is how many
+  // tokens we report to our peer that we're willing to accept.
+  // Instantiators *must* call MakeAction before destruction of this value.
+  class IncomingUpdateContext {
+   public:
+    explicit IncomingUpdateContext(TransportFlowControl* tfc) : tfc_(tfc) {}
+    ~IncomingUpdateContext() { GPR_ASSERT(tfc_ == nullptr); }
+
+    IncomingUpdateContext(const IncomingUpdateContext&) = delete;
+    IncomingUpdateContext& operator=(const IncomingUpdateContext&) = delete;
+
+    // Reads the flow control data and returns an actionable struct that will
+    // tell chttp2 exactly what it needs to do
+    FlowControlAction MakeAction() {
+      return absl::exchange(tfc_, nullptr)->UpdateAction(FlowControlAction());
+    }
 
-  // Reads the flow control data and returns and actionable struct that will
-  // tell chttp2 exactly what it needs to do
-  FlowControlAction MakeAction() override {
-    return UpdateAction(FlowControlAction());
-  }
+    // Notify of data receipt. Returns OkStatus if the data was accepted,
+    // else an error status if the connection should be closed.
+    absl::Status RecvData(
+        int64_t incoming_frame_size, absl::FunctionRef<absl::Status()> stream =
+                                         []() { return absl::OkStatus(); });
+
+    // Update a stream announce window delta, keeping track of how much total
+    // positive delta is present on the transport.
+    void UpdateAnnouncedWindowDelta(int64_t* delta, int64_t change) {
+      if (change == 0) return;
+      if (*delta > 0) {
+        tfc_->announced_stream_total_over_incoming_window_ -= *delta;
+      }
+      *delta += change;
+      if (*delta > 0) {
+        tfc_->announced_stream_total_over_incoming_window_ += *delta;
+      }
+    }
+
+   private:
+    TransportFlowControl* tfc_;
+  };
+
+  // Track an update to the outgoing flow control counters - that is how many
+  // tokens our peer has said we can send.
+  class OutgoingUpdateContext {
+   public:
+    explicit OutgoingUpdateContext(TransportFlowControl* tfc) : tfc_(tfc) {}
+    void StreamSentData(int64_t size) { tfc_->remote_window_ -= size; }
+
+    // we have received a WINDOW_UPDATE frame for a transport
+    void RecvUpdate(uint32_t size) { tfc_->remote_window_ += size; }
+
+    // Finish the update and check whether we became stalled or unstalled.
+    StallEdge Finish() {
+      bool is_stalled = tfc_->remote_window_ <= 0;
+      if (is_stalled != was_stalled_) {
+        return is_stalled ? StallEdge::kStalled : StallEdge::kUnstalled;
+      } else {
+        return StallEdge::kNoChange;
+      }
+    }
+
+   private:
+    TransportFlowControl* tfc_;
+    const bool was_stalled_ = tfc_->remote_window_ <= 0;
+  };
 
   // Call periodically (at a low-ish rate, 100ms - 10s makes sense)
   // to perform more complex flow control calculations and return an action
   // to let chttp2 change its parameters
-  FlowControlAction PeriodicUpdate() override;
+  FlowControlAction PeriodicUpdate();
 
-  void StreamSentData(int64_t size) override { remote_window_ -= size; }
+  int64_t target_window() const;
+  int64_t target_frame_size() const { return target_frame_size_; }
 
-  grpc_error_handle ValidateRecvData(int64_t incoming_frame_size);
-  void CommitRecvData(int64_t incoming_frame_size) {
-    announced_window_ -= incoming_frame_size;
-  }
+  BdpEstimator* bdp_estimator() { return &bdp_estimator_; }
 
-  grpc_error_handle RecvData(int64_t incoming_frame_size) override {
-    FlowControlTrace trace("  data recv", this, nullptr);
-    grpc_error_handle error = ValidateRecvData(incoming_frame_size);
-    if (error != GRPC_ERROR_NONE) return error;
-    CommitRecvData(incoming_frame_size);
-    return GRPC_ERROR_NONE;
-  }
+  uint32_t acked_init_window() const { return acked_init_window_; }
 
-  // we have received a WINDOW_UPDATE frame for a transport
-  void RecvUpdate(uint32_t size) override {
-    FlowControlTrace trace("t updt recv", this, nullptr);
-    remote_window_ += size;
-  }
-
-  // See comment above announced_stream_total_over_incoming_window_ for the
-  // logic behind this decision.
-  int64_t target_window() const override {
-    return static_cast<uint32_t>(
-        std::min(static_cast<int64_t>((1u << 31) - 1),
-                 announced_stream_total_over_incoming_window_ +
-                     target_initial_window_size_));
-  }
+  void SetAckedInitialWindow(uint32_t value) { acked_init_window_ = value; }
 
-  const grpc_chttp2_transport* transport() const { return t_; }
+  // Getters
+  int64_t remote_window() const { return remote_window_; }
+  int64_t announced_window() const { return announced_window_; }
 
-  void PreUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) {
-    if (delta > 0) {
-      announced_stream_total_over_incoming_window_ -= delta;
-    }
+  int64_t announced_stream_total_over_incoming_window() const {
+    return announced_stream_total_over_incoming_window_;
   }
 
-  void PostUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) {
+  void RemoveAnnouncedWindowDelta(int64_t delta) {
     if (delta > 0) {
-      announced_stream_total_over_incoming_window_ += delta;
+      announced_stream_total_over_incoming_window_ -= delta;
     }
   }
 
-  BdpEstimator* bdp_estimator() override { return &bdp_estimator_; }
-
-  void TestOnlyForceHugeWindow() override {
-    announced_window_ = 1024 * 1024 * 1024;
-    remote_window_ = 1024 * 1024 * 1024;
-  }
-
  private:
   double TargetLogBdp();
   double SmoothLogBdp(double value);
-  FlowControlAction::Urgency DeltaUrgency(int64_t value,
-                                          grpc_chttp2_setting_id setting_id);
+  double TargetInitialWindowSizeBasedOnMemoryPressureAndBdp() const;
+  static void UpdateSetting(int64_t* desired_value, int64_t new_desired_value,
+                            FlowControlAction* action,
+                            FlowControlAction& (FlowControlAction::*set)(
+                                FlowControlAction::Urgency, uint32_t));
 
-  FlowControlAction UpdateAction(FlowControlAction action) {
-    if (announced_window_ < target_window() / 2) {
-      action.set_send_transport_update(
-          FlowControlAction::Urgency::UPDATE_IMMEDIATELY);
-    }
-    return action;
-  }
+  FlowControlAction UpdateAction(FlowControlAction action);
 
-  const grpc_chttp2_transport* const t_;
+  MemoryOwner* const memory_owner_;
 
   /** calculating what we should give for local window:
       we track the total amount of flow control over initial window size
@@ -340,140 +281,87 @@ class TransportFlowControl final : public TransportFlowControlBase {
   /* pid controller */
   PidController pid_controller_;
   Timestamp last_pid_update_;
+
+  int64_t remote_window_ = kDefaultWindow;
+  int64_t target_initial_window_size_ = kDefaultWindow;
+  int64_t target_frame_size_ = kDefaultFrameSize;
+  int64_t announced_window_ = kDefaultWindow;
+  uint32_t acked_init_window_ = kDefaultWindow;
 };
 
-// Fat interface with all methods a stream flow control implementation needs
-// to support.
-class StreamFlowControlBase {
+// Implementation of flow control that abides to HTTP/2 spec and attempts
+// to be as performant as possible.
+class StreamFlowControl final {
  public:
-  StreamFlowControlBase() {}
-  virtual ~StreamFlowControlBase() {}
-
-  // Updates an action using the protected members.
-  virtual FlowControlAction UpdateAction(FlowControlAction /* action */) {
-    abort();
+  explicit StreamFlowControl(TransportFlowControl* tfc);
+  ~StreamFlowControl() {
+    tfc_->RemoveAnnouncedWindowDelta(announced_window_delta_);
   }
 
-  // Using the protected members, returns an Action for this stream to be
-  // taken by the tranport.
-  virtual FlowControlAction MakeAction() = 0;
-
-  // Bookkeeping for when data is sent on this stream.
-  virtual void SentData(int64_t /* outgoing_frame_size */) = 0;
-
-  // Bookkeeping and error checking for when data is received by this stream.
-  virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0;
+  // Track an update to the incoming flow control counters - that is how many
+  // tokens we report to our peer that we're willing to accept.
+  // Instantiators *must* call MakeAction before destruction of this value.
+  class IncomingUpdateContext {
+   public:
+    explicit IncomingUpdateContext(StreamFlowControl* sfc)
+        : tfc_upd_(sfc->tfc_), sfc_(sfc) {}
 
-  // Called to check if this stream needs to send a WINDOW_UPDATE frame.
-  virtual uint32_t MaybeSendUpdate() = 0;
-
-  // Bookkeeping for receiving a WINDOW_UPDATE from for this stream.
-  virtual void RecvUpdate(uint32_t /* size */) = 0;
-
-  // Bookkeeping for when a call pulls bytes out of the transport. At this
-  // point we consider the data 'used' and can thus let out peer know we are
-  // ready for more data.
-  virtual void IncomingByteStreamUpdate(size_t /* max_size_hint */,
-                                        size_t /* have_already */) {
-    abort();
-  }
-
-  // Used in certain benchmarks in which we don't want FlowControl to be a
-  // factor
-  virtual void TestOnlyForceHugeWindow() {}
+    FlowControlAction MakeAction() {
+      return sfc_->UpdateAction(tfc_upd_.MakeAction());
+    }
 
-  // Getters
-  int64_t remote_window_delta() const { return remote_window_delta_; }
-  int64_t local_window_delta() const { return local_window_delta_; }
-  int64_t announced_window_delta() const { return announced_window_delta_; }
+    // we have received data from the wire
+    absl::Status RecvData(int64_t incoming_frame_size);
 
- protected:
-  friend class grpc::testing::TrickledCHTTP2;
-  int64_t remote_window_delta_ = 0;
-  int64_t local_window_delta_ = 0;
-  int64_t announced_window_delta_ = 0;
-};
-
-// Implementation of flow control that does NOTHING. Always returns maximum
-// values, never initiates writes, and assumes that the remote peer is doing
-// the same. To be used to narrow down on flow control as the cause of negative
-// performance.
-class StreamFlowControlDisabled : public StreamFlowControlBase {
- public:
-  FlowControlAction UpdateAction(FlowControlAction action) override {
-    return action;
-  }
-  FlowControlAction MakeAction() override { return FlowControlAction(); }
-  void SentData(int64_t /* outgoing_frame_size */) override {}
-  grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override {
-    return GRPC_ERROR_NONE;
-  }
-  uint32_t MaybeSendUpdate() override { return 0; }
-  void RecvUpdate(uint32_t /* size */) override {}
-  void IncomingByteStreamUpdate(size_t /* max_size_hint */,
-                                size_t /* have_already */) override {}
-};
+    // the application is asking for a certain amount of bytes
+    void SetMinProgressSize(uint32_t min_progress_size) {
+      sfc_->min_progress_size_ = min_progress_size;
+    }
 
-// Implementation of flow control that abides to HTTP/2 spec and attempts
-// to be as performant as possible.
-class StreamFlowControl final : public StreamFlowControlBase {
- public:
-  StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s);
-  ~StreamFlowControl() override {
-    tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
-  }
+    void SetPendingSize(int64_t pending_size);
 
-  FlowControlAction UpdateAction(FlowControlAction action) override;
-  FlowControlAction MakeAction() override {
-    return UpdateAction(tfc_->MakeAction());
-  }
+   private:
+    TransportFlowControl::IncomingUpdateContext tfc_upd_;
+    StreamFlowControl* const sfc_;
+  };
 
-  // we have sent data on the wire, we must track this in our bookkeeping for
-  // the remote peer's flow control.
-  void SentData(int64_t outgoing_frame_size) override {
-    FlowControlTrace tracer("  data sent", tfc_, this);
-    tfc_->StreamSentData(outgoing_frame_size);
-    remote_window_delta_ -= outgoing_frame_size;
-  }
+  // Track an update to the outgoing flow control counters - that is how many
+  // tokens our peer has said we can send.
+  class OutgoingUpdateContext {
+   public:
+    explicit OutgoingUpdateContext(StreamFlowControl* sfc)
+        : tfc_upd_(sfc->tfc_), sfc_(sfc) {}
+    // we have received a WINDOW_UPDATE frame for a stream
+    void RecvUpdate(uint32_t size) { sfc_->remote_window_delta_ += size; }
+    // we have sent data on the wire, we must track this in our bookkeeping for
+    // the remote peer's flow control.
+    void SentData(int64_t outgoing_frame_size) {
+      tfc_upd_.StreamSentData(outgoing_frame_size);
+      sfc_->remote_window_delta_ -= outgoing_frame_size;
+    }
 
-  // we have received data from the wire
-  grpc_error_handle RecvData(int64_t incoming_frame_size) override;
+   private:
+    TransportFlowControl::OutgoingUpdateContext tfc_upd_;
+    StreamFlowControl* const sfc_;
+  };
 
   // returns an announce if we should send a stream update to our peer, else
   // returns zero
-  uint32_t MaybeSendUpdate() override;
-
-  // we have received a WINDOW_UPDATE frame for a stream
-  void RecvUpdate(uint32_t size) override {
-    FlowControlTrace trace("s updt recv", tfc_, this);
-    remote_window_delta_ += size;
-  }
-
-  // the application is asking for a certain amount of bytes
-  void IncomingByteStreamUpdate(size_t max_size_hint,
-                                size_t have_already) override;
+  uint32_t MaybeSendUpdate();
 
   int64_t remote_window_delta() const { return remote_window_delta_; }
-  int64_t local_window_delta() const { return local_window_delta_; }
   int64_t announced_window_delta() const { return announced_window_delta_; }
-
-  const grpc_chttp2_stream* stream() const { return s_; }
-
-  void TestOnlyForceHugeWindow() override {
-    announced_window_delta_ = 1024 * 1024 * 1024;
-    local_window_delta_ = 1024 * 1024 * 1024;
-    remote_window_delta_ = 1024 * 1024 * 1024;
-  }
+  uint32_t min_progress_size() const { return min_progress_size_; }
 
  private:
   TransportFlowControl* const tfc_;
-  const grpc_chttp2_stream* const s_;
+  int64_t min_progress_size_ = 0;
+  int64_t remote_window_delta_ = 0;
+  int64_t announced_window_delta_ = 0;
+  absl::optional<int64_t> pending_size_;
 
-  void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change) {
-    tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
-    announced_window_delta_ += change;
-    tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
-  }
+  FlowControlAction UpdateAction(FlowControlAction action);
+  uint32_t DesiredAnnounceSize() const;
 };
 
 class TestOnlyTransportTargetWindowEstimatesMocker {