grpc 1.83.0 → 1.84.0.pre1

data/src/core/ext/transport/chttp2/transport/flow_control.h

@@ -24,6 +24,7 @@
 #include <stdint.h>
 
 #include <algorithm>
+#include <cstddef>
 #include <iosfwd>
 #include <optional>
 #include <string>
@@ -370,6 +371,7 @@ class TransportFlowControl final {
     int64_t bdp_accumulator;
     int64_t bdp_estimate;
     double bdp_bw_est;
+    bool bdp_ping_blocked;
 
     std::string ToString() const;
     channelz::PropertyList ChannelzProperties() const {
@@ -386,6 +388,7 @@ class TransportFlowControl final {
           .Set("announced_stream_total_over_incoming_window",
                announced_stream_total_over_incoming_window)
           .Set("bdp_accumulator", bdp_accumulator)
+          .Set("bdp_ping_blocked", bdp_ping_blocked)
           .Set("bdp_estimate", bdp_estimate)
           .Set("bdp_bw_est", bdp_bw_est);
     }
@@ -405,6 +408,7 @@ class TransportFlowControl final {
     stats.announced_stream_total_over_incoming_window =
         announced_stream_total_over_incoming_window();
     stats.bdp_accumulator = bdp_estimator_.accumulator();
+    stats.bdp_ping_blocked = bdp_ping_blocked_;
     stats.bdp_estimate = bdp_estimator_.EstimateBdp();
     stats.bdp_bw_est = bdp_estimator_.EstimateBandwidth();
     return stats;
@@ -418,6 +422,67 @@ class TransportFlowControl final {
   }
   size_t window_update_list_size() const { return window_update_list_.size(); }
 
+  //////////////////////////////////////////////////////////////////////////////
+  // BDP Ping related functions.
+
+  // The current implementation of BDP relies on the assumption that our
+  // transport will not send out a new ping till it has received an ACK for the
+  // previous ping.
+
+  // CHTTP2 reference : schedule_bdp_ping_locked
+  void ScheduleBdpPing() {
+    GRPC_DCHECK(enable_bdp_probe_);
+    bdp_estimator_.SchedulePing();
+  }
+
+  // CHTTP2 reference : start_bdp_ping_locked
+  void StartBdpPing() {
+    // TODO(tjagtap) : [PH2][P1] : Reset the keepalive ping timer
+    bdp_estimator_.StartPing();
+  }
+
+  // CHTTP2 reference : finish_bdp_ping_locked
+  Duration CompleteBdpPing() {
+    return bdp_estimator_.CompletePing() - Timestamp::Now();
+  }
+
+  // CHTTP2 reference : init_data_frame_parser
+  void OnReceiveDataFrame(const uint32_t bytes) {
+    if (enable_bdp_probe_) {
+      bdp_estimator_.AddIncomingBytes(bytes);
+      if (bdp_ping_blocked_ && (bytes > 0)) {
+        // Once we receive a data frame, we can start sending BDP pings again.
+        bdp_ping_blocked_ = false;
+        bdp_waker_.Wakeup();
+      }
+    }
+  }
+
+  // CHTTP2 reference : grpc_chttp2_transport::bdp_ping_blocked
+  bool IsBdpPingBlocked() const { return bdp_ping_blocked_; }
+
+  auto WaitForBdpActivation() {
+    GRPC_HTTP2_FLOW_CONTROL_DLOG
+        << "TransportFlowControl::WaitForBdpActivation Factory";
+    return [this]() -> Poll<absl::Status> {
+      GRPC_DCHECK(enable_bdp_probe_);
+      if (bdp_estimator_.accumulator() == 0) {
+        // Block BDP ping till we receive data in a DATA frame on the transport.
+        bdp_ping_blocked_ = true;
+      }
+      if (bdp_ping_blocked_) {
+        bdp_waker_ = GetContext<Activity>()->MakeNonOwningWaker();
+        GRPC_HTTP2_FLOW_CONTROL_DLOG
+            << "TransportFlowControl::WaitForBdpActivation Pending";
+        return Pending{};
+      }
+      GRPC_HTTP2_FLOW_CONTROL_DLOG
+          << "TransportFlowControl::WaitForBdpActivation Resolved";
+      ScheduleBdpPing();
+      return absl::OkStatus();
+    };
+  }
+
  private:
   friend class StreamFlowControl;
 
@@ -461,10 +526,12 @@ class TransportFlowControl final {
   /// incoming_window = total_over - max(bdp - total_under, 0)
   int64_t announced_stream_total_over_incoming_window_ = 0;
 
-  /// should we probe bdp?
+  //  Transport level channel argument GRPC_ARG_HTTP2_BDP_PROBE decides if we
+  //  should probe for bdp.
   const bool enable_bdp_probe_;
-
-  // bdp estimation
+  // CHTTP2 reference : grpc_chttp2_transport::bdp_ping_blocked
+  bool bdp_ping_blocked_;
+  Waker bdp_waker_;
   BdpEstimator bdp_estimator_;
 
   int64_t remote_window_ = kDefaultWindow;
@@ -490,10 +557,17 @@ class TransportFlowControl final {
 class StreamFlowControl final {
  public:
   explicit StreamFlowControl(TransportFlowControl* tfc);
+  // TODO(ritulb) : [PH2][P5] :  When CHTTP2 is removed, delete this
+  // destructor(no-op).
   ~StreamFlowControl() {
     tfc_->RemoveAnnouncedWindowDelta(announced_window_delta_);
   }
 
+  void OnStreamClosed() {
+    tfc_->RemoveAnnouncedWindowDelta(announced_window_delta_);
+    announced_window_delta_ = 0;
+  }
+
   // Track an update to the incoming flow control counters - that is how many
   // tokens we report to our peer that we're willing to accept.
   // Instantiators *must* call MakeAction before destruction of this value.
@@ -526,7 +600,7 @@ class StreamFlowControl final {
     // `min_progress_size` bytes to make progress on reading the current stream.
     // An example usage of this would be, say we receive the first 1000 bytes of
     // a 2000 byte gRPC message, we can call SetMinProgressSize(1000)
-    // TODO(tjagtap) [PH2][P2] Plumb with PH2 flow control.
+    // TODO(tjagtap) [PH2][P2][FCV3] Plumb with PH2 flow control.
     void SetMinProgressSize(int64_t min_progress_size) {
       sfc_->min_progress_size_ = min_progress_size;
     }

data/src/core/ext/transport/chttp2/transport/frame.h

@@ -398,6 +398,8 @@ inline constexpr absl::string_view kSettingsTimeout =
     "timeout. Connection will be closed";
 inline constexpr absl::string_view kLastStreamClosed =
     "Closing last stream and cannot create any more streams.";
+inline constexpr absl::string_view kReceivedStreamAfterGoaway =
+    "Received new stream after GOAWAY was received.";
 inline constexpr absl::string_view kFirstSettingsFrameClient =
     "RFC9113: The server connection preface consists of a potentially empty "
     "SETTINGS frame that MUST be the first frame the server sends in the "

data/src/core/ext/transport/chttp2/transport/frame_data.cc

@@ -48,6 +48,7 @@ absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags,
   } else {
     s->received_last_frame = false;
   }
+  ++s->num_frames;
 
   return absl::OkStatus();
 }
@@ -123,6 +124,23 @@ grpc_core::Poll<grpc_error_handle> grpc_deframe_unprocessed_incoming_frames(
                   (static_cast<uint32_t>(header[3]) << 8) |
                   static_cast<uint32_t>(header[4]);
 
+  if (grpc_core::IsMessageSizeRefactoringEnabled()) {
+    if (s->max_recv_message_length.has_value() &&
+        length > *(s->max_recv_message_length)) {
+      error = GRPC_ERROR_CREATE(
+          absl::StrFormat("%s: Received message larger than max (%d vs. %u)",
+                          s->t->is_client ? "CLIENT" : "SERVER", length,
+                          *(s->max_recv_message_length)));
+      error = grpc_error_set_int(error, grpc_core::StatusIntProperty::kStreamId,
+                                 static_cast<intptr_t>(s->id));
+      // Attach the explicit gRPC status code to fail the RPC correctly
+      error = grpc_core::ReplaceStatusCode(
+          error, absl::StatusCode::kResourceExhausted);
+      s->message_size_limit_exceeded = true;
+      return error;
+    }
+  }
+
   if (slices->length < length + GRPC_HEADER_SIZE_IN_BYTES) {
     if (min_progress_size != nullptr) {
       *min_progress_size = length + GRPC_HEADER_SIZE_IN_BYTES - slices->length;
@@ -138,6 +156,7 @@ grpc_core::Poll<grpc_error_handle> grpc_deframe_unprocessed_incoming_frames(
     grpc_slice_buffer_move_first_into_buffer(slices, GRPC_HEADER_SIZE_IN_BYTES,
                                              header);
     grpc_slice_buffer_move_first(slices, length, stream_out->c_slice_buffer());
+    s->num_frames = 0;
   }
 
   return absl::OkStatus();
@@ -148,8 +167,20 @@ grpc_error_handle grpc_chttp2_data_parser_parse(void* /*parser*/,
                                                 grpc_chttp2_stream* s,
                                                 const grpc_slice& slice,
                                                 int is_last) {
-  grpc_core::CSliceRef(slice);
-  grpc_slice_buffer_add(&s->frame_storage, slice);
+  const size_t slice_len = GRPC_SLICE_LENGTH(slice);
+  bool is_small_frame = 0 < slice_len && slice_len < GRPC_SLICE_INLINED_SIZE;
+  bool multiple_small_frames =
+      (s->num_frames >= 64) &&
+      ((s->frame_storage.length / s->num_frames) < GRPC_SLICE_INLINED_SIZE);
+  if (GPR_UNLIKELY(multiple_small_frames && is_small_frame &&
+                   grpc_core::IsHeaderDataFrameEnabled())) {
+    uint8_t* append_ptr =
+        grpc_slice_buffer_tiny_add(&s->frame_storage, slice_len);
+    memcpy(append_ptr, GRPC_SLICE_START_PTR(slice), slice_len);
+  } else {
+    grpc_core::CSliceRef(slice);
+    grpc_slice_buffer_add(&s->frame_storage, slice);
+  }
   grpc_chttp2_maybe_complete_recv_message(t, s);
 
   if (is_last) {

data/src/core/ext/transport/chttp2/transport/frame_goaway.cc

@@ -31,11 +31,15 @@
 #include "absl/strings/string_view.h"
 
 void grpc_chttp2_goaway_parser_init(grpc_chttp2_goaway_parser* p) {
-  p->debug_data = nullptr;
+  if (!grpc_core::IsPh2Perf01Enabled()) {
+    p->debug_data = nullptr;
+  }
 }
 
 void grpc_chttp2_goaway_parser_destroy(grpc_chttp2_goaway_parser* p) {
-  gpr_free(p->debug_data);
+  if (!grpc_core::IsPh2Perf01Enabled()) {
+    gpr_free(p->debug_data);
+  }
 }
 
 grpc_error_handle grpc_chttp2_goaway_parser_begin_frame(
@@ -45,10 +49,14 @@ grpc_error_handle grpc_chttp2_goaway_parser_begin_frame(
         absl::StrFormat("goaway frame too short (%d bytes)", length));
   }
 
-  gpr_free(p->debug_data);
   p->debug_length = length - 8;
-  p->debug_data = static_cast<char*>(gpr_malloc(p->debug_length));
-  p->debug_pos = 0;
+  if (grpc_core::IsPh2Perf01Enabled()) {
+    p->debug_slice_buffer.Clear();
+  } else {
+    gpr_free(p->debug_data);
+    p->debug_data = static_cast<char*>(gpr_malloc(p->debug_length));
+    p->debug_pos = 0;
+  }
   p->state = GRPC_CHTTP2_GOAWAY_LSI0;
   return absl::OkStatus();
 }
@@ -130,24 +138,48 @@ grpc_error_handle grpc_chttp2_goaway_parser_parse(void* parser,
       ++cur;
       [[fallthrough]];
     case GRPC_CHTTP2_GOAWAY_DEBUG:
-      if (end != cur) {
-        memcpy(p->debug_data + p->debug_pos, cur,
-               static_cast<size_t>(end - cur));
-      }
-      GRPC_CHECK((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
-      p->debug_pos += static_cast<uint32_t>(end - cur);
-      p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
-      if (is_last) {
-        t->http2_ztrace_collector.Append([p]() {
-          return grpc_core::H2GoAwayTrace<true>{
-              p->last_stream_id, p->error_code,
-              std::string(absl::string_view(p->debug_data, p->debug_length))};
-        });
-        grpc_chttp2_add_incoming_goaway(
-            t, p->error_code, p->last_stream_id,
-            absl::string_view(p->debug_data, p->debug_length));
-        gpr_free(p->debug_data);
-        p->debug_data = nullptr;
+      if (grpc_core::IsPh2Perf01Enabled()) {
+        if (end != cur) {
+          p->debug_slice_buffer.AppendIndexed(
+              grpc_core::Slice::FromCopiedBuffer(
+                  cur, static_cast<size_t>(end - cur)));
+        }
+        GRPC_CHECK(p->debug_slice_buffer.Length() <= p->debug_length);
+        p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
+        if (is_last) {
+          std::string debug_str = p->debug_slice_buffer.JoinIntoString();
+
+          grpc_chttp2_add_incoming_goaway(t, p->error_code, p->last_stream_id,
+                                          absl::string_view(debug_str));
+
+          t->http2_ztrace_collector.Append(
+              [last_stream_id = p->last_stream_id, error_code = p->error_code,
+               debug_str = std::move(debug_str)]() mutable {
+                return grpc_core::H2GoAwayTrace<true>{
+                    last_stream_id, error_code, std::move(debug_str)};
+              });
+          p->debug_slice_buffer.Clear();
+        }
+      } else {
+        if (end != cur) {
+          memcpy(p->debug_data + p->debug_pos, cur,
+                 static_cast<size_t>(end - cur));
+        }
+        GRPC_CHECK((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
+        p->debug_pos += static_cast<uint32_t>(end - cur);
+        p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
+        if (is_last) {
+          t->http2_ztrace_collector.Append([p]() {
+            return grpc_core::H2GoAwayTrace<true>{
+                p->last_stream_id, p->error_code,
+                std::string(absl::string_view(p->debug_data, p->debug_length))};
+          });
+          grpc_chttp2_add_incoming_goaway(
+              t, p->error_code, p->last_stream_id,
+              absl::string_view(p->debug_data, p->debug_length));
+          gpr_free(p->debug_data);
+          p->debug_data = nullptr;
+        }
       }
       return absl::OkStatus();
   }

data/src/core/ext/transport/chttp2/transport/frame_goaway.h

@@ -44,6 +44,7 @@ struct grpc_chttp2_goaway_parser {
   uint32_t last_stream_id;
   uint32_t error_code;
   char* debug_data;
+  grpc_core::SliceBuffer debug_slice_buffer;
   uint32_t debug_length;
   uint32_t debug_pos;
 };

data/src/core/ext/transport/chttp2/transport/goaway.h

@@ -365,6 +365,13 @@ class GoawayManager {
            !context_->was_immediate;
   }
 
+  bool IsGracefulGoawayInProgress() const {
+    return context_->goaway_state ==
+               GoawayState::kInitialGracefulGoawayScheduled ||
+           context_->goaway_state == GoawayState::kInitialGracefulGoawaySent ||
+           context_->goaway_state == GoawayState::kFinalGracefulGoawayScheduled;
+  }
+
   // Called from the transport write cycle to notify the GOAWAY manager that a
   // GOAWAY frame may have been sent. If a GOAWAY frame is sent in current
   // write cycle, this function handles the needed state transition.

data/src/core/ext/transport/chttp2/transport/hpack_parser.cc

@@ -1009,7 +1009,7 @@ class HPackParser::Parser {
         key_string.size() + value.wire_size + hpack_constants::kEntryOverhead;
     if (state_.mitigation_engine != nullptr) {
       auto action = state_.mitigation_engine->EvaluateIncomingMetadata(
-          key_string, value_slice.as_string_view());
+          key_string, value_slice.as_string_view(), state_.peer_address);
       if (action.has_value()) {
         if (action.value() == MitigationEngine::Action::kCloseConnection) {
           input_->SetErrorAndStopParsing(
@@ -1118,7 +1118,8 @@ void HPackParser::BeginFrame(grpc_metadata_batch* metadata_buffer,
                              uint32_t metadata_size_hard_limit,
                              Boundary boundary, Priority priority,
                              LogInfo log_info,
-                             MitigationEngine* mitigation_engine) {
+                             MitigationEngine* mitigation_engine,
+                             absl::string_view peer_address) {
   metadata_buffer_ = metadata_buffer;
   if (metadata_buffer != nullptr) {
     metadata_buffer->Set(GrpcStatusFromWire(), true);
@@ -1126,6 +1127,7 @@ void HPackParser::BeginFrame(grpc_metadata_batch* metadata_buffer,
   boundary_ = boundary;
   priority_ = priority;
   state_.mitigation_engine = mitigation_engine;
+  state_.peer_address = peer_address;
   state_.dynamic_table_updates_allowed = 2;
   state_.metadata_early_detection.SetLimits(
       /*soft_limit=*/metadata_size_soft_limit,
@@ -1162,7 +1164,7 @@ grpc_error_handle HPackParser::ParseInput(Input input, bool is_last,
   if (is_last && is_boundary()) {
     if (state_.mitigation_engine != nullptr && metadata_buffer_ != nullptr) {
       auto action = state_.mitigation_engine->EvaluateAllIncomingMetadata(
-          *metadata_buffer_);
+          *metadata_buffer_, state_.peer_address);
       if (action.has_value()) {
         if (action.value() == MitigationEngine::Action::kCloseConnection) {
           input.SetErrorAndStopParsing(HpackParseResult::MitigationEngineError(

data/src/core/ext/transport/chttp2/transport/hpack_parser.h

@@ -101,7 +101,8 @@ class HPackParser {
                   uint32_t metadata_size_soft_limit,
                   uint32_t metadata_size_hard_limit, Boundary boundary,
                   Priority priority, LogInfo log_info,
-                  MitigationEngine* mitigation_engine);
+                  MitigationEngine* mitigation_engine,
+                  absl::string_view peer_address = "");
   // Start throwing away any received headers after parsing them.
   void StopBufferingFrame() { metadata_buffer_ = nullptr; }
   // Parse one slice worth of data
@@ -255,6 +256,7 @@ class HPackParser {
     // Current parse state
     ParseState parse_state = ParseState::kTop;
     MitigationEngine* mitigation_engine = nullptr;
+    absl::string_view peer_address;
     std::variant<const HPackTable::Memento*, Slice> key;
   };