grpc 1.81.1 → 1.83.0

data/src/core/ext/transport/chttp2/transport/stream_data_queue.h

@@ -26,12 +26,10 @@
 #include <cstdint>
 #include <limits>
 #include <optional>
-#include <queue>
 #include <string>
 #include <type_traits>
 #include <utility>
 #include <variant>
-#include <vector>
 
 #include "src/core/call/message.h"
 #include "src/core/ext/transport/chttp2/transport/frame.h"
@@ -103,7 +101,7 @@ class SimpleQueue {
   }
 
   // Returns true if the queue is empty. This function is NOT thread safe.
-  inline bool IsEmpty() { return queue_.Peek() == nullptr; }
+  bool IsEmpty() { return queue_.Peek() == nullptr; }
 
   // Clears the queue. This function is NOT thread safe.
   void Clear() {
@@ -112,7 +110,7 @@ class SimpleQueue {
     GRPC_DCHECK(IsEmpty());
   }
 
-  inline std::optional<uint32_t> GetNextEntryTokens() {
+  std::optional<uint32_t> GetNextEntryTokens() {
     Entry* front = queue_.Peek();
     return front == nullptr ? std::nullopt : std::make_optional(front->tokens);
   }
@@ -141,8 +139,7 @@ class SimpleQueue {
     return Pending{};
   }
 
-  inline absl::StatusOr<bool> ImmediateEnqueueInternal(T data,
-                                                       const uint32_t tokens) {
+  absl::StatusOr<bool> ImmediateEnqueueInternal(T data, const uint32_t tokens) {
     GRPC_DCHECK_LE(tokens_consumed_,
                    std::numeric_limits<uint32_t>::max() - tokens);
     if (tokens_consumed_ > std::numeric_limits<uint32_t>::max() - tokens) {
@@ -456,7 +453,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
 
     GRPC_STREAM_DATA_QUEUE_DEBUG
         << "Immediate enqueueing reset stream with error code: " << error_code;
-    reset_stream_state_ = RstStreamState::kQueued;
+    reset_stream_state_ = ResetStreamState::kQueued;
     reset_stream_error_code_ = error_code;
 
     // became_non_empty is set to true if the queue is empty because we are not
@@ -487,6 +484,10 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
     bool IsInitialMetadataDequeued() const {
       return flags.IsInitialMetadataDequeued();
     }
+
+    bool IsTrailingMetadataDequeued() const {
+      return flags.IsTrailingMetadataDequeued();
+    }
   };
 
   // TODO(akshitpatel) : [PH2][P4] : Measure the performance of this function
@@ -564,8 +565,6 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
         /*flags=*/handle_dequeue.GetDequeueFlags()};
   }
 
-  // TODO(tjagtap) : [PH2][P1][FlowControl] : Call this while processing
-  // window update frame.
   // Needs to be invoked when the peer sends stream flow control window update.
   // stream_fc_tokens represents the stream flow control (delta) window +
   // intial_window_size.
@@ -603,8 +602,8 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
    public:
     HandleDequeue(const uint32_t max_tokens, const uint32_t max_frame_length,
                   HPackCompressor& encoder, FrameSender& frame_sender,
-                  StreamDataQueue& queue)
-        : queue_(queue),
+                  StreamDataQueue& stream_data_queue)
+        : stream_data_queue_(stream_data_queue),
           max_frame_length_(max_frame_length),
           max_tokens_available_(max_tokens),
           flow_control_tokens_consumed_(0),
@@ -613,7 +612,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
 
     void operator()(InitialMetadataType initial_metadata) {
       GRPC_STREAM_DATA_QUEUE_DEBUG << "Preparing initial metadata for sending";
-      queue_.initial_metadata_disassembler_.PrepareForSending(
+      stream_data_queue_.initial_metadata_disassembler_.PrepareForSending(
           std::move(initial_metadata.metadata), encoder_);
       dequeue_flags_.set_initial_metadata_dequeued();
       MaybeAppendInitialMetadataFrames();
@@ -621,14 +620,14 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
 
     void operator()(TrailingMetadataType trailing_metadata) {
       GRPC_STREAM_DATA_QUEUE_DEBUG << "Preparing trailing metadata for sending";
-      queue_.trailing_metadata_disassembler_.PrepareForSending(
+      stream_data_queue_.trailing_metadata_disassembler_.PrepareForSending(
           std::move(trailing_metadata.metadata), encoder_);
       dequeue_flags_.set_trailing_metadata_dequeued();
     }
 
     void operator()(MessageHandle message) {
       GRPC_STREAM_DATA_QUEUE_DEBUG << "Preparing message for sending";
-      queue_.message_disassembler_.PrepareBatchedMessageForSending(
+      stream_data_queue_.message_disassembler_.PrepareBatchedMessageForSending(
           std::move(message));
     }
 
@@ -657,39 +656,52 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
     DequeueFlags GetDequeueFlags() const { return dequeue_flags_; }
 
    private:
-    inline void MaybeAppendInitialMetadataFrames() {
-      while (queue_.initial_metadata_disassembler_.HasMoreData()) {
+    void MaybeAppendInitialMetadataFrames() {
+      while (stream_data_queue_.initial_metadata_disassembler_.HasMoreData()) {
         GRPC_DCHECK(!dequeue_flags_.IsHalfCloseDequeued());
         GRPC_DCHECK(!dequeue_flags_.IsResetStreamDequeued());
         // TODO(akshitpatel) : [PH2][P2] : I do not think we need this.
         // HasMoreData() should be enough.
         bool is_end_headers = false;
-        AppendFrame(queue_.initial_metadata_disassembler_.GetNextFrame(
-            max_frame_length_, is_end_headers));
+        AppendFrame(
+            stream_data_queue_.initial_metadata_disassembler_.GetNextFrame(
+                max_frame_length_, is_end_headers));
       }
     }
 
-    inline void MaybeAppendTrailingMetadataFrames() {
-      while (queue_.trailing_metadata_disassembler_.HasMoreData()) {
+    void MaybeAppendTrailingMetadataFrames() {
+      while (stream_data_queue_.trailing_metadata_disassembler_.HasMoreData()) {
         GRPC_DCHECK(!dequeue_flags_.IsHalfCloseDequeued());
-        GRPC_DCHECK_EQ(queue_.message_disassembler_.GetBufferedLength(), 0u);
         GRPC_DCHECK_EQ(
-            queue_.initial_metadata_disassembler_.GetBufferedLength(), 0u);
+            stream_data_queue_.message_disassembler_.GetBufferedLength(), 0u);
+        GRPC_DCHECK_EQ(stream_data_queue_.initial_metadata_disassembler_
+                           .GetBufferedLength(),
+                       0u);
+        GRPC_DCHECK(dequeue_flags_.IsTrailingMetadataDequeued());
         // TODO(akshitpatel) : [PH2][P2] : I do not think we need this.
         // HasMoreData() should be enough.
         bool is_end_headers = false;
-        AppendFrame(queue_.trailing_metadata_disassembler_.GetNextFrame(
-            max_frame_length_, is_end_headers));
+        AppendFrame(
+            stream_data_queue_.trailing_metadata_disassembler_.GetNextFrame(
+                max_frame_length_, is_end_headers));
+      }
+
+      // In gRPC C++, TrailingMetadata is always followed by RST_STREAM.
+      if (dequeue_flags_.IsTrailingMetadataDequeued()) {
+        AppendResetStreamFrame(Http2ErrorCode::kNoError);
       }
     }
 
-    inline void MaybeAppendEndOfStreamFrame() {
+    void MaybeAppendEndOfStreamFrame() {
       if (dequeue_flags_.IsHalfCloseDequeued()) {
-        GRPC_DCHECK_EQ(queue_.message_disassembler_.GetBufferedLength(), 0u);
-        GRPC_DCHECK_EQ(
-            queue_.initial_metadata_disassembler_.GetBufferedLength(), 0u);
         GRPC_DCHECK_EQ(
-            queue_.trailing_metadata_disassembler_.GetBufferedLength(), 0u);
+            stream_data_queue_.message_disassembler_.GetBufferedLength(), 0u);
+        GRPC_DCHECK_EQ(stream_data_queue_.initial_metadata_disassembler_
+                           .GetBufferedLength(),
+                       0u);
+        GRPC_DCHECK_EQ(stream_data_queue_.trailing_metadata_disassembler_
+                           .GetBufferedLength(),
+                       0u);
         Http2Frame* last_frame = frame_sender_.MutableLastRegularFrame();
 
         // On the Server side, trailing metadata acts as end of stream and is
@@ -709,21 +721,23 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
               *last_frame);
           if (merged) return;
         }
-        AppendFrame(Http2DataFrame{/*stream_id=*/queue_.stream_id_,
+        AppendFrame(Http2DataFrame{/*stream_id=*/stream_data_queue_.stream_id_,
                                    /*end_stream=*/true,
                                    /*payload=*/SliceBuffer()});
       }
     }
 
-    inline void MaybeAppendMessageFrames() {
-      while (queue_.message_disassembler_.GetBufferedLength() > 0 &&
+    void MaybeAppendMessageFrames() {
+      while (stream_data_queue_.message_disassembler_.GetBufferedLength() > 0 &&
              (max_tokens_available_ - flow_control_tokens_consumed_) > 0) {
-        GRPC_DCHECK_EQ(
-            queue_.initial_metadata_disassembler_.GetBufferedLength(), 0u);
-        Http2DataFrame frame = queue_.message_disassembler_.GenerateNextFrame(
-            queue_.stream_id_,
-            std::min(max_tokens_available_ - flow_control_tokens_consumed_,
-                     max_frame_length_));
+        GRPC_DCHECK_EQ(stream_data_queue_.initial_metadata_disassembler_
+                           .GetBufferedLength(),
+                       0u);
+        Http2DataFrame frame =
+            stream_data_queue_.message_disassembler_.GenerateNextFrame(
+                stream_data_queue_.stream_id_,
+                std::min(max_tokens_available_ - flow_control_tokens_consumed_,
+                         max_frame_length_));
         flow_control_tokens_consumed_ += frame.payload.Length();
         GRPC_STREAM_DATA_QUEUE_DEBUG
             << "Appending message frame with length " << frame.payload.Length()
@@ -734,31 +748,35 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
       }
     }
 
-    inline void MaybeAppendResetStreamFrame() {
-      if (dequeue_flags_.IsResetStreamDequeued()) {
-        // TODO(akshitpatel) : [PH2][P2] : Consider if we can send reset stream
-        // frame without flushing all the messages enqueued until now.
-        GRPC_DCHECK_EQ(queue_.message_disassembler_.GetBufferedLength(), 0u);
-        GRPC_DCHECK_EQ(
-            queue_.initial_metadata_disassembler_.GetBufferedLength(), 0u);
-        GRPC_DCHECK_EQ(
-            queue_.trailing_metadata_disassembler_.GetBufferedLength(), 0u);
-        AppendFrame(Http2RstStreamFrame{queue_.stream_id_, error_code_});
-      }
+    void AppendResetStreamFrame(Http2ErrorCode error_code)
+        ABSL_NO_THREAD_SAFETY_ANALYSIS {
+      // The caller is guaranteed to hold the StreamDataQueue mutex.
+      GRPC_DCHECK_EQ(
+          stream_data_queue_.message_disassembler_.GetBufferedLength(), 0u);
+      GRPC_DCHECK_EQ(
+          stream_data_queue_.initial_metadata_disassembler_.GetBufferedLength(),
+          0u);
+      GRPC_DCHECK_EQ(stream_data_queue_.trailing_metadata_disassembler_
+                         .GetBufferedLength(),
+                     0u);
+      AppendFrame(
+          Http2RstStreamFrame{stream_data_queue_.stream_id_,
+                              Http2ErrorCodeToFrameErrorCode(error_code)});
+      stream_data_queue_.reset_stream_state_ = ResetStreamState::kDequeued;
+      dequeue_flags_.set_reset_stream_dequeued();
     }
 
-    inline void AppendFrame(Http2Frame&& frame) {
+    void AppendFrame(Http2Frame&& frame) {
       // FrameSender automatically accounts for the frame size in write quota.
-      // Maybe not be extremely accurate but should be good enough for our
-      // purposes.
+      // We do tend to overestimate the bytes consumed here but should be good
+      // enough for our purposes.
       frame_sender_.AddRegularFrame(std::forward<Http2Frame>(frame));
     }
 
-    StreamDataQueue& queue_;
+    StreamDataQueue& stream_data_queue_;
     const uint32_t max_frame_length_;
     const uint32_t max_tokens_available_;
     uint32_t flow_control_tokens_consumed_;
-    uint32_t error_code_ = static_cast<uint32_t>(Http2ErrorCode::kNoError);
     HPackCompressor& encoder_;
     FrameSender& frame_sender_;
     DequeueFlags dequeue_flags_;
@@ -839,7 +857,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
     return is_writable_;
   }
 
-  inline bool IsNextQueueEntryMessage() {
+  bool IsNextQueueEntryMessage() {
     std::optional<size_t> next_entry_tokens = queue_.GetNextEntryTokens();
     return next_entry_tokens.has_value() && *next_entry_tokens > 0;
   }
@@ -852,7 +870,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
       const bool can_send_reset_stream, FrameSender& frame_sender)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
     switch (reset_stream_state_) {
-      case RstStreamState::kDequeued:
+      case ResetStreamState::kDequeued:
         GRPC_STREAM_DATA_QUEUE_DEBUG
             << "Reset stream is already dequeued. Returning empty frames.";
         GRPC_DCHECK(queue_.IsEmpty());
@@ -860,7 +878,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
         return DequeueResult{is_writable_, priority_,
                              /*flow_control_tokens_consumed=*/0u,
                              /*flags=*/DequeueFlags{}};
-      case RstStreamState::kQueued: {
+      case ResetStreamState::kQueued: {
         GRPC_STREAM_DATA_QUEUE_DEBUG
             << "Reset stream is queued. Skipping all frames (if any) for "
                "dequeuing.";
@@ -872,11 +890,11 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
           flags.set_reset_stream_dequeued();
         }
         queue_.Clear();
-        reset_stream_state_ = RstStreamState::kDequeued;
+        reset_stream_state_ = ResetStreamState::kDequeued;
         return DequeueResult{is_writable_, priority_,
                              /*flow_control_tokens_consumed=*/0u, flags};
       }
-      case RstStreamState::kNotQueued:
+      case ResetStreamState::kNotQueued:
         return std::nullopt;
       default:
         GRPC_CHECK(false) << "Invalid reset stream state: "
@@ -887,10 +905,10 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
 
   bool IsEnqueueClosed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
     switch (reset_stream_state_) {
-      case RstStreamState::kNotQueued:
+      case ResetStreamState::kNotQueued:
         return false;
-      case RstStreamState::kQueued:
-      case RstStreamState::kDequeued:
+      case ResetStreamState::kQueued:
+      case ResetStreamState::kDequeued:
         // This can happen when the transport tries to close the stream and the
         // stream is cancelled from the call stack.
         GRPC_STREAM_DATA_QUEUE_DEBUG << "Reset stream already queued.";
@@ -908,7 +926,7 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
   // This is only used for DCHECKs. Not actually used for any business logic.
   const bool is_client_;
 
-  enum class RstStreamState : uint8_t {
+  enum class ResetStreamState : uint8_t {
     kNotQueued = 0,
     kQueued,
     kDequeued,
@@ -925,8 +943,8 @@ class StreamDataQueue : public RefCounted<StreamDataQueue<MetadataHandle>> {
   // (if needed) to send them. This variable also has 1-1 correspondence with
   // whether the stream is in the list of writable streams in the transport.
   bool is_writable_ ABSL_GUARDED_BY(mu_) = false;
-  RstStreamState reset_stream_state_ ABSL_GUARDED_BY(mu_) =
-      RstStreamState::kNotQueued;
+  ResetStreamState reset_stream_state_ ABSL_GUARDED_BY(mu_) =
+      ResetStreamState::kNotQueued;
   SimpleQueue<QueueEntry> queue_;
   WritableStreamPriority priority_ ABSL_GUARDED_BY(mu_) =
       WritableStreamPriority::kDefault;

data/src/core/ext/transport/chttp2/transport/write_cycle.h

@@ -30,11 +30,15 @@
 #include "src/core/ext/transport/chttp2/transport/write_size_policy.h"
 #include "src/core/lib/slice/slice_buffer.h"
 #include "src/core/lib/transport/promise_endpoint.h"
+#include "src/core/util/debug_location.h"
 #include "absl/container/inlined_vector.h"
 
 namespace grpc_core {
 namespace http2 {
 
+#define GRPC_HTTP2_WRITE_CYCLE_DEBUG \
+  DLOG_IF(INFO, GRPC_TRACE_FLAG_ENABLED(http2_ph2_transport))
+
 // Tracks the number of bytes that can be written in the current write
 // attempt.
 class WriteQuota {
@@ -329,7 +333,11 @@ class TransportWriteContext {
 
   // Calls to this function MUST only be made between StartWriteCycle and
   // EndWriteCycle.
-  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION WriteCycle& GetWriteCycle() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION WriteCycle& GetWriteCycle(
+      DebugLocation whence = {}) {
+    GRPC_HTTP2_WRITE_CYCLE_DEBUG << "GetWriteCycle whence: " << whence.file()
+                                 << ":" << whence.line()
+                                 << " Debug: " << DebugString();
     return *write_cycle_;
   }
 

data/src/core/ext/transport/inproc/legacy_inproc_transport.cc

@@ -40,6 +40,7 @@
 #include "src/core/config/core_configuration.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_args_preconditioning.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/error.h"
@@ -68,6 +69,7 @@
 namespace {
 struct inproc_stream;
 bool cancel_stream_locked(inproc_stream* s, grpc_error_handle error);
+bool cancel_stream(inproc_stream* s, grpc_error_handle error);
 void maybe_process_ops_locked(inproc_stream* s, grpc_error_handle error);
 void op_state_machine_locked(inproc_stream* s, grpc_error_handle error);
 void log_metadata(const grpc_metadata_batch* md_batch, bool is_client,
@@ -195,8 +197,12 @@ struct inproc_stream {
       other_side = nullptr;  // will get filled in soon
       inproc_transport* st = t->other_side;
       if (st->accept_stream_cb == nullptr) {
-        cancel_stream_locked(this,
-                             absl::UnavailableError("inproc server closed"));
+        if (grpc_core::IsInprocCancelStreamEnabled()) {
+          cancel_stream(this, absl::UnavailableError("inproc server closed"));
+        } else {
+          cancel_stream_locked(this,
+                               absl::UnavailableError("inproc server closed"));
+        }
       } else {
         st->ref();
         // Pass the client-side stream address to the server-side for a ref
@@ -952,6 +958,14 @@ bool cancel_stream_locked(inproc_stream* s, grpc_error_handle error) {
   return ret;
 }
 
+bool cancel_stream(inproc_stream* s, grpc_error_handle error) {
+  inproc_transport* t = s->t;
+  gpr_mu_lock(&t->mu->mu);
+  bool ret = cancel_stream_locked(s, std::move(error));
+  gpr_mu_unlock(&t->mu->mu);
+  return ret;
+}
+
 void inproc_transport::PerformStreamOp(grpc_stream* gs,
                                        grpc_transport_stream_op_batch* op) {
   GRPC_TRACE_LOG(inproc, INFO)