grpc 1.69.0 → 1.70.1

data/src/core/lib/promise/map.h

@@ -21,6 +21,9 @@
 #include <tuple>
 #include <utility>
 
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/str_cat.h"
 #include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/poll.h"
 
@@ -155,6 +158,45 @@ struct JustElem {
   }
 };
 
+namespace promise_detail {
+template <typename Fn>
+class MapError {
+ public:
+  explicit MapError(Fn fn) : fn_(std::move(fn)) {}
+  absl::Status operator()(absl::Status status) {
+    if (status.ok()) return status;
+    return fn_(std::move(status));
+  }
+  template <typename T>
+  absl::StatusOr<T> operator()(absl::StatusOr<T> status) {
+    if (status.ok()) return status;
+    return fn_(std::move(status.status()));
+  }
+
+ private:
+  Fn fn_;
+};
+}  // namespace promise_detail
+
+// Map status->better status in the case of errors
+template <typename Promise, typename Fn>
+auto MapErrors(Promise promise, Fn fn) {
+  return Map(std::move(promise), promise_detail::MapError<Fn>(std::move(fn)));
+}
+
+// Simple mapper to add a prefix to the message of an error
+template <typename Promise>
+auto AddErrorPrefix(absl::string_view prefix, Promise promise) {
+  return MapErrors(std::move(promise), [prefix](absl::Status status) {
+    absl::Status out(status.code(), absl::StrCat(prefix, status.message()));
+    status.ForEachPayload(
+        [&out](absl::string_view name, const absl::Cord& value) {
+          out.SetPayload(name, value);
+        });
+    return out;
+  });
+}
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_MAP_H

data/src/core/lib/promise/party.cc

@@ -30,6 +30,7 @@
 #include "src/core/util/sync.h"
 
 #ifdef GRPC_MAXIMIZE_THREADYNESS
+#include "absl/random/random.h"           // IWYU pragma: keep
 #include "src/core/lib/iomgr/exec_ctx.h"  // IWYU pragma: keep
 #include "src/core/util/thd.h"            // IWYU pragma: keep
 #endif
@@ -359,6 +360,40 @@ void Party::RunPartyAndUnref(uint64_t prev_state) {
   }
 }
 
+// Given a bitmask indicating allocation status of promises, return the index of
+// the next slot to allocate.
+// By default we use a deterministic and fast algorithm (fit-first), but we
+// don't want to guarantee that this is the order of spawning -- if a promise is
+// locked by another thread (for instance) a sequence of spawns may be reordered
+// for initial execution.
+// So for thready-tsan we provide an alternative implementation that
+// additionally reorders promises.
+#ifndef GRPC_MAXIMIZE_THREADYNESS
+uint64_t Party::NextAllocationMask(uint64_t current_allocation_mask) {
+  return LowestOneBit(~current_allocation_mask);
+}
+#else
+uint64_t Party::NextAllocationMask(uint64_t current_allocation_mask) {
+  CHECK_EQ(current_allocation_mask & ~kWakeupMask, 0);
+  if (current_allocation_mask == kWakeupMask) return kWakeupMask + 1;
+  // Count number of unset bits in the wakeup mask
+  size_t unset_bits = 0;
+  for (size_t i = 0; i < party_detail::kMaxParticipants; i++) {
+    if (current_allocation_mask & (1ull << i)) continue;
+    ++unset_bits;
+  }
+  CHECK_GT(unset_bits, 0);
+  absl::BitGen bitgen;
+  size_t selected = absl::Uniform<size_t>(bitgen, 0, unset_bits);
+  for (size_t i = 0; i < party_detail::kMaxParticipants; i++) {
+    if (current_allocation_mask & (1ull << i)) continue;
+    if (selected == 0) return 1ull << i;
+    --selected;
+  }
+  LOG(FATAL) << "unreachable";
+}
+#endif
+
 void Party::AddParticipant(Participant* participant) {
   GRPC_LATENT_SEE_INNER_SCOPE("Party::AddParticipant");
   uint64_t state = state_.load(std::memory_order_acquire);
@@ -372,7 +407,7 @@ void Party::AddParticipant(Participant* participant) {
   uint64_t new_state;
   do {
     allocated = (state & kAllocatedMask) >> kAllocatedShift;
-    wakeup_mask = LowestOneBit(~allocated);
+    wakeup_mask = NextAllocationMask(allocated);
     if (GPR_UNLIKELY((wakeup_mask & kWakeupMask) == 0)) {
       DelayAddParticipant(participant);
       return;

data/src/core/lib/promise/party.h

@@ -25,11 +25,10 @@
 #include <utility>
 
 #include "absl/base/attributes.h"
-#include "absl/base/thread_annotations.h"
 #include "absl/log/check.h"
-#include "absl/log/log.h"
 #include "absl/strings/string_view.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/event_engine_context.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
@@ -39,8 +38,6 @@
 #include "src/core/util/crash.h"
 #include "src/core/util/ref_counted.h"
 #include "src/core/util/ref_counted_ptr.h"
-#include "src/core/util/sync.h"
-#include "src/core/util/useful.h"
 
 namespace grpc_core {
 
@@ -179,13 +176,22 @@ class Party : public Activity, private Wakeable {
     return RefCountedPtr<Party>(this);
   }
 
+  template <typename T>
+  RefCountedPtr<T> RefAsSubclass() {
+    IncrementRefCount();
+    return RefCountedPtr<T>(DownCast<T*>(this));
+  }
+
   Arena* arena() { return arena_.get(); }
 
  protected:
   friend class Arena;
 
   // Derived types should be constructed upon `arena`.
-  explicit Party(RefCountedPtr<Arena> arena) : arena_(std::move(arena)) {}
+  explicit Party(RefCountedPtr<Arena> arena) : arena_(std::move(arena)) {
+    CHECK(arena_->GetContext<grpc_event_engine::experimental::EventEngine>() !=
+          nullptr);
+  }
   ~Party() override;
 
   // Main run loop. Must be locked.
@@ -418,6 +424,8 @@ class Party : public Activity, private Wakeable {
   void AddParticipant(Participant* participant);
   void DelayAddParticipant(Participant* participant);
 
+  static uint64_t NextAllocationMask(uint64_t current_allocation_mask);
+
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void LogStateChange(
       const char* op, uint64_t prev_state, uint64_t new_state,
       DebugLocation loc = {}) {

data/src/core/lib/promise/sleep.h

@@ -32,6 +32,7 @@ namespace grpc_core {
 class Sleep final {
  public:
   explicit Sleep(Timestamp deadline);
+  explicit Sleep(Duration timeout) : Sleep(Timestamp::Now() + timeout) {}
   ~Sleep();
 
   Sleep(const Sleep&) = delete;

data/src/core/lib/promise/status_flag.h

@@ -183,6 +183,16 @@ struct StatusCastImpl<StatusFlag, Success> {
   static StatusFlag Cast(Success) { return StatusFlag(true); }
 };
 
+template <>
+struct StatusCastImpl<StatusFlag, Failure> {
+  static StatusFlag Cast(Failure) { return StatusFlag(false); }
+};
+
+template <>
+struct FailureStatusCastImpl<StatusFlag, Failure> {
+  static StatusFlag Cast(Failure) { return StatusFlag(false); }
+};
+
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, StatusFlag> {
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(

data/src/core/lib/resource_quota/arena.h

@@ -190,6 +190,14 @@ class Arena final : public RefCounted<Arena, NonPolymorphicRefCount,
     return &p->t;
   }
 
+  template <typename T, typename... Args>
+  absl::enable_if_t<std::is_same<typename T::RefCountedUnrefBehaviorType,
+                                 UnrefCallDtor>::value,
+                    RefCountedPtr<T>>
+  MakeRefCounted(Args&&... args) {
+    return RefCountedPtr<T>(New<T>(std::forward<Args>(args)...));
+  }
+
   class PooledDeleter {
    public:
     PooledDeleter() = default;

data/src/core/lib/resource_quota/connection_quota.h

@@ -48,6 +48,10 @@ class ConnectionQuota : public RefCounted<ConnectionQuota> {
   // Mark connections as closed.
   void ReleaseConnections(int num_connections);
 
+  int TestOnlyActiveIncomingConnections() const {
+    return active_incoming_connections_;
+  }
+
  private:
   std::atomic<int> active_incoming_connections_{0};
   std::atomic<int> max_incoming_connections_{std::numeric_limits<int>::max()};

data/src/core/lib/surface/call_utils.h

@@ -273,6 +273,8 @@ class BatchOpIndex {
     return idxs_[op_type] == 255 ? nullptr : &ops_[idxs_[op_type]];
   }
 
+  bool has_op(grpc_op_type op_type) const { return idxs_[op_type] != 255; }
+
  private:
   const grpc_op* const ops_;
   std::array<uint8_t, 8> idxs_{255, 255, 255, 255, 255, 255, 255, 255};

data/src/core/lib/surface/client_call.cc

@@ -348,42 +348,19 @@ void ClientCall::CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
     auto out_error_string = op->data.recv_status_on_client.error_string;
     auto out_trailing_metadata =
         op->data.recv_status_on_client.trailing_metadata;
-    auto make_read_trailing_metadata = [this, out_status, out_status_details,
-                                        out_error_string,
+    auto make_read_trailing_metadata = [self = WeakRef(), out_status,
+                                        out_status_details, out_error_string,
                                         out_trailing_metadata]() {
-      return Map(
-          started_call_initiator_.PullServerTrailingMetadata(),
-          [this, out_status, out_status_details, out_error_string,
-           out_trailing_metadata](
-              ServerMetadataHandle server_trailing_metadata) {
-            saw_trailing_metadata_.store(true, std::memory_order_relaxed);
-            ResetDeadline();
-            GRPC_TRACE_LOG(call, INFO)
-                << DebugTag() << "RecvStatusOnClient "
-                << server_trailing_metadata->DebugString();
-            const auto status =
-                server_trailing_metadata->get(GrpcStatusMetadata())
-                    .value_or(GRPC_STATUS_UNKNOWN);
-            *out_status = status;
-            Slice message_slice;
-            if (Slice* message = server_trailing_metadata->get_pointer(
-                    GrpcMessageMetadata())) {
-              message_slice = message->Ref();
-            }
-            *out_status_details = message_slice.TakeCSlice();
-            if (out_error_string != nullptr) {
-              if (status != GRPC_STATUS_OK) {
-                *out_error_string = gpr_strdup(
-                    MakeErrorString(server_trailing_metadata.get()).c_str());
-              } else {
-                *out_error_string = nullptr;
-              }
-            }
-            PublishMetadataArray(server_trailing_metadata.get(),
-                                 out_trailing_metadata, true);
-            received_trailing_metadata_ = std::move(server_trailing_metadata);
-            return Success{};
-          });
+      return Map(self->started_call_initiator_.PullServerTrailingMetadata(),
+                 [self, out_status, out_status_details, out_error_string,
+                  out_trailing_metadata](
+                     ServerMetadataHandle server_trailing_metadata) {
+                   self->OnReceivedStatus(std::move(server_trailing_metadata),
+                                          out_status, out_status_details,
+                                          out_error_string,
+                                          out_trailing_metadata);
+                   return Success{};
+                 });
     };
     ScheduleCommittedBatch(InfallibleBatch(
         std::move(primary_ops),
@@ -408,6 +385,37 @@ void ClientCall::CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
   }
 }
 
+void ClientCall::OnReceivedStatus(ServerMetadataHandle server_trailing_metadata,
+                                  grpc_status_code* out_status,
+                                  grpc_slice* out_status_details,
+                                  const char** out_error_string,
+                                  grpc_metadata_array* out_trailing_metadata) {
+  saw_trailing_metadata_.store(true, std::memory_order_relaxed);
+  ResetDeadline();
+  GRPC_TRACE_LOG(call, INFO) << DebugTag() << "RecvStatusOnClient "
+                             << server_trailing_metadata->DebugString();
+  const auto status = server_trailing_metadata->get(GrpcStatusMetadata())
+                          .value_or(GRPC_STATUS_UNKNOWN);
+  *out_status = status;
+  Slice message_slice;
+  if (Slice* message =
+          server_trailing_metadata->get_pointer(GrpcMessageMetadata())) {
+    message_slice = message->Ref();
+  }
+  *out_status_details = message_slice.TakeCSlice();
+  if (out_error_string != nullptr) {
+    if (status != GRPC_STATUS_OK) {
+      *out_error_string =
+          gpr_strdup(MakeErrorString(server_trailing_metadata.get()).c_str());
+    } else {
+      *out_error_string = nullptr;
+    }
+  }
+  PublishMetadataArray(server_trailing_metadata.get(), out_trailing_metadata,
+                       true);
+  received_trailing_metadata_ = std::move(server_trailing_metadata);
+}
+
 char* ClientCall::GetPeer() {
   Slice peer_slice = GetPeerString();
   if (!peer_slice.empty()) {

data/src/core/lib/surface/client_call.h

@@ -139,6 +139,11 @@ class ClientCall final
                                  UnstartedCallHandler& handler);
 
   std::string DebugTag() { return absl::StrFormat("CLIENT_CALL[%p]: ", this); }
+  void OnReceivedStatus(ServerMetadataHandle server_trailing_metadata,
+                        grpc_status_code* out_status,
+                        grpc_slice* out_status_details,
+                        const char** out_error_string,
+                        grpc_metadata_array* out_trailing_metadata);
 
   // call_state_ is one of:
   // 1. kUnstarted - call has not yet been started

data/src/core/lib/surface/event_string.cc

@@ -27,7 +27,13 @@
 #include "absl/strings/str_join.h"
 
 static void addhdr(grpc_event* ev, std::vector<std::string>* buf) {
-  buf->push_back(absl::StrFormat("tag:%p", ev->tag));
+  if (reinterpret_cast<intptr_t>(ev->tag) < 1024 &&
+      reinterpret_cast<intptr_t>(ev->tag) > -1024) {
+    buf->push_back(
+        absl::StrFormat("tag:%d", reinterpret_cast<int64_t>(ev->tag)));
+  } else {
+    buf->push_back(absl::StrFormat("tag:%p", ev->tag));
+  }
 }
 
 static const char* errstr(int success) { return success ? "OK" : "ERROR"; }

data/src/core/lib/surface/init_internally.h

@@ -26,10 +26,21 @@ extern bool (*IsInitializedInternally)();
 
 class KeepsGrpcInitialized {
  public:
-  KeepsGrpcInitialized() { InitInternally(); }
-  ~KeepsGrpcInitialized() { ShutdownInternally(); }
+  explicit KeepsGrpcInitialized(bool enabled = true) : enabled_(enabled) {
+    if (enabled_) {
+      InitInternally();
+    }
+  }
+  ~KeepsGrpcInitialized() {
+    if (enabled_) {
+      ShutdownInternally();
+    }
+  }
   KeepsGrpcInitialized(const KeepsGrpcInitialized&) = delete;
   KeepsGrpcInitialized& operator=(const KeepsGrpcInitialized&) = delete;
+
+ private:
+  bool enabled_;
 };
 
 }  // namespace grpc_core

data/src/core/lib/surface/server_call.cc

@@ -119,93 +119,108 @@ void ServerCall::CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
                              bool is_notify_tag_closure) {
   BatchOpIndex op_index(ops, nops);
   if (!is_notify_tag_closure) grpc_cq_begin_op(cq_, notify_tag);
-  auto send_initial_metadata =
-      op_index.OpHandler<GRPC_OP_SEND_INITIAL_METADATA>([this](
-                                                            const grpc_op& op) {
-        auto metadata = arena()->MakePooled<ServerMetadata>();
-        PrepareOutgoingInitialMetadata(op, *metadata);
-        CToMetadata(op.data.send_initial_metadata.metadata,
-                    op.data.send_initial_metadata.count, metadata.get());
-        GRPC_TRACE_LOG(call, INFO)
-            << DebugTag() << "[call] Send initial metadata";
-        return [this, metadata = std::move(metadata)]() mutable {
-          return call_handler_.PushServerInitialMetadata(std::move(metadata));
-        };
-      });
-  auto send_message =
-      op_index.OpHandler<GRPC_OP_SEND_MESSAGE>([this](const grpc_op& op) {
-        SliceBuffer send;
-        grpc_slice_buffer_swap(
-            &op.data.send_message.send_message->data.raw.slice_buffer,
-            send.c_slice_buffer());
-        auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
-        return [this, msg = std::move(msg)]() mutable {
-          return call_handler_.PushMessage(std::move(msg));
-        };
-      });
-  auto send_trailing_metadata =
-      op_index.OpHandler<GRPC_OP_SEND_STATUS_FROM_SERVER>(
-          [this](const grpc_op& op) {
-            auto metadata = arena()->MakePooled<ServerMetadata>();
-            CToMetadata(op.data.send_status_from_server.trailing_metadata,
-                        op.data.send_status_from_server.trailing_metadata_count,
-                        metadata.get());
-            metadata->Set(GrpcStatusMetadata(),
-                          op.data.send_status_from_server.status);
-            if (auto* details =
-                    op.data.send_status_from_server.status_details) {
-              // TODO(ctiller): this should not be a copy, but we have
-              // callers that allocate and pass in a slice created with
-              // grpc_slice_from_static_string and then delete the string
-              // after passing it in, which shouldn't be a supported API.
-              metadata->Set(GrpcMessageMetadata(),
-                            Slice(grpc_slice_copy(*details)));
-            }
-            CHECK(metadata != nullptr);
-            return [this, metadata = std::move(metadata)]() mutable {
-              CHECK(metadata != nullptr);
-              return [this, metadata = std::move(
-                                metadata)]() mutable -> Poll<Success> {
-                CHECK(metadata != nullptr);
-                call_handler_.PushServerTrailingMetadata(std::move(metadata));
-                return Success{};
-              };
-            };
-          });
-  auto recv_message =
-      op_index.OpHandler<GRPC_OP_RECV_MESSAGE>([this](const grpc_op& op) {
-        return message_receiver_.MakeBatchOp(op, &call_handler_);
-      });
-  auto primary_ops = AllOk<StatusFlag>(
-      TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
-                               std::move(send_message)),
-             std::move(send_trailing_metadata)),
-      std::move(recv_message));
-  if (auto* op = op_index.op(GRPC_OP_RECV_CLOSE_ON_SERVER)) {
-    auto recv_trailing_metadata = OpHandler<GRPC_OP_RECV_CLOSE_ON_SERVER>(
-        [this, cancelled = op->data.recv_close_on_server.cancelled]() {
-          return Map(call_handler_.WasCancelled(),
-                     [cancelled, this](bool result) -> Success {
-                       saw_was_cancelled_.store(true,
-                                                std::memory_order_relaxed);
-                       ResetDeadline();
-                       *cancelled = result ? 1 : 0;
-                       return Success{};
-                     });
+
+  auto commit_with_send_ops = [&](auto send_ops) {
+    auto recv_message =
+        op_index.OpHandler<GRPC_OP_RECV_MESSAGE>([this](const grpc_op& op) {
+          return message_receiver_.MakeBatchOp(op, &call_handler_);
         });
-    call_handler_.SpawnInfallible(
-        "final-batch",
-        GRPC_LATENT_SEE_PROMISE(
-            "ServerCallBatch",
-            InfallibleBatch(std::move(primary_ops),
-                            std::move(recv_trailing_metadata),
-                            is_notify_tag_closure, notify_tag, cq_)));
+    auto primary_ops =
+        AllOk<StatusFlag>(std::move(send_ops), std::move(recv_message));
+    if (auto* op = op_index.op(GRPC_OP_RECV_CLOSE_ON_SERVER)) {
+      auto recv_trailing_metadata = OpHandler<GRPC_OP_RECV_CLOSE_ON_SERVER>(
+          [this, cancelled = op->data.recv_close_on_server.cancelled]() {
+            return Map(call_handler_.WasCancelled(),
+                       [cancelled, this](bool result) -> Success {
+                         saw_was_cancelled_.store(true,
+                                                  std::memory_order_relaxed);
+                         ResetDeadline();
+                         *cancelled = result ? 1 : 0;
+                         return Success{};
+                       });
+          });
+      call_handler_.SpawnInfallible(
+          "final-batch",
+          GRPC_LATENT_SEE_PROMISE(
+              "ServerCallBatch",
+              InfallibleBatch(std::move(primary_ops),
+                              std::move(recv_trailing_metadata),
+                              is_notify_tag_closure, notify_tag, cq_)));
+    } else {
+      call_handler_.SpawnInfallible(
+          "batch", GRPC_LATENT_SEE_PROMISE(
+                       "ServerCallBatch",
+                       FallibleBatch(std::move(primary_ops),
+                                     is_notify_tag_closure, notify_tag, cq_)));
+    }
+  };
+
+  auto make_send_trailing_metadata = [this](const grpc_op& op) {
+    auto metadata = arena()->MakePooled<ServerMetadata>();
+    CToMetadata(op.data.send_status_from_server.trailing_metadata,
+                op.data.send_status_from_server.trailing_metadata_count,
+                metadata.get());
+    metadata->Set(GrpcStatusMetadata(), op.data.send_status_from_server.status);
+    if (auto* details = op.data.send_status_from_server.status_details) {
+      // TODO(ctiller): this should not be a copy, but we have
+      // callers that allocate and pass in a slice created with
+      // grpc_slice_from_static_string and then delete the string
+      // after passing it in, which shouldn't be a supported API.
+      metadata->Set(GrpcMessageMetadata(), Slice(grpc_slice_copy(*details)));
+    }
+    CHECK(metadata != nullptr);
+    return [this, metadata = std::move(metadata)]() mutable {
+      CHECK(metadata != nullptr);
+      return [this, metadata = std::move(metadata)]() mutable -> Poll<Success> {
+        CHECK(metadata != nullptr);
+        call_handler_.PushServerTrailingMetadata(std::move(metadata));
+        return Success{};
+      };
+    };
+  };
+
+  if (op_index.has_op(GRPC_OP_SEND_INITIAL_METADATA) &&
+      op_index.has_op(GRPC_OP_SEND_STATUS_FROM_SERVER) &&
+      !op_index.has_op(GRPC_OP_SEND_MESSAGE) &&
+      op_index.op(GRPC_OP_SEND_INITIAL_METADATA)
+              ->data.send_initial_metadata.count == 0) {
+    const auto& trailing_metadata =
+        *op_index.op(GRPC_OP_SEND_STATUS_FROM_SERVER);
+    commit_with_send_ops(OpHandler<GRPC_OP_SEND_STATUS_FROM_SERVER>(
+        make_send_trailing_metadata(trailing_metadata)));
   } else {
-    call_handler_.SpawnInfallible(
-        "batch", GRPC_LATENT_SEE_PROMISE(
-                     "ServerCallBatch",
-                     FallibleBatch(std::move(primary_ops),
-                                   is_notify_tag_closure, notify_tag, cq_)));
+    auto send_initial_metadata =
+        op_index.OpHandler<GRPC_OP_SEND_INITIAL_METADATA>(
+            [this](const grpc_op& op) {
+              auto metadata = arena()->MakePooled<ServerMetadata>();
+              PrepareOutgoingInitialMetadata(op, *metadata);
+              CToMetadata(op.data.send_initial_metadata.metadata,
+                          op.data.send_initial_metadata.count, metadata.get());
+              GRPC_TRACE_LOG(call, INFO)
+                  << DebugTag() << "[call] Send initial metadata";
+              return [this, metadata = std::move(metadata)]() mutable {
+                return call_handler_.PushServerInitialMetadata(
+                    std::move(metadata));
+              };
+            });
+    auto send_message =
+        op_index.OpHandler<GRPC_OP_SEND_MESSAGE>([this](const grpc_op& op) {
+          SliceBuffer send;
+          grpc_slice_buffer_swap(
+              &op.data.send_message.send_message->data.raw.slice_buffer,
+              send.c_slice_buffer());
+          auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
+          return [this, msg = std::move(msg)]() mutable {
+            return call_handler_.PushMessage(std::move(msg));
+          };
+        });
+    auto send_trailing_metadata =
+        op_index.OpHandler<GRPC_OP_SEND_STATUS_FROM_SERVER>(
+            make_send_trailing_metadata);
+    commit_with_send_ops(
+        TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
+                                 std::move(send_message)),
+               std::move(send_trailing_metadata)));
   }
 }
 

data/src/core/lib/surface/version.cc

@@ -22,6 +22,6 @@
 #include <grpc/grpc.h>
 #include <grpc/support/port_platform.h>
 
-const char* grpc_version_string(void) { return "44.2.0"; }
+const char* grpc_version_string(void) { return "45.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gridiron"; }
+const char* grpc_g_stands_for(void) { return "genius"; }

data/src/core/lib/transport/call_filters.cc

@@ -79,10 +79,7 @@ void CallFilters::CancelDueToFailedPipeOperation(SourceLocation but_where) {
   GRPC_TRACE_VLOG(promise_primitives, 2)
           .AtLocation(but_where.file(), but_where.line())
       << "Cancelling due to failed pipe operation: " << DebugString();
-  auto status =
-      ServerMetadataFromStatus(GRPC_STATUS_CANCELLED, "Failed pipe operation");
-  status->Set(GrpcCallWasCancelled(), true);
-  PushServerTrailingMetadata(std::move(status));
+  Cancel();
 }
 
 void CallFilters::PushServerTrailingMetadata(ServerMetadataHandle md) {
@@ -97,6 +94,15 @@ void CallFilters::PushServerTrailingMetadata(ServerMetadataHandle md) {
   }
 }
 
+void CallFilters::Cancel() {
+  GRPC_TRACE_LOG(call, INFO) << GetContext<Activity>()->DebugTag() << " Cancel["
+                             << this << "]: into " << DebugString();
+  if (call_state_.PushServerTrailingMetadata(true)) {
+    push_server_trailing_metadata_ =
+        CancelledServerMetadataFromStatus(GRPC_STATUS_CANCELLED);
+  }
+}
+
 std::string CallFilters::DebugString() const {
   std::vector<std::string> components = {
       absl::StrFormat("this:%p", this),

data/src/core/lib/transport/call_filters.h

@@ -1811,6 +1811,8 @@ class CallFilters {
   // If no server initial metadata has been sent, implies
   // NoServerInitialMetadata() called.
   void PushServerTrailingMetadata(ServerMetadataHandle md);
+  // Optimized path to push cancellation onto the call
+  void Cancel();
   // Client: Fetch server trailing metadata
   // Returns a promise that resolves to ServerMetadataHandle
   GRPC_MUST_USE_RESULT auto PullServerTrailingMetadata() {
@@ -1837,6 +1839,12 @@ class CallFilters {
   GRPC_MUST_USE_RESULT auto WasCancelled() {
     return [this]() { return call_state_.PollWasCancelled(); };
   }
+  // Client & server: wait for server trailing metadata to be available, and
+  // resolve to empty when it is.
+  GRPC_MUST_USE_RESULT auto ServerTrailingMetadataWasPushed() {
+    return
+        [this]() { return call_state_.PollServerTrailingMetadataWasPushed(); };
+  }
   // Client & server: returns true if server trailing metadata has been pushed
   // *and* contained a cancellation, false otherwise.
   GRPC_MUST_USE_RESULT bool WasCancelledPushed() const {