grpc 1.78.1 → 1.80.0

data/src/core/call/call_spine.h

@@ -17,6 +17,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
+#include <utility>
+
 #include "src/core/call/call_arena_allocator.h"
 #include "src/core/call/call_filters.h"
 #include "src/core/call/channelz_context.h"
@@ -61,15 +64,15 @@ class CallSpine final : public Party, public channelz::DataSource {
   // Add a callback to be called when server trailing metadata is received and
   // return true.
   // If CallOnDone has already been invoked, does nothing and returns false.
-  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)> fn) {
+  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)>&& fn) {
     if (call_filters().WasServerTrailingMetadataPulled()) {
       return false;
     }
     if (on_done_ == nullptr) {
-      on_done_ = std::move(fn);
+      on_done_ = std::forward<absl::AnyInvocable<void(bool)>>(fn);
       return true;
     }
-    on_done_ = [first = std::move(fn),
+    on_done_ = [first = std::forward<absl::AnyInvocable<void(bool)>>(fn),
                 next = std::move(on_done_)](bool cancelled) mutable {
       first(cancelled);
       next(cancelled);
@@ -136,11 +139,11 @@ class CallSpine final : public Party, public channelz::DataSource {
   // the rest of the call.
   // The resulting (returned) promise will resolve to Empty.
   template <typename Promise>
-  auto CancelIfFails(Promise promise) {
+  auto CancelIfFails(Promise&& promise) {
     GRPC_DCHECK(GetContext<Activity>() == this);
-    using P = promise_detail::PromiseLike<Promise>;
+    using P = promise_detail::PromiseLike<std::decay_t<Promise>>;
     using ResultType = typename P::Result;
-    return Map(std::move(promise),
+    return Map(std::forward<Promise>(promise),
                [self = RefAsSubclass<CallSpine>()](ResultType r) {
                  self->CancelIfFailed(r);
                });
@@ -160,48 +163,54 @@ class CallSpine final : public Party, public channelz::DataSource {
   // Spawn a promise that returns Empty{} and save some boilerplate handling
   // that detail.
   template <typename PromiseFactory>
-  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
-    Spawn(name, std::move(promise_factory), [](Empty) {});
+  void SpawnInfallible(absl::string_view name,
+                       PromiseFactory&& promise_factory) {
+    Spawn(name, std::forward<PromiseFactory>(promise_factory), [](Empty) {});
   }
 
   // Spawn a promise that returns some status-like type; if the status
   // represents failure automatically cancel the rest of the call.
   template <typename PromiseFactory>
-  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory,
+  void SpawnGuarded(absl::string_view name, PromiseFactory&& promise_factory,
                     DebugLocation whence = {}) {
     using FactoryType =
-        promise_detail::OncePromiseFactory<void, PromiseFactory>;
+        promise_detail::OncePromiseFactory<void, std::decay_t<PromiseFactory>>;
     using PromiseType = typename FactoryType::Promise;
     using ResultType = typename PromiseType::Result;
     static_assert(
         std::is_same<bool,
                      decltype(IsStatusOk(std::declval<ResultType>()))>::value,
         "SpawnGuarded promise must return a status-like object");
-    Spawn(name, std::move(promise_factory), [this, whence](ResultType r) {
-      if (!IsStatusOk(r)) {
-        GRPC_TRACE_LOG(promise_primitives, INFO)
-            << "SpawnGuarded sees failure: " << r
-            << " (source: " << whence.file() << ":" << whence.line() << ")";
-        auto status = StatusCast<ServerMetadataHandle>(std::move(r));
-        status->Set(GrpcCallWasCancelled(), true);
-        PushServerTrailingMetadata(std::move(status));
-      }
-    });
+    Spawn(name, std::forward<PromiseFactory>(promise_factory),
+          [this, whence](ResultType&& r) {
+            if (!IsStatusOk(r)) {
+              GRPC_TRACE_LOG(promise_primitives, INFO)
+                  << "SpawnGuarded sees failure: " << r
+                  << " (source: " << whence.file() << ":" << whence.line()
+                  << ")";
+              auto status =
+                  StatusCast<ServerMetadataHandle>(std::forward<ResultType>(r));
+              status->Set(GrpcCallWasCancelled(), true);
+              PushServerTrailingMetadata(std::move(status));
+            }
+          });
   }
 
   // Wrap a promise so that if the call completes that promise is cancelled.
   template <typename Promise>
-  auto UntilCallCompletes(Promise promise) {
-    using Result = PromiseResult<Promise>;
-    return PrioritizedRace(std::move(promise), Map(WasCancelled(), [](bool) {
+  auto UntilCallCompletes(Promise&& promise) {
+    using Result = PromiseResult<std::decay_t<Promise>>;
+    return PrioritizedRace(std::forward<Promise>(promise),
+                           Map(WasCancelled(), [](bool) {
                              return FailureStatusCast<Result>(Failure{});
                            }));
   }
 
   template <typename PromiseFactory>
   void SpawnGuardedUntilCallCompletes(absl::string_view name,
-                                      PromiseFactory promise_factory) {
-    SpawnGuarded(name, [this, promise_factory]() mutable {
+                                      PromiseFactory&& promise_factory) {
+    SpawnGuarded(name, [this, promise_factory = std::forward<PromiseFactory>(
+                                  promise_factory)]() mutable {
       return UntilCallCompletes(promise_factory());
     });
   }
@@ -353,9 +362,9 @@ class CallInitiator {
   // the rest of the call.
   // The resulting (returned) promise will resolve to Empty.
   template <typename Promise>
-  auto CancelIfFails(Promise promise) {
+  auto CancelIfFails(Promise&& promise) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    return spine_->CancelIfFails(std::move(promise));
+    return spine_->CancelIfFails(std::forward<Promise>(promise));
   }
 
   auto PullServerInitialMetadata() {
@@ -419,40 +428,44 @@ class CallInitiator {
     spine_->SpawnCancel();
   }
 
-  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)> fn) {
+  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)>&& fn) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    return spine_->OnDone(std::move(fn));
+    return spine_->OnDone(std::forward<absl::AnyInvocable<void(bool)>>(fn));
   }
 
   template <typename Promise>
-  auto UntilCallCompletes(Promise promise) {
+  auto UntilCallCompletes(Promise&& promise) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    return spine_->UntilCallCompletes(std::move(promise));
+    return spine_->UntilCallCompletes(std::forward<Promise>(promise));
   }
 
   template <typename PromiseFactory>
-  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) {
+  void SpawnGuarded(absl::string_view name, PromiseFactory&& promise_factory) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    spine_->SpawnGuarded(name, std::move(promise_factory));
+    spine_->SpawnGuarded(name, std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
   void SpawnGuardedUntilCallCompletes(absl::string_view name,
-                                      PromiseFactory promise_factory) {
+                                      PromiseFactory&& promise_factory) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    spine_->SpawnGuardedUntilCallCompletes(name, std::move(promise_factory));
+    spine_->SpawnGuardedUntilCallCompletes(
+        name, std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
+  void SpawnInfallible(absl::string_view name,
+                       PromiseFactory&& promise_factory) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    spine_->SpawnInfallible(name, std::move(promise_factory));
+    spine_->SpawnInfallible(name,
+                            std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  auto SpawnWaitable(absl::string_view name, PromiseFactory promise_factory) {
+  auto SpawnWaitable(absl::string_view name, PromiseFactory&& promise_factory) {
     GRPC_DCHECK_NE(spine_.get(), nullptr);
-    return spine_->SpawnWaitable(name, std::move(promise_factory));
+    return spine_->SpawnWaitable(name,
+                                 std::forward<PromiseFactory>(promise_factory));
   }
 
   bool WasCancelledPushed() const {
@@ -501,16 +514,16 @@ class CallHandler {
     spine_->SpawnPushServerTrailingMetadata(std::move(status));
   }
 
-  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)> fn) {
-    return spine_->OnDone(std::move(fn));
+  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)>&& fn) {
+    return spine_->OnDone(std::forward<absl::AnyInvocable<void(bool)>>(fn));
   }
 
   // Wrap a promise so that if it returns failure it automatically cancels
   // the rest of the call.
   // The resulting (returned) promise will resolve to Empty.
   template <typename Promise>
-  auto CancelIfFails(Promise promise) {
-    return spine_->CancelIfFails(std::move(promise));
+  auto CancelIfFails(Promise&& promise) {
+    return spine_->CancelIfFails(std::forward<Promise>(promise));
   }
 
   auto PushMessage(MessageHandle message) {
@@ -530,30 +543,35 @@ class CallHandler {
   }
 
   template <typename Promise>
-  auto UntilCallCompletes(Promise promise) {
-    return spine_->UntilCallCompletes(std::move(promise));
+  auto UntilCallCompletes(Promise&& promise) {
+    return spine_->UntilCallCompletes(std::forward<Promise>(promise));
   }
 
   template <typename PromiseFactory>
-  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory,
+  void SpawnGuarded(absl::string_view name, PromiseFactory&& promise_factory,
                     DebugLocation whence = {}) {
-    spine_->SpawnGuarded(name, std::move(promise_factory), whence);
+    spine_->SpawnGuarded(name, std::forward<PromiseFactory>(promise_factory),
+                         whence);
   }
 
   template <typename PromiseFactory>
   void SpawnGuardedUntilCallCompletes(absl::string_view name,
-                                      PromiseFactory promise_factory) {
-    spine_->SpawnGuardedUntilCallCompletes(name, std::move(promise_factory));
+                                      PromiseFactory&& promise_factory) {
+    spine_->SpawnGuardedUntilCallCompletes(
+        name, std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
-    spine_->SpawnInfallible(name, std::move(promise_factory));
+  void SpawnInfallible(absl::string_view name,
+                       PromiseFactory&& promise_factory) {
+    spine_->SpawnInfallible(name,
+                            std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  auto SpawnWaitable(absl::string_view name, PromiseFactory promise_factory) {
-    return spine_->SpawnWaitable(name, std::move(promise_factory));
+  auto SpawnWaitable(absl::string_view name, PromiseFactory&& promise_factory) {
+    return spine_->SpawnWaitable(name,
+                                 std::forward<PromiseFactory>(promise_factory));
   }
 
   void AddChildCall(const CallInitiator& initiator) {
@@ -577,35 +595,40 @@ class UnstartedCallHandler {
     spine_->PushServerTrailingMetadata(std::move(status));
   }
 
-  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)> fn) {
-    return spine_->OnDone(std::move(fn));
+  GRPC_MUST_USE_RESULT bool OnDone(absl::AnyInvocable<void(bool)>&& fn) {
+    return spine_->OnDone(std::forward<absl::AnyInvocable<void(bool)>>(fn));
   }
 
   template <typename Promise>
-  auto CancelIfFails(Promise promise) {
-    return spine_->CancelIfFails(std::move(promise));
+  auto CancelIfFails(Promise&& promise) {
+    return spine_->CancelIfFails(std::forward<Promise>(promise));
   }
 
   template <typename PromiseFactory>
-  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory,
+  void SpawnGuarded(absl::string_view name, PromiseFactory&& promise_factory,
                     DebugLocation whence = {}) {
-    spine_->SpawnGuarded(name, std::move(promise_factory), whence);
+    spine_->SpawnGuarded(name, std::forward<PromiseFactory>(promise_factory),
+                         whence);
   }
 
   template <typename PromiseFactory>
   void SpawnGuardedUntilCallCompletes(absl::string_view name,
-                                      PromiseFactory promise_factory) {
-    spine_->SpawnGuardedUntilCallCompletes(name, std::move(promise_factory));
+                                      PromiseFactory&& promise_factory) {
+    spine_->SpawnGuardedUntilCallCompletes(
+        name, std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
-    spine_->SpawnInfallible(name, std::move(promise_factory));
+  void SpawnInfallible(absl::string_view name,
+                       PromiseFactory&& promise_factory) {
+    spine_->SpawnInfallible(name,
+                            std::forward<PromiseFactory>(promise_factory));
   }
 
   template <typename PromiseFactory>
-  auto SpawnWaitable(absl::string_view name, PromiseFactory promise_factory) {
-    return spine_->SpawnWaitable(name, std::move(promise_factory));
+  auto SpawnWaitable(absl::string_view name, PromiseFactory&& promise_factory) {
+    return spine_->SpawnWaitable(name,
+                                 std::forward<PromiseFactory>(promise_factory));
   }
 
   ClientMetadata& UnprocessedClientInitialMetadata() {
@@ -636,12 +659,12 @@ CallInitiatorAndHandler MakeCallPair(
     ClientMetadataHandle client_initial_metadata, RefCountedPtr<Arena> arena);
 
 template <typename CallHalf>
-auto MessagesFrom(CallHalf h) {
+auto MessagesFrom(CallHalf&& h) {
   struct Wrapper {
-    CallHalf h;
+    std::decay_t<CallHalf> h;
     auto Next() { return h.PullMessage(); }
   };
-  return Wrapper{std::move(h)};
+  return Wrapper{std::forward<CallHalf>(h)};
 }
 
 template <typename CallHalf>

data/src/core/call/call_state.h

@@ -92,6 +92,9 @@ class CallState {
   // PULL: server -> client
 
   // Poll for initial metadata to be available.
+  // One and only one call to GRPC_OP_RECV_INITIAL_METADATA MUST be made on the
+  // client. By extension, this function is also expected to be called exactly
+  // once.
   Poll<bool> PollPullServerInitialMetadataAvailable();
   // Finish pulling server initial metadata.
   void FinishPullServerInitialMetadata();
@@ -209,6 +212,7 @@ class CallState {
     kReading,
     // Main call loop: processing one message
     kProcessingServerToClientMessage,
+    kDone,
     kTerminated,
   };
   static const char* ServerToClientPullStateString(
@@ -232,6 +236,8 @@ class CallState {
         return "Reading";
       case ServerToClientPullState::kProcessingServerToClientMessage:
         return "ProcessingServerToClientMessage";
+      case ServerToClientPullState::kDone:
+        return "Done";
       case ServerToClientPullState::kTerminated:
         return "Terminated";
     }
@@ -354,6 +360,7 @@ GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline void CallState::Start() {
     case ServerToClientPullState::kIdle:
     case ServerToClientPullState::kReading:
     case ServerToClientPullState::kProcessingServerToClientMessage:
+    case ServerToClientPullState::kDone:
       LOG(FATAL) << "Start called twice; "
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_);
     case ServerToClientPullState::kTerminated:
@@ -747,7 +754,12 @@ CallState::PollPullServerInitialMetadataAvailable() {
     case ServerToClientPullState::kUnstartedReading:
       if (server_to_client_push_state_ ==
           ServerToClientPushState::kTrailersOnly) {
+        GRPC_DCHECK(server_trailing_metadata_state_ ==
+                        ServerTrailingMetadataState::kPushedCancel ||
+                    server_trailing_metadata_state_ ==
+                        ServerTrailingMetadataState::kPulledCancel);
         server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
+        server_to_client_pull_waiter_.Wake();
         return false;
       }
       server_to_client_push_waiter_.pending();
@@ -766,6 +778,8 @@ CallState::PollPullServerInitialMetadataAvailable() {
       LOG(FATAL) << "PollPullServerInitialMetadataAvailable called twice; "
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_,
                                    server_to_client_push_state_);
+    // If trailing metadata has been pulled, initial metadata cannot be pulled.
+    case ServerToClientPullState::kDone:
     case ServerToClientPullState::kTerminated:
       return false;
   }
@@ -829,6 +843,7 @@ CallState::FinishPullServerInitialMetadata() {
       LOG(FATAL) << "Out of order FinishPullServerInitialMetadata; "
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_,
                                    server_to_client_push_state_);
+    case ServerToClientPullState::kDone:
     case ServerToClientPullState::kTerminated:
       return;
   }
@@ -870,12 +885,28 @@ CallState::PollPullServerToClientMessageAvailable() {
   switch (server_to_client_pull_state_) {
     case ServerToClientPullState::kUnstarted:
       server_to_client_pull_state_ = ServerToClientPullState::kUnstartedReading;
+      [[fallthrough]];
+    case ServerToClientPullState::kUnstartedReading:
+      if (server_to_client_push_state_ ==
+          ServerToClientPushState::kTrailersOnly) {
+        // Here we do not transition to kDone because this poll would return a
+        // null message (based on whether the trailing metadata was cancelled)
+        // which MUST be interpreted as end of stream.
+        server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
+        server_to_client_pull_waiter_.Wake();
+        GRPC_DCHECK(server_trailing_metadata_state_ !=
+                    ServerTrailingMetadataState::kNotPushed);
+        return WasCancelledPushed() ? Failure{} : ValueOrFailure<bool>{false};
+      }
+      // When TrailersOnly is pushed, server_to_client_push_state_ is modified
+      // and the server_to_client_push_waiter_ is woken up. If this promise only
+      // waits on server_to_client_pull_waiter_, it may never be polled.
+      server_to_client_push_waiter_.pending();
       return server_to_client_pull_waiter_.pending();
     case ServerToClientPullState::kProcessingServerInitialMetadata:
       server_to_client_pull_state_ =
           ServerToClientPullState::kProcessingServerInitialMetadataReading;
       return server_to_client_pull_waiter_.pending();
-    case ServerToClientPullState::kUnstartedReading:
     case ServerToClientPullState::kProcessingServerInitialMetadataReading:
       return server_to_client_pull_waiter_.pending();
     case ServerToClientPullState::kStarted:
@@ -884,8 +915,19 @@ CallState::PollPullServerToClientMessageAvailable() {
     case ServerToClientPullState::kStartedReading:
       if (server_to_client_push_state_ ==
           ServerToClientPushState::kTrailersOnly) {
-        return false;
+        // Here we do not transition to kDone because this poll would return a
+        // null message (based on whether the trailing metadata was cancelled)
+        // which MUST be interpreted as end of stream.
+        server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
+        server_to_client_pull_waiter_.Wake();
+        GRPC_DCHECK(server_trailing_metadata_state_ !=
+                    ServerTrailingMetadataState::kNotPushed);
+        return WasCancelledPushed() ? Failure{} : ValueOrFailure<bool>{false};
       }
+      // When TrailersOnly is pushed, server_to_client_push_state_ is modified
+      // and the server_to_client_push_waiter_ is woken up. If this promise only
+      // waits on server_to_client_pull_waiter_, it may never be polled.
+      server_to_client_push_waiter_.pending();
       return server_to_client_pull_waiter_.pending();
     case ServerToClientPullState::kIdle:
       server_to_client_pull_state_ = ServerToClientPullState::kReading;
@@ -898,6 +940,12 @@ CallState::PollPullServerToClientMessageAvailable() {
                     "processing a message; "
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_,
                                    server_to_client_push_state_);
+    case ServerToClientPullState::kDone:
+      // Return EndOfStream to the upper layer. Any further attempts to read
+      // message will be treated as failures.
+      server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
+      server_to_client_pull_waiter_.Wake();
+      return false;
     case ServerToClientPullState::kTerminated:
       return Failure{};
   }
@@ -949,6 +997,7 @@ CallState::PollPullServerToClientMessageStarted() {
     case ServerToClientPullState::kReading:
     case ServerToClientPullState::kProcessingServerToClientMessage:
       return Success{};
+    case ServerToClientPullState::kDone:
     case ServerToClientPullState::kTerminated:
       return Failure{};
   }
@@ -977,6 +1026,7 @@ CallState::FinishPullServerToClientMessage() {
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_,
                                    server_to_client_push_state_);
     case ServerToClientPullState::kReading:
+    case ServerToClientPullState::kDone:
       LOG(FATAL) << "FinishPullServerToClientMessage called before "
                  << "PollPullServerToClientMessageAvailable; "
                  << GRPC_DUMP_ARGS(server_to_client_pull_state_,
@@ -1027,6 +1077,7 @@ CallState::PollServerTrailingMetadataAvailable() {
     case ServerToClientPullState::kProcessingServerToClientMessage:
     case ServerToClientPullState::kProcessingServerInitialMetadataReading:
     case ServerToClientPullState::kUnstartedReading:
+      server_trailing_metadata_waiter_.pending();
       return server_to_client_pull_waiter_.pending();
     case ServerToClientPullState::kStartedReading:
     case ServerToClientPullState::kReading:
@@ -1045,6 +1096,7 @@ CallState::PollServerTrailingMetadataAvailable() {
         case ServerToClientPushState::
             kPushedServerInitialMetadataAndPushedMessage:
         case ServerToClientPushState::kPushedMessage:
+          server_trailing_metadata_waiter_.pending();
           server_to_client_push_waiter_.pending();
           return server_to_client_pull_waiter_.pending();
       }
@@ -1057,19 +1109,23 @@ CallState::PollServerTrailingMetadataAvailable() {
         break;  // Ready for processing
       }
       return server_trailing_metadata_waiter_.pending();
+    case ServerToClientPullState::kDone:
     case ServerToClientPullState::kTerminated:
       break;
   }
-  server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
-  server_to_client_pull_waiter_.Wake();
+
   switch (server_trailing_metadata_state_) {
     case ServerTrailingMetadataState::kPushed:
       server_trailing_metadata_state_ = ServerTrailingMetadataState::kPulled;
+      server_to_client_pull_state_ = ServerToClientPullState::kDone;
+      server_to_client_pull_waiter_.Wake();
       server_trailing_metadata_waiter_.Wake();
       break;
     case ServerTrailingMetadataState::kPushedCancel:
       server_trailing_metadata_state_ =
           ServerTrailingMetadataState::kPulledCancel;
+      server_to_client_pull_state_ = ServerToClientPullState::kTerminated;
+      server_to_client_pull_waiter_.Wake();
       server_trailing_metadata_waiter_.Wake();
       break;
     case ServerTrailingMetadataState::kNotPushed:

data/src/core/call/client_call.cc

@@ -211,7 +211,7 @@ void ClientCall::CancelWithError(grpc_error_handle error) {
 }
 
 template <typename Batch>
-void ClientCall::ScheduleCommittedBatch(Batch batch) {
+void ClientCall::ScheduleCommittedBatch(Batch&& batch) {
   GRPC_LATENT_SEE_SCOPE("ClientCall::ScheduleCommittedBatch");
   auto cur_state = call_state_.load(std::memory_order_acquire);
   while (true) {
@@ -219,12 +219,12 @@ void ClientCall::ScheduleCommittedBatch(Batch batch) {
       case kUnstarted:
       default: {  // UnorderedStart
         auto pending = std::make_unique<UnorderedStart>();
-        pending->start_pending_batch = [this,
-                                        batch = std::move(batch)]() mutable {
-          started_call_initiator_.SpawnInfallible(
-              "batch",
-              GRPC_LATENT_SEE_PROMISE("ClientCallBatch", std::move(batch)));
-        };
+        pending->start_pending_batch =
+            [this, batch = std::forward<Batch>(batch)]() mutable {
+              started_call_initiator_.SpawnInfallible(
+                  "batch", GRPC_LATENT_SEE_PROMISE("ClientCallBatch",
+                                                   std::forward<Batch>(batch)));
+            };
         while (true) {
           pending->next = reinterpret_cast<UnorderedStart*>(cur_state);
           if (call_state_.compare_exchange_strong(
@@ -244,8 +244,8 @@ void ClientCall::ScheduleCommittedBatch(Batch batch) {
       }
       case kStarted:
         started_call_initiator_.SpawnInfallible(
-            "batch",
-            GRPC_LATENT_SEE_PROMISE("ClientCallBatch", std::move(batch)));
+            "batch", GRPC_LATENT_SEE_PROMISE("ClientCallBatch",
+                                             std::forward<Batch>(batch)));
         return;
       case kCancelled:
         return;

data/src/core/call/client_call.h

@@ -130,7 +130,7 @@ class ClientCall final
   void CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
                    bool is_notify_tag_closure);
   template <typename Batch>
-  void ScheduleCommittedBatch(Batch batch);
+  void ScheduleCommittedBatch(Batch&& batch);
   Party::WakeupHold StartCall(const grpc_op& send_initial_metadata_op);
   // Attempt to start the call and send handler down the stack; returns true if
   // state was updated, false otherwise (with cur_state updated to the new

data/src/core/call/metadata_batch.cc

@@ -59,6 +59,8 @@ bool IsMetadataKeyAllowedInDebugOutput(absl::string_view key) {
   if (key == UserAgentMetadata::key()) return true;
   if (key == W3CTraceParentMetadata::key()) return true;
   if (key == XEnvoyPeerMetadata::key()) return true;
+  if (key == XForwardedForMetadata::key()) return true;
+  if (key == XForwardedHostMetadata::key()) return true;
   // go/keep-sorted end
   // go/keep-sorted start
   if (key == GrpcCallWasCancelled::DebugKey()) return true;