grpc 1.64.0 → 1.65.2

data/src/core/lib/transport/call_filters.h

@@ -23,6 +23,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/gprpp/dump_args.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/promise/latch.h"
@@ -43,6 +44,7 @@
 // - OnServerInitialMetadata  - $VALUE_TYPE = ServerMetadata
 // - OnServerToClientMessage  - $VALUE_TYPE = Message
 // - OnClientToServerMessage  - $VALUE_TYPE = Message
+// - OnClientToServerHalfClose - no value
 // - OnServerTrailingMetadata - $VALUE_TYPE = ServerMetadata
 // - OnFinalize               - special, see below
 // These members define an interception point for a particular event in
@@ -192,6 +194,16 @@ struct Operator {
   void (*early_destroy)(void* promise_data);
 };
 
+struct HalfCloseOperator {
+  // Pointer to corresponding channel data for this filter
+  void* channel_data;
+  // Offset of the call data for this filter within the call data memory
+  size_t call_offset;
+  void (*half_close)(void* call_data, void* channel_data);
+};
+
+void RunHalfClose(absl::Span<const HalfCloseOperator> ops, void* call_data);
+
 // We divide operations into fallible and infallible.
 // Fallible operations can fail, and that failure terminates the call.
 // Infallible operations cannot fail.
@@ -265,6 +277,32 @@ void AddOp(FilterType* channel_data, size_t call_offset,
                                                     to);
 }
 
+template <typename FilterType>
+void AddHalfClose(FilterType* channel_data, size_t call_offset,
+                  void (FilterType::Call::*)(),
+                  std::vector<HalfCloseOperator>& to) {
+  to.push_back(
+      HalfCloseOperator{channel_data, call_offset, [](void* call_data, void*) {
+                          static_cast<typename FilterType::Call*>(call_data)
+                              ->OnClientToServerHalfClose();
+                        }});
+}
+
+template <typename FilterType>
+void AddHalfClose(FilterType* channel_data, size_t call_offset,
+                  void (FilterType::Call::*)(FilterType*),
+                  std::vector<HalfCloseOperator>& to) {
+  to.push_back(HalfCloseOperator{
+      channel_data, call_offset, [](void* call_data, void* channel_data) {
+        static_cast<typename FilterType::Call*>(call_data)
+            ->OnClientToServerHalfClose(static_cast<FilterType*>(channel_data));
+      }});
+}
+
+template <typename FilterType>
+void AddHalfClose(FilterType*, size_t, const NoInterceptor*,
+                  std::vector<HalfCloseOperator>&) {}
+
 // const NoInterceptor $EVENT
 // These do nothing, and specifically DO NOT add an operation to the layout.
 // Supported for fallible & infallible operations.
@@ -852,6 +890,7 @@ struct StackData {
   Layout<FallibleOperator<ClientMetadataHandle>> client_initial_metadata;
   Layout<FallibleOperator<ServerMetadataHandle>> server_initial_metadata;
   Layout<FallibleOperator<MessageHandle>> client_to_server_messages;
+  std::vector<HalfCloseOperator> client_to_server_half_close;
   Layout<FallibleOperator<MessageHandle>> server_to_client_messages;
   Layout<InfallibleOperator<ServerMetadataHandle>> server_trailing_metadata;
   // A list of finalizers for this call.
@@ -972,6 +1011,14 @@ struct StackData {
         channel_data, call_offset, client_to_server_messages);
   }
 
+  template <typename FilterType>
+  void AddClientToServerHalfClose(FilterType* channel_data,
+                                  size_t call_offset) {
+    AddHalfClose(channel_data, call_offset,
+                 &FilterType::Call::OnClientToServerHalfClose,
+                 client_to_server_half_close);
+  }
+
   template <typename FilterType>
   void AddServerToClientMessageOp(FilterType* channel_data,
                                   size_t call_offset) {
@@ -1217,6 +1264,7 @@ class ServerTrailingMetadataInterceptor {
     static const NoInterceptor OnClientInitialMetadata;
     static const NoInterceptor OnServerInitialMetadata;
     static const NoInterceptor OnClientToServerMessage;
+    static const NoInterceptor OnClientToServerHalfClose;
     static const NoInterceptor OnServerToClientMessage;
     static const NoInterceptor OnFinalize;
     void OnServerTrailingMetadata(ServerMetadata& md,
@@ -1240,11 +1288,54 @@ template <typename Fn>
 const NoInterceptor
     ServerTrailingMetadataInterceptor<Fn>::Call::OnClientToServerMessage;
 template <typename Fn>
+const NoInterceptor
+    ServerTrailingMetadataInterceptor<Fn>::Call::OnClientToServerHalfClose;
+template <typename Fn>
 const NoInterceptor
     ServerTrailingMetadataInterceptor<Fn>::Call::OnServerToClientMessage;
 template <typename Fn>
 const NoInterceptor ServerTrailingMetadataInterceptor<Fn>::Call::OnFinalize;
 
+template <typename Fn>
+class ClientInitialMetadataInterceptor {
+ public:
+  class Call {
+   public:
+    auto OnClientInitialMetadata(ClientMetadata& md,
+                                 ClientInitialMetadataInterceptor* filter) {
+      return filter->fn_(md);
+    }
+    static const NoInterceptor OnServerInitialMetadata;
+    static const NoInterceptor OnClientToServerMessage;
+    static const NoInterceptor OnClientToServerHalfClose;
+    static const NoInterceptor OnServerToClientMessage;
+    static const NoInterceptor OnServerTrailingMetadata;
+    static const NoInterceptor OnFinalize;
+  };
+
+  explicit ClientInitialMetadataInterceptor(Fn fn) : fn_(std::move(fn)) {}
+
+ private:
+  GPR_NO_UNIQUE_ADDRESS Fn fn_;
+};
+template <typename Fn>
+const NoInterceptor
+    ClientInitialMetadataInterceptor<Fn>::Call::OnServerInitialMetadata;
+template <typename Fn>
+const NoInterceptor
+    ClientInitialMetadataInterceptor<Fn>::Call::OnClientToServerMessage;
+template <typename Fn>
+const NoInterceptor
+    ClientInitialMetadataInterceptor<Fn>::Call::OnClientToServerHalfClose;
+template <typename Fn>
+const NoInterceptor
+    ClientInitialMetadataInterceptor<Fn>::Call::OnServerToClientMessage;
+template <typename Fn>
+const NoInterceptor
+    ClientInitialMetadataInterceptor<Fn>::Call::OnServerTrailingMetadata;
+template <typename Fn>
+const NoInterceptor ClientInitialMetadataInterceptor<Fn>::Call::OnFinalize;
+
 }  // namespace filters_detail
 
 // Execution environment for a stack of filters.
@@ -1283,6 +1374,7 @@ class CallFilters {
       data_.AddClientInitialMetadataOp(filter, call_offset);
       data_.AddServerInitialMetadataOp(filter, call_offset);
       data_.AddClientToServerMessageOp(filter, call_offset);
+      data_.AddClientToServerHalfClose(filter, call_offset);
       data_.AddServerToClientMessageOp(filter, call_offset);
       data_.AddServerTrailingMetadataOp(filter, call_offset);
       data_.AddFinalizer(filter, call_offset, &FilterType::Call::OnFinalize);
@@ -1302,6 +1394,14 @@ class CallFilters {
       AddOwnedObject([](void* p) { delete static_cast<T*>(p); }, p.release());
     }
 
+    template <typename Fn>
+    void AddOnClientInitialMetadata(Fn fn) {
+      auto filter = std::make_unique<
+          filters_detail::ClientInitialMetadataInterceptor<Fn>>(std::move(fn));
+      Add(filter.get());
+      AddOwnedObject(std::move(filter));
+    }
+
     template <typename Fn>
     void AddOnServerTrailingMetadata(Fn fn) {
       auto filter = std::make_unique<
@@ -1391,7 +1491,7 @@ class CallFilters {
       Push(CallFilters* filters, T x)
           : filters_(filters), value_(std::move(x)) {
         CHECK(value_ != nullptr);
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "BeginPush[%p|%p]: %s", &state(), this,
                   state().DebugString().c_str());
         }
@@ -1425,17 +1525,17 @@ class CallFilters {
       Poll<StatusFlag> operator()() {
         if (value_ == nullptr) {
           CHECK_EQ(filters_, nullptr);
-          if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+          if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
             gpr_log(GPR_INFO, "Push[|%p]: already done", this);
           }
           return Success{};
         }
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "Push[%p|%p]: %s", &state(), this,
                   state().DebugString().c_str());
         }
         auto r = state().PollPush();
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           if (r.pending()) {
             gpr_log(GPR_INFO, "Push[%p|%p]: pending; %s", &state(), this,
                     state().DebugString().c_str());
@@ -1455,7 +1555,7 @@ class CallFilters {
       }
 
       T TakeValue() {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "Push[%p|%p]: take value; %s", &state(), this,
                   state().DebugString().c_str());
         }
@@ -1502,7 +1602,7 @@ class CallFilters {
       PullMaybe& operator=(PullMaybe&&) = delete;
 
       Poll<ValueOrFailure<absl::optional<T>>> operator()() {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "PullMaybe[%p|%p]: %s executor:%d", &state(), this,
                   state().DebugString().c_str(), executor_.IsRunning());
         }
@@ -1549,6 +1649,8 @@ class CallFilters {
       filters_detail::OperationExecutor<T> executor_;
     };
 
+    template <std::vector<filters_detail::HalfCloseOperator>(
+        filters_detail::StackData::*half_close_layout_ptr)>
     class PullMessage {
      public:
       explicit PullMessage(CallFilters* filters) : filters_(filters) {}
@@ -1567,7 +1669,7 @@ class CallFilters {
 
       Poll<ValueOrFailure<absl::optional<MessageHandle>>> operator()() {
         CHECK(filters_ != nullptr);
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "PullMessage[%p|%p]: %s executor:%d", &state(),
                   this, state().DebugString().c_str(), executor_.IsRunning());
         }
@@ -1582,7 +1684,7 @@ class CallFilters {
         auto p = state().PollPull();
         auto* r = p.value_if_ready();
         if (r == nullptr) {
-          if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+          if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
             gpr_log(GPR_INFO, "PullMessage[%p] pending: %s executor:%d",
                     &state(), state().DebugString().c_str(),
                     executor_.IsRunning());
@@ -1593,7 +1695,14 @@ class CallFilters {
           filters_->CancelDueToFailedPipeOperation();
           return Failure{};
         }
-        if (!**r) return absl::nullopt;
+        if (!**r) {
+          if (half_close_layout_ptr != nullptr) {
+            filters_detail::RunHalfClose(
+                filters_->stack_->data_.*half_close_layout_ptr,
+                filters_->call_data_);
+          }
+          return absl::nullopt;
+        }
         CHECK(filters_ != nullptr);
         return FinishOperationExecutor(executor_.Start(
             layout(), push()->TakeValue(), filters_->call_data_));
@@ -1612,7 +1721,7 @@ class CallFilters {
         auto* r = p.value_if_ready();
         if (r == nullptr) return Pending{};
         DCHECK(!executor_.IsRunning());
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO, "PullMessage[%p|%p] executor done: %s", &state(),
                   this, state().DebugString().c_str());
         }
@@ -1649,7 +1758,7 @@ class CallFilters {
       }
       auto p = state().PollPull();
       auto* r = p.value_if_ready();
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+      if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
         gpr_log(GPR_INFO, "%s",
                 r == nullptr
                     ? "PENDING"
@@ -1705,7 +1814,7 @@ class CallFilters {
     Poll<ServerMetadataHandle> operator()() {
       if (executor_.IsRunning()) {
         auto r = executor_.Step(filters_->call_data_);
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           if (r.pending()) {
             gpr_log(GPR_INFO,
                     "%s PullServerTrailingMetadata[%p]: Pending(but executing)",
@@ -1719,7 +1828,7 @@ class CallFilters {
         return r;
       }
       if (filters_->server_trailing_metadata_ == nullptr) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO,
                   "%s PullServerTrailingMetadata[%p]: Pending(not pushed)",
                   GetContext<Activity>()->DebugTag().c_str(), filters_);
@@ -1728,7 +1837,7 @@ class CallFilters {
       }
       // If no stack has been set, we can just return the result of the call
       if (filters_->stack_ == nullptr) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+        if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
           gpr_log(GPR_INFO,
                   "%s PullServerTrailingMetadata[%p]: Ready(no-stack): %s",
                   GetContext<Activity>()->DebugTag().c_str(), filters_,
@@ -1740,7 +1849,7 @@ class CallFilters {
       auto r = executor_.Start(
           &filters_->stack_->data_.server_trailing_metadata,
           std::move(filters_->server_trailing_metadata_), filters_->call_data_);
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+      if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
         if (r.pending()) {
           gpr_log(GPR_INFO,
                   "%s PullServerTrailingMetadata[%p]: Pending(but executing)",
@@ -1821,7 +1930,8 @@ inline auto CallFilters::PushClientToServerMessage(MessageHandle message) {
 }
 
 inline auto CallFilters::PullClientToServerMessage() {
-  return ClientToServerMessagePromises::PullMessage{this};
+  return ClientToServerMessagePromises::PullMessage<
+      &filters_detail::StackData::client_to_server_half_close>{this};
 }
 
 inline auto CallFilters::PushServerToClientMessage(MessageHandle message) {
@@ -1831,15 +1941,11 @@ inline auto CallFilters::PushServerToClientMessage(MessageHandle message) {
 }
 
 inline auto CallFilters::PullServerToClientMessage() {
-  return ServerToClientMessagePromises::PullMessage{this};
+  return ServerToClientMessagePromises::PullMessage<nullptr>{this};
 }
 
 inline auto CallFilters::PullServerTrailingMetadata() {
-  return Map(PullServerTrailingMetadataPromise(this),
-             [this](ServerMetadataHandle h) {
-               cancelled_.Set(h->get(GrpcCallWasCancelled()).value_or(false));
-               return h;
-             });
+  return PullServerTrailingMetadataPromise(this);
 }
 
 inline auto CallFilters::WasCancelled() { return cancelled_.Wait(); }

data/src/core/lib/transport/call_spine.cc

@@ -94,12 +94,11 @@ void ForwardCall(CallHandler call_handler, CallInitiator call_initiator) {
 
 CallInitiatorAndHandler MakeCallPair(
     ClientMetadataHandle client_initial_metadata,
-    grpc_event_engine::experimental::EventEngine* event_engine, Arena* arena,
-    RefCountedPtr<CallArenaAllocator> call_arena_allocator_if_arena_is_owned,
-    grpc_call_context_element* legacy_context) {
-  auto spine = CallSpine::Create(
-      std::move(client_initial_metadata), event_engine, arena,
-      std::move(call_arena_allocator_if_arena_is_owned), legacy_context);
+    grpc_event_engine::experimental::EventEngine* event_engine,
+    RefCountedPtr<Arena> arena) {
+  CHECK_NE(arena.get(), nullptr);
+  auto spine = CallSpine::Create(std::move(client_initial_metadata),
+                                 event_engine, std::move(arena));
   return {CallInitiator(spine), UnstartedCallHandler(spine)};
 }