grpc 1.81.1 → 1.83.1

data/src/core/lib/channel/promise_based_filter.h

@@ -38,7 +38,6 @@
 #include "src/core/call/message.h"
 #include "src/core/call/metadata.h"
 #include "src/core/call/metadata_batch.h"
-#include "src/core/filter/blackboard.h"
 #include "src/core/filter/filter_args.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_fwd.h"
@@ -71,6 +70,8 @@
 #include "src/core/util/debug_location.h"
 #include "src/core/util/grpc_check.h"
 #include "src/core/util/match.h"
+#include "src/core/util/orphanable.h"
+#include "src/core/util/ref_counted_ptr.h"
 #include "src/core/util/time.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/functional/function_ref.h"
@@ -163,7 +164,7 @@ struct HasAsyncErrorInterceptor<Promise (T::*)(A...),
   static constexpr bool value = true;
 };
 
-// For the list case we do two interceptors to avoid amiguities with the single
+// For the list case we do two interceptors to avoid ambiguities with the single
 // argument forms above.
 template <typename... Interceptors>
 inline constexpr bool HasAnyAsyncErrorInterceptor(Interceptors...) {
@@ -1175,7 +1176,7 @@ MakeFilterCall(Derived* derived) {
 //   useful for cases where the exact metadata returned needs to be customized.
 // It's also acceptable to return a promise that resolves to the
 // relevant return type listed above.
-// Finally, OnFinalize can be added to intecept call finalization.
+// Finally, OnFinalize can be added to intercept call finalization.
 // It must have one of the signatures:
 // - static inline const NoInterceptor OnFinalize:
 //   the filter does not intercept call finalization.
@@ -1274,192 +1275,241 @@ class V3InterceptorToV2Bridge : public ChannelFilter, public Interceptor {
       InterActivityLatch<std::optional<ServerMetadataHandle>>
           server_initial_metadata;
       InterActivityPipe<MessageHandle, 1> server_to_client_messages;
+      InterActivityLatch<ServerMetadataHandle> server_trailing_metadata;
     };
     auto* pipe_owner = GetContext<Arena>()->ManagedNew<PipeOwner>();
+    if (IsV2NonOwningWakerImplementationEnabled()) {
+      // We need to start polling the initiator for server trailing
+      // metadata immediately, since the v3 interceptor may generate a
+      // failure before any of the other promises resolve.
+      //
+      // Spawn a promise on the initiator's activity to pull server trailing
+      // metadata and pass it to the v2 activity via an inter-activity latch.
+      initiator.SpawnInfallible(
+          "pull_server_trailing_metadata",
+          [initiator = initiator, pipe_owner]() mutable {
+            return Map(
+                initiator.PullServerTrailingMetadata(),
+                [pipe_owner](ServerMetadataHandle metadata) {
+                  pipe_owner->server_trailing_metadata.Set(std::move(metadata));
+                  return Empty{};
+                });
+          });
+    }
     // Now return a promise that does all the things.
-    return Race(
-        // We need to start polling the initiator for server trailing
-        // metadata immediately, since the v3 interceptor may generate a
-        // failure before any of the other promises resolve.
-        initiator.PullServerTrailingMetadata(),
-        // This promise does the rest of the things, but it will always
-        // return pending, because the promise can't actually finish
-        // until the initiator returns trailing metadata above.
-        TrySeq(
-            state->call_handler_latch.Wait(),
-            [initiator = initiator, pipe_owner,
-             call_args = std::move(call_args),
-             next_promise_factory =
-                 std::move(next_promise_factory)](CallHandler handler) mutable {
-              // Intercept all pipes from v2 API.
-              //
-              // For client-to-server messages, we do the following:
-              // 1. Push the message into the v3 initiator, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 initiator's activity.
-              // 2. Pull the message from the v3 handler, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    handler's activity.  We then push the message into
-              //    an inter-activity pipe to return it to the v2 activity.
-              // 3. In the v2 activity, read the message from the
-              //    inter-activity pipe and send it on to the next
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull messages from the v3
-              // handler and push them into an inter-activity pipe to
-              // return them to the v2 activity.
-              handler.SpawnGuarded(
-                  "pull_client_to_server_message",
-                  [handler, pipe_owner]() mutable {
-                    return ForEach(
-                        MessagesFrom(handler),
-                        [pipe_owner](MessageHandle message) {
-                          return Map(
-                              pipe_owner->client_to_server_messages.sender.Push(
-                                  std::move(message)),
-                              [](bool x) { return StatusFlag(x); });
-                        });
-                  });
-              call_args.client_to_server_messages->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](MessageHandle message) mutable {
-                    // Step 1: Push the message onto the v3 initiator in
-                    // its activity.
-                    initiator.SpawnPushMessage(std::move(message));
-                    // Step 3: Here in the v2 activity, read the message
-                    // from the inter-activity pipe and return it.
-                    return Map(
-                        pipe_owner->client_to_server_messages.receiver.Next(),
-                        [](InterActivityPipe<MessageHandle, 1>::NextResult
-                               message) -> std::optional<MessageHandle> {
-                          if (!message.has_value()) return std::nullopt;
-                          return std::move(*message);
-                        });
-                  });
-              // For server initial metadata, we do a similar thing, but
-              // in the opposite direction, and using an inter-activity
-              // latch instead of a pipe:
-              // 1. Push the metadata into the v3 handler, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 handler's activity.
-              // 2. Pull the metadata from the v3 initiator, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    initiator's activity.  We then use an inter-activity
-              //    latch to return the metadata to the v2 activity.
-              // 3. In the v2 activity, read the metadata from the
-              //    inter-activity latch and send it on to the previous
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull the metadata from the v3
-              // initiator and use an inter-activity latch to return it to
-              // the v2 activity.
-              initiator.SpawnGuarded(
-                  "pull_server_initial_metadata",
-                  [initiator, pipe_owner]() mutable {
-                    return TrySeq(
-                        initiator.PullServerInitialMetadata(),
-                        [pipe_owner](
-                            std::optional<ServerMetadataHandle> metadata) {
-                          pipe_owner->server_initial_metadata.Set(
-                              std::move(metadata));
-                        });
-                  });
-              call_args.server_initial_metadata->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](ServerMetadataHandle metadata) mutable {
-                    // Step 1: Push the metadata onto the v3 handler in
-                    // its activity.
-                    handler.SpawnPushServerInitialMetadata(std::move(metadata));
-                    // Step 3: Here in the v2 activity, read from the
-                    // inter-activity latch and return the metadata.
-                    return pipe_owner->server_initial_metadata.Wait();
-                  });
-              // We handle server-to-client messages the same as
-              // client-to-server messages, except in the opposite
-              // direction:
-              // 1. Push the message into the v3 handler, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 handler's activity.
-              // 2. Pull the message from the v3 initiator, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    initiator's activity.  We then push the message into
-              //    an inter-activity pipe to return it to the v2 activity.
-              // 3. In the v2 activity, read the message from the
-              //    inter-activity pipe and send it on to the next
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull messages from the v3
-              // initiator and push them into an inter-activity pipe to
-              // return them to the v2 activity.
-              initiator.SpawnGuarded(
-                  "pull_server_to_client_message",
-                  [initiator, pipe_owner]() mutable {
-                    return ForEach(
-                        MessagesFrom(initiator),
-                        [pipe_owner](MessageHandle message) {
-                          return Map(
-                              pipe_owner->server_to_client_messages.sender.Push(
-                                  std::move(message)),
-                              [](bool x) { return StatusFlag(x); });
-                        });
-                  });
-              call_args.server_to_client_messages->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](MessageHandle message) mutable {
-                    // Step 1: Push the message onto the v3 handler in
-                    // its activity.
-                    handler.SpawnPushMessage(std::move(message));
-                    // Step 3: Here in the v2 activity, read from the
-                    // inter-activity pipe and return the messages.
-                    return Map(
-                        pipe_owner->server_to_client_messages.receiver.Next(),
-                        [](InterActivityPipe<MessageHandle, 1>::NextResult
-                               message) -> std::optional<MessageHandle> {
-                          if (!message.has_value()) return std::nullopt;
-                          return std::move(*message);
-                        });
-                  });
-              // In the v3 handler's activity, pull client initial metadata.
-              // Use an inter-acitivity latch to get it back to the v2
-              // activity.
-              handler.SpawnGuarded(
-                  "pull_client_initial_metadata",
-                  [handler, pipe_owner]() mutable {
-                    return TrySeq(handler.PullClientInitialMetadata(),
-                                  [pipe_owner](ClientMetadataHandle metadata) {
-                                    pipe_owner->client_initial_metadata.Set(
-                                        std::move(metadata));
-                                  });
-                  });
-              // A wrapper for next_promise_factory that does the following:
-              // - Pulls client initial metadata from the V3 handler via
-              //   the inter-activity latch and injects it into the next
-              //   V2 filter via CallArgs.
-              // - Polls the next promise to get server trailing metadata
-              //   from the next V2 filter and feeds it into the V3 handler.
-              // Note that this does not actually pull the trailing metadata
-              // from the V3 initiator; instead, we do that in a separate
-              // promise above.  That promise will always complete at the
-              // end of the call, so we always return pending here.
-              return Seq(
-                  pipe_owner->client_initial_metadata.Wait(),
-                  [next_promise_factory = std::move(next_promise_factory),
-                   call_args = std::move(call_args),
-                   handler](ClientMetadataHandle metadata) mutable {
-                    call_args.client_initial_metadata = std::move(metadata);
-                    return Seq(next_promise_factory(std::move(call_args)),
-                               [handler](ServerMetadataHandle metadata) mutable
-                                   -> Poll<ServerMetadataHandle> {
-                                 handler.SpawnPushServerTrailingMetadata(
-                                     std::move(metadata));
-                                 // We always lose the race.
-                                 return Pending{};
-                               });
-                  });
-            }));
+    // The returned promise is wrapped in OnCancel so that, if the v2 stack
+    // destroys it before it completes (e.g. the forced-cancellation path in
+    // ServerCallData::Completed, which resets promise_ to an empty
+    // ArenaPromise), we actively propagate a cancellation into the v3 call
+    // pair.
+    return OnCancel(
+        Race(
+            // Get server trailing metadata from the v3 promise via the
+            // inter-activity latch.
+            If(
+                IsV2NonOwningWakerImplementationEnabled(),
+                [pipe_owner]() {
+                  return pipe_owner->server_trailing_metadata.Wait();
+                },
+                [initiator = initiator]() mutable {
+                  return initiator.PullServerTrailingMetadata();
+                }),
+            // This promise does the rest of the things, but it will always
+            // return pending, because the promise can't actually finish
+            // until the initiator returns trailing metadata above.
+            TrySeq(
+                state->call_handler_latch.Wait(),
+                [initiator = initiator, pipe_owner,
+                 call_args = std::move(call_args),
+                 next_promise_factory = std::move(next_promise_factory)](
+                    CallHandler handler) mutable {
+                  // Intercept all pipes from v2 API.
+                  //
+                  // For client-to-server messages, we do the following:
+                  // 1. Push the message into the v3 initiator, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 initiator's activity.
+                  // 2. Pull the message from the v3 handler, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    handler's activity.  We then push the message into
+                  //    an inter-activity pipe to return it to the v2 activity.
+                  // 3. In the v2 activity, read the message from the
+                  //    inter-activity pipe and send it on to the next
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull messages from the v3
+                  // handler and push them into an inter-activity pipe to
+                  // return them to the v2 activity.
+                  handler.SpawnGuarded(
+                      "pull_client_to_server_message",
+                      [handler, pipe_owner]() mutable {
+                        return ForEach(
+                            MessagesFrom(handler),
+                            [pipe_owner](MessageHandle message) {
+                              return Map(pipe_owner->client_to_server_messages
+                                             .sender.Push(std::move(message)),
+                                         [](bool x) { return StatusFlag(x); });
+                            });
+                      });
+                  call_args.client_to_server_messages->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](MessageHandle message) mutable {
+                        // Step 1: Push the message onto the v3 initiator in
+                        // its activity.
+                        initiator.SpawnPushMessage(std::move(message));
+                        // Step 3: Here in the v2 activity, read the message
+                        // from the inter-activity pipe and return it.
+                        return Map(
+                            pipe_owner->client_to_server_messages.receiver
+                                .Next(),
+                            [](InterActivityPipe<MessageHandle, 1>::NextResult
+                                   message) -> std::optional<MessageHandle> {
+                              if (!message.has_value()) return std::nullopt;
+                              return std::move(*message);
+                            });
+                      });
+                  // For server initial metadata, we do a similar thing, but
+                  // in the opposite direction, and using an inter-activity
+                  // latch instead of a pipe:
+                  // 1. Push the metadata into the v3 handler, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 handler's activity.
+                  // 2. Pull the metadata from the v3 initiator, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    initiator's activity.  We then use an inter-activity
+                  //    latch to return the metadata to the v2 activity.
+                  // 3. In the v2 activity, read the metadata from the
+                  //    inter-activity latch and send it on to the previous
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull the metadata from the v3
+                  // initiator and use an inter-activity latch to return it to
+                  // the v2 activity.
+                  initiator.SpawnGuarded(
+                      "pull_server_initial_metadata",
+                      [initiator, pipe_owner]() mutable {
+                        return TrySeq(
+                            initiator.PullServerInitialMetadata(),
+                            [pipe_owner](
+                                std::optional<ServerMetadataHandle> metadata) {
+                              pipe_owner->server_initial_metadata.Set(
+                                  std::move(metadata));
+                            });
+                      });
+                  call_args.server_initial_metadata->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](ServerMetadataHandle metadata) mutable {
+                        // Step 1: Push the metadata onto the v3 handler in
+                        // its activity.
+                        handler.SpawnPushServerInitialMetadata(
+                            std::move(metadata));
+                        // Step 3: Here in the v2 activity, read from the
+                        // inter-activity latch and return the metadata.
+                        return pipe_owner->server_initial_metadata.Wait();
+                      });
+                  // We handle server-to-client messages the same as
+                  // client-to-server messages, except in the opposite
+                  // direction:
+                  // 1. Push the message into the v3 handler, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 handler's activity.
+                  // 2. Pull the message from the v3 initiator, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    initiator's activity.  We then push the message into
+                  //    an inter-activity pipe to return it to the v2 activity.
+                  // 3. In the v2 activity, read the message from the
+                  //    inter-activity pipe and send it on to the next
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull messages from the v3
+                  // initiator and push them into an inter-activity pipe to
+                  // return them to the v2 activity.
+                  initiator.SpawnGuarded(
+                      "pull_server_to_client_message",
+                      [initiator, pipe_owner]() mutable {
+                        return ForEach(
+                            MessagesFrom(initiator),
+                            [pipe_owner](MessageHandle message) {
+                              return Map(pipe_owner->server_to_client_messages
+                                             .sender.Push(std::move(message)),
+                                         [](bool x) { return StatusFlag(x); });
+                            });
+                      });
+                  call_args.server_to_client_messages->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](MessageHandle message) mutable {
+                        // Step 1: Push the message onto the v3 handler in
+                        // its activity.
+                        handler.SpawnPushMessage(std::move(message));
+                        // Step 3: Here in the v2 activity, read from the
+                        // inter-activity pipe and return the messages.
+                        return Map(
+                            pipe_owner->server_to_client_messages.receiver
+                                .Next(),
+                            [](InterActivityPipe<MessageHandle, 1>::NextResult
+                                   message) -> std::optional<MessageHandle> {
+                              if (!message.has_value()) return std::nullopt;
+                              return std::move(*message);
+                            });
+                      });
+                  // In the v3 handler's activity, pull client initial metadata.
+                  // Use an inter-activity latch to get it back to the v2
+                  // activity.
+                  handler.SpawnGuarded(
+                      "pull_client_initial_metadata",
+                      [handler, pipe_owner]() mutable {
+                        return TrySeq(
+                            handler.PullClientInitialMetadata(),
+                            [pipe_owner](ClientMetadataHandle metadata) {
+                              pipe_owner->client_initial_metadata.Set(
+                                  std::move(metadata));
+                            });
+                      });
+                  // A wrapper for next_promise_factory that does the following:
+                  // - Pulls client initial metadata from the V3 handler via
+                  //   the inter-activity latch and injects it into the next
+                  //   V2 filter via CallArgs.
+                  // - Polls the next promise to get server trailing metadata
+                  //   from the next V2 filter and feeds it into the V3 handler.
+                  // Note that this does not actually pull the trailing metadata
+                  // from the V3 initiator; instead, we do that in a separate
+                  // promise above.  That promise will always complete at the
+                  // end of the call, so we always return pending here.
+                  return Seq(
+                      pipe_owner->client_initial_metadata.Wait(),
+                      [next_promise_factory = std::move(next_promise_factory),
+                       call_args = std::move(call_args),
+                       handler](ClientMetadataHandle metadata) mutable {
+                        call_args.client_initial_metadata = std::move(metadata);
+                        return Seq(
+                            next_promise_factory(std::move(call_args)),
+                            [handler](ServerMetadataHandle metadata) mutable
+                                -> Poll<ServerMetadataHandle> {
+                              handler.SpawnPushServerTrailingMetadata(
+                                  std::move(metadata));
+                              // We always lose the race.
+                              return Pending{};
+                            });
+                      });
+                })),
+        [initiator = initiator]() mutable {
+          // The v2 promise was destroyed before completing: propagate the
+          // cancellation into the v3 call pair so its CallSpine is torn down
+          // and the v3 interceptor is notified.
+          if (IsV2NonOwningWakerImplementationEnabled()) {
+            initiator.SpawnCancel(
+                absl::CancelledError("call cancelled by v2 filter stack"));
+          } else {
+            initiator.SpawnInfallible("v2-force-cancel", [initiator]() mutable {
+              initiator.Cancel();
+              return Map(initiator.PullServerTrailingMetadata(),
+                         [](ServerMetadataHandle) { return Empty{}; });
+            });
+          }
+        });
   }
 
  protected:
@@ -1918,9 +1968,11 @@ class BaseCallData : public Activity,
   std::string LogTag() const;
 
  private:
+  class WeakWakerHandle;
+
   // Wakeable implementation.
   void Wakeup(WakeupMask) final;
-  void WakeupAsync(WakeupMask) final { Crash("not implemented"); }
+  void WakeupAsync(WakeupMask) final;
   void Drop(WakeupMask) final;
 
   virtual void OnWakeup() = 0;
@@ -1932,6 +1984,7 @@ class BaseCallData : public Activity,
   const Timestamp deadline_;
   CallFinalization finalization_;
   std::atomic<grpc_polling_entity*> pollent_{nullptr};
+  OrphanablePtr<WeakWakerHandle> handle_;
   Pipe<ServerMetadataHandle>* const server_initial_metadata_pipe_;
   SendMessage* const send_message_;
   ReceiveMessage* const receive_message_;
@@ -2268,7 +2321,7 @@ struct ChannelFilterWithFlagsMethods {
         F::Create(args->channel_args,
                   ChannelFilter::Args(args->channel_stack, elem,
                                       grpc_channel_stack_filter_instance_number,
-                                      args->config, args->blackboard));
+                                      args->config));
     if (!status.ok()) {
       new (elem->channel_data) F*(nullptr);
       return absl_status_to_grpc_error(status.status());