grpc 1.82.0 → 1.83.0.pre1

data/src/core/lib/channel/promise_based_filter.h

@@ -70,6 +70,8 @@
 #include "src/core/util/debug_location.h"
 #include "src/core/util/grpc_check.h"
 #include "src/core/util/match.h"
+#include "src/core/util/orphanable.h"
+#include "src/core/util/ref_counted_ptr.h"
 #include "src/core/util/time.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/functional/function_ref.h"
@@ -1273,192 +1275,241 @@ class V3InterceptorToV2Bridge : public ChannelFilter, public Interceptor {
       InterActivityLatch<std::optional<ServerMetadataHandle>>
           server_initial_metadata;
       InterActivityPipe<MessageHandle, 1> server_to_client_messages;
+      InterActivityLatch<ServerMetadataHandle> server_trailing_metadata;
     };
     auto* pipe_owner = GetContext<Arena>()->ManagedNew<PipeOwner>();
+    if (IsV2NonOwningWakerImplementationEnabled()) {
+      // We need to start polling the initiator for server trailing
+      // metadata immediately, since the v3 interceptor may generate a
+      // failure before any of the other promises resolve.
+      //
+      // Spawn a promise on the initiator's activity to pull server trailing
+      // metadata and pass it to the v2 activity via an inter-activity latch.
+      initiator.SpawnInfallible(
+          "pull_server_trailing_metadata",
+          [initiator = initiator, pipe_owner]() mutable {
+            return Map(
+                initiator.PullServerTrailingMetadata(),
+                [pipe_owner](ServerMetadataHandle metadata) {
+                  pipe_owner->server_trailing_metadata.Set(std::move(metadata));
+                  return Empty{};
+                });
+          });
+    }
     // Now return a promise that does all the things.
-    return Race(
-        // We need to start polling the initiator for server trailing
-        // metadata immediately, since the v3 interceptor may generate a
-        // failure before any of the other promises resolve.
-        initiator.PullServerTrailingMetadata(),
-        // This promise does the rest of the things, but it will always
-        // return pending, because the promise can't actually finish
-        // until the initiator returns trailing metadata above.
-        TrySeq(
-            state->call_handler_latch.Wait(),
-            [initiator = initiator, pipe_owner,
-             call_args = std::move(call_args),
-             next_promise_factory =
-                 std::move(next_promise_factory)](CallHandler handler) mutable {
-              // Intercept all pipes from v2 API.
-              //
-              // For client-to-server messages, we do the following:
-              // 1. Push the message into the v3 initiator, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 initiator's activity.
-              // 2. Pull the message from the v3 handler, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    handler's activity.  We then push the message into
-              //    an inter-activity pipe to return it to the v2 activity.
-              // 3. In the v2 activity, read the message from the
-              //    inter-activity pipe and send it on to the next
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull messages from the v3
-              // handler and push them into an inter-activity pipe to
-              // return them to the v2 activity.
-              handler.SpawnGuarded(
-                  "pull_client_to_server_message",
-                  [handler, pipe_owner]() mutable {
-                    return ForEach(
-                        MessagesFrom(handler),
-                        [pipe_owner](MessageHandle message) {
-                          return Map(
-                              pipe_owner->client_to_server_messages.sender.Push(
-                                  std::move(message)),
-                              [](bool x) { return StatusFlag(x); });
-                        });
-                  });
-              call_args.client_to_server_messages->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](MessageHandle message) mutable {
-                    // Step 1: Push the message onto the v3 initiator in
-                    // its activity.
-                    initiator.SpawnPushMessage(std::move(message));
-                    // Step 3: Here in the v2 activity, read the message
-                    // from the inter-activity pipe and return it.
-                    return Map(
-                        pipe_owner->client_to_server_messages.receiver.Next(),
-                        [](InterActivityPipe<MessageHandle, 1>::NextResult
-                               message) -> std::optional<MessageHandle> {
-                          if (!message.has_value()) return std::nullopt;
-                          return std::move(*message);
-                        });
-                  });
-              // For server initial metadata, we do a similar thing, but
-              // in the opposite direction, and using an inter-activity
-              // latch instead of a pipe:
-              // 1. Push the metadata into the v3 handler, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 handler's activity.
-              // 2. Pull the metadata from the v3 initiator, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    initiator's activity.  We then use an inter-activity
-              //    latch to return the metadata to the v2 activity.
-              // 3. In the v2 activity, read the metadata from the
-              //    inter-activity latch and send it on to the previous
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull the metadata from the v3
-              // initiator and use an inter-activity latch to return it to
-              // the v2 activity.
-              initiator.SpawnGuarded(
-                  "pull_server_initial_metadata",
-                  [initiator, pipe_owner]() mutable {
-                    return TrySeq(
-                        initiator.PullServerInitialMetadata(),
-                        [pipe_owner](
-                            std::optional<ServerMetadataHandle> metadata) {
-                          pipe_owner->server_initial_metadata.Set(
-                              std::move(metadata));
-                        });
-                  });
-              call_args.server_initial_metadata->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](ServerMetadataHandle metadata) mutable {
-                    // Step 1: Push the metadata onto the v3 handler in
-                    // its activity.
-                    handler.SpawnPushServerInitialMetadata(std::move(metadata));
-                    // Step 3: Here in the v2 activity, read from the
-                    // inter-activity latch and return the metadata.
-                    return pipe_owner->server_initial_metadata.Wait();
-                  });
-              // We handle server-to-client messages the same as
-              // client-to-server messages, except in the opposite
-              // direction:
-              // 1. Push the message into the v3 handler, which sends
-              //    it to the v3 interceptor.  Note that this needs to
-              //    be done inside of the v3 handler's activity.
-              // 2. Pull the message from the v3 initiator, where it will
-              //    arrive when the v3 interceptor is done with it.
-              //    Note that this needs to be done inside of the v3
-              //    initiator's activity.  We then push the message into
-              //    an inter-activity pipe to return it to the v2 activity.
-              // 3. In the v2 activity, read the message from the
-              //    inter-activity pipe and send it on to the next
-              //    filter.
-              //
-              // Step 2: Spawn a promise to pull messages from the v3
-              // initiator and push them into an inter-activity pipe to
-              // return them to the v2 activity.
-              initiator.SpawnGuarded(
-                  "pull_server_to_client_message",
-                  [initiator, pipe_owner]() mutable {
-                    return ForEach(
-                        MessagesFrom(initiator),
-                        [pipe_owner](MessageHandle message) {
-                          return Map(
-                              pipe_owner->server_to_client_messages.sender.Push(
-                                  std::move(message)),
-                              [](bool x) { return StatusFlag(x); });
-                        });
-                  });
-              call_args.server_to_client_messages->InterceptAndMap(
-                  [initiator, handler,
-                   pipe_owner](MessageHandle message) mutable {
-                    // Step 1: Push the message onto the v3 handler in
-                    // its activity.
-                    handler.SpawnPushMessage(std::move(message));
-                    // Step 3: Here in the v2 activity, read from the
-                    // inter-activity pipe and return the messages.
-                    return Map(
-                        pipe_owner->server_to_client_messages.receiver.Next(),
-                        [](InterActivityPipe<MessageHandle, 1>::NextResult
-                               message) -> std::optional<MessageHandle> {
-                          if (!message.has_value()) return std::nullopt;
-                          return std::move(*message);
-                        });
-                  });
-              // In the v3 handler's activity, pull client initial metadata.
-              // Use an inter-activity latch to get it back to the v2
-              // activity.
-              handler.SpawnGuarded(
-                  "pull_client_initial_metadata",
-                  [handler, pipe_owner]() mutable {
-                    return TrySeq(handler.PullClientInitialMetadata(),
-                                  [pipe_owner](ClientMetadataHandle metadata) {
-                                    pipe_owner->client_initial_metadata.Set(
-                                        std::move(metadata));
-                                  });
-                  });
-              // A wrapper for next_promise_factory that does the following:
-              // - Pulls client initial metadata from the V3 handler via
-              //   the inter-activity latch and injects it into the next
-              //   V2 filter via CallArgs.
-              // - Polls the next promise to get server trailing metadata
-              //   from the next V2 filter and feeds it into the V3 handler.
-              // Note that this does not actually pull the trailing metadata
-              // from the V3 initiator; instead, we do that in a separate
-              // promise above.  That promise will always complete at the
-              // end of the call, so we always return pending here.
-              return Seq(
-                  pipe_owner->client_initial_metadata.Wait(),
-                  [next_promise_factory = std::move(next_promise_factory),
-                   call_args = std::move(call_args),
-                   handler](ClientMetadataHandle metadata) mutable {
-                    call_args.client_initial_metadata = std::move(metadata);
-                    return Seq(next_promise_factory(std::move(call_args)),
-                               [handler](ServerMetadataHandle metadata) mutable
-                                   -> Poll<ServerMetadataHandle> {
-                                 handler.SpawnPushServerTrailingMetadata(
-                                     std::move(metadata));
-                                 // We always lose the race.
-                                 return Pending{};
-                               });
-                  });
-            }));
+    // The returned promise is wrapped in OnCancel so that, if the v2 stack
+    // destroys it before it completes (e.g. the forced-cancellation path in
+    // ServerCallData::Completed, which resets promise_ to an empty
+    // ArenaPromise), we actively propagate a cancellation into the v3 call
+    // pair.
+    return OnCancel(
+        Race(
+            // Get server trailing metadata from the v3 promise via the
+            // inter-activity latch.
+            If(
+                IsV2NonOwningWakerImplementationEnabled(),
+                [pipe_owner]() {
+                  return pipe_owner->server_trailing_metadata.Wait();
+                },
+                [initiator = initiator]() mutable {
+                  return initiator.PullServerTrailingMetadata();
+                }),
+            // This promise does the rest of the things, but it will always
+            // return pending, because the promise can't actually finish
+            // until the initiator returns trailing metadata above.
+            TrySeq(
+                state->call_handler_latch.Wait(),
+                [initiator = initiator, pipe_owner,
+                 call_args = std::move(call_args),
+                 next_promise_factory = std::move(next_promise_factory)](
+                    CallHandler handler) mutable {
+                  // Intercept all pipes from v2 API.
+                  //
+                  // For client-to-server messages, we do the following:
+                  // 1. Push the message into the v3 initiator, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 initiator's activity.
+                  // 2. Pull the message from the v3 handler, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    handler's activity.  We then push the message into
+                  //    an inter-activity pipe to return it to the v2 activity.
+                  // 3. In the v2 activity, read the message from the
+                  //    inter-activity pipe and send it on to the next
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull messages from the v3
+                  // handler and push them into an inter-activity pipe to
+                  // return them to the v2 activity.
+                  handler.SpawnGuarded(
+                      "pull_client_to_server_message",
+                      [handler, pipe_owner]() mutable {
+                        return ForEach(
+                            MessagesFrom(handler),
+                            [pipe_owner](MessageHandle message) {
+                              return Map(pipe_owner->client_to_server_messages
+                                             .sender.Push(std::move(message)),
+                                         [](bool x) { return StatusFlag(x); });
+                            });
+                      });
+                  call_args.client_to_server_messages->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](MessageHandle message) mutable {
+                        // Step 1: Push the message onto the v3 initiator in
+                        // its activity.
+                        initiator.SpawnPushMessage(std::move(message));
+                        // Step 3: Here in the v2 activity, read the message
+                        // from the inter-activity pipe and return it.
+                        return Map(
+                            pipe_owner->client_to_server_messages.receiver
+                                .Next(),
+                            [](InterActivityPipe<MessageHandle, 1>::NextResult
+                                   message) -> std::optional<MessageHandle> {
+                              if (!message.has_value()) return std::nullopt;
+                              return std::move(*message);
+                            });
+                      });
+                  // For server initial metadata, we do a similar thing, but
+                  // in the opposite direction, and using an inter-activity
+                  // latch instead of a pipe:
+                  // 1. Push the metadata into the v3 handler, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 handler's activity.
+                  // 2. Pull the metadata from the v3 initiator, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    initiator's activity.  We then use an inter-activity
+                  //    latch to return the metadata to the v2 activity.
+                  // 3. In the v2 activity, read the metadata from the
+                  //    inter-activity latch and send it on to the previous
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull the metadata from the v3
+                  // initiator and use an inter-activity latch to return it to
+                  // the v2 activity.
+                  initiator.SpawnGuarded(
+                      "pull_server_initial_metadata",
+                      [initiator, pipe_owner]() mutable {
+                        return TrySeq(
+                            initiator.PullServerInitialMetadata(),
+                            [pipe_owner](
+                                std::optional<ServerMetadataHandle> metadata) {
+                              pipe_owner->server_initial_metadata.Set(
+                                  std::move(metadata));
+                            });
+                      });
+                  call_args.server_initial_metadata->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](ServerMetadataHandle metadata) mutable {
+                        // Step 1: Push the metadata onto the v3 handler in
+                        // its activity.
+                        handler.SpawnPushServerInitialMetadata(
+                            std::move(metadata));
+                        // Step 3: Here in the v2 activity, read from the
+                        // inter-activity latch and return the metadata.
+                        return pipe_owner->server_initial_metadata.Wait();
+                      });
+                  // We handle server-to-client messages the same as
+                  // client-to-server messages, except in the opposite
+                  // direction:
+                  // 1. Push the message into the v3 handler, which sends
+                  //    it to the v3 interceptor.  Note that this needs to
+                  //    be done inside of the v3 handler's activity.
+                  // 2. Pull the message from the v3 initiator, where it will
+                  //    arrive when the v3 interceptor is done with it.
+                  //    Note that this needs to be done inside of the v3
+                  //    initiator's activity.  We then push the message into
+                  //    an inter-activity pipe to return it to the v2 activity.
+                  // 3. In the v2 activity, read the message from the
+                  //    inter-activity pipe and send it on to the next
+                  //    filter.
+                  //
+                  // Step 2: Spawn a promise to pull messages from the v3
+                  // initiator and push them into an inter-activity pipe to
+                  // return them to the v2 activity.
+                  initiator.SpawnGuarded(
+                      "pull_server_to_client_message",
+                      [initiator, pipe_owner]() mutable {
+                        return ForEach(
+                            MessagesFrom(initiator),
+                            [pipe_owner](MessageHandle message) {
+                              return Map(pipe_owner->server_to_client_messages
+                                             .sender.Push(std::move(message)),
+                                         [](bool x) { return StatusFlag(x); });
+                            });
+                      });
+                  call_args.server_to_client_messages->InterceptAndMap(
+                      [initiator, handler,
+                       pipe_owner](MessageHandle message) mutable {
+                        // Step 1: Push the message onto the v3 handler in
+                        // its activity.
+                        handler.SpawnPushMessage(std::move(message));
+                        // Step 3: Here in the v2 activity, read from the
+                        // inter-activity pipe and return the messages.
+                        return Map(
+                            pipe_owner->server_to_client_messages.receiver
+                                .Next(),
+                            [](InterActivityPipe<MessageHandle, 1>::NextResult
+                                   message) -> std::optional<MessageHandle> {
+                              if (!message.has_value()) return std::nullopt;
+                              return std::move(*message);
+                            });
+                      });
+                  // In the v3 handler's activity, pull client initial metadata.
+                  // Use an inter-activity latch to get it back to the v2
+                  // activity.
+                  handler.SpawnGuarded(
+                      "pull_client_initial_metadata",
+                      [handler, pipe_owner]() mutable {
+                        return TrySeq(
+                            handler.PullClientInitialMetadata(),
+                            [pipe_owner](ClientMetadataHandle metadata) {
+                              pipe_owner->client_initial_metadata.Set(
+                                  std::move(metadata));
+                            });
+                      });
+                  // A wrapper for next_promise_factory that does the following:
+                  // - Pulls client initial metadata from the V3 handler via
+                  //   the inter-activity latch and injects it into the next
+                  //   V2 filter via CallArgs.
+                  // - Polls the next promise to get server trailing metadata
+                  //   from the next V2 filter and feeds it into the V3 handler.
+                  // Note that this does not actually pull the trailing metadata
+                  // from the V3 initiator; instead, we do that in a separate
+                  // promise above.  That promise will always complete at the
+                  // end of the call, so we always return pending here.
+                  return Seq(
+                      pipe_owner->client_initial_metadata.Wait(),
+                      [next_promise_factory = std::move(next_promise_factory),
+                       call_args = std::move(call_args),
+                       handler](ClientMetadataHandle metadata) mutable {
+                        call_args.client_initial_metadata = std::move(metadata);
+                        return Seq(
+                            next_promise_factory(std::move(call_args)),
+                            [handler](ServerMetadataHandle metadata) mutable
+                                -> Poll<ServerMetadataHandle> {
+                              handler.SpawnPushServerTrailingMetadata(
+                                  std::move(metadata));
+                              // We always lose the race.
+                              return Pending{};
+                            });
+                      });
+                })),
+        [initiator = initiator]() mutable {
+          // The v2 promise was destroyed before completing: propagate the
+          // cancellation into the v3 call pair so its CallSpine is torn down
+          // and the v3 interceptor is notified.
+          if (IsV2NonOwningWakerImplementationEnabled()) {
+            initiator.SpawnCancel(
+                absl::CancelledError("call cancelled by v2 filter stack"));
+          } else {
+            initiator.SpawnInfallible("v2-force-cancel", [initiator]() mutable {
+              initiator.Cancel();
+              return Map(initiator.PullServerTrailingMetadata(),
+                         [](ServerMetadataHandle) { return Empty{}; });
+            });
+          }
+        });
   }
 
  protected:
@@ -1917,9 +1968,11 @@ class BaseCallData : public Activity,
   std::string LogTag() const;
 
  private:
+  class WeakWakerHandle;
+
   // Wakeable implementation.
   void Wakeup(WakeupMask) final;
-  void WakeupAsync(WakeupMask) final { Crash("not implemented"); }
+  void WakeupAsync(WakeupMask) final;
   void Drop(WakeupMask) final;
 
   virtual void OnWakeup() = 0;
@@ -1931,6 +1984,7 @@ class BaseCallData : public Activity,
   const Timestamp deadline_;
   CallFinalization finalization_;
   std::atomic<grpc_polling_entity*> pollent_{nullptr};
+  OrphanablePtr<WeakWakerHandle> handle_;
   Pipe<ServerMetadataHandle>* const server_initial_metadata_pipe_;
   SendMessage* const send_message_;
   ReceiveMessage* const receive_message_;

data/src/core/lib/event_engine/extensions/receive_coalescing_extension.h

@@ -37,10 +37,6 @@ class ReceiveCoalescingExtension {
   /// It is safe to call this only when there are no outstanding Reads on
   /// the Endpoint.
   virtual void DisableRpcReceiveCoalescing() = 0;
-
-  /// If invoked, the endpoint tries to preserve proper order and alignment of
-  /// any memory that maybe shared across reads.
-  virtual void EnforceRxMemoryAlignment() = 0;
 };
 
 }  // namespace grpc_event_engine::experimental

data/src/core/lib/event_engine/extensions/tcp_trace.h

@@ -36,7 +36,7 @@ class TcpTraceExtension {
   // Enable TCP telemetry collection using the Instrumentation API.
   virtual void EnableTcpTelemetry(
       grpc_core::RefCountedPtr<grpc_core::CollectionScope> collection_scope,
-      bool is_control_endpoint) = 0;
+      bool is_control_endpoint, bool trace_full_buffer) = 0;
 };
 
 }  // namespace grpc_event_engine::experimental

data/src/core/lib/event_engine/posix_engine/posix_engine.cc

@@ -76,14 +76,6 @@ namespace grpc_event_engine::experimental {
 
 namespace {
 
-bool ShouldUsePosixPoller() {
-#if defined(GRPC_PYTHON_BUILD)
-  return grpc_core::IsEventEnginePollerForPythonEnabled();
-#else
-  return true;
-#endif
-}
-
 #if GRPC_ENABLE_FORK_SUPPORT && GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK
 
 // Thread pool can outlive EE but we need to ensure the ordering if both
@@ -434,10 +426,8 @@ PosixEventEngine::PosixEventEngine(const Options& options)
     : connection_shards_(options.connection_shards),
       executor_(MakeThreadPool(options.reserve_threads)),
       timer_manager_(std::make_shared<TimerManager>(executor_)) {
-  if (ShouldUsePosixPoller()) {
-    poller_ = grpc_event_engine::experimental::MakeDefaultPoller(executor_);
-    SchedulePoller();
-  }
+  poller_ = grpc_event_engine::experimental::MakeDefaultPoller(executor_);
+  SchedulePoller();
 }
 
 #endif  // GRPC_POSIX_SOCKET_TCP

data/src/core/lib/event_engine/shim.cc

@@ -21,14 +21,10 @@
 
 namespace grpc_event_engine::experimental {
 
-bool UseEventEngineClient() {
-  return !EventEngineExperimentDisabledForPython() &&
-         grpc_core::IsEventEngineClientEnabled();
-}
+bool UseEventEngineClient() { return grpc_core::IsEventEngineClientEnabled(); }
 
 bool UseEventEngineListener() {
-  return !EventEngineExperimentDisabledForPython() &&
-         grpc_core::IsEventEngineListenerEnabled();
+  return grpc_core::IsEventEngineListenerEnabled();
 }
 
 bool UsePollsetAlternative() {
@@ -36,14 +32,4 @@ bool UsePollsetAlternative() {
          grpc_core::IsPollsetAlternativeEnabled();
 }
 
-// Returns true if the poller is disabled by build configuration or experiment
-// flags.
-bool EventEngineExperimentDisabledForPython() {
-#ifdef GRPC_PYTHON_BUILD
-  return !grpc_core::IsEventEnginePollerForPythonEnabled();
-#else
-  return false;
-#endif
-}
-
 }  // namespace grpc_event_engine::experimental

data/src/core/lib/event_engine/shim.h

@@ -35,10 +35,6 @@ bool UseEventEngineListener();
 // may disable the poller in some builds.
 bool UsePollsetAlternative();
 
-// Returns true if the poller is disabled by build configuration or experiment
-// flags.
-bool EventEngineExperimentDisabledForPython();
-
 }  // namespace grpc_event_engine::experimental
 
 #endif  // GRPC_SRC_CORE_LIB_EVENT_ENGINE_SHIM_H