grpc 1.68.1 → 1.71.0

data/src/core/lib/promise/party.h

@@ -21,15 +21,15 @@
 #include <stdint.h>
 
 #include <atomic>
+#include <limits>
 #include <string>
 #include <utility>
 
 #include "absl/base/attributes.h"
-#include "absl/base/thread_annotations.h"
 #include "absl/log/check.h"
-#include "absl/log/log.h"
 #include "absl/strings/string_view.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/event_engine_context.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
@@ -39,29 +39,127 @@
 #include "src/core/util/crash.h"
 #include "src/core/util/ref_counted.h"
 #include "src/core/util/ref_counted_ptr.h"
-#include "src/core/util/sync.h"
-#include "src/core/util/useful.h"
 
 namespace grpc_core {
 
+// Terms Used:
+//
+// Concurrent promises: Promises that may be in progress at a given instant
+// of time, but they may or may not execute simultaneously.
+// Parallel promises: Two or more Promises that are getting executed
+// simultaneously at the same instant of time.
+//
+// Promise Party
+// A Promise Party forms the execution environment for Promises. It runs up
+// to sixteen concurrent party participants and ensures none of the participants
+// run in parallel with each other.
+//
+// Party Participant
+// One Promise Party has 16 Participant slots. Each Participant slot can hold
+// one of the three
+// 1. A Promise
+// 2. Or a Promise Factory
+// 3. or any type of Participant object, such as SpawnSerializer.
+//
+// Using a Party
+// A party should be used when
+// 1. You need many promises to be run in a way that is concurrent but not
+// parallel.
+// 2. These promises have their own complex sleep and wake mechanisms.
+// 3. You need a way to run the pending promises to completion by repolling them
+// as needed.
+//
+// Creating a new Party
+// A Party must only be created using Party::Make function.
+//
+// Spawning a promise on a Party
+// 1. A promise can be spawned on a party using either Spawn or SpawnWaitable
+// method.
+// 2. If you want to bulk spawn promises on a party before any thread
+// starts executing them, use Party::WakeupHold.
+//
+// Execution of Spawned Promises
+// A Participant spawned on a party will get executed by that party.
+// Whenever the party wakes up, it will execute all unresolved Party
+// Participants at least once. After any Party Participant is resolved, its slot
+// is freed up to make place for new participants.
+//
+// When these Party Participants are executed (polled), they can either
+// 1. Resolve by returning a value
+// 2. Return Pending{}
+// 3. Wait for a certain event to happen using either a Notification or a Latch.
+//
+// When a promise factory is passed as a Party Participant, the promise factory
+// is used to generate the promise once, and then the promise is executed once
+// each time the party is polled till it resolves.
+//
+// Sleep mechanism of a Party
+// A party will Sleep/Quiece if all Participants Spawned on the party are in any
+// of the following states
+// 1. Return Pending{}
+// 2. Resolve
+// 3. Are waiting because of a Latch.
+// If a party is currently running a Participant, it is said to be active/awake.
+// Otherwise it is said to be Sleeping or Quieced.
+//
+// Wake mechanism of a Party
+// To wake up a sleeping party you can use the Waker object. Once the Party is
+// woken, it will be executed as mentioned above.
+//
+// Party Cancellation
+// A Party can be cancelled using party_.reset() method.
+//
+// Guarantees of a Party
+// 1. All Participant spawned on one party are guaranteed to be run serially.
+// Their execution will not happen in parallel.
+// 2. If a promise is executed, its on_complete is guaranteed to be executed as
+// long as the party is not cancelled.
+// 3. Once a party is cancelled, Participants that were Spawned onto the party,
+// but not yet executed, will not get executed.
+// 4. Promise spawned on a party will never be repolled after it is resolved.
+// 5. A promise spawned on a party, can in turn spawn another promise either on
+// the same party or on another party. We allow nesting of Spawn function.
+// 6. A promise or promise factory that is passed to a Spawn function could
+// either be a single simple promise, or it could be a promise combinator such
+// as TrySeq, TryJoin, Loop or any such promise combinator. Nesting of these
+// promise combinators is allowed.
+// 7. You can re-use the same party to spawn new Participants as long as the
+// older Participants have been resolved.
+// 8. We guarantee safe working of up to 16 un-resolved participants
+// on a party at a time.
+//
+// Non-Guarantees of a Party
+// 1. Promises spawned on one party are not guaranteed to execute in the same
+// order. They can execute in any order. If you need the promises to be executed
+// in a specific order, use a SpawnSerializer. If that is not feasible for some
+// reason, then either consider the use of a promise combinator, or order the
+// execution of promises using Notifications or Latches.
+// 2. A party cannot guarantee which thread a party Participant will execute on.
+// It could either execute on the current thread, or an event engine thread or
+// any other thread.
+// 3. Say, we spawned promises P1_1, P1_2, P1_3 on party1. Then promise P1_1 in
+// turn spawns promise P2_1, P2_2, P2_3 on party2. In such cases, party1 and
+// party2 may either execute on the same thread, or they may both execute on
+// different threads. party1 and party2 may or may not run in parallel.
+
 namespace party_detail {
 
 // Number of bits reserved for wakeups gives us the maximum number of
-// participants.
+// participants. This can change in the future and we don't guarantee this
+// number to be 16 always.
 static constexpr size_t kMaxParticipants = 16;
 
 }  // namespace party_detail
 
-// A Party is an Activity with multiple participant promises.
 class Party : public Activity, private Wakeable {
  private:
   // Non-owning wakeup handle.
   class Handle;
 
-  // One participant in the party.
+  // One promise participant in the party.
   class Participant {
    public:
-    // Poll the participant. Return true if complete.
+    // Poll the participant promise. Return true if complete.
     // Participant should take care of its own deallocation in this case.
     virtual bool PollParticipantPromise() = 0;
 
@@ -82,22 +180,140 @@ class Party : public Activity, private Wakeable {
   Party(const Party&) = delete;
   Party& operator=(const Party&) = delete;
 
+  // A Party object must be created only by using this method.
   static RefCountedPtr<Party> Make(RefCountedPtr<Arena> arena) {
     auto* arena_ptr = arena.get();
     return RefCountedPtr<Party>(arena_ptr->New<Party>(std::move(arena)));
   }
 
+  // When calling into a Party from outside the promises system we often would
+  // like to perform more than one action.
+  // This class tries to acquire the party lock just once - if it succeeds then
+  // it runs the party in its destructor, effectively holding all wakeups of the
+  // party until it goes out of scope.
+  // If it fails, presumably some other thread holds the lock - and in this case
+  // we don't attempt to do any buffering.
+  class WakeupHold {
+   public:
+    WakeupHold() = default;
+    explicit WakeupHold(Party* party)
+        : prev_state_(party->state_.load(std::memory_order_relaxed)) {
+      // Try to lock
+      if ((prev_state_ & kLocked) == 0 &&
+          party->state_.compare_exchange_weak(prev_state_,
+                                              (prev_state_ | kLocked) + kOneRef,
+                                              std::memory_order_relaxed)) {
+        DCHECK_EQ(prev_state_ & ~(kRefMask | kAllocatedMask), 0u)
+            << "Party should have contained no wakeups on lock";
+        // If we win, record that fact for the destructor
+        party->LogStateChange("WakeupHold", prev_state_,
+                              (prev_state_ | kLocked) + kOneRef);
+        party_ = party;
+      }
+    }
+    WakeupHold(const WakeupHold&) = delete;
+    WakeupHold& operator=(const WakeupHold&) = delete;
+    WakeupHold(WakeupHold&& other) noexcept
+        : party_(std::exchange(other.party_, nullptr)),
+          prev_state_(other.prev_state_) {}
+    WakeupHold& operator=(WakeupHold&& other) noexcept {
+      std::swap(party_, other.party_);
+      std::swap(prev_state_, other.prev_state_);
+      return *this;
+    }
+
+    ~WakeupHold() {
+      if (party_ == nullptr) return;
+      party_->RunLockedAndUnref(party_, prev_state_);
+    }
+
+   private:
+    Party* party_ = nullptr;
+    uint64_t prev_state_;
+  };
+
+  // SpawnSerializer is a helper class to serialize the execution of multiple
+  // promises on a party.
+  //
+  // Provides the guarantee that given:
+  //   SpawnSerializer* serializer = party->MakeSpawnSerializer();
+  //   serializer->Spawn([] { /* promise 1 */; });
+  //   serializer->Spawn([] { /* promise 2 */; });
+  // 1. promise 1 will be resolved before promise 2 is started.
+  // 2. promise 1 and promise 2 will execute on `party`.
+  //
+  // It's possible to have multiple SpawnSerializer instances on a party.
+  // Once created, the SpawnSerializer is only valid until the party is
+  // released - note that SpawnSerializer itself does not hold a ref to the
+  // party.
+  //
+  // Each SpawnSerializer consumes one slot in the party's participant array for
+  // the lifetime of the party - that is that a SpawnSerializer counts as one of
+  // the sixteen promises executing on a Party.
+  //
+  // The promises spawned by SpawnSerializer *DO NOT* count towards the sixteen
+  // promise limit.
+  //
+  // The size of this type matters, and so we leverage private inheritance to
+  // minimize the number of pointers needed to be kept per instance.
+  class SpawnSerializer final : private Participant {
+   public:
+    // Spawn a promise into the party.
+    //
+    // The promise will be polled until it is resolved, or until the party is
+    // shut down.
+    //
+    // Later promises spawned by this serializer will not be polled until this
+    // promise is resolved, and then one at a time in the order they were
+    // spawned.
+    //
+    // Spawn itself is not thread safe: at most one thread can call Spawn at a
+    // time. Just as the execution of promises is serialized by this type, the
+    // spawning of promises is expected to be serialized by some external entity
+    // (usually this is a Seq running on a different party).
+    template <class Factory>
+    void Spawn(Factory factory) {
+      auto empty_completion = [](Empty) {};
+      next_.Push(new ParticipantImpl<Factory, decltype(empty_completion)>(
+          "SpawnSerializer", std::move(factory), empty_completion));
+      party_->WakeupFromState<false>(
+          party_->state_.load(std::memory_order_relaxed), wakeup_mask_);
+    }
+
+   private:
+    friend class Party;
+    friend class Arena;
+
+    bool PollParticipantPromise() override;
+    void Destroy() override;
+
+    explicit SpawnSerializer(Party* party)
+        : next_(party->arena()), party_(party) {}
+
+    // Queue of promises to be executed after the active promise resolves.
+    ArenaSpsc<Participant*, false> next_;
+    // The promise currently being executed.
+    Participant* active_ = nullptr;
+    // The wakeup mask for this serializers participant.
+    // Allows us to wake up the party when a new promise is added to the queue.
+    WakeupMask wakeup_mask_;
+    // The party this serializer is running on.
+    Party* const party_;
+  };
+
   // Spawn one promise into the party.
-  // The promise will be polled until it is resolved, or until the party is shut
-  // down.
-  // The on_complete callback will be called with the result of the promise if
-  // it completes.
-  // A maximum of sixteen promises can be spawned onto a party.
+  // The party can poll the promise until it is resolved, or until the party is
+  // shut down.
+  // The on_complete callback will be called with the result of the
+  // promise if it completes.
   // promise_factory called to create the promise with the party lock taken;
-  // after the promise is created the factory is destroyed.
-  // This means that pointers or references to factory members will be
-  // invalidated after the promise is created - so the promise should not retain
-  // any of these.
+  // after the promise is created the factory is destroyed. This means that
+  // pointers or references to factory members will be invalidated after the
+  // promise is created - so the promise should not retain any of these.
+  // This function is thread safe. We can Spawn different promises onto the
+  // same party from different threads.
+  // A party can hold upto 16 unresolved promises at a time. However, this
+  // number might change in the future.
   template <typename Factory, typename OnComplete>
   void Spawn(absl::string_view name, Factory promise_factory,
              OnComplete on_complete);
@@ -133,30 +349,30 @@ class Party : public Activity, private Wakeable {
     return RefCountedPtr<Party>(this);
   }
 
-  Arena* arena() { return arena_.get(); }
-
-  class BulkSpawner {
-   public:
-    explicit BulkSpawner(Party* party) : party_(party) {}
-    ~BulkSpawner() {
-      party_->AddParticipants(participants_, num_participants_);
-    }
+  template <typename T>
+  RefCountedPtr<T> RefAsSubclass() {
+    IncrementRefCount();
+    return RefCountedPtr<T>(DownCast<T*>(this));
+  }
 
-    template <typename Factory, typename OnComplete>
-    void Spawn(absl::string_view name, Factory promise_factory,
-               OnComplete on_complete);
+  Arena* arena() { return arena_.get(); }
 
-   private:
-    Party* const party_;
-    size_t num_participants_ = 0;
-    Participant* participants_[party_detail::kMaxParticipants];
-  };
+  SpawnSerializer* MakeSpawnSerializer() {
+    auto* const serializer = arena_->New<SpawnSerializer>(this);
+    const size_t slot = AddParticipant(serializer);
+    DCHECK_NE(slot, std::numeric_limits<size_t>::max());
+    serializer->wakeup_mask_ = 1ull << slot;
+    return serializer;
+  }
 
  protected:
   friend class Arena;
 
   // Derived types should be constructed upon `arena`.
-  explicit Party(RefCountedPtr<Arena> arena) : arena_(std::move(arena)) {}
+  explicit Party(RefCountedPtr<Arena> arena) : arena_(std::move(arena)) {
+    CHECK(arena_->GetContext<grpc_event_engine::experimental::EventEngine>() !=
+          nullptr);
+  }
   ~Party() override;
 
   // Main run loop. Must be locked.
@@ -252,7 +468,7 @@ class Party : public Activity, private Wakeable {
           Construct(&promise_, std::move(p));
           state_.store(State::kPromise, std::memory_order_relaxed);
         }
-          ABSL_FALLTHROUGH_INTENDED;
+          [[fallthrough]];
         case State::kPromise: {
           auto p = promise_();
           if (auto* r = p.value_if_ready()) {
@@ -342,16 +558,19 @@ class Party : public Activity, private Wakeable {
 
   // Wakeable implementation
   void Wakeup(WakeupMask wakeup_mask) final {
+    GRPC_LATENT_SEE_INNER_SCOPE("Party::Wakeup");
     if (Activity::current() == this) {
       wakeup_mask_ |= wakeup_mask;
       Unref();
       return;
     }
-    WakeupFromState(state_.load(std::memory_order_relaxed), wakeup_mask);
+    WakeupFromState<true>(state_.load(std::memory_order_relaxed), wakeup_mask);
   }
 
+  template <bool kReffed>
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void WakeupFromState(
       uint64_t cur_state, WakeupMask wakeup_mask) {
+    GRPC_LATENT_SEE_INNER_SCOPE("Party::WakeupFromState");
     DCHECK_NE(wakeup_mask & kWakeupMask, 0u)
         << "Wakeup mask must be non-zero: " << wakeup_mask;
     while (true) {
@@ -359,8 +578,13 @@ class Party : public Activity, private Wakeable {
         // If the party is locked, we need to set the wakeup bits, and then
         // we'll immediately unref. Since something is running this should never
         // bring the refcount to zero.
-        DCHECK_GT(cur_state & kRefMask, kOneRef);
-        auto new_state = (cur_state | wakeup_mask) - kOneRef;
+        if (kReffed) {
+          DCHECK_GT(cur_state & kRefMask, kOneRef);
+        } else {
+          DCHECK_GE(cur_state & kRefMask, kOneRef);
+        }
+        const uint64_t new_state =
+            (cur_state | wakeup_mask) - (kReffed ? kOneRef : 0);
         if (state_.compare_exchange_weak(cur_state, new_state,
                                          std::memory_order_release)) {
           LogStateChange("Wakeup", cur_state, cur_state | wakeup_mask);
@@ -369,9 +593,11 @@ class Party : public Activity, private Wakeable {
       } else {
         // If the party is not locked, we need to lock it and run.
         DCHECK_EQ(cur_state & kWakeupMask, 0u);
-        if (state_.compare_exchange_weak(cur_state, cur_state | kLocked,
+        const uint64_t new_state =
+            (cur_state | kLocked) + (kReffed ? 0 : kOneRef);
+        if (state_.compare_exchange_weak(cur_state, new_state,
                                          std::memory_order_acq_rel)) {
-          LogStateChange("WakeupAndRun", cur_state, cur_state | kLocked);
+          LogStateChange("WakeupAndRun", cur_state, new_state);
           wakeup_mask_ |= wakeup_mask;
           RunLockedAndUnref(this, cur_state);
           return;
@@ -384,9 +610,10 @@ class Party : public Activity, private Wakeable {
   void Drop(WakeupMask wakeup_mask) final;
 
   // Add a participant (backs Spawn, after type erasure to ParticipantFactory).
-  void AddParticipants(Participant** participant, size_t count);
-  void AddParticipant(Participant* participant);
-  void DelayAddParticipants(Participant** participant, size_t count);
+  size_t AddParticipant(Participant* participant);
+  void MaybeAsyncAddParticipant(Participant* participant);
+
+  static uint64_t NextAllocationMask(uint64_t current_allocation_mask);
 
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void LogStateChange(
       const char* op, uint64_t prev_state, uint64_t new_state,
@@ -397,7 +624,7 @@ class Party : public Activity, private Wakeable {
                            new_state);
   }
 
-  // Sentinal value for currently_polling_ when no participant is being polled.
+  // Sentinel value for currently_polling_ when no participant is being polled.
   static constexpr uint8_t kNotPolling = 255;
 
   std::atomic<uint64_t> state_{kOneRef};
@@ -415,21 +642,11 @@ struct ContextSubclass<Party> {
   using Base = Activity;
 };
 
-template <typename Factory, typename OnComplete>
-void Party::BulkSpawner::Spawn(absl::string_view name, Factory promise_factory,
-                               OnComplete on_complete) {
-  GRPC_TRACE_LOG(promise_primitives, INFO)
-      << party_->DebugTag() << "[bulk_spawn] On " << this << " queue " << name
-      << " (" << sizeof(ParticipantImpl<Factory, OnComplete>) << " bytes)";
-  participants_[num_participants_++] = new ParticipantImpl<Factory, OnComplete>(
-      name, std::move(promise_factory), std::move(on_complete));
-}
-
 template <typename Factory, typename OnComplete>
 void Party::Spawn(absl::string_view name, Factory promise_factory,
                   OnComplete on_complete) {
   GRPC_TRACE_LOG(party_state, INFO) << "PARTY[" << this << "]: spawn " << name;
-  AddParticipant(new ParticipantImpl<Factory, OnComplete>(
+  MaybeAsyncAddParticipant(new ParticipantImpl<Factory, OnComplete>(
       name, std::move(promise_factory), std::move(on_complete)));
 }
 
@@ -439,7 +656,7 @@ auto Party::SpawnWaitable(absl::string_view name, Factory promise_factory) {
   auto participant = MakeRefCounted<PromiseParticipantImpl<Factory>>(
       name, std::move(promise_factory));
   Participant* p = participant->Ref().release();
-  AddParticipant(p);
+  MaybeAsyncAddParticipant(p);
   return [participant = std::move(participant)]() mutable {
     return participant->PollCompletion();
   };

data/src/core/lib/promise/pipe.h

@@ -20,14 +20,14 @@
 #include <stdlib.h>
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
+#include <variant>
 
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
-#include "absl/types/optional.h"
-#include "absl/types/variant.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/if.h"
@@ -189,7 +189,7 @@ class Center : public InterceptorList<T> {
   // Return Pending if there is no value.
   // Return the value if one was retrieved.
   // Return nullopt if the send end is closed and no value had been pushed.
-  Poll<absl::optional<T>> Next() {
+  Poll<std::optional<T>> Next() {
     GRPC_TRACE_LOG(promise_primitives, INFO) << DebugOpString("Next");
     DCHECK_NE(refs_, 0);
     switch (value_state_) {
@@ -206,9 +206,9 @@ class Center : public InterceptorList<T> {
         return std::move(value_);
       case ValueState::kClosed:
       case ValueState::kCancelled:
-        return absl::nullopt;
+        return std::nullopt;
     }
-    GPR_UNREACHABLE_CODE(return absl::nullopt);
+    GPR_UNREACHABLE_CODE(return std::nullopt);
   }
 
   // Check if the pipe is closed for sending (if there is a value still queued
@@ -517,8 +517,8 @@ class Next {
   Next(Next&& other) noexcept = default;
   Next& operator=(Next&& other) noexcept = default;
 
-  Poll<absl::optional<T>> operator()() {
-    return center_ == nullptr ? absl::nullopt : center_->Next();
+  Poll<std::optional<T>> operator()() {
+    return center_ == nullptr ? std::nullopt : center_->Next();
   }
 
  private:
@@ -551,7 +551,7 @@ class PipeReceiver {
   // available.
   auto Next() {
     return Seq(pipe_detail::Next<T>(center_), [center = center_](
-                                                  absl::optional<T> value) {
+                                                  std::optional<T> value) {
       bool open = value.has_value();
       bool cancelled = center == nullptr ? true : center->cancelled();
       return If(
@@ -559,7 +559,7 @@ class PipeReceiver {
           [center = std::move(center), value = std::move(value)]() mutable {
             auto run = center->Run(std::move(value));
             return Map(std::move(run), [center = std::move(center)](
-                                           absl::optional<T> value) mutable {
+                                           std::optional<T> value) mutable {
               if (value.has_value()) {
                 center->value() = std::move(*value);
                 return NextResult<T>(std::move(center));
@@ -637,7 +637,7 @@ class Push {
           << " Pipe push has a null center";
       return false;
     }
-    if (auto* p = absl::get_if<T>(&state_)) {
+    if (auto* p = std::get_if<T>(&state_)) {
       auto r = center_->Push(p);
       if (auto* ok = r.value_if_ready()) {
         state_.template emplace<AwaitingAck>();
@@ -646,7 +646,7 @@ class Push {
         return Pending{};
       }
     }
-    DCHECK(absl::holds_alternative<AwaitingAck>(state_));
+    DCHECK(std::holds_alternative<AwaitingAck>(state_));
     return center_->PollAck();
   }
 
@@ -658,7 +658,7 @@ class Push {
       : center_(std::move(center)), state_(std::move(push)) {}
 
   RefCountedPtr<Center<T>> center_;
-  absl::variant<T, AwaitingAck> state_;
+  std::variant<T, AwaitingAck> state_;
 };
 
 }  // namespace pipe_detail

data/src/core/lib/promise/poll.h

@@ -17,12 +17,12 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <optional>
 #include <string>
 #include <utility>
 
 #include "absl/log/check.h"
 #include "absl/strings/str_format.h"
-#include "absl/types/optional.h"
 #include "src/core/util/construct_destruct.h"
 
 namespace grpc_core {
@@ -135,8 +135,8 @@ class Poll {
   //
   // Why not optional<T>?
   //
-  // We have cases where we want to return absl::nullopt{} from a promise, and
-  // have that upgraded to a Poll<absl::nullopt_t> prior to a cast to some
+  // We have cases where we want to return std::nullopt{} from a promise, and
+  // have that upgraded to a Poll<std::nullopt_t> prior to a cast to some
   // Poll<optional<T>>.
   //
   // Since optional<nullopt_t> is not allowed, we'd not be allowed to make
@@ -152,6 +152,9 @@ class Poll {
   };
 };
 
+template <typename T>
+class Poll<T&&>;
+
 template <>
 class Poll<Empty> {
  public:
@@ -287,7 +290,20 @@ void AbslStringify(Sink& sink, const Poll<T>& poll) {
 }
 
 template <typename Sink, typename T>
-void AbslStringify(Sink& sink, const Poll<absl::optional<T>>& poll) {
+void AbslStringify(Sink& sink, const Poll<absl::StatusOr<T>>& poll) {
+  if (poll.pending()) {
+    absl::Format(&sink, "<<pending>>");
+    return;
+  }
+  if (poll.value().ok()) {
+    absl::Format(&sink, "%v", *poll.value());
+  } else {
+    absl::Format(&sink, "%v", poll.value().status());
+  }
+}
+
+template <typename Sink, typename T>
+void AbslStringify(Sink& sink, const Poll<std::optional<T>>& poll) {
   if (poll.pending()) {
     absl::Format(&sink, "<<pending>>");
     return;
@@ -303,7 +319,7 @@ void AbslStringify(Sink& sink, const Poll<absl::optional<T>>& poll) {
 // Hack to get metadata printing
 template <typename Sink, typename T, typename Deleter>
 void AbslStringify(
-    Sink& sink, const Poll<absl::optional<std::unique_ptr<T, Deleter>>>& poll) {
+    Sink& sink, const Poll<std::optional<std::unique_ptr<T, Deleter>>>& poll) {
   if (poll.pending()) {
     absl::Format(&sink, "<<pending>>");
     return;