grpc 1.70.1 → 1.71.0

data/src/core/lib/promise/party.cc

@@ -18,6 +18,7 @@
 
 #include <atomic>
 #include <cstdint>
+#include <limits>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/log/check.h"
@@ -131,6 +132,9 @@ class Party::Handle final : public Wakeable {
   Party* party_ ABSL_GUARDED_BY(mu_);
 };
 
+///////////////////////////////////////////////////////////////////////////////
+// Party::Participant
+
 Wakeable* Party::Participant::MakeNonOwningWakeable(Party* party) {
   if (handle_ == nullptr) {
     handle_ = new Handle(party);
@@ -146,6 +150,41 @@ Party::Participant::~Participant() {
   }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+// Party::SpawnSerializer
+
+bool Party::SpawnSerializer::PollParticipantPromise() {
+  if (active_ == nullptr) {
+    active_ = next_.Pop().value_or(nullptr);
+  }
+  while (active_ != nullptr) {
+    // If the active participant is incomplete, we are also incomplete.
+    if (!active_->PollParticipantPromise()) return false;
+    // Otherwise, yank the next participant from the queue.
+    active_ = next_.Pop().value_or(nullptr);
+  }
+  // If we have no active participant and no more participants in the queue, we
+  // can return. `true` here indicates this participant is done - we never
+  // return that value as we'd not be able to continue polling.
+  // TODO(ctiller): if ArenaSpsc had a way to signal 'first item added' to the
+  // queue, we could use that as a signal to re-add the participant and allow
+  // ourselves to be removed. Consider if that would help.
+  return false;
+}
+
+void Party::SpawnSerializer::Destroy() {
+  if (active_ != nullptr) {
+    active_->Destroy();
+  }
+  while (auto* p = next_.Pop().value_or(nullptr)) {
+    p->Destroy();
+  }
+  Destruct(this);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Party
+
 Party::~Party() {}
 
 void Party::CancelRemainingParticipants() {
@@ -198,7 +237,6 @@ void Party::RunLockedAndUnref(Party* party, uint64_t prev_state) {
   Thread thd(
       "RunParty",
       [party, prev_state]() {
-        ApplicationCallbackExecCtx app_exec_ctx;
         ExecCtx exec_ctx;
         party->RunPartyAndUnref(prev_state);
       },
@@ -258,7 +296,6 @@ void Party::RunLockedAndUnref(Party* party, uint64_t prev_state) {
       CHECK(event_engine != nullptr) << "; " << GRPC_DUMP_ARGS(party, arena);
       event_engine->Run([wakeup]() {
         GRPC_LATENT_SEE_PARENT_SCOPE("Party::RunLocked offload");
-        ApplicationCallbackExecCtx app_exec_ctx;
         ExecCtx exec_ctx;
         RunState{wakeup}.Run();
       });
@@ -281,11 +318,8 @@ void Party::RunPartyAndUnref(uint64_t prev_state) {
   DCHECK_EQ(prev_state & ~(kRefMask | kAllocatedMask), 0u)
       << "Party should have contained no wakeups on lock";
   prev_state |= kLocked;
-  absl::optional<ScopedTimeCache> time_cache;
 #if !TARGET_OS_IPHONE
-  if (IsTimeCachingInPartyEnabled()) {
-    time_cache.emplace();
-  }
+  ScopedTimeCache time_cache;
 #endif
   for (;;) {
     uint64_t keep_allocated_mask = kAllocatedMask;
@@ -394,7 +428,7 @@ uint64_t Party::NextAllocationMask(uint64_t current_allocation_mask) {
 }
 #endif
 
-void Party::AddParticipant(Participant* participant) {
+size_t Party::AddParticipant(Participant* participant) {
   GRPC_LATENT_SEE_INNER_SCOPE("Party::AddParticipant");
   uint64_t state = state_.load(std::memory_order_acquire);
   uint64_t allocated;
@@ -409,8 +443,7 @@ void Party::AddParticipant(Participant* participant) {
     allocated = (state & kAllocatedMask) >> kAllocatedShift;
     wakeup_mask = NextAllocationMask(allocated);
     if (GPR_UNLIKELY((wakeup_mask & kWakeupMask) == 0)) {
-      DelayAddParticipant(participant);
-      return;
+      return std::numeric_limits<size_t>::max();
     }
     DCHECK_NE(wakeup_mask & kWakeupMask, 0u)
         << "No available slots for new participant; allocated=" << allocated
@@ -429,19 +462,21 @@ void Party::AddParticipant(Participant* participant) {
       << " [participant=" << participant << "]";
   participants_[slot].store(participant, std::memory_order_release);
   // Now we need to wake up the party.
-  WakeupFromState(new_state, wakeup_mask);
+  WakeupFromState<true>(new_state, wakeup_mask);
+  return slot;
 }
 
-void Party::DelayAddParticipant(Participant* participant) {
+void Party::MaybeAsyncAddParticipant(Participant* participant) {
+  const size_t slot = AddParticipant(participant);
+  if (slot != std::numeric_limits<size_t>::max()) return;
   // We need to delay the addition of participants.
   IncrementRefCount();
   VLOG_EVERY_N_SEC(2, 10) << "Delaying addition of participant to party "
                           << this << " because it is full.";
   arena_->GetContext<grpc_event_engine::experimental::EventEngine>()->Run(
       [this, participant]() mutable {
-        ApplicationCallbackExecCtx app_exec_ctx;
         ExecCtx exec_ctx;
-        AddParticipant(participant);
+        MaybeAsyncAddParticipant(participant);
         Unref();
       });
 }
@@ -461,7 +496,6 @@ void Party::WakeupAsync(WakeupMask wakeup_mask) {
         arena_->GetContext<grpc_event_engine::experimental::EventEngine>()->Run(
             [this, prev_state]() {
               GRPC_LATENT_SEE_PARENT_SCOPE("Party::WakeupAsync");
-              ApplicationCallbackExecCtx app_exec_ctx;
               ExecCtx exec_ctx;
               RunLockedAndUnref(this, prev_state);
             });

data/src/core/lib/promise/party.h

@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 #include <atomic>
+#include <limits>
 #include <string>
 #include <utility>
 
@@ -41,24 +42,124 @@
 
 namespace grpc_core {
 
+// Terms Used:
+//
+// Concurrent promises: Promises that may be in progress at a given instant
+// of time, but they may or may not execute simultaneously.
+// Parallel promises: Two or more Promises that are getting executed
+// simultaneously at the same instant of time.
+//
+// Promise Party
+// A Promise Party forms the execution environment for Promises. It runs up
+// to sixteen concurrent party participants and ensures none of the participants
+// run in parallel with each other.
+//
+// Party Participant
+// One Promise Party has 16 Participant slots. Each Participant slot can hold
+// one of the three
+// 1. A Promise
+// 2. Or a Promise Factory
+// 3. or any type of Participant object, such as SpawnSerializer.
+//
+// Using a Party
+// A party should be used when
+// 1. You need many promises to be run in a way that is concurrent but not
+// parallel.
+// 2. These promises have their own complex sleep and wake mechanisms.
+// 3. You need a way to run the pending promises to completion by repolling them
+// as needed.
+//
+// Creating a new Party
+// A Party must only be created using Party::Make function.
+//
+// Spawning a promise on a Party
+// 1. A promise can be spawned on a party using either Spawn or SpawnWaitable
+// method.
+// 2. If you want to bulk spawn promises on a party before any thread
+// starts executing them, use Party::WakeupHold.
+//
+// Execution of Spawned Promises
+// A Participant spawned on a party will get executed by that party.
+// Whenever the party wakes up, it will execute all unresolved Party
+// Participants at least once. After any Party Participant is resolved, its slot
+// is freed up to make place for new participants.
+//
+// When these Party Participants are executed (polled), they can either
+// 1. Resolve by returning a value
+// 2. Return Pending{}
+// 3. Wait for a certain event to happen using either a Notification or a Latch.
+//
+// When a promise factory is passed as a Party Participant, the promise factory
+// is used to generate the promise once, and then the promise is executed once
+// each time the party is polled till it resolves.
+//
+// Sleep mechanism of a Party
+// A party will Sleep/Quiece if all Participants Spawned on the party are in any
+// of the following states
+// 1. Return Pending{}
+// 2. Resolve
+// 3. Are waiting because of a Latch.
+// If a party is currently running a Participant, it is said to be active/awake.
+// Otherwise it is said to be Sleeping or Quieced.
+//
+// Wake mechanism of a Party
+// To wake up a sleeping party you can use the Waker object. Once the Party is
+// woken, it will be executed as mentioned above.
+//
+// Party Cancellation
+// A Party can be cancelled using party_.reset() method.
+//
+// Guarantees of a Party
+// 1. All Participant spawned on one party are guaranteed to be run serially.
+// Their execution will not happen in parallel.
+// 2. If a promise is executed, its on_complete is guaranteed to be executed as
+// long as the party is not cancelled.
+// 3. Once a party is cancelled, Participants that were Spawned onto the party,
+// but not yet executed, will not get executed.
+// 4. Promise spawned on a party will never be repolled after it is resolved.
+// 5. A promise spawned on a party, can in turn spawn another promise either on
+// the same party or on another party. We allow nesting of Spawn function.
+// 6. A promise or promise factory that is passed to a Spawn function could
+// either be a single simple promise, or it could be a promise combinator such
+// as TrySeq, TryJoin, Loop or any such promise combinator. Nesting of these
+// promise combinators is allowed.
+// 7. You can re-use the same party to spawn new Participants as long as the
+// older Participants have been resolved.
+// 8. We guarantee safe working of up to 16 un-resolved participants
+// on a party at a time.
+//
+// Non-Guarantees of a Party
+// 1. Promises spawned on one party are not guaranteed to execute in the same
+// order. They can execute in any order. If you need the promises to be executed
+// in a specific order, use a SpawnSerializer. If that is not feasible for some
+// reason, then either consider the use of a promise combinator, or order the
+// execution of promises using Notifications or Latches.
+// 2. A party cannot guarantee which thread a party Participant will execute on.
+// It could either execute on the current thread, or an event engine thread or
+// any other thread.
+// 3. Say, we spawned promises P1_1, P1_2, P1_3 on party1. Then promise P1_1 in
+// turn spawns promise P2_1, P2_2, P2_3 on party2. In such cases, party1 and
+// party2 may either execute on the same thread, or they may both execute on
+// different threads. party1 and party2 may or may not run in parallel.
+
 namespace party_detail {
 
 // Number of bits reserved for wakeups gives us the maximum number of
-// participants.
+// participants. This can change in the future and we don't guarantee this
+// number to be 16 always.
 static constexpr size_t kMaxParticipants = 16;
 
 }  // namespace party_detail
 
-// A Party is an Activity with multiple participant promises.
 class Party : public Activity, private Wakeable {
  private:
   // Non-owning wakeup handle.
   class Handle;
 
-  // One participant in the party.
+  // One promise participant in the party.
   class Participant {
    public:
-    // Poll the participant. Return true if complete.
+    // Poll the participant promise. Return true if complete.
     // Participant should take care of its own deallocation in this case.
     virtual bool PollParticipantPromise() = 0;
 
@@ -79,6 +180,12 @@ class Party : public Activity, private Wakeable {
   Party(const Party&) = delete;
   Party& operator=(const Party&) = delete;
 
+  // A Party object must be created only by using this method.
+  static RefCountedPtr<Party> Make(RefCountedPtr<Arena> arena) {
+    auto* arena_ptr = arena.get();
+    return RefCountedPtr<Party>(arena_ptr->New<Party>(std::move(arena)));
+  }
+
   // When calling into a Party from outside the promises system we often would
   // like to perform more than one action.
   // This class tries to acquire the party lock just once - if it succeeds then
@@ -125,22 +232,88 @@ class Party : public Activity, private Wakeable {
     uint64_t prev_state_;
   };
 
-  static RefCountedPtr<Party> Make(RefCountedPtr<Arena> arena) {
-    auto* arena_ptr = arena.get();
-    return RefCountedPtr<Party>(arena_ptr->New<Party>(std::move(arena)));
-  }
+  // SpawnSerializer is a helper class to serialize the execution of multiple
+  // promises on a party.
+  //
+  // Provides the guarantee that given:
+  //   SpawnSerializer* serializer = party->MakeSpawnSerializer();
+  //   serializer->Spawn([] { /* promise 1 */; });
+  //   serializer->Spawn([] { /* promise 2 */; });
+  // 1. promise 1 will be resolved before promise 2 is started.
+  // 2. promise 1 and promise 2 will execute on `party`.
+  //
+  // It's possible to have multiple SpawnSerializer instances on a party.
+  // Once created, the SpawnSerializer is only valid until the party is
+  // released - note that SpawnSerializer itself does not hold a ref to the
+  // party.
+  //
+  // Each SpawnSerializer consumes one slot in the party's participant array for
+  // the lifetime of the party - that is that a SpawnSerializer counts as one of
+  // the sixteen promises executing on a Party.
+  //
+  // The promises spawned by SpawnSerializer *DO NOT* count towards the sixteen
+  // promise limit.
+  //
+  // The size of this type matters, and so we leverage private inheritance to
+  // minimize the number of pointers needed to be kept per instance.
+  class SpawnSerializer final : private Participant {
+   public:
+    // Spawn a promise into the party.
+    //
+    // The promise will be polled until it is resolved, or until the party is
+    // shut down.
+    //
+    // Later promises spawned by this serializer will not be polled until this
+    // promise is resolved, and then one at a time in the order they were
+    // spawned.
+    //
+    // Spawn itself is not thread safe: at most one thread can call Spawn at a
+    // time. Just as the execution of promises is serialized by this type, the
+    // spawning of promises is expected to be serialized by some external entity
+    // (usually this is a Seq running on a different party).
+    template <class Factory>
+    void Spawn(Factory factory) {
+      auto empty_completion = [](Empty) {};
+      next_.Push(new ParticipantImpl<Factory, decltype(empty_completion)>(
+          "SpawnSerializer", std::move(factory), empty_completion));
+      party_->WakeupFromState<false>(
+          party_->state_.load(std::memory_order_relaxed), wakeup_mask_);
+    }
+
+   private:
+    friend class Party;
+    friend class Arena;
+
+    bool PollParticipantPromise() override;
+    void Destroy() override;
+
+    explicit SpawnSerializer(Party* party)
+        : next_(party->arena()), party_(party) {}
+
+    // Queue of promises to be executed after the active promise resolves.
+    ArenaSpsc<Participant*, false> next_;
+    // The promise currently being executed.
+    Participant* active_ = nullptr;
+    // The wakeup mask for this serializers participant.
+    // Allows us to wake up the party when a new promise is added to the queue.
+    WakeupMask wakeup_mask_;
+    // The party this serializer is running on.
+    Party* const party_;
+  };
 
   // Spawn one promise into the party.
-  // The promise will be polled until it is resolved, or until the party is shut
-  // down.
-  // The on_complete callback will be called with the result of the promise if
-  // it completes.
-  // A maximum of sixteen promises can be spawned onto a party.
+  // The party can poll the promise until it is resolved, or until the party is
+  // shut down.
+  // The on_complete callback will be called with the result of the
+  // promise if it completes.
   // promise_factory called to create the promise with the party lock taken;
-  // after the promise is created the factory is destroyed.
-  // This means that pointers or references to factory members will be
-  // invalidated after the promise is created - so the promise should not retain
-  // any of these.
+  // after the promise is created the factory is destroyed. This means that
+  // pointers or references to factory members will be invalidated after the
+  // promise is created - so the promise should not retain any of these.
+  // This function is thread safe. We can Spawn different promises onto the
+  // same party from different threads.
+  // A party can hold upto 16 unresolved promises at a time. However, this
+  // number might change in the future.
   template <typename Factory, typename OnComplete>
   void Spawn(absl::string_view name, Factory promise_factory,
              OnComplete on_complete);
@@ -184,6 +357,14 @@ class Party : public Activity, private Wakeable {
 
   Arena* arena() { return arena_.get(); }
 
+  SpawnSerializer* MakeSpawnSerializer() {
+    auto* const serializer = arena_->New<SpawnSerializer>(this);
+    const size_t slot = AddParticipant(serializer);
+    DCHECK_NE(slot, std::numeric_limits<size_t>::max());
+    serializer->wakeup_mask_ = 1ull << slot;
+    return serializer;
+  }
+
  protected:
   friend class Arena;
 
@@ -287,7 +468,7 @@ class Party : public Activity, private Wakeable {
           Construct(&promise_, std::move(p));
           state_.store(State::kPromise, std::memory_order_relaxed);
         }
-          ABSL_FALLTHROUGH_INTENDED;
+          [[fallthrough]];
         case State::kPromise: {
           auto p = promise_();
           if (auto* r = p.value_if_ready()) {
@@ -383,9 +564,10 @@ class Party : public Activity, private Wakeable {
       Unref();
       return;
     }
-    WakeupFromState(state_.load(std::memory_order_relaxed), wakeup_mask);
+    WakeupFromState<true>(state_.load(std::memory_order_relaxed), wakeup_mask);
   }
 
+  template <bool kReffed>
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void WakeupFromState(
       uint64_t cur_state, WakeupMask wakeup_mask) {
     GRPC_LATENT_SEE_INNER_SCOPE("Party::WakeupFromState");
@@ -396,8 +578,13 @@ class Party : public Activity, private Wakeable {
         // If the party is locked, we need to set the wakeup bits, and then
         // we'll immediately unref. Since something is running this should never
         // bring the refcount to zero.
-        DCHECK_GT(cur_state & kRefMask, kOneRef);
-        auto new_state = (cur_state | wakeup_mask) - kOneRef;
+        if (kReffed) {
+          DCHECK_GT(cur_state & kRefMask, kOneRef);
+        } else {
+          DCHECK_GE(cur_state & kRefMask, kOneRef);
+        }
+        const uint64_t new_state =
+            (cur_state | wakeup_mask) - (kReffed ? kOneRef : 0);
         if (state_.compare_exchange_weak(cur_state, new_state,
                                          std::memory_order_release)) {
           LogStateChange("Wakeup", cur_state, cur_state | wakeup_mask);
@@ -406,9 +593,11 @@ class Party : public Activity, private Wakeable {
       } else {
         // If the party is not locked, we need to lock it and run.
         DCHECK_EQ(cur_state & kWakeupMask, 0u);
-        if (state_.compare_exchange_weak(cur_state, cur_state | kLocked,
+        const uint64_t new_state =
+            (cur_state | kLocked) + (kReffed ? 0 : kOneRef);
+        if (state_.compare_exchange_weak(cur_state, new_state,
                                          std::memory_order_acq_rel)) {
-          LogStateChange("WakeupAndRun", cur_state, cur_state | kLocked);
+          LogStateChange("WakeupAndRun", cur_state, new_state);
           wakeup_mask_ |= wakeup_mask;
           RunLockedAndUnref(this, cur_state);
           return;
@@ -421,8 +610,8 @@ class Party : public Activity, private Wakeable {
   void Drop(WakeupMask wakeup_mask) final;
 
   // Add a participant (backs Spawn, after type erasure to ParticipantFactory).
-  void AddParticipant(Participant* participant);
-  void DelayAddParticipant(Participant* participant);
+  size_t AddParticipant(Participant* participant);
+  void MaybeAsyncAddParticipant(Participant* participant);
 
   static uint64_t NextAllocationMask(uint64_t current_allocation_mask);
 
@@ -457,7 +646,7 @@ template <typename Factory, typename OnComplete>
 void Party::Spawn(absl::string_view name, Factory promise_factory,
                   OnComplete on_complete) {
   GRPC_TRACE_LOG(party_state, INFO) << "PARTY[" << this << "]: spawn " << name;
-  AddParticipant(new ParticipantImpl<Factory, OnComplete>(
+  MaybeAsyncAddParticipant(new ParticipantImpl<Factory, OnComplete>(
       name, std::move(promise_factory), std::move(on_complete)));
 }
 
@@ -467,7 +656,7 @@ auto Party::SpawnWaitable(absl::string_view name, Factory promise_factory) {
   auto participant = MakeRefCounted<PromiseParticipantImpl<Factory>>(
       name, std::move(promise_factory));
   Participant* p = participant->Ref().release();
-  AddParticipant(p);
+  MaybeAsyncAddParticipant(p);
   return [participant = std::move(participant)]() mutable {
     return participant->PollCompletion();
   };

data/src/core/lib/promise/pipe.h

@@ -20,14 +20,14 @@
 #include <stdlib.h>
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
+#include <variant>
 
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
-#include "absl/types/optional.h"
-#include "absl/types/variant.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/if.h"
@@ -189,7 +189,7 @@ class Center : public InterceptorList<T> {
   // Return Pending if there is no value.
   // Return the value if one was retrieved.
   // Return nullopt if the send end is closed and no value had been pushed.
-  Poll<absl::optional<T>> Next() {
+  Poll<std::optional<T>> Next() {
     GRPC_TRACE_LOG(promise_primitives, INFO) << DebugOpString("Next");
     DCHECK_NE(refs_, 0);
     switch (value_state_) {
@@ -206,9 +206,9 @@ class Center : public InterceptorList<T> {
         return std::move(value_);
       case ValueState::kClosed:
       case ValueState::kCancelled:
-        return absl::nullopt;
+        return std::nullopt;
     }
-    GPR_UNREACHABLE_CODE(return absl::nullopt);
+    GPR_UNREACHABLE_CODE(return std::nullopt);
   }
 
   // Check if the pipe is closed for sending (if there is a value still queued
@@ -517,8 +517,8 @@ class Next {
   Next(Next&& other) noexcept = default;
   Next& operator=(Next&& other) noexcept = default;
 
-  Poll<absl::optional<T>> operator()() {
-    return center_ == nullptr ? absl::nullopt : center_->Next();
+  Poll<std::optional<T>> operator()() {
+    return center_ == nullptr ? std::nullopt : center_->Next();
   }
 
  private:
@@ -551,7 +551,7 @@ class PipeReceiver {
   // available.
   auto Next() {
     return Seq(pipe_detail::Next<T>(center_), [center = center_](
-                                                  absl::optional<T> value) {
+                                                  std::optional<T> value) {
       bool open = value.has_value();
       bool cancelled = center == nullptr ? true : center->cancelled();
       return If(
@@ -559,7 +559,7 @@ class PipeReceiver {
           [center = std::move(center), value = std::move(value)]() mutable {
             auto run = center->Run(std::move(value));
             return Map(std::move(run), [center = std::move(center)](
-                                           absl::optional<T> value) mutable {
+                                           std::optional<T> value) mutable {
               if (value.has_value()) {
                 center->value() = std::move(*value);
                 return NextResult<T>(std::move(center));
@@ -637,7 +637,7 @@ class Push {
           << " Pipe push has a null center";
       return false;
     }
-    if (auto* p = absl::get_if<T>(&state_)) {
+    if (auto* p = std::get_if<T>(&state_)) {
       auto r = center_->Push(p);
       if (auto* ok = r.value_if_ready()) {
         state_.template emplace<AwaitingAck>();
@@ -646,7 +646,7 @@ class Push {
         return Pending{};
       }
     }
-    DCHECK(absl::holds_alternative<AwaitingAck>(state_));
+    DCHECK(std::holds_alternative<AwaitingAck>(state_));
     return center_->PollAck();
   }
 
@@ -658,7 +658,7 @@ class Push {
       : center_(std::move(center)), state_(std::move(push)) {}
 
   RefCountedPtr<Center<T>> center_;
-  absl::variant<T, AwaitingAck> state_;
+  std::variant<T, AwaitingAck> state_;
 };
 
 }  // namespace pipe_detail

data/src/core/lib/promise/poll.h

@@ -17,12 +17,12 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <optional>
 #include <string>
 #include <utility>
 
 #include "absl/log/check.h"
 #include "absl/strings/str_format.h"
-#include "absl/types/optional.h"
 #include "src/core/util/construct_destruct.h"
 
 namespace grpc_core {
@@ -135,8 +135,8 @@ class Poll {
   //
   // Why not optional<T>?
   //
-  // We have cases where we want to return absl::nullopt{} from a promise, and
-  // have that upgraded to a Poll<absl::nullopt_t> prior to a cast to some
+  // We have cases where we want to return std::nullopt{} from a promise, and
+  // have that upgraded to a Poll<std::nullopt_t> prior to a cast to some
   // Poll<optional<T>>.
   //
   // Since optional<nullopt_t> is not allowed, we'd not be allowed to make
@@ -152,6 +152,9 @@ class Poll {
   };
 };
 
+template <typename T>
+class Poll<T&&>;
+
 template <>
 class Poll<Empty> {
  public:
@@ -300,7 +303,7 @@ void AbslStringify(Sink& sink, const Poll<absl::StatusOr<T>>& poll) {
 }
 
 template <typename Sink, typename T>
-void AbslStringify(Sink& sink, const Poll<absl::optional<T>>& poll) {
+void AbslStringify(Sink& sink, const Poll<std::optional<T>>& poll) {
   if (poll.pending()) {
     absl::Format(&sink, "<<pending>>");
     return;
@@ -316,7 +319,7 @@ void AbslStringify(Sink& sink, const Poll<absl::optional<T>>& poll) {
 // Hack to get metadata printing
 template <typename Sink, typename T, typename Deleter>
 void AbslStringify(
-    Sink& sink, const Poll<absl::optional<std::unique_ptr<T, Deleter>>>& poll) {
+    Sink& sink, const Poll<std::optional<std::unique_ptr<T, Deleter>>>& poll) {
   if (poll.pending()) {
     absl::Format(&sink, "<<pending>>");
     return;