grpc 1.46.3 → 1.53.1

data/src/core/lib/promise/activity.h

@@ -12,94 +12,129 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_PROMISE_ACTIVITY_H
-#define GRPC_CORE_LIB_PROMISE_ACTIVITY_H
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H
+#define GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H
 
 #include <grpc/support/port_platform.h>
 
-#include <stddef.h>
 #include <stdint.h>
 
 #include <algorithm>
 #include <atomic>
-#include <functional>
 #include <memory>
-#include <type_traits>
+#include <string>
 #include <utility>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/status/status.h"
 #include "absl/types/optional.h"
-#include "absl/types/variant.h"
-#include "absl/utility/utility.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/detail/status.h"
-#include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 
+class Activity;
+
 // A Wakeable object is used by queues to wake activities.
 class Wakeable {
  public:
   // Wake up the underlying activity.
   // After calling, this Wakeable cannot be used again.
-  virtual void Wakeup() = 0;
+  // arg comes from the Waker object and allows one Wakeable instance to be used
+  // for multiple disjoint subparts of an Activity.
+  virtual void Wakeup(void* arg) = 0;
   // Drop this wakeable without waking up the underlying activity.
-  virtual void Drop() = 0;
+  virtual void Drop(void* arg) = 0;
+
+  // Return the underlying activity debug tag, or "<unknown>" if not available.
+  virtual std::string ActivityDebugTag(void* arg) const = 0;
 
  protected:
   inline ~Wakeable() {}
 };
 
+namespace promise_detail {
+struct Unwakeable final : public Wakeable {
+  void Wakeup(void*) override {}
+  void Drop(void*) override {}
+  std::string ActivityDebugTag(void*) const override;
+};
+static Unwakeable* unwakeable() {
+  return NoDestructSingleton<Unwakeable>::Get();
+}
+}  // namespace promise_detail
+
 // An owning reference to a Wakeable.
 // This type is non-copyable but movable.
 class Waker {
  public:
-  explicit Waker(Wakeable* wakeable) : wakeable_(wakeable) {}
-  Waker() : wakeable_(&unwakeable_) {}
-  ~Waker() { wakeable_->Drop(); }
+  Waker(Wakeable* wakeable, void* arg) : wakeable_and_arg_{wakeable, arg} {}
+  Waker() : Waker(promise_detail::unwakeable(), nullptr) {}
+  ~Waker() { wakeable_and_arg_.Drop(); }
   Waker(const Waker&) = delete;
   Waker& operator=(const Waker&) = delete;
-  Waker(Waker&& other) noexcept : wakeable_(other.wakeable_) {
-    other.wakeable_ = &unwakeable_;
-  }
+  Waker(Waker&& other) noexcept : wakeable_and_arg_(other.Take()) {}
   Waker& operator=(Waker&& other) noexcept {
-    std::swap(wakeable_, other.wakeable_);
+    std::swap(wakeable_and_arg_, other.wakeable_and_arg_);
     return *this;
   }
 
   // Wake the underlying activity.
-  void Wakeup() {
-    wakeable_->Wakeup();
-    wakeable_ = &unwakeable_;
-  }
+  void Wakeup() { Take().Wakeup(); }
 
   template <typename H>
   friend H AbslHashValue(H h, const Waker& w) {
-    return H::combine(std::move(h), w.wakeable_);
+    return H::combine(H::combine(std::move(h), w.wakeable_and_arg_.wakeable),
+                      w.wakeable_and_arg_.arg);
   }
 
   bool operator==(const Waker& other) const noexcept {
-    return wakeable_ == other.wakeable_;
+    return wakeable_and_arg_ == other.wakeable_and_arg_;
+  }
+
+  bool operator!=(const Waker& other) const noexcept {
+    return !operator==(other);
+  }
+
+  std::string ActivityDebugTag() {
+    return wakeable_and_arg_.ActivityDebugTag();
+  }
+
+  // This is for tests to assert that a waker is occupied or not.
+  bool is_unwakeable() const {
+    return wakeable_and_arg_.wakeable == promise_detail::unwakeable();
   }
 
  private:
-  class Unwakeable final : public Wakeable {
-   public:
-    void Wakeup() final {}
-    void Drop() final {}
+  struct WakeableAndArg {
+    Wakeable* wakeable;
+    void* arg;
+
+    void Wakeup() { wakeable->Wakeup(arg); }
+    void Drop() { wakeable->Drop(arg); }
+    std::string ActivityDebugTag() const {
+      return wakeable == nullptr ? "<unknown>"
+                                 : wakeable->ActivityDebugTag(arg);
+    }
+    bool operator==(const WakeableAndArg& other) const noexcept {
+      return wakeable == other.wakeable && arg == other.arg;
+    }
   };
 
-  Wakeable* wakeable_;
-  static Unwakeable unwakeable_;
+  WakeableAndArg Take() {
+    return std::exchange(wakeable_and_arg_,
+                         {promise_detail::unwakeable(), nullptr});
+  }
+
+  WakeableAndArg wakeable_and_arg_;
 };
 
 // An Activity tracks execution of a single promise.
@@ -141,6 +176,9 @@ class Activity : public Orphanable {
   // delivered until long after the activity should be destroyed.
   virtual Waker MakeNonOwningWaker() = 0;
 
+  // Some descriptive text to add to log messages to identify this activity.
+  virtual std::string DebugTag() const;
+
  protected:
   // Check if this activity is the current activity executing on the current
   // thread.
@@ -150,19 +188,22 @@ class Activity : public Orphanable {
   // Set the current activity at construction, clean it up at destruction.
   class ScopedActivity {
    public:
-    explicit ScopedActivity(Activity* activity) {
-      GPR_ASSERT(g_current_activity_ == nullptr);
+    explicit ScopedActivity(Activity* activity)
+        : prior_activity_(g_current_activity_) {
       g_current_activity_ = activity;
     }
-    ~ScopedActivity() { g_current_activity_ = nullptr; }
+    ~ScopedActivity() { g_current_activity_ = prior_activity_; }
     ScopedActivity(const ScopedActivity&) = delete;
     ScopedActivity& operator=(const ScopedActivity&) = delete;
+
+   private:
+    Activity* const prior_activity_;
   };
 
  private:
   // Set during RunLoop to the Activity that's executing.
   // Being set implies that mu_ is held.
-  static GPR_THREAD_LOCAL(Activity*) g_current_activity_;
+  static thread_local Activity* g_current_activity_;
 };
 
 // Owned pointer to one Activity.
@@ -221,7 +262,10 @@ class ActivityContexts : public ContextHolder<Contexts>... {
     explicit ScopedContext(ActivityContexts* contexts)
         : Context<ContextTypeFromHeld<Contexts>>(
               static_cast<ContextHolder<Contexts>*>(contexts)
-                  ->GetContext())... {}
+                  ->GetContext())... {
+      // Silence `unused-but-set-parameter` in case of Contexts = {}
+      (void)contexts;
+    }
   };
 };
 
@@ -240,7 +284,7 @@ class FreestandingActivity : public Activity, private Wakeable {
  public:
   Waker MakeOwningWaker() final {
     Ref();
-    return Waker(this);
+    return Waker(this, nullptr);
   }
   Waker MakeNonOwningWaker() final;
 
@@ -273,7 +317,7 @@ class FreestandingActivity : public Activity, private Wakeable {
   // Check if we got an internal wakeup since the last time this function was
   // called.
   ActionDuringRun GotActionDuringRun() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-    return absl::exchange(action_during_run_, ActionDuringRun::kNone);
+    return std::exchange(action_during_run_, ActionDuringRun::kNone);
   }
 
   // Implementors of Wakeable::Wakeup should call this after the wakeup has
@@ -289,6 +333,8 @@ class FreestandingActivity : public Activity, private Wakeable {
 
   Mutex* mu() ABSL_LOCK_RETURNED(mu_) { return &mu_; }
 
+  std::string ActivityDebugTag(void*) const override { return DebugTag(); }
+
  private:
   class Handle;
 
@@ -327,32 +373,42 @@ class FreestandingActivity : public Activity, private Wakeable {
 };
 
 // Implementation details for an Activity of an arbitrary type of promise.
-// There should exist a static function:
+// There should exist an inner template class `BoundScheduler` that provides
+// the following interface:
 // struct WakeupScheduler {
 //   template <typename ActivityType>
-//   void ScheduleWakeup(ActivityType* activity);
+//   class BoundScheduler {
+//    public:
+//     BoundScheduler(WakeupScheduler);
+//     void ScheduleWakeup();
+//   };
 // };
-// This function should arrange that activity->RunScheduledWakeup() be invoked
-// at the earliest opportunity.
+// The ScheduleWakeup function should arrange that
+// static_cast<ActivityType*>(this)->RunScheduledWakeup() be invoked at the
+// earliest opportunity.
 // It can assume that activity will remain live until RunScheduledWakeup() is
 // invoked, and that a given activity will not be concurrently scheduled again
 // until its RunScheduledWakeup() has been invoked.
-// We use private inheritance here as a way of getting private members for
-// each of the contexts.
+// We use private inheritance here as a way of getting private members for each
+// of the contexts.
 // TODO(ctiller): We can probably reconsider the private inheritance here
 // when we move away from C++11 and have more powerful template features.
 template <class F, class WakeupScheduler, class OnDone, typename... Contexts>
-class PromiseActivity final : public FreestandingActivity,
-                              private ActivityContexts<Contexts...> {
+class PromiseActivity final
+    : public FreestandingActivity,
+      public WakeupScheduler::template BoundScheduler<
+          PromiseActivity<F, WakeupScheduler, OnDone, Contexts...>>,
+      private ActivityContexts<Contexts...> {
  public:
-  using Factory = PromiseFactory<void, F>;
+  using Factory = OncePromiseFactory<void, F>;
   using ResultType = typename Factory::Promise::Result;
 
   PromiseActivity(F promise_factory, WakeupScheduler wakeup_scheduler,
                   OnDone on_done, Contexts&&... contexts)
       : FreestandingActivity(),
+        WakeupScheduler::template BoundScheduler<PromiseActivity>(
+            std::move(wakeup_scheduler)),
         ActivityContexts<Contexts...>(std::forward<Contexts>(contexts)...),
-        wakeup_scheduler_(std::move(wakeup_scheduler)),
         on_done_(std::move(on_done)) {
     // Lock, construct an initial promise from the factory, and step it.
     // This may hit a waiter, which could expose our this pointer to other
@@ -394,7 +450,11 @@ class PromiseActivity final : public FreestandingActivity,
       MutexLock lock(mu());
       // Check if we were done, and flag done.
       was_done = done_;
-      if (!done_) MarkDone();
+      if (!done_) {
+        ScopedActivity scoped_activity(this);
+        ScopedContext contexts(this);
+        MarkDone();
+      }
     }
     // If we were not done, then call the on_done callback.
     if (!was_done) {
@@ -407,7 +467,7 @@ class PromiseActivity final : public FreestandingActivity,
   // the activity to an external threadpool to run. If the activity is already
   // running on this thread, a note is taken of such and the activity is
   // repolled if it doesn't complete.
-  void Wakeup() final {
+  void Wakeup(void*) final {
     // If there is an active activity, but hey it's us, flag that and we'll loop
     // in RunLoop (that's calling from above here!).
     if (Activity::is_current()) {
@@ -418,7 +478,7 @@ class PromiseActivity final : public FreestandingActivity,
     }
     if (!wakeup_scheduled_.exchange(true, std::memory_order_acq_rel)) {
       // Can't safely run, so ask to run later.
-      wakeup_scheduler_.ScheduleWakeup(this);
+      this->ScheduleWakeup();
     } else {
       // Already a wakeup scheduled for later, drop ref.
       WakeupComplete();
@@ -426,13 +486,13 @@ class PromiseActivity final : public FreestandingActivity,
   }
 
   // Drop a wakeup
-  void Drop() final { this->WakeupComplete(); }
+  void Drop(void*) final { this->WakeupComplete(); }
 
   // Notification that we're no longer executing - it's ok to destruct the
   // promise.
   void MarkDone() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
-    GPR_ASSERT(!done_);
-    done_ = true;
+    GPR_ASSERT(!std::exchange(done_, true));
+    ScopedContext contexts(this);
     Destruct(&promise_holder_.promise);
   }
 
@@ -469,7 +529,7 @@ class PromiseActivity final : public FreestandingActivity,
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
     ScopedActivity scoped_activity(this);
     ScopedContext contexts(this);
-    Construct(&promise_holder_.promise, promise_factory.Once());
+    Construct(&promise_holder_.promise, promise_factory.Make());
     return StepLoop();
   }
 
@@ -481,7 +541,7 @@ class PromiseActivity final : public FreestandingActivity,
       // Run the promise.
       GPR_ASSERT(!done_);
       auto r = promise_holder_.promise();
-      if (auto* status = absl::get_if<kPollReadyIdx>(&r)) {
+      if (auto* status = r.value_if_ready()) {
         // If complete, destroy the promise, flag done, and exit this loop.
         MarkDone();
         return IntoStatus(status);
@@ -500,8 +560,6 @@ class PromiseActivity final : public FreestandingActivity,
   }
 
   using Promise = typename Factory::Promise;
-  // Scheduler for wakeups
-  GPR_NO_UNIQUE_ADDRESS WakeupScheduler wakeup_scheduler_;
   // Callback on completion of the promise.
   GPR_NO_UNIQUE_ADDRESS OnDone on_done_;
   // Has execution completed?
@@ -537,4 +595,4 @@ ActivityPtr MakeActivity(Factory promise_factory,
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_PROMISE_ACTIVITY_H
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H

data/src/core/lib/promise/arena_promise.h

@@ -12,13 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H
-#define GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H
+#define GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/support/log.h>
+#include <stdlib.h>
 
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+#include "absl/meta/type_traits.h"
+
+#include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/resource_quota/arena.h"
 
@@ -26,60 +34,82 @@ namespace grpc_core {
 
 namespace arena_promise_detail {
 
-// Type erased promise stored in the arena.
+using ArgType = std::aligned_storage_t<sizeof(void*)>;
 template <typename T>
-class ImplInterface {
- public:
+T*& ArgAsPtr(ArgType* arg) {
+  static_assert(sizeof(ArgType) >= sizeof(T**),
+                "Must have ArgType of at least one pointer size");
+  return *reinterpret_cast<T**>(arg);
+}
+
+template <typename T>
+struct Vtable {
   // Poll the promise, once.
-  virtual Poll<T> PollOnce() = 0;
+  Poll<T> (*poll_once)(ArgType* arg);
   // Destroy the underlying callable object if there is one.
   // Since we don't delete (the arena owns the memory) but we may need to call a
   // destructor, we expose this for when the ArenaPromise object is destroyed.
-  virtual void Destroy() = 0;
+  void (*destroy)(ArgType* arg);
+};
 
- protected:
-  ~ImplInterface() = default;
+template <typename T>
+struct VtableAndArg {
+  const Vtable<T>* vtable;
+  ArgType arg;
 };
 
-// Implementation of ImplInterface for an empty object.
+// Implementation of Vtable for an empty object.
 // Used when an empty ArenaPromise is created, or when the ArenaPromise is moved
 // from. Since in either case these objects should not be polled, we simply
 // crash if it is.
 template <typename T>
-class NullImpl final : public ImplInterface<T> {
- public:
-  Poll<T> PollOnce() override {
+struct Null {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType*) {
     abort();
     GPR_UNREACHABLE_CODE(return Pending{});
   }
-  void Destroy() override {}
 
-  static ImplInterface<T>* Get() {
-    static NullImpl<T> instance;
-    return &instance;
+  static void Destroy(ArgType*) {}
+};
+
+template <typename T>
+const Vtable<T> Null<T>::vtable = {PollOnce, Destroy};
+
+// Implementation of ImplInterface for a callable object.
+template <typename T, typename Callable>
+struct AllocatedCallable {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*ArgAsPtr<Callable>(arg))());
   }
 
- private:
-  ~NullImpl() = default;
+  static void Destroy(ArgType* arg) { Destruct(ArgAsPtr<Callable>(arg)); }
 };
 
-// Implementation of ImplInterface for a callable object.
 template <typename T, typename Callable>
-class CallableImpl final : public ImplInterface<T> {
- public:
-  explicit CallableImpl(Callable&& callable) : callable_(std::move(callable)) {}
-  // Forward polls to the callable object.
-  Poll<T> PollOnce() override { return poll_cast<T>(callable_()); }
-  // Destroy destructs the callable object.
-  void Destroy() override { this->~CallableImpl(); }
+const Vtable<T> AllocatedCallable<T, Callable>::vtable = {PollOnce, Destroy};
 
- private:
-  // Should only be called by Destroy().
-  ~CallableImpl() = default;
+// Implementation of ImplInterface for a small callable object (one that fits
+// within the ArgType arg)
+template <typename T, typename Callable>
+struct Inlined {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*reinterpret_cast<Callable*>(arg))());
+  }
 
-  Callable callable_;
+  static void Destroy(ArgType* arg) {
+    Destruct(reinterpret_cast<Callable*>(arg));
+  }
 };
 
+template <typename T, typename Callable>
+const Vtable<T> Inlined<T, Callable>::vtable = {PollOnce, Destroy};
+
 // If a callable object is empty we can substitute any instance of that callable
 // for the one we call (for how could we tell the difference)?
 // Since this corresponds to a lambda with no fields, and we expect these to be
@@ -88,28 +118,18 @@ class CallableImpl final : public ImplInterface<T> {
 // (this comes up often when the promise only accesses context data from the
 // containing activity).
 template <typename T, typename Callable>
-class SharedImpl final : public ImplInterface<T>, private Callable {
- public:
-  // Call the callable, or at least an exact duplicate of it - if you have no
-  // members, all your instances look the same.
-  Poll<T> PollOnce() override { return Callable::operator()(); }
-  // Nothing to destroy.
-  void Destroy() override {}
-  // Return a pointer to the shared instance - these are singletons, and are
-  // needed just to get the vtable in place.
-  static SharedImpl* Get(Callable&& callable) {
-    static_assert(sizeof(SharedImpl) == sizeof(void*),
-                  "SharedImpl should be pointer sized");
-    static SharedImpl impl(std::forward<Callable>(callable));
-    return &impl;
-  }
+struct SharedCallable {
+  static const Vtable<T> vtable;
 
- private:
-  explicit SharedImpl(Callable&& callable)
-      : Callable(std::forward<Callable>(callable)) {}
-  ~SharedImpl() = default;
+  static Poll<T> PollOnce(ArgType* arg) {
+    return (*reinterpret_cast<Callable*>(arg))();
+  }
 };
 
+template <typename T, typename Callable>
+const Vtable<T> SharedCallable<T, Callable>::vtable = {PollOnce,
+                                                       Null<T>::Destroy};
+
 // Redirector type: given a callable type, expose a Make() function that creates
 // the appropriate underlying implementation.
 template <typename T, typename Callable, typename Ignored = void>
@@ -117,26 +137,41 @@ struct ChooseImplForCallable;
 
 template <typename T, typename Callable>
 struct ChooseImplForCallable<
-    T, Callable, absl::enable_if_t<!std::is_empty<Callable>::value>> {
-  static ImplInterface<T>* Make(Callable&& callable) {
-    return GetContext<Arena>()->template New<CallableImpl<T, Callable>>(
+    T, Callable,
+    absl::enable_if_t<!std::is_empty<Callable>::value &&
+                      (sizeof(Callable) > sizeof(ArgType))>> {
+  static void Make(Callable&& callable, VtableAndArg<T>* out) {
+    out->vtable = &AllocatedCallable<T, Callable>::vtable;
+    ArgAsPtr<Callable>(&out->arg) = GetContext<Arena>()->template New<Callable>(
         std::forward<Callable>(callable));
   }
 };
 
+template <typename T, typename Callable>
+struct ChooseImplForCallable<
+    T, Callable,
+    absl::enable_if_t<!std::is_empty<Callable>::value &&
+                      (sizeof(Callable) <= sizeof(ArgType))>> {
+  static void Make(Callable&& callable, VtableAndArg<T>* out) {
+    out->vtable = &Inlined<T, Callable>::vtable;
+    Construct(reinterpret_cast<Callable*>(&out->arg),
+              std::forward<Callable>(callable));
+  }
+};
+
 template <typename T, typename Callable>
 struct ChooseImplForCallable<
     T, Callable, absl::enable_if_t<std::is_empty<Callable>::value>> {
-  static ImplInterface<T>* Make(Callable&& callable) {
-    return SharedImpl<T, Callable>::Get(std::forward<Callable>(callable));
+  static void Make(Callable&&, VtableAndArg<T>* out) {
+    out->vtable = &SharedCallable<T, Callable>::vtable;
   }
 };
 
 // Wrap ChooseImplForCallable with a friend approachable syntax.
 template <typename T, typename Callable>
-ImplInterface<T>* MakeImplForCallable(Callable&& callable) {
-  return ChooseImplForCallable<T, Callable>::Make(
-      std::forward<Callable>(callable));
+void MakeImplForCallable(Callable&& callable, VtableAndArg<T>* out) {
+  ChooseImplForCallable<T, Callable>::Make(std::forward<Callable>(callable),
+                                           out);
 }
 
 }  // namespace arena_promise_detail
@@ -153,36 +188,44 @@ class ArenaPromise {
             typename Ignored =
                 absl::enable_if_t<!std::is_same<Callable, ArenaPromise>::value>>
   // NOLINTNEXTLINE(google-explicit-constructor)
-  ArenaPromise(Callable&& callable)
-      : impl_(arena_promise_detail::MakeImplForCallable<T>(
-            std::forward<Callable>(callable))) {}
+  ArenaPromise(Callable&& callable) {
+    arena_promise_detail::MakeImplForCallable(std::forward<Callable>(callable),
+                                              &vtable_and_arg_);
+  }
 
   // ArenaPromise is not copyable.
   ArenaPromise(const ArenaPromise&) = delete;
   ArenaPromise& operator=(const ArenaPromise&) = delete;
   // ArenaPromise is movable.
-  ArenaPromise(ArenaPromise&& other) noexcept : impl_(other.impl_) {
-    other.impl_ = arena_promise_detail::NullImpl<T>::Get();
+  ArenaPromise(ArenaPromise&& other) noexcept
+      : vtable_and_arg_(other.vtable_and_arg_) {
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
   }
   ArenaPromise& operator=(ArenaPromise&& other) noexcept {
-    impl_->Destroy();
-    impl_ = other.impl_;
-    other.impl_ = arena_promise_detail::NullImpl<T>::Get();
+    vtable_and_arg_.vtable->destroy(&vtable_and_arg_.arg);
+    vtable_and_arg_ = other.vtable_and_arg_;
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
     return *this;
   }
 
   // Destruction => call Destroy on the underlying impl object.
-  ~ArenaPromise() { impl_->Destroy(); }
+  ~ArenaPromise() { vtable_and_arg_.vtable->destroy(&vtable_and_arg_.arg); }
 
   // Expose the promise interface: a call operator that returns Poll<T>.
-  Poll<T> operator()() { return impl_->PollOnce(); }
+  Poll<T> operator()() {
+    return vtable_and_arg_.vtable->poll_once(&vtable_and_arg_.arg);
+  }
+
+  bool has_value() const {
+    return vtable_and_arg_.vtable != &arena_promise_detail::Null<T>::vtable;
+  }
 
  private:
   // Underlying impl object.
-  arena_promise_detail::ImplInterface<T>* impl_ =
-      arena_promise_detail::NullImpl<T>::Get();
+  arena_promise_detail::VtableAndArg<T> vtable_and_arg_ = {
+      &arena_promise_detail::Null<T>::vtable, {}};
 };
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_PROMISE_ARENA_PROMISE_H */
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_ARENA_PROMISE_H