grpc 1.50.0 → 1.51.0

data/src/core/lib/matchers/matchers.cc

@@ -18,7 +18,6 @@
 
 #include <utility>
 
-#include "absl/memory/memory.h"
 #include "absl/status/status.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/match.h"
@@ -35,7 +34,7 @@ absl::StatusOr<StringMatcher> StringMatcher::Create(Type type,
                                                     absl::string_view matcher,
                                                     bool case_sensitive) {
   if (type == Type::kSafeRegex) {
-    auto regex_matcher = absl::make_unique<RE2>(std::string(matcher));
+    auto regex_matcher = std::make_unique<RE2>(std::string(matcher));
     if (!regex_matcher->ok()) {
       return absl::InvalidArgumentError(
           "Invalid regex string specified in matcher.");
@@ -56,7 +55,7 @@ StringMatcher::StringMatcher(std::unique_ptr<RE2> regex_matcher)
 StringMatcher::StringMatcher(const StringMatcher& other)
     : type_(other.type_), case_sensitive_(other.case_sensitive_) {
   if (type_ == Type::kSafeRegex) {
-    regex_matcher_ = absl::make_unique<RE2>(other.regex_matcher_->pattern());
+    regex_matcher_ = std::make_unique<RE2>(other.regex_matcher_->pattern());
   } else {
     string_matcher_ = other.string_matcher_;
   }
@@ -65,7 +64,7 @@ StringMatcher::StringMatcher(const StringMatcher& other)
 StringMatcher& StringMatcher::operator=(const StringMatcher& other) {
   type_ = other.type_;
   if (type_ == Type::kSafeRegex) {
-    regex_matcher_ = absl::make_unique<RE2>(other.regex_matcher_->pattern());
+    regex_matcher_ = std::make_unique<RE2>(other.regex_matcher_->pattern());
   } else {
     string_matcher_ = other.string_matcher_;
   }

data/src/core/lib/promise/activity.cc

@@ -18,6 +18,8 @@
 
 #include <stddef.h>
 
+#include "absl/strings/str_format.h"
+
 #include "src/core/lib/gprpp/atomic_utils.h"
 
 namespace grpc_core {
@@ -25,13 +27,15 @@ namespace grpc_core {
 ///////////////////////////////////////////////////////////////////////////////
 // GLOBALS
 
-GPR_THREAD_LOCAL(Activity*) Activity::g_current_activity_{nullptr};
+thread_local Activity* Activity::g_current_activity_{nullptr};
 
 namespace promise_detail {
 
 ///////////////////////////////////////////////////////////////////////////////
 // HELPER TYPES
 
+std::string Unwakeable::ActivityDebugTag() const { return "<unknown>"; }
+
 // Weak handle to an Activity.
 // Handle can persist while Activity goes away.
 class FreestandingActivity::Handle final : public Wakeable {
@@ -74,6 +78,11 @@ class FreestandingActivity::Handle final : public Wakeable {
 
   void Drop() override { Unref(); }
 
+  std::string ActivityDebugTag() const override {
+    MutexLock lock(&mu_);
+    return activity_ == nullptr ? "<unknown>" : activity_->DebugTag();
+  }
+
  private:
   // Unref the Handle (not the activity).
   void Unref() {
@@ -85,7 +94,7 @@ class FreestandingActivity::Handle final : public Wakeable {
   // Two initial refs: one for the waiter that caused instantiation, one for the
   // activity.
   std::atomic<size_t> refs_{2};
-  Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_);
+  mutable Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_);
   FreestandingActivity* activity_ ABSL_GUARDED_BY(mu_);
 };
 
@@ -117,4 +126,9 @@ Waker FreestandingActivity::MakeNonOwningWaker() {
 }
 
 }  // namespace promise_detail
+
+std::string Activity::DebugTag() const {
+  return absl::StrFormat("ACTIVITY[%p]", this);
+}
+
 }  // namespace grpc_core

data/src/core/lib/promise/activity.h

@@ -22,16 +22,17 @@
 #include <algorithm>
 #include <atomic>
 #include <memory>
+#include <string>
 #include <utility>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/status/status.h"
 #include "absl/types/optional.h"
 #include "absl/types/variant.h"
+#include "absl/utility/utility.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/orphanable.h"
@@ -43,6 +44,8 @@
 
 namespace grpc_core {
 
+class Activity;
+
 // A Wakeable object is used by queues to wake activities.
 class Wakeable {
  public:
@@ -52,19 +55,23 @@ class Wakeable {
   // Drop this wakeable without waking up the underlying activity.
   virtual void Drop() = 0;
 
+  // Return the underlying activity debug tag, or "<unknown>" if not available.
+  virtual std::string ActivityDebugTag() const = 0;
+
  protected:
   inline ~Wakeable() {}
 };
 
-namespace activity_detail {
+namespace promise_detail {
 struct Unwakeable final : public Wakeable {
   void Wakeup() override {}
   void Drop() override {}
+  std::string ActivityDebugTag() const override;
 };
 static Unwakeable* unwakeable() {
   return NoDestructSingleton<Unwakeable>::Get();
 }
-}  // namespace activity_detail
+}  // namespace promise_detail
 
 class AtomicWaker;
 
@@ -73,7 +80,7 @@ class AtomicWaker;
 class Waker {
  public:
   explicit Waker(Wakeable* wakeable) : wakeable_(wakeable) {}
-  Waker() : Waker(activity_detail::unwakeable()) {}
+  Waker() : Waker(promise_detail::unwakeable()) {}
   ~Waker() { wakeable_->Drop(); }
   Waker(const Waker&) = delete;
   Waker& operator=(const Waker&) = delete;
@@ -95,11 +102,15 @@ class Waker {
     return wakeable_ == other.wakeable_;
   }
 
+  std::string ActivityDebugTag() {
+    return wakeable_ == nullptr ? "<unknown>" : wakeable_->ActivityDebugTag();
+  }
+
  private:
   friend class AtomicWaker;
 
   Wakeable* Take() {
-    return std::exchange(wakeable_, activity_detail::unwakeable());
+    return std::exchange(wakeable_, promise_detail::unwakeable());
   }
 
   Wakeable* wakeable_;
@@ -109,7 +120,7 @@ class Waker {
 class AtomicWaker {
  public:
   explicit AtomicWaker(Wakeable* wakeable) : wakeable_(wakeable) {}
-  AtomicWaker() : AtomicWaker(activity_detail::unwakeable()) {}
+  AtomicWaker() : AtomicWaker(promise_detail::unwakeable()) {}
   explicit AtomicWaker(Waker waker) : AtomicWaker(waker.Take()) {}
   ~AtomicWaker() { wakeable_.load(std::memory_order_acquire)->Drop(); }
   AtomicWaker(const AtomicWaker&) = delete;
@@ -123,7 +134,7 @@ class AtomicWaker {
   // Return true if there is a not-unwakeable wakeable present.
   bool Armed() const noexcept {
     return wakeable_.load(std::memory_order_relaxed) !=
-           activity_detail::unwakeable();
+           promise_detail::unwakeable();
   }
 
   // Set to some new waker
@@ -133,7 +144,7 @@ class AtomicWaker {
 
  private:
   Wakeable* Take() {
-    return wakeable_.exchange(activity_detail::unwakeable(),
+    return wakeable_.exchange(promise_detail::unwakeable(),
                               std::memory_order_acq_rel);
   }
 
@@ -179,6 +190,9 @@ class Activity : public Orphanable {
   // delivered until long after the activity should be destroyed.
   virtual Waker MakeNonOwningWaker() = 0;
 
+  // Some descriptive text to add to log messages to identify this activity.
+  virtual std::string DebugTag() const;
+
  protected:
   // Check if this activity is the current activity executing on the current
   // thread.
@@ -203,7 +217,7 @@ class Activity : public Orphanable {
  private:
   // Set during RunLoop to the Activity that's executing.
   // Being set implies that mu_ is held.
-  static GPR_THREAD_LOCAL(Activity*) g_current_activity_;
+  static thread_local Activity* g_current_activity_;
 };
 
 // Owned pointer to one Activity.
@@ -262,7 +276,10 @@ class ActivityContexts : public ContextHolder<Contexts>... {
     explicit ScopedContext(ActivityContexts* contexts)
         : Context<ContextTypeFromHeld<Contexts>>(
               static_cast<ContextHolder<Contexts>*>(contexts)
-                  ->GetContext())... {}
+                  ->GetContext())... {
+      // Silence `unused-but-set-parameter` in case of Contexts = {}
+      (void)contexts;
+    }
   };
 };
 
@@ -330,6 +347,8 @@ class FreestandingActivity : public Activity, private Wakeable {
 
   Mutex* mu() ABSL_LOCK_RETURNED(mu_) { return &mu_; }
 
+  std::string ActivityDebugTag() const override { return DebugTag(); }
+
  private:
   class Handle;
 
@@ -386,7 +405,7 @@ template <class F, class WakeupScheduler, class OnDone, typename... Contexts>
 class PromiseActivity final : public FreestandingActivity,
                               private ActivityContexts<Contexts...> {
  public:
-  using Factory = PromiseFactory<void, F>;
+  using Factory = OncePromiseFactory<void, F>;
   using ResultType = typename Factory::Promise::Result;
 
   PromiseActivity(F promise_factory, WakeupScheduler wakeup_scheduler,
@@ -435,7 +454,11 @@ class PromiseActivity final : public FreestandingActivity,
       MutexLock lock(mu());
       // Check if we were done, and flag done.
       was_done = done_;
-      if (!done_) MarkDone();
+      if (!done_) {
+        ScopedActivity scoped_activity(this);
+        ScopedContext contexts(this);
+        MarkDone();
+      }
     }
     // If we were not done, then call the on_done callback.
     if (!was_done) {
@@ -472,8 +495,8 @@ class PromiseActivity final : public FreestandingActivity,
   // Notification that we're no longer executing - it's ok to destruct the
   // promise.
   void MarkDone() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
-    GPR_ASSERT(!done_);
-    done_ = true;
+    GPR_ASSERT(!absl::exchange(done_, true));
+    ScopedContext contexts(this);
     Destruct(&promise_holder_.promise);
   }
 
@@ -510,7 +533,7 @@ class PromiseActivity final : public FreestandingActivity,
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
     ScopedActivity scoped_activity(this);
     ScopedContext contexts(this);
-    Construct(&promise_holder_.promise, promise_factory.Once());
+    Construct(&promise_holder_.promise, promise_factory.Make());
     return StepLoop();
   }
 

data/src/core/lib/promise/arena_promise.h

@@ -19,11 +19,12 @@
 
 #include <stdlib.h>
 
-#include <new>
+#include <memory>
 #include <utility>
 
 #include "absl/meta/type_traits.h"
 
+#include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/resource_quota/arena.h"
@@ -32,20 +33,28 @@ namespace grpc_core {
 
 namespace arena_promise_detail {
 
+using ArgType = std::aligned_storage_t<sizeof(void*)>;
+template <typename T>
+T*& ArgAsPtr(ArgType* arg) {
+  static_assert(sizeof(ArgType) >= sizeof(T**),
+                "Must have ArgType of at least one pointer size");
+  return *reinterpret_cast<T**>(arg);
+}
+
 template <typename T>
 struct Vtable {
   // Poll the promise, once.
-  Poll<T> (*poll_once)(void** arg);
+  Poll<T> (*poll_once)(ArgType* arg);
   // Destroy the underlying callable object if there is one.
   // Since we don't delete (the arena owns the memory) but we may need to call a
   // destructor, we expose this for when the ArenaPromise object is destroyed.
-  void (*destroy)(void** arg);
+  void (*destroy)(ArgType* arg);
 };
 
 template <typename T>
 struct VtableAndArg {
   const Vtable<T>* vtable;
-  void* arg;
+  ArgType arg;
 };
 
 // Implementation of Vtable for an empty object.
@@ -53,37 +62,52 @@ struct VtableAndArg {
 // from. Since in either case these objects should not be polled, we simply
 // crash if it is.
 template <typename T>
-inline const Vtable<T>* null_impl() {
-  static const Vtable<T> vtable = {[](void**) -> Poll<T> {
-                                     abort();
-                                     GPR_UNREACHABLE_CODE(return Pending{});
-                                   },
-                                   [](void**) {}};
-  return &vtable;
-}
+struct Null {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType*) {
+    abort();
+    GPR_UNREACHABLE_CODE(return Pending{});
+  }
+
+  static void Destroy(ArgType*) {}
+};
+
+template <typename T>
+const Vtable<T> Null<T>::vtable = {PollOnce, Destroy};
 
 // Implementation of ImplInterface for a callable object.
 template <typename T, typename Callable>
-inline const Vtable<T>* allocated_callable_impl() {
-  static const Vtable<T> vtable = {
-      [](void** arg) -> Poll<T> {
-        return poll_cast<T>((*static_cast<Callable*>(*arg))());
-      },
-      [](void** arg) { static_cast<Callable*>(*arg)->~Callable(); }};
-  return &vtable;
-}
+struct AllocatedCallable {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*ArgAsPtr<Callable>(arg))());
+  }
+
+  static void Destroy(ArgType* arg) { Destruct(ArgAsPtr<Callable>(arg)); }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> AllocatedCallable<T, Callable>::vtable = {PollOnce, Destroy};
 
 // Implementation of ImplInterface for a small callable object (one that fits
-// within the void* arg)
+// within the ArgType arg)
 template <typename T, typename Callable>
-inline const Vtable<T>* inlined_callable_impl() {
-  static const Vtable<T> vtable = {
-      [](void** arg) -> Poll<T> {
-        return poll_cast<T>((*reinterpret_cast<Callable*>(arg))());
-      },
-      [](void** arg) { reinterpret_cast<Callable*>(arg)->~Callable(); }};
-  return &vtable;
-}
+struct Inlined {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return poll_cast<T>((*reinterpret_cast<Callable*>(arg))());
+  }
+
+  static void Destroy(ArgType* arg) {
+    Destruct(reinterpret_cast<Callable*>(arg));
+  }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> Inlined<T, Callable>::vtable = {PollOnce, Destroy};
 
 // If a callable object is empty we can substitute any instance of that callable
 // for the one we call (for how could we tell the difference)?
@@ -93,12 +117,17 @@ inline const Vtable<T>* inlined_callable_impl() {
 // (this comes up often when the promise only accesses context data from the
 // containing activity).
 template <typename T, typename Callable>
-inline const Vtable<T>* shared_callable_impl(Callable&& callable) {
-  static Callable instance = std::forward<Callable>(callable);
-  static const Vtable<T> vtable = {[](void**) -> Poll<T> { return instance(); },
-                                   [](void**) {}};
-  return &vtable;
-}
+struct SharedCallable {
+  static const Vtable<T> vtable;
+
+  static Poll<T> PollOnce(ArgType* arg) {
+    return (*reinterpret_cast<Callable*>(arg))();
+  }
+};
+
+template <typename T, typename Callable>
+const Vtable<T> SharedCallable<T, Callable>::vtable = {PollOnce,
+                                                       Null<T>::Destroy};
 
 // Redirector type: given a callable type, expose a Make() function that creates
 // the appropriate underlying implementation.
@@ -109,11 +138,11 @@ template <typename T, typename Callable>
 struct ChooseImplForCallable<
     T, Callable,
     absl::enable_if_t<!std::is_empty<Callable>::value &&
-                      (sizeof(Callable) > sizeof(void*))>> {
+                      (sizeof(Callable) > sizeof(ArgType))>> {
   static void Make(Callable&& callable, VtableAndArg<T>* out) {
-    *out = {allocated_callable_impl<T, Callable>(),
-            GetContext<Arena>()->template New<Callable>(
-                std::forward<Callable>(callable))};
+    out->vtable = &AllocatedCallable<T, Callable>::vtable;
+    ArgAsPtr<Callable>(&out->arg) = GetContext<Arena>()->template New<Callable>(
+        std::forward<Callable>(callable));
   }
 };
 
@@ -121,19 +150,19 @@ template <typename T, typename Callable>
 struct ChooseImplForCallable<
     T, Callable,
     absl::enable_if_t<!std::is_empty<Callable>::value &&
-                      (sizeof(Callable) <= sizeof(void*))>> {
+                      (sizeof(Callable) <= sizeof(ArgType))>> {
   static void Make(Callable&& callable, VtableAndArg<T>* out) {
-    out->vtable = inlined_callable_impl<T, Callable>();
-    new (&out->arg) Callable(std::forward<Callable>(callable));
+    out->vtable = &Inlined<T, Callable>::vtable;
+    Construct(reinterpret_cast<Callable*>(&out->arg),
+              std::forward<Callable>(callable));
   }
 };
 
 template <typename T, typename Callable>
 struct ChooseImplForCallable<
     T, Callable, absl::enable_if_t<std::is_empty<Callable>::value>> {
-  static void Make(Callable&& callable, VtableAndArg<T>* out) {
-    out->vtable =
-        shared_callable_impl<T, Callable>(std::forward<Callable>(callable));
+  static void Make(Callable&&, VtableAndArg<T>* out) {
+    out->vtable = &SharedCallable<T, Callable>::vtable;
   }
 };
 
@@ -159,8 +188,8 @@ class ArenaPromise {
                 absl::enable_if_t<!std::is_same<Callable, ArenaPromise>::value>>
   // NOLINTNEXTLINE(google-explicit-constructor)
   ArenaPromise(Callable&& callable) {
-    arena_promise_detail::MakeImplForCallable<T>(
-        std::forward<Callable>(callable), &vtable_and_arg_);
+    arena_promise_detail::MakeImplForCallable(std::forward<Callable>(callable),
+                                              &vtable_and_arg_);
   }
 
   // ArenaPromise is not copyable.
@@ -169,12 +198,12 @@ class ArenaPromise {
   // ArenaPromise is movable.
   ArenaPromise(ArenaPromise&& other) noexcept
       : vtable_and_arg_(other.vtable_and_arg_) {
-    other.vtable_and_arg_.vtable = arena_promise_detail::null_impl<T>();
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
   }
   ArenaPromise& operator=(ArenaPromise&& other) noexcept {
     vtable_and_arg_.vtable->destroy(&vtable_and_arg_.arg);
     vtable_and_arg_ = other.vtable_and_arg_;
-    other.vtable_and_arg_.vtable = arena_promise_detail::null_impl<T>();
+    other.vtable_and_arg_.vtable = &arena_promise_detail::Null<T>::vtable;
     return *this;
   }
 
@@ -187,13 +216,13 @@ class ArenaPromise {
   }
 
   bool has_value() const {
-    return vtable_and_arg_.vtable != arena_promise_detail::null_impl<T>();
+    return vtable_and_arg_.vtable != &arena_promise_detail::Null<T>::vtable;
   }
 
  private:
   // Underlying impl object.
   arena_promise_detail::VtableAndArg<T> vtable_and_arg_ = {
-      arena_promise_detail::null_impl<T>(), nullptr};
+      &arena_promise_detail::Null<T>::vtable, {}};
 };
 
 }  // namespace grpc_core

data/src/core/lib/promise/context.h

@@ -19,7 +19,7 @@
 
 #include <utility>
 
-#include "src/core/lib/gpr/tls.h"
+#include <grpc/support/log.h>
 
 namespace grpc_core {
 
@@ -44,12 +44,11 @@ class Context : public ContextType<T> {
 
  private:
   T* const old_;
-  static GPR_THREAD_LOCAL(T*) current_;
+  static thread_local T* current_;
 };
 
 template <typename T>
-GPR_THREAD_LOCAL(T*)
-Context<T>::current_;
+thread_local T* Context<T>::current_;
 
 template <typename T, typename F>
 class WithContext {
@@ -68,10 +67,18 @@ class WithContext {
 
 }  // namespace promise_detail
 
-// Retrieve the current value of a context.
+// Return true if a context of type T is currently active.
+template <typename T>
+bool HasContext() {
+  return promise_detail::Context<T>::get() != nullptr;
+}
+
+// Retrieve the current value of a context, or abort if the value is unset.
 template <typename T>
 T* GetContext() {
-  return promise_detail::Context<T>::get();
+  auto* p = promise_detail::Context<T>::get();
+  GPR_ASSERT(p != nullptr);
+  return p;
 }
 
 // Given a promise and a context, return a promise that has that context set.

data/src/core/lib/promise/detail/basic_seq.h

@@ -17,8 +17,6 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stddef.h>
-
 #include <array>
 #include <cassert>
 #include <new>
@@ -32,29 +30,12 @@
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/detail/promise_like.h"
+#include "src/core/lib/promise/detail/switch.h"
 #include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 namespace promise_detail {
 
-// Given f0, ..., fn, call function idx and return the result.
-template <typename R, typename A, R (*... f)(A* arg)>
-class JumpTable {
- public:
-  JumpTable() = delete;
-  JumpTable(const JumpTable&) = delete;
-
-  static R Run(size_t idx, A* arg) { return fs_[idx](arg); }
-
- private:
-  using Fn = R (*)(A* arg);
-  static const Fn fs_[sizeof...(f)];
-};
-
-template <typename R, typename A, R (*... f)(A* arg)>
-const typename JumpTable<R, A, f...>::Fn
-    JumpTable<R, A, f...>::fs_[sizeof...(f)] = {f...};
-
 // Helper for SeqState to evaluate some common types to all partial
 // specializations.
 template <template <typename> class Traits, typename FPromise, typename FNext>
@@ -68,7 +49,7 @@ struct SeqStateTypes {
   // Wrap the factory callable in our factory wrapper to deal with common edge
   // cases. We use the 'unwrapped type' from the traits, so for instance, TrySeq
   // can pass back a T from a StatusOr<T>.
-  using Next = promise_detail::PromiseFactory<
+  using Next = promise_detail::OncePromiseFactory<
       typename PromiseResultTraits::UnwrappedType, FNext>;
 };
 
@@ -353,14 +334,16 @@ class BasicSeq {
   // parameter unpacking can work.
   template <char I>
   struct RunStateStruct {
-    static Poll<Result> Run(BasicSeq* s) { return s->RunState<I>(); }
+    BasicSeq* s;
+    Poll<Result> operator()() { return s->RunState<I>(); }
   };
 
   // Similarly placate those compilers for
   // DestructCurrentPromiseAndSubsequentFactories
   template <char I>
   struct DestructCurrentPromiseAndSubsequentFactoriesStruct {
-    static void Run(BasicSeq* s) {
+    BasicSeq* s;
+    void operator()() {
       return s->DestructCurrentPromiseAndSubsequentFactories<I>();
     }
   };
@@ -373,8 +356,7 @@ class BasicSeq {
   // Duff's device like mechanic for evaluating sequences.
   template <char... I>
   Poll<Result> Run(absl::integer_sequence<char, I...>) {
-    return JumpTable<Poll<Result>, BasicSeq, RunStateStruct<I>::Run...>::Run(
-        state_, this);
+    return Switch<Poll<Result>>(state_, RunStateStruct<I>{this}...);
   }
 
   // Run the appropriate destructors for a given state.
@@ -384,9 +366,8 @@ class BasicSeq {
   // which can choose the correct instance at runtime to destroy everything.
   template <char... I>
   void RunDestruct(absl::integer_sequence<char, I...>) {
-    JumpTable<void, BasicSeq,
-              DestructCurrentPromiseAndSubsequentFactoriesStruct<I>::Run...>::
-        Run(state_, this);
+    Switch<void>(
+        state_, DestructCurrentPromiseAndSubsequentFactoriesStruct<I>{this}...);
   }
 
  public: