grpc 1.48.0 → 1.56.0

data/src/core/lib/promise/activity.cc

@@ -18,20 +18,32 @@
 
 #include <stddef.h>
 
+#include <initializer_list>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+
 #include "src/core/lib/gprpp/atomic_utils.h"
+#include "src/core/lib/gprpp/crash.h"
 
 namespace grpc_core {
 
 ///////////////////////////////////////////////////////////////////////////////
 // GLOBALS
 
-GPR_THREAD_LOCAL(Activity*) Activity::g_current_activity_{nullptr};
+thread_local Activity* Activity::g_current_activity_{nullptr};
 
 namespace promise_detail {
 
 ///////////////////////////////////////////////////////////////////////////////
 // HELPER TYPES
 
+std::string Unwakeable::ActivityDebugTag(WakeupMask) const {
+  return "<unknown>";
+}
+
 // Weak handle to an Activity.
 // Handle can persist while Activity goes away.
 class FreestandingActivity::Handle final : public Wakeable {
@@ -52,7 +64,7 @@ class FreestandingActivity::Handle final : public Wakeable {
 
   // Activity needs to wake up (if it still exists!) - wake it up, and drop the
   // ref that was kept for this handle.
-  void Wakeup() override ABSL_LOCKS_EXCLUDED(mu_) {
+  void Wakeup(WakeupMask) override ABSL_LOCKS_EXCLUDED(mu_) {
     mu_.Lock();
     // Note that activity refcount can drop to zero, but we could win the lock
     // against DropActivity, so we need to only increase activities refcount if
@@ -62,7 +74,7 @@ class FreestandingActivity::Handle final : public Wakeable {
       mu_.Unlock();
       // Activity still exists and we have a reference: wake it up, which will
       // drop the ref.
-      activity->Wakeup();
+      activity->Wakeup(0);
     } else {
       // Could not get the activity - it's either gone or going. No need to wake
       // it up!
@@ -72,7 +84,16 @@ class FreestandingActivity::Handle final : public Wakeable {
     Unref();
   }
 
-  void Drop() override { Unref(); }
+  void WakeupAsync(WakeupMask) override ABSL_LOCKS_EXCLUDED(mu_) {
+    Crash("not implemented");
+  }
+
+  void Drop(WakeupMask) override { Unref(); }
+
+  std::string ActivityDebugTag(WakeupMask) const override {
+    MutexLock lock(&mu_);
+    return activity_ == nullptr ? "<unknown>" : activity_->DebugTag();
+  }
 
  private:
   // Unref the Handle (not the activity).
@@ -85,7 +106,7 @@ class FreestandingActivity::Handle final : public Wakeable {
   // Two initial refs: one for the waiter that caused instantiation, one for the
   // activity.
   std::atomic<size_t> refs_{2};
-  Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_);
+  mutable Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_);
   FreestandingActivity* activity_ ABSL_GUARDED_BY(mu_);
 };
 
@@ -113,8 +134,24 @@ void FreestandingActivity::DropHandle() {
 
 Waker FreestandingActivity::MakeNonOwningWaker() {
   mu_.AssertHeld();
-  return Waker(RefHandle());
+  return Waker(RefHandle(), 0);
 }
 
 }  // namespace promise_detail
+
+std::string Activity::DebugTag() const {
+  return absl::StrFormat("ACTIVITY[%p]", this);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// INTRA ACTIVITY WAKER IMPLEMENTATION
+
+std::string IntraActivityWaiter::DebugString() const {
+  std::vector<int> bits;
+  for (size_t i = 0; i < 8 * sizeof(WakeupMask); i++) {
+    if (wakeups_ & (1 << i)) bits.push_back(i);
+  }
+  return absl::StrCat("{", absl::StrJoin(bits, ","), "}");
+}
+
 }  // namespace grpc_core

data/src/core/lib/promise/activity.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_PROMISE_ACTIVITY_H
-#define GRPC_CORE_LIB_PROMISE_ACTIVITY_H
+#ifndef GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H
+#define GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H
 
 #include <grpc/support/port_platform.h>
 
@@ -22,18 +22,18 @@
 #include <algorithm>
 #include <atomic>
 #include <memory>
+#include <string>
 #include <utility>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/status/status.h"
 #include "absl/types/optional.h"
-#include "absl/types/variant.h"
-#include "absl/utility/utility.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/gprpp/crash.h"
+#include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/promise/context.h"
@@ -43,54 +43,127 @@
 
 namespace grpc_core {
 
+class Activity;
+
+// WakeupMask is a bitfield representing which parts of an activity should be
+// woken up.
+using WakeupMask = uint16_t;
+
 // A Wakeable object is used by queues to wake activities.
 class Wakeable {
  public:
   // Wake up the underlying activity.
   // After calling, this Wakeable cannot be used again.
-  virtual void Wakeup() = 0;
+  // WakeupMask comes from the activity that created this Wakeable and specifies
+  // the set of promises that should be awoken.
+  virtual void Wakeup(WakeupMask wakeup_mask) = 0;
+  // Per Wakeup, but guarantee that the activity will be woken up out-of-line.
+  // Useful if there may be mutexes or the like held by the current thread.
+  virtual void WakeupAsync(WakeupMask wakeup_mask) = 0;
   // Drop this wakeable without waking up the underlying activity.
-  virtual void Drop() = 0;
+  virtual void Drop(WakeupMask wakeup_mask) = 0;
+
+  // Return the underlying activity debug tag, or "<unknown>" if not available.
+  virtual std::string ActivityDebugTag(WakeupMask wakeup_mask) const = 0;
 
  protected:
   inline ~Wakeable() {}
 };
 
+namespace promise_detail {
+struct Unwakeable final : public Wakeable {
+  void Wakeup(WakeupMask) override {}
+  void WakeupAsync(WakeupMask) override {}
+  void Drop(WakeupMask) override {}
+  std::string ActivityDebugTag(WakeupMask) const override;
+};
+static Unwakeable* unwakeable() {
+  return NoDestructSingleton<Unwakeable>::Get();
+}
+}  // namespace promise_detail
+
 // An owning reference to a Wakeable.
 // This type is non-copyable but movable.
 class Waker {
  public:
-  explicit Waker(Wakeable* wakeable) : wakeable_(wakeable) {}
-  Waker() : Waker(nullptr) {}
-  ~Waker() {
-    if (wakeable_ != nullptr) wakeable_->Drop();
-  }
+  Waker(Wakeable* wakeable, WakeupMask wakeup_mask)
+      : wakeable_and_arg_{wakeable, wakeup_mask} {}
+  Waker() : Waker(promise_detail::unwakeable(), 0) {}
+  ~Waker() { wakeable_and_arg_.Drop(); }
   Waker(const Waker&) = delete;
   Waker& operator=(const Waker&) = delete;
-  Waker(Waker&& other) noexcept : wakeable_(other.Take()) {}
+  Waker(Waker&& other) noexcept : wakeable_and_arg_(other.Take()) {}
   Waker& operator=(Waker&& other) noexcept {
-    std::swap(wakeable_, other.wakeable_);
+    std::swap(wakeable_and_arg_, other.wakeable_and_arg_);
     return *this;
   }
 
   // Wake the underlying activity.
-  void Wakeup() {
-    if (auto* wakeable = Take()) wakeable->Wakeup();
-  }
+  void Wakeup() { Take().Wakeup(); }
+
+  void WakeupAsync() { Take().WakeupAsync(); }
 
   template <typename H>
   friend H AbslHashValue(H h, const Waker& w) {
-    return H::combine(std::move(h), w.wakeable_);
+    return H::combine(H::combine(std::move(h), w.wakeable_and_arg_.wakeable),
+                      w.wakeable_and_arg_.wakeup_mask);
   }
 
   bool operator==(const Waker& other) const noexcept {
-    return wakeable_ == other.wakeable_;
+    return wakeable_and_arg_ == other.wakeable_and_arg_;
+  }
+
+  bool operator!=(const Waker& other) const noexcept {
+    return !operator==(other);
+  }
+
+  std::string ActivityDebugTag() {
+    return wakeable_and_arg_.ActivityDebugTag();
+  }
+
+  // This is for tests to assert that a waker is occupied or not.
+  bool is_unwakeable() const {
+    return wakeable_and_arg_.wakeable == promise_detail::unwakeable();
   }
 
  private:
-  Wakeable* Take() { return absl::exchange(wakeable_, nullptr); }
+  struct WakeableAndArg {
+    Wakeable* wakeable;
+    WakeupMask wakeup_mask;
+
+    void Wakeup() { wakeable->Wakeup(wakeup_mask); }
+    void WakeupAsync() { wakeable->WakeupAsync(wakeup_mask); }
+    void Drop() { wakeable->Drop(wakeup_mask); }
+    std::string ActivityDebugTag() const {
+      return wakeable == nullptr ? "<unknown>"
+                                 : wakeable->ActivityDebugTag(wakeup_mask);
+    }
+    bool operator==(const WakeableAndArg& other) const noexcept {
+      return wakeable == other.wakeable && wakeup_mask == other.wakeup_mask;
+    }
+  };
 
-  Wakeable* wakeable_;
+  WakeableAndArg Take() {
+    return std::exchange(wakeable_and_arg_, {promise_detail::unwakeable(), 0});
+  }
+
+  WakeableAndArg wakeable_and_arg_;
+};
+
+// Helper type to track wakeups between objects in the same activity.
+// Can be fairly fast as no ref counting or locking needs to occur.
+class IntraActivityWaiter {
+ public:
+  // Register for wakeup, return Pending(). If state is not ready to proceed,
+  // Promises should bottom out here.
+  Pending pending();
+  // Wake the activity
+  void Wake();
+
+  std::string DebugString() const;
+
+ private:
+  WakeupMask wakeups_ = 0;
 };
 
 // An Activity tracks execution of a single promise.
@@ -112,7 +185,13 @@ class Activity : public Orphanable {
   void ForceWakeup() { MakeOwningWaker().Wakeup(); }
 
   // Force the current activity to immediately repoll if it doesn't complete.
-  virtual void ForceImmediateRepoll() = 0;
+  virtual void ForceImmediateRepoll(WakeupMask mask) = 0;
+  // Legacy version of ForceImmediateRepoll() that uses the current participant.
+  // Will go away once Party gets merged with Activity. New usage is banned.
+  void ForceImmediateRepoll() { ForceImmediateRepoll(CurrentParticipant()); }
+
+  // Return the current part of the activity as a bitmask
+  virtual WakeupMask CurrentParticipant() const { return 1; }
 
   // Return the current activity.
   // Additionally:
@@ -132,6 +211,9 @@ class Activity : public Orphanable {
   // delivered until long after the activity should be destroyed.
   virtual Waker MakeNonOwningWaker() = 0;
 
+  // Some descriptive text to add to log messages to identify this activity.
+  virtual std::string DebugTag() const;
+
  protected:
   // Check if this activity is the current activity executing on the current
   // thread.
@@ -156,7 +238,7 @@ class Activity : public Orphanable {
  private:
   // Set during RunLoop to the Activity that's executing.
   // Being set implies that mu_ is held.
-  static GPR_THREAD_LOCAL(Activity*) g_current_activity_;
+  static thread_local Activity* g_current_activity_;
 };
 
 // Owned pointer to one Activity.
@@ -215,7 +297,10 @@ class ActivityContexts : public ContextHolder<Contexts>... {
     explicit ScopedContext(ActivityContexts* contexts)
         : Context<ContextTypeFromHeld<Contexts>>(
               static_cast<ContextHolder<Contexts>*>(contexts)
-                  ->GetContext())... {}
+                  ->GetContext())... {
+      // Silence `unused-but-set-parameter` in case of Contexts = {}
+      (void)contexts;
+    }
   };
 };
 
@@ -234,7 +319,7 @@ class FreestandingActivity : public Activity, private Wakeable {
  public:
   Waker MakeOwningWaker() final {
     Ref();
-    return Waker(this);
+    return Waker(this, 0);
   }
   Waker MakeNonOwningWaker() final;
 
@@ -243,7 +328,7 @@ class FreestandingActivity : public Activity, private Wakeable {
     Unref();
   }
 
-  void ForceImmediateRepoll() final {
+  void ForceImmediateRepoll(WakeupMask) final {
     mu_.AssertHeld();
     SetActionDuringRun(ActionDuringRun::kWakeup);
   }
@@ -267,7 +352,7 @@ class FreestandingActivity : public Activity, private Wakeable {
   // Check if we got an internal wakeup since the last time this function was
   // called.
   ActionDuringRun GotActionDuringRun() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-    return absl::exchange(action_during_run_, ActionDuringRun::kNone);
+    return std::exchange(action_during_run_, ActionDuringRun::kNone);
   }
 
   // Implementors of Wakeable::Wakeup should call this after the wakeup has
@@ -283,6 +368,8 @@ class FreestandingActivity : public Activity, private Wakeable {
 
   Mutex* mu() ABSL_LOCK_RETURNED(mu_) { return &mu_; }
 
+  std::string ActivityDebugTag(WakeupMask) const override { return DebugTag(); }
+
  private:
   class Handle;
 
@@ -321,32 +408,42 @@ class FreestandingActivity : public Activity, private Wakeable {
 };
 
 // Implementation details for an Activity of an arbitrary type of promise.
-// There should exist a static function:
+// There should exist an inner template class `BoundScheduler` that provides
+// the following interface:
 // struct WakeupScheduler {
 //   template <typename ActivityType>
-//   void ScheduleWakeup(ActivityType* activity);
+//   class BoundScheduler {
+//    public:
+//     BoundScheduler(WakeupScheduler);
+//     void ScheduleWakeup();
+//   };
 // };
-// This function should arrange that activity->RunScheduledWakeup() be invoked
-// at the earliest opportunity.
+// The ScheduleWakeup function should arrange that
+// static_cast<ActivityType*>(this)->RunScheduledWakeup() be invoked at the
+// earliest opportunity.
 // It can assume that activity will remain live until RunScheduledWakeup() is
 // invoked, and that a given activity will not be concurrently scheduled again
 // until its RunScheduledWakeup() has been invoked.
-// We use private inheritance here as a way of getting private members for
-// each of the contexts.
+// We use private inheritance here as a way of getting private members for each
+// of the contexts.
 // TODO(ctiller): We can probably reconsider the private inheritance here
 // when we move away from C++11 and have more powerful template features.
 template <class F, class WakeupScheduler, class OnDone, typename... Contexts>
-class PromiseActivity final : public FreestandingActivity,
-                              private ActivityContexts<Contexts...> {
+class PromiseActivity final
+    : public FreestandingActivity,
+      public WakeupScheduler::template BoundScheduler<
+          PromiseActivity<F, WakeupScheduler, OnDone, Contexts...>>,
+      private ActivityContexts<Contexts...> {
  public:
-  using Factory = PromiseFactory<void, F>;
+  using Factory = OncePromiseFactory<void, F>;
   using ResultType = typename Factory::Promise::Result;
 
   PromiseActivity(F promise_factory, WakeupScheduler wakeup_scheduler,
                   OnDone on_done, Contexts&&... contexts)
       : FreestandingActivity(),
+        WakeupScheduler::template BoundScheduler<PromiseActivity>(
+            std::move(wakeup_scheduler)),
         ActivityContexts<Contexts...>(std::forward<Contexts>(contexts)...),
-        wakeup_scheduler_(std::move(wakeup_scheduler)),
         on_done_(std::move(on_done)) {
     // Lock, construct an initial promise from the factory, and step it.
     // This may hit a waiter, which could expose our this pointer to other
@@ -388,7 +485,11 @@ class PromiseActivity final : public FreestandingActivity,
       MutexLock lock(mu());
       // Check if we were done, and flag done.
       was_done = done_;
-      if (!done_) MarkDone();
+      if (!done_) {
+        ScopedActivity scoped_activity(this);
+        ScopedContext contexts(this);
+        MarkDone();
+      }
     }
     // If we were not done, then call the on_done callback.
     if (!was_done) {
@@ -401,7 +502,7 @@ class PromiseActivity final : public FreestandingActivity,
   // the activity to an external threadpool to run. If the activity is already
   // running on this thread, a note is taken of such and the activity is
   // repolled if it doesn't complete.
-  void Wakeup() final {
+  void Wakeup(WakeupMask) final {
     // If there is an active activity, but hey it's us, flag that and we'll loop
     // in RunLoop (that's calling from above here!).
     if (Activity::is_current()) {
@@ -412,21 +513,23 @@ class PromiseActivity final : public FreestandingActivity,
     }
     if (!wakeup_scheduled_.exchange(true, std::memory_order_acq_rel)) {
       // Can't safely run, so ask to run later.
-      wakeup_scheduler_.ScheduleWakeup(this);
+      this->ScheduleWakeup();
     } else {
       // Already a wakeup scheduled for later, drop ref.
       WakeupComplete();
     }
   }
 
+  void WakeupAsync(WakeupMask) final { Crash("not implemented"); }
+
   // Drop a wakeup
-  void Drop() final { this->WakeupComplete(); }
+  void Drop(WakeupMask) final { this->WakeupComplete(); }
 
   // Notification that we're no longer executing - it's ok to destruct the
   // promise.
   void MarkDone() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
-    GPR_ASSERT(!done_);
-    done_ = true;
+    GPR_ASSERT(!std::exchange(done_, true));
+    ScopedContext contexts(this);
     Destruct(&promise_holder_.promise);
   }
 
@@ -463,7 +566,7 @@ class PromiseActivity final : public FreestandingActivity,
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) {
     ScopedActivity scoped_activity(this);
     ScopedContext contexts(this);
-    Construct(&promise_holder_.promise, promise_factory.Once());
+    Construct(&promise_holder_.promise, promise_factory.Make());
     return StepLoop();
   }
 
@@ -475,7 +578,7 @@ class PromiseActivity final : public FreestandingActivity,
       // Run the promise.
       GPR_ASSERT(!done_);
       auto r = promise_holder_.promise();
-      if (auto* status = absl::get_if<kPollReadyIdx>(&r)) {
+      if (auto* status = r.value_if_ready()) {
         // If complete, destroy the promise, flag done, and exit this loop.
         MarkDone();
         return IntoStatus(status);
@@ -494,8 +597,6 @@ class PromiseActivity final : public FreestandingActivity,
   }
 
   using Promise = typename Factory::Promise;
-  // Scheduler for wakeups
-  GPR_NO_UNIQUE_ADDRESS WakeupScheduler wakeup_scheduler_;
   // Callback on completion of the promise.
   GPR_NO_UNIQUE_ADDRESS OnDone on_done_;
   // Has execution completed?
@@ -529,6 +630,16 @@ ActivityPtr MakeActivity(Factory promise_factory,
           std::move(on_done), std::forward<Contexts>(contexts)...));
 }
 
+inline Pending IntraActivityWaiter::pending() {
+  wakeups_ |= Activity::current()->CurrentParticipant();
+  return Pending();
+}
+
+inline void IntraActivityWaiter::Wake() {
+  if (wakeups_ == 0) return;
+  Activity::current()->ForceImmediateRepoll(std::exchange(wakeups_, 0));
+}
+
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_PROMISE_ACTIVITY_H
+#endif  // GRPC_SRC_CORE_LIB_PROMISE_ACTIVITY_H