grpc 1.65.2 → 1.66.0.pre3

data/src/core/lib/promise/party.cc

@@ -18,14 +18,16 @@
 
 #include "absl/base/thread_annotations.h"
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/strings/str_format.h"
 
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/event_engine/event_engine_context.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/promise/activity.h"
+#include "src/core/util/latent_see.h"
 
 #ifdef GRPC_MAXIMIZE_THREADYNESS
 #include "src/core/lib/gprpp/thd.h"       // IWYU pragma: keep
@@ -34,11 +36,6 @@
 
 namespace grpc_core {
 
-namespace {
-// TODO(ctiller): Once all activities are parties we can remove this.
-thread_local Party** g_current_party_run_next = nullptr;
-}  // namespace
-
 ///////////////////////////////////////////////////////////////////////////////
 // PartySyncUsingAtomics
 
@@ -178,7 +175,9 @@ Party::Participant::~Participant() {
 Party::~Party() {}
 
 void Party::CancelRemainingParticipants() {
+  if (!sync_.has_participants()) return;
   ScopedActivity activity(this);
+  promise_detail::Context<Arena> arena_ctx(arena_.get());
   for (size_t i = 0; i < party_detail::kMaxParticipants; i++) {
     if (auto* p =
             participants_[i].exchange(nullptr, std::memory_order_acquire)) {
@@ -210,77 +209,92 @@ void Party::ForceImmediateRepoll(WakeupMask mask) {
   sync_.ForceImmediateRepoll(mask);
 }
 
-void Party::RunLocked() {
-  // If there is a party running, then we don't run it immediately
-  // but instead add it to the end of the list of parties to run.
-  // This enables a fairly straightforward batching of work from a
-  // call to a transport (or back again).
-  if (g_current_party_run_next != nullptr) {
-    if (*g_current_party_run_next == nullptr) {
-      *g_current_party_run_next = this;
-    } else {
-      // But if there's already a party queued, we're better off asking event
-      // engine to run it so we can spread load.
-      event_engine()->Run([this]() {
-        ApplicationCallbackExecCtx app_exec_ctx;
-        ExecCtx exec_ctx;
-        RunLocked();
-      });
-    }
-    return;
-  }
-  auto body = [this]() {
-    DCHECK_EQ(g_current_party_run_next, nullptr);
-    Party* run_next = nullptr;
-    g_current_party_run_next = &run_next;
-    const bool done = RunParty();
-    DCHECK(g_current_party_run_next == &run_next);
-    g_current_party_run_next = nullptr;
-    if (done) {
-      ScopedActivity activity(this);
-      PartyOver();
-    }
-    if (run_next != nullptr) {
-      run_next->RunLocked();
-    }
-  };
+void Party::RunLocked(Party* party) {
+  GRPC_LATENT_SEE_PARENT_SCOPE("Party::RunLocked");
 #ifdef GRPC_MAXIMIZE_THREADYNESS
   Thread thd(
       "RunParty",
-      [body]() {
+      [party]() {
         ApplicationCallbackExecCtx app_exec_ctx;
         ExecCtx exec_ctx;
-        body();
+        if (party->RunParty()) party->PartyIsOver();
       },
       nullptr, Thread::Options().set_joinable(false));
   thd.Start();
 #else
-  body();
+  struct RunState;
+  static thread_local RunState* g_run_state = nullptr;
+  struct RunState {
+    explicit RunState(Party* party) : running(party), next(nullptr) {}
+    Party* running;
+    Party* next;
+    void Run() {
+      g_run_state = this;
+      do {
+        GRPC_LATENT_SEE_INNER_SCOPE("run_one_party");
+        if (running->RunParty()) {
+          running->PartyIsOver();
+        }
+        running = std::exchange(next, nullptr);
+      } while (running != nullptr);
+      DCHECK(g_run_state == this);
+      g_run_state = nullptr;
+    }
+  };
+  // If there is a party running, then we don't run it immediately
+  // but instead add it to the end of the list of parties to run.
+  // This enables a fairly straightforward batching of work from a
+  // call to a transport (or back again).
+  if (g_run_state != nullptr) {
+    if (g_run_state->running == party || g_run_state->next == party) {
+      // Already running or already queued.
+      return;
+    }
+    if (g_run_state->next != nullptr) {
+      // If there's already a different party queued, we're better off asking
+      // event engine to run it so we can spread load.
+      // We swap the oldest party to run on the event engine so that we don't
+      // accidentally end up with a tail latency problem whereby one party
+      // gets held for a really long time.
+      std::swap(g_run_state->next, party);
+      party->arena_->GetContext<grpc_event_engine::experimental::EventEngine>()
+          ->Run([party]() {
+            GRPC_LATENT_SEE_PARENT_SCOPE("Party::RunLocked offload");
+            ApplicationCallbackExecCtx app_exec_ctx;
+            ExecCtx exec_ctx;
+            RunState{party}.Run();
+          });
+      return;
+    }
+    g_run_state->next = party;
+    return;
+  }
+  RunState{party}.Run();
 #endif
 }
 
 bool Party::RunParty() {
   ScopedActivity activity(this);
+  promise_detail::Context<Arena> arena_ctx(arena_.get());
   return sync_.RunParty([this](int i) { return RunOneParticipant(i); });
 }
 
 bool Party::RunOneParticipant(int i) {
+  GRPC_LATENT_SEE_INNER_SCOPE("Party::RunOneParticipant");
   // If the participant is null, skip.
   // This allows participants to complete whilst wakers still exist
   // somewhere.
   auto* participant = participants_[i].load(std::memory_order_acquire);
   if (participant == nullptr) {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%s[party] wakeup %d already complete",
-              DebugTag().c_str(), i);
+      LOG(INFO) << DebugTag() << "[party] wakeup " << i << " already complete";
     }
     return false;
   }
   absl::string_view name;
   if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
     name = participant->name();
-    gpr_log(GPR_INFO, "%s[%s] begin job %d", DebugTag().c_str(),
-            std::string(name).c_str(), i);
+    LOG(INFO) << DebugTag() << "[" << name << "] begin job " << i;
   }
   // Poll the participant.
   currently_polling_ = i;
@@ -304,32 +318,31 @@ void Party::AddParticipants(Participant** participants, size_t count) {
                                                        count](size_t* slots) {
     for (size_t i = 0; i < count; i++) {
       if (GRPC_TRACE_FLAG_ENABLED(party_state)) {
-        gpr_log(GPR_INFO,
-                "Party %p                 AddParticipant: %s @ %" PRIdPTR
-                " [participant=%p]",
-                &sync_, std::string(participants[i]->name()).c_str(), slots[i],
-                participants[i]);
+        LOG(INFO) << "Party " << &sync_ << "                 AddParticipant: "
+                  << participants[i]->name() << " @ " << slots[i]
+                  << " [participant=" << participants[i] << "]";
       }
       participants_[slots[i]].store(participants[i], std::memory_order_release);
     }
   });
-  if (run_party) RunLocked();
+  if (run_party) RunLocked(this);
   Unref();
 }
 
 void Party::Wakeup(WakeupMask wakeup_mask) {
-  if (sync_.ScheduleWakeup(wakeup_mask)) RunLocked();
+  if (sync_.ScheduleWakeup(wakeup_mask)) RunLocked(this);
   Unref();
 }
 
 void Party::WakeupAsync(WakeupMask wakeup_mask) {
   if (sync_.ScheduleWakeup(wakeup_mask)) {
-    event_engine()->Run([this]() {
-      ApplicationCallbackExecCtx app_exec_ctx;
-      ExecCtx exec_ctx;
-      RunLocked();
-      Unref();
-    });
+    arena_->GetContext<grpc_event_engine::experimental::EventEngine>()->Run(
+        [this]() {
+          ApplicationCallbackExecCtx app_exec_ctx;
+          ExecCtx exec_ctx;
+          RunLocked(this);
+          Unref();
+        });
   } else {
     Unref();
   }
@@ -338,8 +351,14 @@ void Party::WakeupAsync(WakeupMask wakeup_mask) {
 void Party::Drop(WakeupMask) { Unref(); }
 
 void Party::PartyIsOver() {
-  ScopedActivity activity(this);
-  PartyOver();
+  auto arena = arena_;
+  {
+    ScopedActivity activity(this);
+    promise_detail::Context<Arena> arena_ctx(arena_.get());
+    CancelRemainingParticipants();
+    arena->DestroyManagedNewObjects();
+  }
+  this->~Party();
 }
 
 }  // namespace grpc_core

data/src/core/lib/promise/party.h

@@ -25,10 +25,10 @@
 #include "absl/base/attributes.h"
 #include "absl/base/thread_annotations.h"
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/strings/string_view.h"
 
 #include <grpc/event_engine/event_engine.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/debug/trace.h"
@@ -41,6 +41,8 @@
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/poll.h"
+#include "src/core/lib/resource_quota/arena.h"
+#include "src/core/util/useful.h"
 
 // Two implementations of party synchronization are provided: one using a single
 // atomic, the other using a mutex and a set of state variables.
@@ -83,7 +85,7 @@ class PartySyncUsingAtomics {
   }
   GRPC_MUST_USE_RESULT bool RefIfNonZero();
   // Returns true if the ref count is now zero and the caller should call
-  // PartyOver
+  // PartyIsOver
   GRPC_MUST_USE_RESULT bool Unref() {
     const uint64_t prev_state =
         state_.fetch_sub(kOneRef, std::memory_order_acq_rel);
@@ -106,31 +108,28 @@ class PartySyncUsingAtomics {
   // participant completed and should be removed from the allocated set.
   template <typename F>
   GRPC_MUST_USE_RESULT bool RunParty(F poll_one_participant) {
-    uint64_t prev_state;
-    iteration_.fetch_add(1, std::memory_order_relaxed);
+    // Grab the current state, and clear the wakeup bits & add flag.
+    uint64_t prev_state = state_.fetch_and(kRefMask | kLocked | kAllocatedMask,
+                                           std::memory_order_acquire);
+    LogStateChange("Run", prev_state,
+                   prev_state & (kRefMask | kLocked | kAllocatedMask));
+    CHECK(prev_state & kLocked);
+    if (prev_state & kDestroying) return true;
+    // From the previous state, extract which participants we're to wakeup.
+    uint64_t wakeups = prev_state & kWakeupMask;
+    // Now update prev_state to be what we want the CAS to see below.
+    prev_state &= kRefMask | kLocked | kAllocatedMask;
     for (;;) {
-      // Grab the current state, and clear the wakeup bits & add flag.
-      prev_state = state_.fetch_and(kRefMask | kLocked | kAllocatedMask,
-                                    std::memory_order_acquire);
-      LogStateChange("Run", prev_state,
-                     prev_state & (kRefMask | kLocked | kAllocatedMask));
-      CHECK(prev_state & kLocked);
-      if (prev_state & kDestroying) return true;
-      // From the previous state, extract which participants we're to wakeup.
-      uint64_t wakeups = prev_state & kWakeupMask;
-      // Now update prev_state to be what we want the CAS to see below.
-      prev_state &= kRefMask | kLocked | kAllocatedMask;
+      uint64_t keep_allocated_mask = kAllocatedMask;
       // For each wakeup bit...
-      for (size_t i = 0; wakeups != 0; i++, wakeups >>= 1) {
+      while (wakeups != 0) {
+        uint64_t t = LowestOneBit(wakeups);
+        const int i = CountTrailingZeros(t);
+        wakeups ^= t;
         // If the bit is not set, skip.
-        if ((wakeups & 1) == 0) continue;
         if (poll_one_participant(i)) {
           const uint64_t allocated_bit = (1u << i << kAllocatedShift);
-          prev_state &= ~allocated_bit;
-          uint64_t finished_prev_state =
-              state_.fetch_and(~allocated_bit, std::memory_order_release);
-          LogStateChange("Run:ParticipantComplete", finished_prev_state,
-                         finished_prev_state & ~allocated_bit);
+          keep_allocated_mask &= ~allocated_bit;
         }
       }
       // Try to CAS the state we expected to have (with no wakeups or adds)
@@ -144,26 +143,27 @@ class PartySyncUsingAtomics {
       // TODO(ctiller): consider mitigations for the accidental wakeup on owning
       // waker creation case -- I currently expect this will be more expensive
       // than this quick loop.
-      if (wake_after_poll_ == 0) {
-        if (state_.compare_exchange_weak(
-                prev_state, (prev_state & (kRefMask | kAllocatedMask)),
-                std::memory_order_acq_rel, std::memory_order_acquire)) {
-          LogStateChange("Run:End", prev_state,
-                         prev_state & (kRefMask | kAllocatedMask));
-          return false;
-        }
-      } else {
-        if (state_.compare_exchange_weak(
-                prev_state,
-                (prev_state & (kRefMask | kAllocatedMask | kLocked)) |
-                    wake_after_poll_,
-                std::memory_order_acq_rel, std::memory_order_acquire)) {
-          LogStateChange("Run:EndIteration", prev_state,
-                         prev_state & (kRefMask | kAllocatedMask));
-          iteration_.fetch_add(1, std::memory_order_relaxed);
-          wake_after_poll_ = 0;
-        }
+      if (state_.compare_exchange_weak(
+              prev_state, (prev_state & (kRefMask | keep_allocated_mask)),
+              std::memory_order_acq_rel, std::memory_order_acquire)) {
+        LogStateChange("Run:End", prev_state,
+                       prev_state & (kRefMask | kAllocatedMask));
+        return false;
+      }
+      while (!state_.compare_exchange_weak(
+          prev_state,
+          prev_state & (kRefMask | kLocked | keep_allocated_mask))) {
+        // Nothing to do here.
       }
+      LogStateChange("Run:Continue", prev_state,
+                     prev_state & (kRefMask | kLocked | keep_allocated_mask));
+      CHECK(prev_state & kLocked);
+      if (prev_state & kDestroying) return true;
+      // From the previous state, extract which participants we're to wakeup.
+      wakeups = prev_state & kWakeupMask;
+      // Now update prev_state to be what we want the CAS to see once wakeups
+      // complete next iteration.
+      prev_state &= kRefMask | kLocked | keep_allocated_mask;
     }
     return false;
   }
@@ -186,15 +186,12 @@ class PartySyncUsingAtomics {
     do {
       wakeup_mask = 0;
       allocated = (state & kAllocatedMask) >> kAllocatedShift;
-      size_t n = 0;
-      for (size_t bit = 0; n < count && bit < party_detail::kMaxParticipants;
-           bit++) {
-        if (allocated & (1 << bit)) continue;
-        wakeup_mask |= (1 << bit);
-        slots[n++] = bit;
-        allocated |= 1 << bit;
+      for (size_t i = 0; i < count; i++) {
+        auto new_mask = LowestOneBit(~allocated);
+        wakeup_mask |= new_mask;
+        allocated |= new_mask;
+        slots[i] = CountTrailingZeros(new_mask);
       }
-      CHECK(n == count);
       // Try to allocate this slot and take a ref (atomically).
       // Ref needs to be taken because once we store the participant it could be
       // spuriously woken up and unref the party.
@@ -219,9 +216,8 @@ class PartySyncUsingAtomics {
   // Returns true if the caller should run the party.
   GRPC_MUST_USE_RESULT bool ScheduleWakeup(WakeupMask mask);
 
-  void WakeAfterPoll(WakeupMask mask) { wake_after_poll_ |= mask; }
-  uint32_t iteration() const {
-    return iteration_.load(std::memory_order_relaxed);
+  bool has_participants() const {
+    return (state_.load(std::memory_order_relaxed) & kAllocatedMask) != 0;
   }
 
  private:
@@ -230,9 +226,9 @@ class PartySyncUsingAtomics {
   void LogStateChange(const char* op, uint64_t prev_state, uint64_t new_state,
                       DebugLocation loc = {}) {
     if (GRPC_TRACE_FLAG_ENABLED(party_state)) {
-      gpr_log(loc.file(), loc.line(), GPR_LOG_SEVERITY_INFO,
-              "Party %p %30s: %016" PRIx64 " -> %016" PRIx64, this, op,
-              prev_state, new_state);
+      LOG(INFO).AtLocation(loc.file(), loc.line())
+          << absl::StrFormat("Party %p %30s: %016" PRIx64 " -> %016" PRIx64,
+                             this, op, prev_state, new_state);
     }
   }
 
@@ -272,8 +268,6 @@ class PartySyncUsingAtomics {
   static constexpr uint64_t kOneRef = 1ull << kRefShift;
 
   std::atomic<uint64_t> state_;
-  std::atomic<uint32_t> iteration_{0};
-  WakeupMask wake_after_poll_ = 0;
 };
 
 class PartySyncUsingMutex {
@@ -371,6 +365,11 @@ class Party : public Activity, private Wakeable {
   Party(const Party&) = delete;
   Party& operator=(const Party&) = delete;
 
+  static RefCountedPtr<Party> Make(RefCountedPtr<Arena> arena) {
+    auto* arena_ptr = arena.get();
+    return RefCountedPtr<Party>(arena_ptr->New<Party>(std::move(arena)));
+  }
+
   // Spawn one promise into the party.
   // The promise will be polled until it is resolved, or until the party is shut
   // down.
@@ -410,19 +409,7 @@ class Party : public Activity, private Wakeable {
     return RefCountedPtr<Party>(this);
   }
 
-  // Return a promise that resolves to Empty{} when the current party poll is
-  // complete.
-  // This is useful for implementing batching and the like: we can hold some
-  // action until the rest of the party resolves itself.
-  auto AfterCurrentPoll() {
-    DCHECK(GetContext<Activity>() == this);
-    sync_.WakeAfterPoll(CurrentParticipant());
-    return [this, iteration = sync_.iteration()]() -> Poll<Empty> {
-      DCHECK(GetContext<Activity>() == this);
-      if (iteration == sync_.iteration()) return Pending{};
-      return Empty{};
-    };
-  }
+  Arena* arena() { return arena_.get(); }
 
   class BulkSpawner {
    public:
@@ -442,26 +429,21 @@ class Party : public Activity, private Wakeable {
   };
 
  protected:
-  explicit Party(size_t initial_refs) : sync_(initial_refs) {}
+  friend class Arena;
+
+  // Derived types should be constructed upon `arena`.
+  explicit Party(RefCountedPtr<Arena> arena)
+      : sync_(1), arena_(std::move(arena)) {}
   ~Party() override;
 
   // Main run loop. Must be locked.
   // Polls participants and drains the add queue until there is no work left to
   // be done.
-  // Derived types will likely want to override this to set up their
-  // contexts before polling.
-  // Should not be called by derived types except as a tail call to the base
-  // class RunParty when overriding this method to add custom context.
   // Returns true if the party is over.
-  GRPC_MUST_USE_RESULT virtual bool RunParty();
+  GRPC_MUST_USE_RESULT bool RunParty();
 
   bool RefIfNonZero() { return sync_.RefIfNonZero(); }
 
-  // Destroy any remaining participants.
-  // Should be called by derived types in response to PartyOver.
-  // Needs to have normal context setup before calling.
-  void CancelRemainingParticipants();
-
  private:
   // Concrete implementation of a participant for some promise & oncomplete
   // type.
@@ -594,13 +576,12 @@ class Party : public Activity, private Wakeable {
     std::atomic<State> state_{State::kFactory};
   };
 
-  // Notification that the party has finished and this instance can be deleted.
-  // Derived types should arrange to call CancelRemainingParticipants during
-  // this sequence.
-  virtual void PartyOver() = 0;
+  // Destroy any remaining participants.
+  // Needs to have normal context setup before calling.
+  void CancelRemainingParticipants();
 
   // Run the locked part of the party until it is unlocked.
-  void RunLocked();
+  static void RunLocked(Party* party);
   // Called in response to Unref() hitting zero - ultimately calls PartyOver,
   // but needs to set some stuff up.
   // Here so it gets compiled out of line.
@@ -615,9 +596,6 @@ class Party : public Activity, private Wakeable {
   void AddParticipants(Participant** participant, size_t count);
   bool RunOneParticipant(int i);
 
-  virtual grpc_event_engine::experimental::EventEngine* event_engine()
-      const = 0;
-
   // Sentinal value for currently_polling_ when no participant is being polled.
   static constexpr uint8_t kNotPolling = 255;
 
@@ -634,6 +612,7 @@ class Party : public Activity, private Wakeable {
   // If the lower bit is unset, then this is a Participant*.
   // If the lower bit is set, then this is a ParticipantFactory*.
   std::atomic<Participant*> participants_[party_detail::kMaxParticipants] = {};
+  RefCountedPtr<Arena> arena_;
 };
 
 template <>

data/src/core/lib/promise/pipe.h

@@ -28,7 +28,6 @@
 #include "absl/types/optional.h"
 #include "absl/types/variant.h"
 
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/debug_location.h"
@@ -636,8 +635,8 @@ class Push {
   Poll<bool> operator()() {
     if (center_ == nullptr) {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_DEBUG, "%s Pipe push has a null center",
-                GetContext<Activity>()->DebugTag().c_str());
+        VLOG(2) << GetContext<Activity>()->DebugTag()
+                << " Pipe push has a null center";
       }
       return false;
     }