grpc 1.56.0 → 1.61.0

data/src/core/lib/promise/detail/status.h

@@ -45,7 +45,12 @@ inline absl::Status IntoStatus(absl::Status* status) {
 // can participate in TrySeq as result types that affect control flow.
 inline bool IsStatusOk(const absl::Status& status) { return status.ok(); }
 
-template <typename To, typename From>
+template <typename T>
+inline bool IsStatusOk(const absl::StatusOr<T>& status) {
+  return status.ok();
+}
+
+template <typename To, typename From, typename SfinaeVoid = void>
 struct StatusCastImpl;
 
 template <typename To>
@@ -59,20 +64,52 @@ struct StatusCastImpl<To, const To&> {
 };
 
 template <typename T>
-struct StatusCastImpl<absl::StatusOr<T>, absl::Status> {
-  static absl::StatusOr<T> Cast(absl::Status&& t) { return std::move(t); }
+struct StatusCastImpl<absl::Status, absl::StatusOr<T>> {
+  static absl::Status Cast(absl::StatusOr<T>&& t) {
+    return std::move(t.status());
+  }
 };
 
 template <typename T>
-struct StatusCastImpl<absl::StatusOr<T>, const absl::Status&> {
-  static absl::StatusOr<T> Cast(const absl::Status& t) { return t; }
+struct StatusCastImpl<absl::Status, absl::StatusOr<T>&> {
+  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
 };
 
+template <typename T>
+struct StatusCastImpl<absl::Status, const absl::StatusOr<T>&> {
+  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
+};
+
+// StatusCast<> allows casting from one status-bearing type to another,
+// regardless of whether the status indicates success or failure.
+// This means that we can go from StatusOr to Status safely, but not in the
+// opposite direction.
+// For cases where the status is guaranteed to be a failure (and hence not
+// needing to preserve values) see FailureStatusCast<> below.
 template <typename To, typename From>
 To StatusCast(From&& from) {
   return StatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }
 
+template <typename To, typename From, typename SfinaeVoid = void>
+struct FailureStatusCastImpl : public StatusCastImpl<To, From> {};
+
+template <typename T>
+struct FailureStatusCastImpl<absl::StatusOr<T>, absl::Status> {
+  static absl::StatusOr<T> Cast(absl::Status&& t) { return std::move(t); }
+};
+
+template <typename T>
+struct FailureStatusCastImpl<absl::StatusOr<T>, const absl::Status&> {
+  static absl::StatusOr<T> Cast(const absl::Status& t) { return t; }
+};
+
+template <typename To, typename From>
+To FailureStatusCast(From&& from) {
+  GPR_DEBUG_ASSERT(!IsStatusOk(from));
+  return FailureStatusCastImpl<To, From>::Cast(std::forward<From>(from));
+}
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_DETAIL_STATUS_H

data/src/core/lib/promise/for_each.h

@@ -20,7 +20,6 @@
 #include <stdint.h>
 
 #include <string>
-#include <type_traits>
 #include <utility>
 
 #include "absl/status/status.h"
@@ -28,11 +27,12 @@
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/detail/status.h"
 #include "src/core/lib/promise/poll.h"
+#include "src/core/lib/promise/status_flag.h"
 #include "src/core/lib/promise/trace.h"
 
 namespace grpc_core {
@@ -50,6 +50,16 @@ struct Done<absl::Status> {
   static absl::Status Make() { return absl::OkStatus(); }
 };
 
+template <>
+struct Done<StatusFlag> {
+  static StatusFlag Make() { return StatusFlag(true); }
+};
+
+template <>
+struct Done<Success> {
+  static Success Make() { return Success{}; }
+};
+
 template <typename Reader, typename Action>
 class ForEach {
  private:
@@ -141,7 +151,7 @@ class ForEach {
     }
     auto r = in_action_.promise();
     if (auto* p = r.value_if_ready()) {
-      if (p->ok()) {
+      if (IsStatusOk(*p)) {
         Destruct(&in_action_);
         Construct(&reader_next_, reader_.Next());
         reading_next_ = true;

data/src/core/lib/promise/if.h

@@ -18,7 +18,6 @@
 #include <grpc/support/port_platform.h>
 
 #include <memory>
-#include <type_traits>
 #include <utility>
 
 #include "absl/status/statusor.h"
@@ -193,6 +192,10 @@ class If<bool, T, F> {
 // If it returns failure, returns failure for the entire combinator.
 // If it returns true, evaluates the second promise.
 // If it returns false, evaluates the third promise.
+// If C is a constant, it's guaranteed that one of the promise factories
+// if_true or if_false will be evaluated before returning from this function.
+// This makes it safe to capture lambda arguments in the promise factory by
+// reference.
 template <typename C, typename T, typename F>
 promise_detail::If<C, T, F> If(C condition, T if_true, F if_false) {
   return promise_detail::If<C, T, F>(std::move(condition), std::move(if_true),

data/src/core/lib/promise/interceptor_list.h

@@ -30,7 +30,6 @@
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/promise/context.h"

data/src/core/lib/promise/latch.h

@@ -21,14 +21,12 @@
 
 #include <atomic>
 #include <string>
-#include <type_traits>
 #include <utility>
 
 #include "absl/strings/str_cat.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/promise/trace.h"
@@ -39,11 +37,13 @@ namespace grpc_core {
 // Initially the Latch is unset.
 // It can be waited upon by the Wait method, which produces a Promise that
 // resolves when the Latch is Set to a value of type T.
+// Latches only work correctly within a single activity.
 template <typename T>
 class Latch {
  public:
   Latch() = default;
   Latch(const Latch&) = delete;
+  explicit Latch(T value) : value_(std::move(value)), has_value_(true) {}
   Latch& operator=(const Latch&) = delete;
   Latch(Latch&& other) noexcept
       : value_(std::move(other.value_)), has_value_(other.has_value_) {
@@ -185,6 +185,8 @@ class Latch<void> {
     waiter_.Wake();
   }
 
+  bool is_set() const { return is_set_; }
+
  private:
   std::string DebugTag() {
     return absl::StrCat(Activity::current()->DebugTag(), " LATCH(void)[0x",
@@ -205,6 +207,9 @@ class Latch<void> {
   IntraActivityWaiter waiter_;
 };
 
+template <typename T>
+using LatchWaitPromise = decltype(std::declval<Latch<T>>().Wait());
+
 // A Latch that can have its value observed by outside threads, but only waited
 // upon from inside a single activity.
 template <typename T>
@@ -269,9 +274,6 @@ class ExternallyObservableLatch<void> {
   IntraActivityWaiter waiter_;
 };
 
-template <typename T>
-using LatchWaitPromise = decltype(std::declval<Latch<T>>().Wait());
-
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_LATCH_H

data/src/core/lib/promise/loop.h

@@ -17,7 +17,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <type_traits>
+#include <utility>
 
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
@@ -54,9 +54,9 @@ struct LoopTraits<absl::StatusOr<LoopCtl<T>>> {
   using Result = absl::StatusOr<T>;
   static LoopCtl<Result> ToLoopCtl(absl::StatusOr<LoopCtl<T>> value) {
     if (!value.ok()) return value.status();
-    const auto& inner = *value;
+    auto& inner = *value;
     if (absl::holds_alternative<Continue>(inner)) return Continue{};
-    return absl::get<T>(inner);
+    return absl::get<T>(std::move(inner));
   }
 };
 
@@ -87,7 +87,10 @@ class Loop {
     if (started_) Destruct(&promise_);
   }
 
-  Loop(Loop&& loop) noexcept : factory_(std::move(loop.factory_)) {}
+  Loop(Loop&& loop) noexcept
+      : factory_(std::move(loop.factory_)), started_(loop.started_) {
+    if (started_) Construct(&promise_, std::move(loop.promise_));
+  }
 
   Loop(const Loop& loop) = delete;
   Loop& operator=(const Loop& loop) = delete;
@@ -104,14 +107,14 @@ class Loop {
       if (auto* p = promise_result.value_if_ready()) {
         //  - then if it's Continue, destroy the promise and recreate a new one
         //  from our factory.
-        auto lc = LoopTraits<PromiseResult>::ToLoopCtl(*p);
+        auto lc = LoopTraits<PromiseResult>::ToLoopCtl(std::move(*p));
         if (absl::holds_alternative<Continue>(lc)) {
           Destruct(&promise_);
           Construct(&promise_, factory_.Make());
           continue;
         }
         //  - otherwise there's our result... return it out.
-        return absl::get<Result>(lc);
+        return absl::get<Result>(std::move(lc));
       } else {
         // Otherwise the inner promise was pending, so we are pending.
         return Pending();

data/src/core/lib/promise/map.h

@@ -20,7 +20,6 @@
 #include <stddef.h>
 
 #include <tuple>
-#include <type_traits>
 #include <utility>
 
 #include "src/core/lib/promise/detail/promise_like.h"

data/src/core/lib/promise/party.cc

@@ -17,14 +17,12 @@
 #include "src/core/lib/promise/party.h"
 
 #include <atomic>
-#include <initializer_list>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/strings/str_format.h"
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/promise/activity.h"
@@ -229,40 +227,42 @@ void Party::RunLocked() {
 bool Party::RunParty() {
   ScopedActivity activity(this);
   promise_detail::Context<Arena> arena_ctx(arena_);
-  return sync_.RunParty([this](int i) {
-    // If the participant is null, skip.
-    // This allows participants to complete whilst wakers still exist
-    // somewhere.
-    auto* participant = participants_[i].load(std::memory_order_acquire);
-    if (participant == nullptr) {
-      if (grpc_trace_promise_primitives.enabled()) {
-        gpr_log(GPR_DEBUG, "%s[party] wakeup %d already complete",
-                DebugTag().c_str(), i);
-      }
-      return false;
-    }
-    absl::string_view name;
+  return sync_.RunParty([this](int i) { return RunOneParticipant(i); });
+}
+
+bool Party::RunOneParticipant(int i) {
+  // If the participant is null, skip.
+  // This allows participants to complete whilst wakers still exist
+  // somewhere.
+  auto* participant = participants_[i].load(std::memory_order_acquire);
+  if (participant == nullptr) {
     if (grpc_trace_promise_primitives.enabled()) {
-      name = participant->name();
-      gpr_log(GPR_DEBUG, "%s[%s] begin job %d", DebugTag().c_str(),
-              std::string(name).c_str(), i);
+      gpr_log(GPR_DEBUG, "%s[party] wakeup %d already complete",
+              DebugTag().c_str(), i);
     }
-    // Poll the participant.
-    currently_polling_ = i;
-    bool done = participant->Poll();
-    currently_polling_ = kNotPolling;
-    if (done) {
-      if (!name.empty()) {
-        gpr_log(GPR_DEBUG, "%s[%s] end poll and finish job %d",
-                DebugTag().c_str(), std::string(name).c_str(), i);
-      }
-      participants_[i].store(nullptr, std::memory_order_relaxed);
-    } else if (!name.empty()) {
-      gpr_log(GPR_DEBUG, "%s[%s] end poll", DebugTag().c_str(),
-              std::string(name).c_str());
+    return false;
+  }
+  absl::string_view name;
+  if (grpc_trace_promise_primitives.enabled()) {
+    name = participant->name();
+    gpr_log(GPR_DEBUG, "%s[%s] begin job %d", DebugTag().c_str(),
+            std::string(name).c_str(), i);
+  }
+  // Poll the participant.
+  currently_polling_ = i;
+  bool done = participant->PollParticipantPromise();
+  currently_polling_ = kNotPolling;
+  if (done) {
+    if (!name.empty()) {
+      gpr_log(GPR_DEBUG, "%s[%s] end poll and finish job %d",
+              DebugTag().c_str(), std::string(name).c_str(), i);
     }
-    return done;
-  });
+    participants_[i].store(nullptr, std::memory_order_relaxed);
+  } else if (!name.empty()) {
+    gpr_log(GPR_DEBUG, "%s[%s] end poll", DebugTag().c_str(),
+            std::string(name).c_str());
+  }
+  return done;
 }
 
 void Party::AddParticipants(Participant** participants, size_t count) {

data/src/core/lib/promise/party.h

@@ -24,13 +24,13 @@
 #include <string>
 #include <utility>
 
+#include "absl/base/attributes.h"
 #include "absl/base/thread_annotations.h"
 #include "absl/strings/string_view.h"
 
 #include <grpc/event_engine/event_engine.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/gprpp/crash.h"
 #include "src/core/lib/gprpp/ref_counted.h"
@@ -39,6 +39,7 @@
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/poll.h"
 #include "src/core/lib/promise/trace.h"
 #include "src/core/lib/resource_quota/arena.h"
 
@@ -101,7 +102,8 @@ class PartySyncUsingAtomics {
   template <typename F>
   GRPC_MUST_USE_RESULT bool RunParty(F poll_one_participant) {
     uint64_t prev_state;
-    do {
+    iteration_.fetch_add(1, std::memory_order_relaxed);
+    for (;;) {
       // Grab the current state, and clear the wakeup bits & add flag.
       prev_state = state_.fetch_and(kRefMask | kLocked | kAllocatedMask,
                                     std::memory_order_acquire);
@@ -132,9 +134,23 @@ class PartySyncUsingAtomics {
       // TODO(ctiller): consider mitigations for the accidental wakeup on owning
       // waker creation case -- I currently expect this will be more expensive
       // than this quick loop.
-    } while (!state_.compare_exchange_weak(
-        prev_state, (prev_state & (kRefMask | kAllocatedMask)),
-        std::memory_order_acq_rel, std::memory_order_acquire));
+      if (wake_after_poll_ == 0) {
+        if (state_.compare_exchange_weak(
+                prev_state, (prev_state & (kRefMask | kAllocatedMask)),
+                std::memory_order_acq_rel, std::memory_order_acquire)) {
+          return false;
+        }
+      } else {
+        if (state_.compare_exchange_weak(
+                prev_state,
+                (prev_state & (kRefMask | kAllocatedMask | kLocked)) |
+                    wake_after_poll_,
+                std::memory_order_acq_rel, std::memory_order_acquire)) {
+          iteration_.fetch_add(1, std::memory_order_relaxed);
+          wake_after_poll_ = 0;
+        }
+      }
+    }
     return false;
   }
 
@@ -185,6 +201,11 @@ class PartySyncUsingAtomics {
   // Returns true if the caller should run the party.
   GRPC_MUST_USE_RESULT bool ScheduleWakeup(WakeupMask mask);
 
+  void WakeAfterPoll(WakeupMask mask) { wake_after_poll_ |= mask; }
+  uint32_t iteration() const {
+    return iteration_.load(std::memory_order_relaxed);
+  }
+
  private:
   bool UnreffedLast();
 
@@ -224,6 +245,8 @@ class PartySyncUsingAtomics {
   static constexpr uint64_t kOneRef = 1ull << kRefShift;
 
   std::atomic<uint64_t> state_;
+  std::atomic<uint32_t> iteration_{0};
+  WakeupMask wake_after_poll_ = 0;
 };
 
 class PartySyncUsingMutex {
@@ -299,7 +322,7 @@ class Party : public Activity, private Wakeable {
     explicit Participant(absl::string_view name) : name_(name) {}
     // Poll the participant. Return true if complete.
     // Participant should take care of its own deallocation in this case.
-    virtual bool Poll() = 0;
+    virtual bool PollParticipantPromise() = 0;
 
     // Destroy the participant before finishing.
     virtual void Destroy() = 0;
@@ -331,6 +354,9 @@ class Party : public Activity, private Wakeable {
   void Spawn(absl::string_view name, Factory promise_factory,
              OnComplete on_complete);
 
+  template <typename Factory>
+  auto SpawnWaitable(absl::string_view name, Factory factory);
+
   void Orphan() final { Crash("unused"); }
 
   // Activity implementation: not allowed to be overridden by derived types.
@@ -354,6 +380,20 @@ class Party : public Activity, private Wakeable {
 
   Arena* arena() const { return arena_; }
 
+  // Return a promise that resolves to Empty{} when the current party poll is
+  // complete.
+  // This is useful for implementing batching and the like: we can hold some
+  // action until the rest of the party resolves itself.
+  auto AfterCurrentPoll() {
+    GPR_DEBUG_ASSERT(Activity::current() == this);
+    sync_.WakeAfterPoll(CurrentParticipant());
+    return [this, iteration = sync_.iteration()]() -> Poll<Empty> {
+      GPR_DEBUG_ASSERT(Activity::current() == this);
+      if (iteration == sync_.iteration()) return Pending{};
+      return Empty{};
+    };
+  }
+
   class BulkSpawner {
    public:
     explicit BulkSpawner(Party* party) : party_(party) {}
@@ -384,7 +424,7 @@ class Party : public Activity, private Wakeable {
   // Should not be called by derived types except as a tail call to the base
   // class RunParty when overriding this method to add custom context.
   // Returns true if the party is over.
-  virtual bool RunParty() GRPC_MUST_USE_RESULT;
+  GRPC_MUST_USE_RESULT virtual bool RunParty();
 
   bool RefIfNonZero() { return sync_.RefIfNonZero(); }
 
@@ -415,7 +455,7 @@ class Party : public Activity, private Wakeable {
       }
     }
 
-    bool Poll() override {
+    bool PollParticipantPromise() override {
       if (!started_) {
         auto p = factory_.Make();
         Destruct(&factory_);
@@ -442,6 +482,89 @@ class Party : public Activity, private Wakeable {
     bool started_ = false;
   };
 
+  template <typename SuppliedFactory>
+  class PromiseParticipantImpl final
+      : public RefCounted<PromiseParticipantImpl<SuppliedFactory>,
+                          NonPolymorphicRefCount>,
+        public Participant {
+    using Factory = promise_detail::OncePromiseFactory<void, SuppliedFactory>;
+    using Promise = typename Factory::Promise;
+    using Result = typename Promise::Result;
+
+   public:
+    PromiseParticipantImpl(absl::string_view name,
+                           SuppliedFactory promise_factory)
+        : Participant(name) {
+      Construct(&factory_, std::move(promise_factory));
+    }
+
+    ~PromiseParticipantImpl() {
+      switch (state_.load(std::memory_order_acquire)) {
+        case State::kFactory:
+          Destruct(&factory_);
+          break;
+        case State::kPromise:
+          Destruct(&promise_);
+          break;
+        case State::kResult:
+          Destruct(&result_);
+          break;
+      }
+    }
+
+    // Inside party poll: drive from factory -> promise -> result
+    bool PollParticipantPromise() override {
+      switch (state_.load(std::memory_order_relaxed)) {
+        case State::kFactory: {
+          auto p = factory_.Make();
+          Destruct(&factory_);
+          Construct(&promise_, std::move(p));
+          state_.store(State::kPromise, std::memory_order_relaxed);
+        }
+          ABSL_FALLTHROUGH_INTENDED;
+        case State::kPromise: {
+          auto p = promise_();
+          if (auto* r = p.value_if_ready()) {
+            Destruct(&promise_);
+            Construct(&result_, std::move(*r));
+            state_.store(State::kResult, std::memory_order_release);
+            waiter_.Wakeup();
+            this->Unref();
+            return true;
+          }
+          return false;
+        }
+        case State::kResult:
+          Crash(
+              "unreachable: promises should not be repolled after completion");
+      }
+    }
+
+    // Outside party poll: check whether the spawning party has completed this
+    // promise.
+    Poll<Result> PollCompletion() {
+      switch (state_.load(std::memory_order_acquire)) {
+        case State::kFactory:
+        case State::kPromise:
+          return Pending{};
+        case State::kResult:
+          return std::move(result_);
+      }
+    }
+
+    void Destroy() override { this->Unref(); }
+
+   private:
+    enum class State : uint8_t { kFactory, kPromise, kResult };
+    union {
+      GPR_NO_UNIQUE_ADDRESS Factory factory_;
+      GPR_NO_UNIQUE_ADDRESS Promise promise_;
+      GPR_NO_UNIQUE_ADDRESS Result result_;
+    };
+    Waker waiter_{Activity::current()->MakeOwningWaker()};
+    std::atomic<State> state_{State::kFactory};
+  };
+
   // Notification that the party has finished and this instance can be deleted.
   // Derived types should arrange to call CancelRemainingParticipants during
   // this sequence.
@@ -461,6 +584,7 @@ class Party : public Activity, private Wakeable {
 
   // Add a participant (backs Spawn, after type erasure to ParticipantFactory).
   void AddParticipants(Participant** participant, size_t count);
+  bool RunOneParticipant(int i);
 
   virtual grpc_event_engine::experimental::EventEngine* event_engine()
       const = 0;
@@ -503,6 +627,17 @@ void Party::Spawn(absl::string_view name, Factory promise_factory,
                           std::move(on_complete));
 }
 
+template <typename Factory>
+auto Party::SpawnWaitable(absl::string_view name, Factory promise_factory) {
+  auto participant = MakeRefCounted<PromiseParticipantImpl<Factory>>(
+      name, std::move(promise_factory));
+  Participant* p = participant->Ref().release();
+  AddParticipants(&p, 1);
+  return [participant = std::move(participant)]() mutable {
+    return participant->PollCompletion();
+  };
+}
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_PARTY_H