grpc 1.62.0 → 1.64.0

data/src/core/lib/promise/detail/status.h

@@ -15,13 +15,14 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_DETAIL_STATUS_H
 #define GRPC_SRC_CORE_LIB_PROMISE_DETAIL_STATUS_H
 
-#include <grpc/support/port_platform.h>
-
 #include <utility>
 
+#include "absl/log/check.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 
+#include <grpc/support/port_platform.h>
+
 // Helpers for dealing with absl::Status/StatusOr generically
 
 namespace grpc_core {
@@ -106,7 +107,7 @@ struct FailureStatusCastImpl<absl::StatusOr<T>, const absl::Status&> {
 
 template <typename To, typename From>
 To FailureStatusCast(From&& from) {
-  GPR_DEBUG_ASSERT(!IsStatusOk(from));
+  DCHECK(!IsStatusOk(from));
   return FailureStatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }
 

data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h

@@ -15,10 +15,10 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_EXEC_CTX_WAKEUP_SCHEDULER_H
 #define GRPC_SRC_CORE_LIB_PROMISE_EXEC_CTX_WAKEUP_SCHEDULER_H
 
-#include <grpc/support/port_platform.h>
-
 #include "absl/status/status.h"
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"

data/src/core/lib/promise/for_each.h

@@ -15,17 +15,17 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_FOR_EACH_H
 #define GRPC_SRC_CORE_LIB_PROMISE_FOR_EACH_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stdint.h>
 
 #include <string>
 #include <utility>
 
+#include "absl/log/check.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/activity.h"
@@ -57,13 +57,52 @@ struct Done<StatusFlag> {
   static StatusFlag Make(bool cancelled) { return StatusFlag(!cancelled); }
 };
 
+template <typename T, typename SfinaeVoid = void>
+struct NextValueTraits;
+
+enum class NextValueType {
+  kValue,
+  kEndOfStream,
+  kError,
+};
+
+template <typename T>
+struct NextValueTraits<T, absl::void_t<typename T::value_type>> {
+  using Value = typename T::value_type;
+
+  static NextValueType Type(const T& t) {
+    if (t.has_value()) return NextValueType::kValue;
+    if (t.cancelled()) return NextValueType::kError;
+    return NextValueType::kEndOfStream;
+  }
+
+  static Value& MutableValue(T& t) { return *t; }
+};
+
+template <typename T>
+struct NextValueTraits<ValueOrFailure<absl::optional<T>>> {
+  using Value = T;
+
+  static NextValueType Type(const ValueOrFailure<absl::optional<T>>& t) {
+    if (t.ok()) {
+      if (t.value().has_value()) return NextValueType::kValue;
+      return NextValueType::kEndOfStream;
+    }
+    return NextValueType::kError;
+  }
+
+  static Value& MutableValue(ValueOrFailure<absl::optional<T>>& t) {
+    return **t;
+  }
+};
+
 template <typename Reader, typename Action>
 class ForEach {
  private:
   using ReaderNext = decltype(std::declval<Reader>().Next());
   using ReaderResult =
       typename PollTraits<decltype(std::declval<ReaderNext>()())>::Type;
-  using ReaderResultValue = typename ReaderResult::value_type;
+  using ReaderResultValue = typename NextValueTraits<ReaderResult>::Value;
   using ActionFactory =
       promise_detail::RepeatedPromiseFactory<ReaderResultValue, Action>;
   using ActionPromise = typename ActionFactory::Promise;
@@ -71,8 +110,10 @@ class ForEach {
  public:
   using Result =
       typename PollTraits<decltype(std::declval<ActionPromise>()())>::Type;
-  ForEach(Reader reader, Action action)
-      : reader_(std::move(reader)), action_factory_(std::move(action)) {
+  ForEach(Reader reader, Action action, DebugLocation whence = {})
+      : reader_(std::move(reader)),
+        action_factory_(std::move(action)),
+        whence_(whence) {
     Construct(&reader_next_, reader_.Next());
   }
   ~ForEach() {
@@ -87,17 +128,19 @@ class ForEach {
   ForEach& operator=(const ForEach&) = delete;
   ForEach(ForEach&& other) noexcept
       : reader_(std::move(other.reader_)),
-        action_factory_(std::move(other.action_factory_)) {
-    GPR_DEBUG_ASSERT(reading_next_);
-    GPR_DEBUG_ASSERT(other.reading_next_);
+        action_factory_(std::move(other.action_factory_)),
+        whence_(other.whence_) {
+    DCHECK(reading_next_);
+    DCHECK(other.reading_next_);
     Construct(&reader_next_, std::move(other.reader_next_));
   }
   ForEach& operator=(ForEach&& other) noexcept {
-    GPR_DEBUG_ASSERT(reading_next_);
-    GPR_DEBUG_ASSERT(other.reading_next_);
+    DCHECK(reading_next_);
+    DCHECK(other.reading_next_);
     reader_ = std::move(other.reader_);
     action_factory_ = std::move(other.action_factory_);
     reader_next_ = std::move(other.reader_next_);
+    whence_ = other.whence_;
     return *this;
   }
 
@@ -116,27 +159,43 @@ class ForEach {
 
   std::string DebugTag() {
     return absl::StrCat(GetContext<Activity>()->DebugTag(), " FOR_EACH[0x",
-                        reinterpret_cast<uintptr_t>(this), "]: ");
+                        reinterpret_cast<uintptr_t>(this), "@", whence_.file(),
+                        ":", whence_.line(), "]: ");
   }
 
   Poll<Result> PollReaderNext() {
     if (grpc_trace_promise_primitives.enabled()) {
-      gpr_log(GPR_DEBUG, "%s PollReaderNext", DebugTag().c_str());
+      gpr_log(GPR_INFO, "%s PollReaderNext", DebugTag().c_str());
     }
     auto r = reader_next_();
     if (auto* p = r.value_if_ready()) {
-      if (grpc_trace_promise_primitives.enabled()) {
-        gpr_log(GPR_DEBUG, "%s PollReaderNext: got has_value=%s",
-                DebugTag().c_str(), p->has_value() ? "true" : "false");
-      }
-      if (p->has_value()) {
-        Destruct(&reader_next_);
-        auto action = action_factory_.Make(std::move(**p));
-        Construct(&in_action_, std::move(action), std::move(*p));
-        reading_next_ = false;
-        return PollAction();
-      } else {
-        return Done<Result>::Make(p->cancelled());
+      switch (NextValueTraits<ReaderResult>::Type(*p)) {
+        case NextValueType::kValue: {
+          if (grpc_trace_promise_primitives.enabled()) {
+            gpr_log(GPR_INFO, "%s PollReaderNext: got value",
+                    DebugTag().c_str());
+          }
+          Destruct(&reader_next_);
+          auto action = action_factory_.Make(
+              std::move(NextValueTraits<ReaderResult>::MutableValue(*p)));
+          Construct(&in_action_, std::move(action), std::move(*p));
+          reading_next_ = false;
+          return PollAction();
+        }
+        case NextValueType::kEndOfStream: {
+          if (grpc_trace_promise_primitives.enabled()) {
+            gpr_log(GPR_INFO, "%s PollReaderNext: got end of stream",
+                    DebugTag().c_str());
+          }
+          return Done<Result>::Make(false);
+        }
+        case NextValueType::kError: {
+          if (grpc_trace_promise_primitives.enabled()) {
+            gpr_log(GPR_INFO, "%s PollReaderNext: got error",
+                    DebugTag().c_str());
+          }
+          return Done<Result>::Make(true);
+        }
       }
     }
     return Pending();
@@ -144,7 +203,7 @@ class ForEach {
 
   Poll<Result> PollAction() {
     if (grpc_trace_promise_primitives.enabled()) {
-      gpr_log(GPR_DEBUG, "%s PollAction", DebugTag().c_str());
+      gpr_log(GPR_INFO, "%s PollAction", DebugTag().c_str());
     }
     auto r = in_action_.promise();
     if (auto* p = r.value_if_ready()) {
@@ -162,6 +221,7 @@ class ForEach {
 
   GPR_NO_UNIQUE_ADDRESS Reader reader_;
   GPR_NO_UNIQUE_ADDRESS ActionFactory action_factory_;
+  GPR_NO_UNIQUE_ADDRESS DebugLocation whence_;
   bool reading_next_ = true;
   union {
     ReaderNext reader_next_;
@@ -173,9 +233,10 @@ class ForEach {
 
 /// For each item acquired by calling Reader::Next, run the promise Action.
 template <typename Reader, typename Action>
-for_each_detail::ForEach<Reader, Action> ForEach(Reader reader, Action action) {
+for_each_detail::ForEach<Reader, Action> ForEach(Reader reader, Action action,
+                                                 DebugLocation whence = {}) {
   return for_each_detail::ForEach<Reader, Action>(std::move(reader),
-                                                  std::move(action));
+                                                  std::move(action), whence);
 }
 
 }  // namespace grpc_core

data/src/core/lib/promise/if.h

@@ -15,14 +15,14 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_IF_H
 #define GRPC_SRC_CORE_LIB_PROMISE_IF_H
 
-#include <grpc/support/port_platform.h>
-
 #include <memory>
 #include <utility>
 
 #include "absl/status/statusor.h"
 #include "absl/types/variant.h"
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/detail/promise_like.h"

data/src/core/lib/promise/interceptor_list.h

@@ -15,8 +15,6 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H
 #define GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stddef.h>
 
 #include <algorithm>
@@ -24,11 +22,13 @@
 #include <string>
 #include <utility>
 
+#include "absl/log/check.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/types/optional.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/gprpp/debug_location.h"
@@ -67,7 +67,7 @@ class InterceptorList {
     // Update the next pointer stored with this map.
     // This is only valid to call once, and only before the map is used.
     void SetNext(Map* next) {
-      GPR_DEBUG_ASSERT(next_ == nullptr);
+      DCHECK_EQ(next_, nullptr);
       next_ = next;
     }
 

data/src/core/lib/promise/latch.h

@@ -15,17 +15,17 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_LATCH_H
 #define GRPC_SRC_CORE_LIB_PROMISE_LATCH_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stdint.h>
 
 #include <atomic>
 #include <string>
 #include <utility>
 
+#include "absl/log/check.h"
 #include "absl/strings/str_cat.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"
@@ -48,12 +48,12 @@ class Latch {
   Latch(Latch&& other) noexcept
       : value_(std::move(other.value_)), has_value_(other.has_value_) {
 #ifndef NDEBUG
-    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+    DCHECK(!other.has_had_waiters_);
 #endif
   }
   Latch& operator=(Latch&& other) noexcept {
 #ifndef NDEBUG
-    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+    DCHECK(!other.has_had_waiters_);
 #endif
     value_ = std::move(other.value_);
     has_value_ = other.has_value_;
@@ -103,7 +103,7 @@ class Latch {
     if (grpc_trace_promise_primitives.enabled()) {
       gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str());
     }
-    GPR_DEBUG_ASSERT(!has_value_);
+    DCHECK(!has_value_);
     value_ = std::move(value);
     has_value_ = true;
     waiter_.Wake();
@@ -146,12 +146,12 @@ class Latch<void> {
   Latch& operator=(const Latch&) = delete;
   Latch(Latch&& other) noexcept : is_set_(other.is_set_) {
 #ifndef NDEBUG
-    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+    DCHECK(!other.has_had_waiters_);
 #endif
   }
   Latch& operator=(Latch&& other) noexcept {
 #ifndef NDEBUG
-    GPR_DEBUG_ASSERT(!other.has_had_waiters_);
+    DCHECK(!other.has_had_waiters_);
 #endif
     is_set_ = other.is_set_;
     return *this;
@@ -180,7 +180,7 @@ class Latch<void> {
     if (grpc_trace_promise_primitives.enabled()) {
       gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str());
     }
-    GPR_DEBUG_ASSERT(!is_set_);
+    DCHECK(!is_set_);
     is_set_ = true;
     waiter_.Wake();
   }

data/src/core/lib/promise/loop.h

@@ -15,14 +15,14 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_LOOP_H
 #define GRPC_SRC_CORE_LIB_PROMISE_LOOP_H
 
-#include <grpc/support/port_platform.h>
-
 #include <utility>
 
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/types/variant.h"
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/poll.h"

data/src/core/lib/promise/map.h

@@ -15,13 +15,13 @@
 #ifndef GRPC_SRC_CORE_LIB_PROMISE_MAP_H
 #define GRPC_SRC_CORE_LIB_PROMISE_MAP_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stddef.h>
 
 #include <tuple>
 #include <utility>
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/poll.h"
 

data/src/core/lib/promise/party.cc

@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/promise/party.h"
 
 #include <atomic>
 
 #include "absl/base/thread_annotations.h"
+#include "absl/log/check.h"
 #include "absl/strings/str_format.h"
 
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
@@ -37,6 +37,11 @@ grpc_core::DebugOnlyTraceFlag grpc_trace_party_state(false, "party_state");
 
 namespace grpc_core {
 
+namespace {
+// TODO(ctiller): Once all activities are parties we can remove this.
+thread_local Party** g_current_party_run_next = nullptr;
+}  // namespace
+
 ///////////////////////////////////////////////////////////////////////////////
 // PartySyncUsingAtomics
 
@@ -98,7 +103,7 @@ class Party::Handle final : public Wakeable {
   // Activity is going away... drop its reference and sever the connection back.
   void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
     mu_.Lock();
-    GPR_ASSERT(party_ != nullptr);
+    CHECK_NE(party_, nullptr);
     party_ = nullptr;
     mu_.Unlock();
     Unref();
@@ -177,7 +182,6 @@ Party::~Party() {}
 
 void Party::CancelRemainingParticipants() {
   ScopedActivity activity(this);
-  promise_detail::Context<Arena> arena_ctx(arena_);
   for (size_t i = 0; i < party_detail::kMaxParticipants; i++) {
     if (auto* p =
             participants_[i].exchange(nullptr, std::memory_order_acquire)) {
@@ -191,13 +195,13 @@ std::string Party::ActivityDebugTag(WakeupMask wakeup_mask) const {
 }
 
 Waker Party::MakeOwningWaker() {
-  GPR_DEBUG_ASSERT(currently_polling_ != kNotPolling);
+  DCHECK(currently_polling_ != kNotPolling);
   IncrementRefCount();
   return Waker(this, 1u << currently_polling_);
 }
 
 Waker Party::MakeNonOwningWaker() {
-  GPR_DEBUG_ASSERT(currently_polling_ != kNotPolling);
+  DCHECK(currently_polling_ != kNotPolling);
   return Waker(participants_[currently_polling_]
                    .load(std::memory_order_relaxed)
                    ->MakeNonOwningWakeable(this),
@@ -205,16 +209,43 @@ Waker Party::MakeNonOwningWaker() {
 }
 
 void Party::ForceImmediateRepoll(WakeupMask mask) {
-  GPR_DEBUG_ASSERT(is_current());
+  DCHECK(is_current());
   sync_.ForceImmediateRepoll(mask);
 }
 
 void Party::RunLocked() {
+  // If there is a party running, then we don't run it immediately
+  // but instead add it to the end of the list of parties to run.
+  // This enables a fairly straightforward batching of work from a
+  // call to a transport (or back again).
+  if (g_current_party_run_next != nullptr) {
+    if (*g_current_party_run_next == nullptr) {
+      *g_current_party_run_next = this;
+    } else {
+      // But if there's already a party queued, we're better off asking event
+      // engine to run it so we can spread load.
+      event_engine()->Run([this]() {
+        ApplicationCallbackExecCtx app_exec_ctx;
+        ExecCtx exec_ctx;
+        RunLocked();
+      });
+    }
+    return;
+  }
   auto body = [this]() {
-    if (RunParty()) {
+    DCHECK_EQ(g_current_party_run_next, nullptr);
+    Party* run_next = nullptr;
+    g_current_party_run_next = &run_next;
+    const bool done = RunParty();
+    DCHECK(g_current_party_run_next == &run_next);
+    g_current_party_run_next = nullptr;
+    if (done) {
       ScopedActivity activity(this);
       PartyOver();
     }
+    if (run_next != nullptr) {
+      run_next->RunLocked();
+    }
   };
 #ifdef GRPC_MAXIMIZE_THREADYNESS
   Thread thd(
@@ -233,7 +264,6 @@ void Party::RunLocked() {
 
 bool Party::RunParty() {
   ScopedActivity activity(this);
-  promise_detail::Context<Arena> arena_ctx(arena_);
   return sync_.RunParty([this](int i) { return RunOneParticipant(i); });
 }
 
@@ -244,7 +274,7 @@ bool Party::RunOneParticipant(int i) {
   auto* participant = participants_[i].load(std::memory_order_acquire);
   if (participant == nullptr) {
     if (grpc_trace_promise_primitives.enabled()) {
-      gpr_log(GPR_DEBUG, "%s[party] wakeup %d already complete",
+      gpr_log(GPR_INFO, "%s[party] wakeup %d already complete",
               DebugTag().c_str(), i);
     }
     return false;
@@ -252,7 +282,7 @@ bool Party::RunOneParticipant(int i) {
   absl::string_view name;
   if (grpc_trace_promise_primitives.enabled()) {
     name = participant->name();
-    gpr_log(GPR_DEBUG, "%s[%s] begin job %d", DebugTag().c_str(),
+    gpr_log(GPR_INFO, "%s[%s] begin job %d", DebugTag().c_str(),
             std::string(name).c_str(), i);
   }
   // Poll the participant.
@@ -261,12 +291,12 @@ bool Party::RunOneParticipant(int i) {
   currently_polling_ = kNotPolling;
   if (done) {
     if (!name.empty()) {
-      gpr_log(GPR_DEBUG, "%s[%s] end poll and finish job %d",
-              DebugTag().c_str(), std::string(name).c_str(), i);
+      gpr_log(GPR_INFO, "%s[%s] end poll and finish job %d", DebugTag().c_str(),
+              std::string(name).c_str(), i);
     }
     participants_[i].store(nullptr, std::memory_order_relaxed);
   } else if (!name.empty()) {
-    gpr_log(GPR_DEBUG, "%s[%s] end poll", DebugTag().c_str(),
+    gpr_log(GPR_INFO, "%s[%s] end poll", DebugTag().c_str(),
             std::string(name).c_str());
   }
   return done;
@@ -277,7 +307,7 @@ void Party::AddParticipants(Participant** participants, size_t count) {
                                                        count](size_t* slots) {
     for (size_t i = 0; i < count; i++) {
       if (grpc_trace_party_state.enabled()) {
-        gpr_log(GPR_DEBUG,
+        gpr_log(GPR_INFO,
                 "Party %p                 AddParticipant: %s @ %" PRIdPTR,
                 &sync_, std::string(participants[i]->name()).c_str(), slots[i]);
       }