grpc 1.65.2 → 1.66.0.pre3

data/src/core/lib/iomgr/timer_generic.cc

@@ -21,12 +21,12 @@
 #include <string>
 
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/cpu.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
 
@@ -335,10 +335,10 @@ static void timer_init(grpc_timer* timer, grpc_core::Timestamp deadline,
 #endif
 
   if (GRPC_TRACE_FLAG_ENABLED(timer)) {
-    gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 " now %" PRId64 " call %p[%p]",
-            timer, deadline.milliseconds_after_process_epoch(),
-            grpc_core::Timestamp::Now().milliseconds_after_process_epoch(),
-            closure, closure->cb);
+    VLOG(2) << "TIMER " << timer << ": SET "
+            << deadline.milliseconds_after_process_epoch() << " now "
+            << grpc_core::Timestamp::Now().milliseconds_after_process_epoch()
+            << " call " << closure << "[" << closure->cb << "]";
   }
 
   if (!g_shared_mutables.initialized) {
@@ -371,12 +371,10 @@ static void timer_init(grpc_timer* timer, grpc_core::Timestamp deadline,
     list_join(&shard->list, timer);
   }
   if (GRPC_TRACE_FLAG_ENABLED(timer)) {
-    gpr_log(GPR_DEBUG,
-            "  .. add to shard %d with queue_deadline_cap=%" PRId64
-            " => is_first_timer=%s",
-            static_cast<int>(shard - g_shards),
-            shard->queue_deadline_cap.milliseconds_after_process_epoch(),
-            is_first_timer ? "true" : "false");
+    VLOG(2) << "  .. add to shard " << (shard - g_shards)
+            << " with queue_deadline_cap="
+            << shard->queue_deadline_cap.milliseconds_after_process_epoch()
+            << " => is_first_timer=" << (is_first_timer ? "true" : "false");
   }
   gpr_mu_unlock(&shard->mu);
 
@@ -394,8 +392,8 @@ static void timer_init(grpc_timer* timer, grpc_core::Timestamp deadline,
   if (is_first_timer) {
     gpr_mu_lock(&g_shared_mutables.mu);
     if (GRPC_TRACE_FLAG_ENABLED(timer)) {
-      gpr_log(GPR_DEBUG, "  .. old shard min_deadline=%" PRId64,
-              shard->min_deadline.milliseconds_after_process_epoch());
+      VLOG(2) << "  .. old shard min_deadline="
+              << shard->min_deadline.milliseconds_after_process_epoch();
     }
     if (deadline < shard->min_deadline) {
       grpc_core::Timestamp old_min_deadline = g_shard_queue[0]->min_deadline;
@@ -436,8 +434,8 @@ static void timer_cancel(grpc_timer* timer) {
   timer_shard* shard = &g_shards[grpc_core::HashPointer(timer, g_num_shards)];
   gpr_mu_lock(&shard->mu);
   if (GRPC_TRACE_FLAG_ENABLED(timer)) {
-    gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer,
-            timer->pending ? "true" : "false");
+    VLOG(2) << "TIMER " << timer
+            << ": CANCEL pending=" << (timer->pending ? "true" : "false");
   }
 
   if (timer->pending) {
@@ -477,9 +475,9 @@ static bool refill_heap(timer_shard* shard, grpc_core::Timestamp now) {
       grpc_core::Duration::FromSecondsAsDouble(deadline_delta);
 
   if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-    gpr_log(GPR_DEBUG, "  .. shard[%d]->queue_deadline_cap --> %" PRId64,
-            static_cast<int>(shard - g_shards),
-            shard->queue_deadline_cap.milliseconds_after_process_epoch());
+    VLOG(2) << "  .. shard[" << (shard - g_shards)
+            << "]->queue_deadline_cap --> "
+            << shard->queue_deadline_cap.milliseconds_after_process_epoch();
   }
   for (timer = shard->list.next; timer != &shard->list; timer = next) {
     next = timer->next;
@@ -489,8 +487,9 @@ static bool refill_heap(timer_shard* shard, grpc_core::Timestamp now) {
 
     if (timer_deadline < shard->queue_deadline_cap) {
       if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-        gpr_log(GPR_DEBUG, "  .. add timer with deadline %" PRId64 " to heap",
-                timer_deadline.milliseconds_after_process_epoch());
+        VLOG(2) << "  .. add timer with deadline "
+                << timer_deadline.milliseconds_after_process_epoch()
+                << " to heap";
       }
       list_remove(timer);
       grpc_timer_heap_add(&shard->heap, timer);
@@ -506,9 +505,8 @@ static grpc_timer* pop_one(timer_shard* shard, grpc_core::Timestamp now) {
   grpc_timer* timer;
   for (;;) {
     if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-      gpr_log(GPR_DEBUG, "  .. shard[%d]: heap_empty=%s",
-              static_cast<int>(shard - g_shards),
-              grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false");
+      VLOG(2) << "  .. shard[" << (shard - g_shards) << "]: heap_empty="
+              << (grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false");
     }
     if (grpc_timer_heap_is_empty(&shard->heap)) {
       if (now < shard->queue_deadline_cap) return nullptr;
@@ -519,15 +517,14 @@ static grpc_timer* pop_one(timer_shard* shard, grpc_core::Timestamp now) {
         grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
             timer->deadline);
     if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-      gpr_log(GPR_DEBUG,
-              "  .. check top timer deadline=%" PRId64 " now=%" PRId64,
-              timer_deadline.milliseconds_after_process_epoch(),
-              now.milliseconds_after_process_epoch());
+      VLOG(2) << "  .. check top timer deadline="
+              << timer_deadline.milliseconds_after_process_epoch()
+              << " now=" << now.milliseconds_after_process_epoch();
     }
     if (timer_deadline > now) return nullptr;
     if (GRPC_TRACE_FLAG_ENABLED(timer)) {
-      gpr_log(GPR_DEBUG, "TIMER %p: FIRE %" PRId64 "ms late", timer,
-              (now - timer_deadline).millis());
+      VLOG(2) << "TIMER " << timer << ": FIRE "
+              << (now - timer_deadline).millis() << "ms late";
     }
     timer->pending = false;
     grpc_timer_heap_pop(&shard->heap);
@@ -550,8 +547,7 @@ static size_t pop_timers(timer_shard* shard, grpc_core::Timestamp now,
   *new_min_deadline = compute_min_deadline(shard);
   gpr_mu_unlock(&shard->mu);
   if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-    gpr_log(GPR_DEBUG, "  .. shard[%d] popped %" PRIdPTR,
-            static_cast<int>(shard - g_shards), n);
+    VLOG(2) << "  .. shard[" << (shard - g_shards) << "] popped " << n;
   }
   return n;
 }
@@ -589,10 +585,10 @@ static grpc_timer_check_result run_some_expired_timers(
     result = GRPC_TIMERS_CHECKED_AND_EMPTY;
 
     if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-      gpr_log(
-          GPR_DEBUG, "  .. shard[%d]->min_deadline = %" PRId64,
-          static_cast<int>(g_shard_queue[0] - g_shards),
-          g_shard_queue[0]->min_deadline.milliseconds_after_process_epoch());
+      VLOG(2)
+          << "  .. shard[" << (g_shard_queue[0] - g_shards)
+          << "]->min_deadline = "
+          << g_shard_queue[0]->min_deadline.milliseconds_after_process_epoch();
     }
 
     while (g_shard_queue[0]->min_deadline < now ||
@@ -608,15 +604,12 @@ static grpc_timer_check_result run_some_expired_timers(
       }
 
       if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-        gpr_log(
-            GPR_DEBUG,
-            "  .. result --> %d"
-            ", shard[%d]->min_deadline %" PRId64 " --> %" PRId64
-            ", now=%" PRId64,
-            result, static_cast<int>(g_shard_queue[0] - g_shards),
-            g_shard_queue[0]->min_deadline.milliseconds_after_process_epoch(),
-            new_min_deadline.milliseconds_after_process_epoch(),
-            now.milliseconds_after_process_epoch());
+        VLOG(2)
+            << "  .. result --> " << result << ", shard["
+            << (g_shard_queue[0] - g_shards) << "]->min_deadline "
+            << g_shard_queue[0]->min_deadline.milliseconds_after_process_epoch()
+            << " --> " << new_min_deadline.milliseconds_after_process_epoch()
+            << ", now=" << now.milliseconds_after_process_epoch();
       }
 
       // An grpc_timer_init() on the shard could intervene here, adding a new
@@ -668,9 +661,9 @@ static grpc_timer_check_result timer_check(grpc_core::Timestamp* next) {
       *next = std::min(*next, min_timer);
     }
     if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-      gpr_log(GPR_DEBUG, "TIMER CHECK SKIP: now=%" PRId64 " min_timer=%" PRId64,
-              now.milliseconds_after_process_epoch(),
-              min_timer.milliseconds_after_process_epoch());
+      VLOG(2) << "TIMER CHECK SKIP: now="
+              << now.milliseconds_after_process_epoch()
+              << " min_timer=" << min_timer.milliseconds_after_process_epoch();
     }
     return GRPC_TIMERS_CHECKED_AND_EMPTY;
   }
@@ -689,20 +682,18 @@ static grpc_timer_check_result timer_check(grpc_core::Timestamp* next) {
       next_str = absl::StrCat(next->milliseconds_after_process_epoch());
     }
 #if GPR_ARCH_64
-    gpr_log(
-        GPR_DEBUG,
-        "TIMER CHECK BEGIN: now=%" PRId64 " next=%s tls_min=%" PRId64
-        " glob_min=%" PRId64,
-        now.milliseconds_after_process_epoch(), next_str.c_str(),
-        min_timer.milliseconds_after_process_epoch(),
-        grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
-            gpr_atm_no_barrier_load((gpr_atm*)(&g_shared_mutables.min_timer)))
-            .milliseconds_after_process_epoch());
+    VLOG(2) << "TIMER CHECK BEGIN: now="
+            << now.milliseconds_after_process_epoch() << " next=" << next_str
+            << " tls_min=" << min_timer.milliseconds_after_process_epoch()
+            << " glob_min="
+            << grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+                   gpr_atm_no_barrier_load(
+                       (gpr_atm*)(&g_shared_mutables.min_timer)))
+                   .milliseconds_after_process_epoch();
 #else
-    gpr_log(GPR_DEBUG,
-            "TIMER CHECK BEGIN: now=%" PRId64 " next=%s min=%" PRId64,
-            now.milliseconds_after_process_epoch(), next_str.c_str(),
-            min_timer.milliseconds_after_process_epoch());
+    VLOG(2) << "TIMER CHECK BEGIN: now="
+            << now.milliseconds_after_process_epoch() << " next=" << next_str
+            << " min=" << min_timer.milliseconds_after_process_epoch();
 #endif
   }
   // actual code
@@ -716,7 +707,7 @@ static grpc_timer_check_result timer_check(grpc_core::Timestamp* next) {
     } else {
       next_str = absl::StrCat(next->milliseconds_after_process_epoch());
     }
-    gpr_log(GPR_DEBUG, "TIMER CHECK END: r=%d; next=%s", r, next_str.c_str());
+    VLOG(2) << "TIMER CHECK END: r=" << r << "; next=" << next_str.c_str();
   }
   return r;
 }

data/src/core/lib/iomgr/timer_manager.cc

@@ -24,7 +24,6 @@
 #include "absl/log/log.h"
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/debug/trace.h"
@@ -182,8 +181,7 @@ static bool wait_until(grpc_core::Timestamp next) {
 
         if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
           grpc_core::Duration wait_time = next - grpc_core::Timestamp::Now();
-          gpr_log(GPR_INFO, "sleep for a %" PRId64 " milliseconds",
-                  wait_time.millis());
+          LOG(INFO) << "sleep for a " << wait_time.millis() << " milliseconds";
         }
       } else {  // g_timed_waiter == true && next >= g_timed_waiter_deadline
         next = grpc_core::Timestamp::InfFuture();
@@ -198,9 +196,9 @@ static bool wait_until(grpc_core::Timestamp next) {
     gpr_cv_wait(&g_cv_wait, &g_mu, next.as_timespec(GPR_CLOCK_MONOTONIC));
 
     if (GRPC_TRACE_FLAG_ENABLED(timer_check)) {
-      gpr_log(GPR_INFO, "wait ended: was_timed:%d kicked:%d",
-              my_timed_waiter_generation == g_timed_waiter_generation,
-              g_kicked);
+      LOG(INFO) << "wait ended: was_timed:"
+                << (my_timed_waiter_generation == g_timed_waiter_generation)
+                << " kicked:" << g_kicked;
     }
     // if this was the timed waiter, then we need to check timers, and flag
     // that there's now no timed waiter... we'll look for a replacement if

data/src/core/lib/promise/activity.h

@@ -32,8 +32,9 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/event_engine/event_engine_context.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
+#include "src/core/lib/gprpp/dump_args.h"
 #include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -41,6 +42,7 @@
 #include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/detail/status.h"
 #include "src/core/lib/promise/poll.h"
+#include "src/core/util/latent_see.h"
 
 namespace grpc_core {
 
@@ -544,6 +546,8 @@ class PromiseActivity final
   }
 
   void WakeupAsync(WakeupMask) final {
+    GRPC_LATENT_SEE_INNER_SCOPE("PromiseActivity::WakeupAsync");
+    wakeup_flow_.Begin(GRPC_LATENT_SEE_METADATA("Activity::Wakeup"));
     if (!wakeup_scheduled_.exchange(true, std::memory_order_acq_rel)) {
       // Can't safely run, so ask to run later.
       this->ScheduleWakeup();
@@ -567,6 +571,8 @@ class PromiseActivity final
   // In response to Wakeup, run the Promise state machine again until it
   // settles. Then check for completion, and if we have completed, call on_done.
   void Step() ABSL_LOCKS_EXCLUDED(mu()) {
+    GRPC_LATENT_SEE_PARENT_SCOPE("PromiseActivity::Step");
+    wakeup_flow_.End();
     // Poll the promise until things settle out under a lock.
     mu()->Lock();
     if (done_) {
@@ -643,6 +649,7 @@ class PromiseActivity final
     GPR_NO_UNIQUE_ADDRESS Promise promise;
   };
   GPR_NO_UNIQUE_ADDRESS PromiseHolder promise_holder_ ABSL_GUARDED_BY(mu());
+  GPR_NO_UNIQUE_ADDRESS latent_see::Flow wakeup_flow_;
 };
 
 }  // namespace promise_detail
@@ -662,7 +669,11 @@ ActivityPtr MakeActivity(Factory promise_factory,
 }
 
 inline Pending IntraActivityWaiter::pending() {
-  wakeups_ |= GetContext<Activity>()->CurrentParticipant();
+  const auto new_wakeups = GetContext<Activity>()->CurrentParticipant();
+  GRPC_TRACE_LOG(promise_primitives, INFO)
+      << "IntraActivityWaiter::pending: "
+      << GRPC_DUMP_ARGS(this, new_wakeups, wakeups_);
+  wakeups_ |= new_wakeups;
   return Pending();
 }
 

data/src/core/lib/promise/all_ok.h

@@ -39,17 +39,21 @@ struct AllOkTraits {
   template <typename T>
   using ResultType = Result;
   template <typename T>
-  static bool IsOk(const T& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(const T& x) {
     return IsStatusOk(x);
   }
-  static Empty Unwrapped(StatusFlag) { return Empty{}; }
-  static Empty Unwrapped(absl::Status) { return Empty{}; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(StatusFlag) {
+    return Empty{};
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Empty Unwrapped(absl::Status) {
+    return Empty{};
+  }
   template <typename R, typename T>
-  static R EarlyReturn(T&& x) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static R EarlyReturn(T&& x) {
     return StatusCast<R>(std::forward<T>(x));
   }
   template <typename... A>
-  static Result FinalReturn(A&&...) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Result FinalReturn(A&&...) {
     return Result{};
   }
 };
@@ -58,8 +62,11 @@ struct AllOkTraits {
 template <typename Result, typename... Promises>
 class AllOk {
  public:
-  explicit AllOk(Promises... promises) : state_(std::move(promises)...) {}
-  auto operator()() { return state_.PollOnce(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit AllOk(Promises... promises)
+      : state_(std::move(promises)...) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()() {
+    return state_.PollOnce();
+  }
 
  private:
   JoinState<AllOkTraits<Result>, Promises...> state_;
@@ -71,7 +78,7 @@ class AllOk {
 // If any fail, cancel the rest and return the failure.
 // If all succeed, return Ok.
 template <typename Result, typename... Promises>
-auto AllOk(Promises... promises) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto AllOk(Promises... promises) {
   return promise_detail::AllOk<Result, Promises...>(std::move(promises)...);
 }
 

data/src/core/lib/promise/cancel_callback.h

@@ -28,26 +28,28 @@ namespace cancel_callback_detail {
 template <typename Fn>
 class Handler {
  public:
-  explicit Handler(Fn fn) : fn_(std::move(fn)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Handler(Fn fn)
+      : fn_(std::move(fn)) {}
   Handler(const Handler&) = delete;
   Handler& operator=(const Handler&) = delete;
-  ~Handler() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Handler() {
     if (!done_) {
       promise_detail::Context<Arena> ctx(arena_.get());
       fn_();
     }
   }
-  Handler(Handler&& other) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Handler(Handler&& other) noexcept
       : fn_(std::move(other.fn_)), done_(other.done_) {
     other.done_ = true;
   }
-  Handler& operator=(Handler&& other) noexcept {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Handler& operator=(
+      Handler&& other) noexcept {
     fn_ = std::move(other.fn_);
     done_ = other.done_;
     other.done_ = true;
   }
 
-  void Done() { done_ = true; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void Done() { done_ = true; }
 
  private:
   Fn fn_;
@@ -67,7 +69,8 @@ class Handler {
 // completion.
 // Returns a promise with the same result type as main_fn.
 template <typename MainFn, typename CancelFn>
-auto OnCancel(MainFn main_fn, CancelFn cancel_fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto OnCancel(MainFn main_fn,
+                                                   CancelFn cancel_fn) {
   return [on_cancel =
               cancel_callback_detail::Handler<CancelFn>(std::move(cancel_fn)),
           main_fn = promise_detail::PromiseLike<MainFn>(
@@ -84,7 +87,8 @@ auto OnCancel(MainFn main_fn, CancelFn cancel_fn) {
 // resulting promise. If the factory is dropped without being called, cancel_fn
 // is called.
 template <typename MainFn, typename CancelFn>
-auto OnCancelFactory(MainFn main_fn, CancelFn cancel_fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto OnCancelFactory(MainFn main_fn,
+                                                          CancelFn cancel_fn) {
   return [on_cancel =
               cancel_callback_detail::Handler<CancelFn>(std::move(cancel_fn)),
           main_fn = std::move(main_fn)]() mutable {

data/src/core/lib/promise/context.h

@@ -62,7 +62,7 @@ class ThreadLocalContext : public ContextType<T> {
   ThreadLocalContext(const ThreadLocalContext&) = delete;
   ThreadLocalContext& operator=(const ThreadLocalContext&) = delete;
 
-  static T* get() { return current_; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T* get() { return current_; }
 
  private:
   T* const old_;
@@ -83,7 +83,7 @@ class Context<T, absl::void_t<typename ContextSubclass<T>::Base>>
     : public Context<typename ContextSubclass<T>::Base> {
  public:
   using Context<typename ContextSubclass<T>::Base>::Context;
-  static T* get() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T* get() {
     return DownCast<T*>(Context<typename ContextSubclass<T>::Base>::get());
   }
 };
@@ -107,26 +107,26 @@ class WithContext {
 
 // Return true if a context of type T is currently active.
 template <typename T>
-bool HasContext() {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION bool HasContext() {
   return promise_detail::Context<T>::get() != nullptr;
 }
 
 // Retrieve the current value of a context, or abort if the value is unset.
 template <typename T>
-T* GetContext() {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T* GetContext() {
   auto* p = promise_detail::Context<T>::get();
-  CHECK_NE(p, nullptr);
+  DCHECK_NE(p, nullptr);
   return p;
 }
 
 // Retrieve the current value of a context, or nullptr if the value is unset.
 template <typename T>
-T* MaybeGetContext() {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T* MaybeGetContext() {
   return promise_detail::Context<T>::get();
 }
 
 template <typename T>
-void SetContext(T* p) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void SetContext(T* p) {
   promise_detail::Context<T>::set(p);
 }