grpc 1.65.1 → 1.66.0.pre3

data/src/core/lib/promise/detail/status.h

@@ -30,12 +30,14 @@ namespace promise_detail {
 
 // Convert with a move the input status to an absl::Status.
 template <typename T>
-absl::Status IntoStatus(absl::StatusOr<T>* status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION absl::Status IntoStatus(
+    absl::StatusOr<T>* status) {
   return std::move(status->status());
 }
 
 // Convert with a move the input status to an absl::Status.
-inline absl::Status IntoStatus(absl::Status* status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline absl::Status IntoStatus(
+    absl::Status* status) {
   return std::move(*status);
 }
 
@@ -44,10 +46,14 @@ inline absl::Status IntoStatus(absl::Status* status) {
 // Return true if the status represented by the argument is ok, false if not.
 // By implementing this function for other, non-absl::Status types, those types
 // can participate in TrySeq as result types that affect control flow.
-inline bool IsStatusOk(const absl::Status& status) { return status.ok(); }
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const absl::Status& status) {
+  return status.ok();
+}
 
 template <typename T>
-inline bool IsStatusOk(const absl::StatusOr<T>& status) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline bool IsStatusOk(
+    const absl::StatusOr<T>& status) {
   return status.ok();
 }
 
@@ -56,29 +62,38 @@ struct StatusCastImpl;
 
 template <typename To>
 struct StatusCastImpl<To, To> {
-  static To Cast(To&& t) { return std::move(t); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static To Cast(To&& t) {
+    return std::move(t);
+  }
 };
 
 template <typename To>
 struct StatusCastImpl<To, const To&> {
-  static To Cast(const To& t) { return t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static To Cast(const To& t) { return t; }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, absl::StatusOr<T>> {
-  static absl::Status Cast(absl::StatusOr<T>&& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      absl::StatusOr<T>&& t) {
     return std::move(t.status());
   }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, absl::StatusOr<T>&> {
-  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      const absl::StatusOr<T>& t) {
+    return t.status();
+  }
 };
 
 template <typename T>
 struct StatusCastImpl<absl::Status, const absl::StatusOr<T>&> {
-  static absl::Status Cast(const absl::StatusOr<T>& t) { return t.status(); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Cast(
+      const absl::StatusOr<T>& t) {
+    return t.status();
+  }
 };
 
 // StatusCast<> allows casting from one status-bearing type to another,
@@ -88,7 +103,7 @@ struct StatusCastImpl<absl::Status, const absl::StatusOr<T>&> {
 // For cases where the status is guaranteed to be a failure (and hence not
 // needing to preserve values) see FailureStatusCast<> below.
 template <typename To, typename From>
-To StatusCast(From&& from) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION To StatusCast(From&& from) {
   return StatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }
 
@@ -97,16 +112,22 @@ struct FailureStatusCastImpl : public StatusCastImpl<To, From> {};
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, absl::Status> {
-  static absl::StatusOr<T> Cast(absl::Status&& t) { return std::move(t); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      absl::Status&& t) {
+    return std::move(t);
+  }
 };
 
 template <typename T>
 struct FailureStatusCastImpl<absl::StatusOr<T>, const absl::Status&> {
-  static absl::StatusOr<T> Cast(const absl::Status& t) { return t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::StatusOr<T> Cast(
+      const absl::Status& t) {
+    return t;
+  }
 };
 
 template <typename To, typename From>
-To FailureStatusCast(From&& from) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION To FailureStatusCast(From&& from) {
   DCHECK(!IsStatusOk(from));
   return FailureStatusCastImpl<To, From>::Cast(std::forward<From>(from));
 }

data/src/core/lib/promise/for_each.h

@@ -21,10 +21,10 @@
 #include <utility>
 
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
 
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/debug/trace.h"
@@ -47,14 +47,17 @@ struct Done;
 
 template <>
 struct Done<absl::Status> {
-  static absl::Status Make(bool cancelled) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static absl::Status Make(
+      bool cancelled) {
     return cancelled ? absl::CancelledError() : absl::OkStatus();
   }
 };
 
 template <>
 struct Done<StatusFlag> {
-  static StatusFlag Make(bool cancelled) { return StatusFlag(!cancelled); }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static StatusFlag Make(bool cancelled) {
+    return StatusFlag(!cancelled);
+  }
 };
 
 template <typename T, typename SfinaeVoid = void>
@@ -70,20 +73,23 @@ template <typename T>
 struct NextValueTraits<T, absl::void_t<typename T::value_type>> {
   using Value = typename T::value_type;
 
-  static NextValueType Type(const T& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static NextValueType Type(const T& t) {
     if (t.has_value()) return NextValueType::kValue;
     if (t.cancelled()) return NextValueType::kError;
     return NextValueType::kEndOfStream;
   }
 
-  static Value& MutableValue(T& t) { return *t; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Value& MutableValue(T& t) {
+    return *t;
+  }
 };
 
 template <typename T>
 struct NextValueTraits<ValueOrFailure<absl::optional<T>>> {
   using Value = T;
 
-  static NextValueType Type(const ValueOrFailure<absl::optional<T>>& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static NextValueType Type(
+      const ValueOrFailure<absl::optional<T>>& t) {
     if (t.ok()) {
       if (t.value().has_value()) return NextValueType::kValue;
       return NextValueType::kEndOfStream;
@@ -91,7 +97,8 @@ struct NextValueTraits<ValueOrFailure<absl::optional<T>>> {
     return NextValueType::kError;
   }
 
-  static Value& MutableValue(ValueOrFailure<absl::optional<T>>& t) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Value& MutableValue(
+      ValueOrFailure<absl::optional<T>>& t) {
     return **t;
   }
 };
@@ -110,13 +117,14 @@ class ForEach {
  public:
   using Result =
       typename PollTraits<decltype(std::declval<ActionPromise>()())>::Type;
-  ForEach(Reader reader, Action action, DebugLocation whence = {})
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ForEach(Reader reader, Action action,
+                                               DebugLocation whence = {})
       : reader_(std::move(reader)),
         action_factory_(std::move(action)),
         whence_(whence) {
     Construct(&reader_next_, reader_.Next());
   }
-  ~ForEach() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~ForEach() {
     if (reading_next_) {
       Destruct(&reader_next_);
     } else {
@@ -163,17 +171,16 @@ class ForEach {
                         ":", whence_.line(), "]: ");
   }
 
-  Poll<Result> PollReaderNext() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> PollReaderNext() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%s PollReaderNext", DebugTag().c_str());
+      LOG(INFO) << DebugTag() << " PollReaderNext";
     }
     auto r = reader_next_();
     if (auto* p = r.value_if_ready()) {
       switch (NextValueTraits<ReaderResult>::Type(*p)) {
         case NextValueType::kValue: {
           if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-            gpr_log(GPR_INFO, "%s PollReaderNext: got value",
-                    DebugTag().c_str());
+            LOG(INFO) << DebugTag() << " PollReaderNext: got value";
           }
           Destruct(&reader_next_);
           auto action = action_factory_.Make(
@@ -184,15 +191,13 @@ class ForEach {
         }
         case NextValueType::kEndOfStream: {
           if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-            gpr_log(GPR_INFO, "%s PollReaderNext: got end of stream",
-                    DebugTag().c_str());
+            LOG(INFO) << DebugTag() << " PollReaderNext: got end of stream";
           }
           return Done<Result>::Make(false);
         }
         case NextValueType::kError: {
           if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-            gpr_log(GPR_INFO, "%s PollReaderNext: got error",
-                    DebugTag().c_str());
+            LOG(INFO) << DebugTag() << " PollReaderNext: got error";
           }
           return Done<Result>::Make(true);
         }
@@ -203,7 +208,7 @@ class ForEach {
 
   Poll<Result> PollAction() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%s PollAction", DebugTag().c_str());
+      LOG(INFO) << DebugTag() << " PollAction";
     }
     auto r = in_action_.promise();
     if (auto* p = r.value_if_ready()) {
@@ -233,8 +238,8 @@ class ForEach {
 
 /// For each item acquired by calling Reader::Next, run the promise Action.
 template <typename Reader, typename Action>
-for_each_detail::ForEach<Reader, Action> ForEach(Reader reader, Action action,
-                                                 DebugLocation whence = {}) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION for_each_detail::ForEach<Reader, Action>
+ForEach(Reader reader, Action action, DebugLocation whence = {}) {
   return for_each_detail::ForEach<Reader, Action>(std::move(reader),
                                                   std::move(action), whence);
 }

data/src/core/lib/promise/if.h

@@ -33,8 +33,8 @@ namespace grpc_core {
 namespace promise_detail {
 
 template <typename CallPoll, typename T, typename F>
-typename CallPoll::PollResult ChooseIf(CallPoll call_poll, bool result,
-                                       T* if_true, F* if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION typename CallPoll::PollResult ChooseIf(
+    CallPoll call_poll, bool result, T* if_true, F* if_false) {
   if (result) {
     auto promise = if_true->Make();
     return call_poll(promise);
@@ -45,9 +45,8 @@ typename CallPoll::PollResult ChooseIf(CallPoll call_poll, bool result,
 }
 
 template <typename CallPoll, typename T, typename F>
-typename CallPoll::PollResult ChooseIf(CallPoll call_poll,
-                                       absl::StatusOr<bool> result, T* if_true,
-                                       F* if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION typename CallPoll::PollResult ChooseIf(
+    CallPoll call_poll, absl::StatusOr<bool> result, T* if_true, F* if_false) {
   if (!result.ok()) {
     return typename CallPoll::PollResult(result.status());
   } else if (*result) {
@@ -71,12 +70,12 @@ class If {
       typename PollTraits<decltype(std::declval<TruePromise>()())>::Type;
 
  public:
-  If(C condition, T if_true, F if_false)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(C condition, T if_true, F if_false)
       : state_(Evaluating{ConditionPromise(std::move(condition)),
                           TrueFactory(std::move(if_true)),
                           FalseFactory(std::move(if_false))}) {}
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     return absl::visit(CallPoll<false>{this}, state_);
   }
 
@@ -95,7 +94,8 @@ class If {
 
     If* const self;
 
-    PollResult operator()(Evaluating& evaluating) const {
+    GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION PollResult
+    operator()(Evaluating& evaluating) const {
       static_assert(
           !kSetState,
           "shouldn't need to set state coming through the initial branch");
@@ -108,7 +108,8 @@ class If {
     }
 
     template <class Promise>
-    PollResult operator()(Promise& promise) const {
+    GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION PollResult
+    operator()(Promise& promise) const {
       auto r = promise();
       if (kSetState && r.pending()) {
         self->state_.template emplace<Promise>(std::move(promise));
@@ -129,7 +130,8 @@ class If<bool, T, F> {
       typename PollTraits<decltype(std::declval<TruePromise>()())>::Type;
 
  public:
-  If(bool condition, T if_true, F if_false) : condition_(condition) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(bool condition, T if_true, F if_false)
+      : condition_(condition) {
     TrueFactory true_factory(std::move(if_true));
     FalseFactory false_factory(std::move(if_false));
     if (condition_) {
@@ -138,7 +140,7 @@ class If<bool, T, F> {
       Construct(&if_false_, false_factory.Make());
     }
   }
-  ~If() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~If() {
     if (condition_) {
       Destruct(&if_true_);
     } else {
@@ -148,21 +150,22 @@ class If<bool, T, F> {
 
   If(const If&) = delete;
   If& operator=(const If&) = delete;
-  If(If&& other) noexcept : condition_(other.condition_) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If(If&& other) noexcept
+      : condition_(other.condition_) {
     if (condition_) {
       Construct(&if_true_, std::move(other.if_true_));
     } else {
       Construct(&if_false_, std::move(other.if_false_));
     }
   }
-  If& operator=(If&& other) noexcept {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION If& operator=(If&& other) noexcept {
     if (&other == this) return *this;
     Destruct(this);
     Construct(this, std::move(other));
     return *this;
   }
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
 #ifndef NDEBUG
     asan_canary_ = std::make_unique<int>(1 + *asan_canary_);
 #endif
@@ -197,7 +200,8 @@ class If<bool, T, F> {
 // This makes it safe to capture lambda arguments in the promise factory by
 // reference.
 template <typename C, typename T, typename F>
-promise_detail::If<C, T, F> If(C condition, T if_true, F if_false) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::If<C, T, F> If(
+    C condition, T if_true, F if_false) {
   return promise_detail::If<C, T, F>(std::move(condition), std::move(if_true),
                                      std::move(if_false));
 }

data/src/core/lib/promise/interceptor_list.h

@@ -23,11 +23,11 @@
 #include <utility>
 
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/types/optional.h"
 
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/gprpp/construct_destruct.h"
@@ -88,8 +88,8 @@ class InterceptorList {
     RunPromise(size_t memory_required, Map** factory, absl::optional<T> value) {
       if (!value.has_value() || *factory == nullptr) {
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-          gpr_log(GPR_DEBUG,
-                  "InterceptorList::RunPromise[%p]: create immediate", this);
+          VLOG(2) << "InterceptorList::RunPromise[" << this
+                  << "]: create immediate";
         }
         is_immediately_resolved_ = true;
         Construct(&result_, std::move(value));
@@ -101,16 +101,15 @@ class InterceptorList {
         async_resolution_.current_factory = *factory;
         async_resolution_.first_factory = factory;
         if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-          gpr_log(GPR_DEBUG,
-                  "InterceptorList::RunPromise[%p]: create async; mem=%p", this,
-                  async_resolution_.space.get());
+          VLOG(2) << "InterceptorList::RunPromise[" << this
+                  << "]: create async; mem=" << async_resolution_.space.get();
         }
       }
     }
 
     ~RunPromise() {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_DEBUG, "InterceptorList::RunPromise[%p]: destroy", this);
+        VLOG(2) << "InterceptorList::RunPromise[" << this << "]: destroy";
       }
       if (is_immediately_resolved_) {
         Destruct(&result_);
@@ -129,8 +128,8 @@ class InterceptorList {
     RunPromise(RunPromise&& other) noexcept
         : is_immediately_resolved_(other.is_immediately_resolved_) {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_DEBUG, "InterceptorList::RunPromise[%p]: move from %p",
-                this, &other);
+        VLOG(2) << "InterceptorList::RunPromise[" << this << "]: move from "
+                << &other;
       }
       if (is_immediately_resolved_) {
         Construct(&result_, std::move(other.result_));
@@ -143,8 +142,8 @@ class InterceptorList {
 
     Poll<absl::optional<T>> operator()() {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_DEBUG, "InterceptorList::RunPromise[%p]: %s", this,
-                DebugString().c_str());
+        VLOG(2) << "InterceptorList::RunPromise[" << this
+                << "]: " << DebugString();
       }
       if (is_immediately_resolved_) return std::move(result_);
       while (true) {
@@ -161,8 +160,8 @@ class InterceptorList {
               async_resolution_.current_factory->next();
           if (!p->has_value()) async_resolution_.current_factory = nullptr;
           if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-            gpr_log(GPR_DEBUG, "InterceptorList::RunPromise[%p]: %s", this,
-                    DebugString().c_str());
+            VLOG(2) << "InterceptorList::RunPromise[" << this
+                    << "]: " << DebugString();
           }
           if (async_resolution_.current_factory == nullptr) {
             return std::move(*p);

data/src/core/lib/promise/latch.h

@@ -22,9 +22,9 @@
 #include <utility>
 
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
 
-#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/debug/trace.h"
@@ -68,8 +68,7 @@ class Latch {
 #endif
     return [this]() -> Poll<T> {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_INFO, "%sWait %s", DebugTag().c_str(),
-                StateString().c_str());
+        LOG(INFO) << DebugTag() << "Wait " << StateString();
       }
       if (has_value_) {
         return std::move(value_);
@@ -87,8 +86,7 @@ class Latch {
 #endif
     return [this]() -> Poll<T> {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_INFO, "%sWaitAndCopy %s", DebugTag().c_str(),
-                StateString().c_str());
+        LOG(INFO) << DebugTag() << "WaitAndCopy " << StateString();
       }
       if (has_value_) {
         return value_;
@@ -101,7 +99,7 @@ class Latch {
   // Set the value of the latch. Can only be called once.
   void Set(T value) {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str());
+      LOG(INFO) << DebugTag() << "Set " << StateString();
     }
     DCHECK(!has_value_);
     value_ = std::move(value);
@@ -164,8 +162,7 @@ class Latch<void> {
 #endif
     return [this]() -> Poll<Empty> {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_INFO, "%sPollWait %s", DebugTag().c_str(),
-                StateString().c_str());
+        LOG(INFO) << DebugTag() << "PollWait " << StateString();
       }
       if (is_set_) {
         return Empty{};
@@ -178,7 +175,7 @@ class Latch<void> {
   // Set the latch. Can only be called once.
   void Set() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str());
+      LOG(INFO) << DebugTag() << "Set " << StateString();
     }
     DCHECK(!is_set_);
     is_set_ = true;
@@ -227,8 +224,7 @@ class ExternallyObservableLatch<void> {
   auto Wait() {
     return [this]() -> Poll<Empty> {
       if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-        gpr_log(GPR_INFO, "%sPollWait %s", DebugTag().c_str(),
-                StateString().c_str());
+        LOG(INFO) << DebugTag() << "PollWait " << StateString();
       }
       if (IsSet()) {
         return Empty{};
@@ -241,7 +237,7 @@ class ExternallyObservableLatch<void> {
   // Set the latch.
   void Set() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str());
+      LOG(INFO) << DebugTag() << "Set " << StateString();
     }
     is_set_.store(true, std::memory_order_relaxed);
     waiter_.Wake();
@@ -251,8 +247,7 @@ class ExternallyObservableLatch<void> {
 
   void Reset() {
     if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
-      gpr_log(GPR_INFO, "%sReset %s", DebugTag().c_str(),
-              StateString().c_str());
+      LOG(INFO) << DebugTag() << "Reset " << StateString();
     }
     is_set_.store(false, std::memory_order_relaxed);
   }

data/src/core/lib/promise/loop.h

@@ -46,13 +46,17 @@ struct LoopTraits;
 template <typename T>
 struct LoopTraits<LoopCtl<T>> {
   using Result = T;
-  static LoopCtl<T> ToLoopCtl(LoopCtl<T> value) { return value; }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<T> ToLoopCtl(
+      LoopCtl<T> value) {
+    return value;
+  }
 };
 
 template <typename T>
 struct LoopTraits<absl::StatusOr<LoopCtl<T>>> {
   using Result = absl::StatusOr<T>;
-  static LoopCtl<Result> ToLoopCtl(absl::StatusOr<LoopCtl<T>> value) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<Result> ToLoopCtl(
+      absl::StatusOr<LoopCtl<T>> value) {
     if (!value.ok()) return value.status();
     auto& inner = *value;
     if (absl::holds_alternative<Continue>(inner)) return Continue{};
@@ -63,7 +67,7 @@ struct LoopTraits<absl::StatusOr<LoopCtl<T>>> {
 template <>
 struct LoopTraits<absl::StatusOr<LoopCtl<absl::Status>>> {
   using Result = absl::Status;
-  static LoopCtl<Result> ToLoopCtl(
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static LoopCtl<Result> ToLoopCtl(
       absl::StatusOr<LoopCtl<absl::Status>> value) {
     if (!value.ok()) return value.status();
     const auto& inner = *value;
@@ -82,12 +86,13 @@ class Loop {
  public:
   using Result = typename LoopTraits<PromiseResult>::Result;
 
-  explicit Loop(F f) : factory_(std::move(f)) {}
-  ~Loop() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Loop(F f)
+      : factory_(std::move(f)) {}
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Loop() {
     if (started_) Destruct(&promise_);
   }
 
-  Loop(Loop&& loop) noexcept
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Loop(Loop&& loop) noexcept
       : factory_(std::move(loop.factory_)), started_(loop.started_) {
     if (started_) Construct(&promise_, std::move(loop.promise_));
   }
@@ -95,7 +100,7 @@ class Loop {
   Loop(const Loop& loop) = delete;
   Loop& operator=(const Loop& loop) = delete;
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     if (!started_) {
       started_ = true;
       Construct(&promise_, factory_.Make());
@@ -136,7 +141,7 @@ class Loop {
 // Expects F returns LoopCtl<T> - if it's Continue, then run the loop again -
 // otherwise yield the returned value as the result of the loop.
 template <typename F>
-promise_detail::Loop<F> Loop(F f) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Loop<F> Loop(F f) {
   return promise_detail::Loop<F>(std::move(f));
 }
 

data/src/core/lib/promise/map.h

@@ -35,7 +35,7 @@ namespace promise_detail {
 template <typename Promise, typename Fn>
 class Map {
  public:
-  Map(Promise promise, Fn fn)
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Map(Promise promise, Fn fn)
       : promise_(std::move(promise)), fn_(std::move(fn)) {}
 
   Map(const Map&) = delete;
@@ -49,7 +49,7 @@ class Map {
   using Result =
       RemoveCVRef<decltype(std::declval<Fn>()(std::declval<PromiseResult>()))>;
 
-  Poll<Result> operator()() {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION Poll<Result> operator()() {
     Poll<PromiseResult> r = promise_();
     if (auto* p = r.value_if_ready()) {
       return fn_(std::move(*p));
@@ -68,7 +68,8 @@ class Map {
 // Takes a promise, and a synchronous function to mutate its result, and
 // returns a promise.
 template <typename Promise, typename Fn>
-promise_detail::Map<Promise, Fn> Map(Promise promise, Fn fn) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION promise_detail::Map<Promise, Fn> Map(
+    Promise promise, Fn fn) {
   return promise_detail::Map<Promise, Fn>(std::move(promise), std::move(fn));
 }
 
@@ -76,7 +77,7 @@ promise_detail::Map<Promise, Fn> Map(Promise promise, Fn fn) {
 // and a bool indicating whether there was ever a Pending{} value observed from
 // polling.
 template <typename Promise>
-auto CheckDelayed(Promise promise) {
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto CheckDelayed(Promise promise) {
   using P = promise_detail::PromiseLike<Promise>;
   return [delayed = false, promise = P(std::move(promise))]() mutable
          -> Poll<std::tuple<typename P::Result, bool>> {
@@ -93,13 +94,13 @@ auto CheckDelayed(Promise promise) {
 template <size_t kElem>
 struct JustElem {
   template <typename... A>
-  auto operator()(std::tuple<A...>&& t) const
-      -> decltype(std::get<kElem>(std::forward<std::tuple<A...>>(t))) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()(std::tuple<A...>&& t)
+      const -> decltype(std::get<kElem>(std::forward<std::tuple<A...>>(t))) {
     return std::get<kElem>(std::forward<std::tuple<A...>>(t));
   }
   template <typename... A>
-  auto operator()(const std::tuple<A...>& t) const
-      -> decltype(std::get<kElem>(t)) {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto operator()(
+      const std::tuple<A...>& t) const -> decltype(std::get<kElem>(t)) {
     return std::get<kElem>(t);
   }
 };