grpc 1.78.0 → 1.80.0.pre1

data/src/core/lib/resource_quota/arena.h

@@ -419,7 +419,7 @@ class ArenaSpsc {
     T result = std::move(next->value);
     Destruct(&next->value);
     tail_.store(next, std::memory_order_release);
-    return result;
+    return std::move(result);
   }
 
   T* Peek() {

data/src/core/lib/resource_quota/memory_quota.cc

@@ -31,6 +31,7 @@
 #include "src/core/channelz/channelz.h"
 #include "src/core/channelz/property_list.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/promise/exec_ctx_wakeup_scheduler.h"
 #include "src/core/lib/promise/loop.h"
 #include "src/core/lib/promise/map.h"
@@ -557,7 +558,9 @@ void BasicMemoryQuota::Take(GrpcMemoryAllocatorImpl* allocator, size_t amount) {
   auto prior = free_bytes_.fetch_sub(amount, std::memory_order_acq_rel);
   // If we push into overcommit, awake the reclaimer.
   if (prior >= 0 && prior < static_cast<intptr_t>(amount)) {
-    if (reclaimer_activity_ != nullptr) reclaimer_activity_->ForceWakeup();
+    if (reclaimer_activity_ != nullptr) {
+      EnsureRunInExecCtx([this]() { reclaimer_activity_->ForceWakeup(); });
+    }
   }
 
   if (IsFreeLargeAllocatorEnabled()) {

data/src/core/lib/resource_quota/resource_quota.cc

@@ -33,7 +33,8 @@ ResourceQuota::ResourceQuota(std::string name)
     : channelz_node_(
           MakeRefCounted<channelz::ResourceQuotaNode>(std::move(name))),
       memory_quota_(MakeMemoryQuota(channelz_node_)),
-      thread_quota_(MakeRefCounted<ThreadQuota>()) {}
+      thread_quota_(MakeRefCounted<ThreadQuota>()),
+      stream_quota_(MakeRefCounted<StreamQuota>()) {}
 
 ResourceQuota::~ResourceQuota() = default;
 

data/src/core/lib/resource_quota/resource_quota.h

@@ -53,6 +53,8 @@ class ResourceQuota : public RefCounted<ResourceQuota>,
 
   const RefCountedPtr<ThreadQuota>& thread_quota() { return thread_quota_; }
 
+  const RefCountedPtr<StreamQuota>& stream_quota() { return stream_quota_; }
+
   // The default global resource quota
   static ResourceQuotaRefPtr Default();
   static void TestOnlyResetDefaultResourceQuota();
@@ -66,6 +68,7 @@ class ResourceQuota : public RefCounted<ResourceQuota>,
   RefCountedPtr<channelz::ResourceQuotaNode> channelz_node_;
   MemoryQuotaRefPtr memory_quota_;
   RefCountedPtr<ThreadQuota> thread_quota_;
+  RefCountedPtr<StreamQuota> stream_quota_;
 };
 
 inline ResourceQuotaRefPtr MakeResourceQuota(std::string name) {

data/src/core/lib/resource_quota/stream_quota.cc

@@ -19,4 +19,80 @@
 #include <cstdint>
 #include <limits>
 
-namespace grpc_core {}  // namespace grpc_core
+namespace grpc_core {
+
+void StreamQuota::SetMaxOutstandingStreams(
+    uint32_t new_max_outstanding_streams) {
+  limiter_.max_outstanding_requests.store(new_max_outstanding_streams,
+                                          std::memory_order_relaxed);
+  UpdatePerConnectionLimits();
+}
+
+uint32_t StreamQuota::GetPerConnectionMaxConcurrentRequests(
+    uint32_t current_open_requests) {
+  if (limiter_.max_outstanding_requests.load(std::memory_order_relaxed) ==
+      std::numeric_limits<uint32_t>::max()) {
+    return std::numeric_limits<uint32_t>::max();
+  }
+
+  limiter_.periodic_update.Tick(
+      [this](Duration) { UpdatePerConnectionLimits(); });
+  const uint64_t allowed_requests_per_channel =
+      limiter_.allowed_requests_per_channel.load(std::memory_order_relaxed);
+  const uint64_t target_mean_requests_per_channel =
+      limiter_.target_mean_requests_per_channel.load(std::memory_order_relaxed);
+
+  if (allowed_requests_per_channel == 0) {
+    // If there are open requests on this channel, but we're past capacity
+    // try to lower the number of requests here. This should slowly force
+    // ramping down to numbers we can cope with.
+    if (current_open_requests > 1) return current_open_requests - 1;
+    return 1;
+  }
+
+  // If there is only one channel, we can allow the target mean.
+  if (limiter_.open_channels.load(std::memory_order_relaxed) <= 1) {
+    return target_mean_requests_per_channel;
+  }
+
+  auto clamp = [](uint64_t x) {
+    return std::min<uint64_t>(x, std::numeric_limits<uint32_t>::max());
+  };
+  if (current_open_requests < target_mean_requests_per_channel) {
+    return std::min(clamp(current_open_requests + allowed_requests_per_channel),
+                    target_mean_requests_per_channel);
+  } else if (current_open_requests < 2 * target_mean_requests_per_channel) {
+    return clamp(current_open_requests + 1);
+  } else {
+    return clamp(2 * target_mean_requests_per_channel);
+  }
+}
+
+void StreamQuota::UpdatePerConnectionLimits() {
+  int64_t outstanding_requests = 0;
+  int64_t open_channels =
+      limiter_.open_channels.load(std::memory_order_relaxed);
+  for (auto& stats : stats_) {
+    outstanding_requests +=
+        stats.outstanding_requests.load(std::memory_order_relaxed);
+  }
+  open_channels = std::max<int64_t>(1, open_channels);
+  outstanding_requests = std::max<int64_t>(0, outstanding_requests);
+  const int64_t max_outstanding_requests =
+      limiter_.max_outstanding_requests.load(std::memory_order_relaxed);
+  const int64_t allowed_requests_per_channel =
+      (max_outstanding_requests - outstanding_requests) / open_channels;
+  const uint64_t target_mean_requests_per_channel =
+      max_outstanding_requests / open_channels;
+  limiter_.allowed_requests_per_channel.store(
+      std::max<int64_t>(0, allowed_requests_per_channel),
+      std::memory_order_relaxed);
+  limiter_.target_mean_requests_per_channel.store(
+      target_mean_requests_per_channel, std::memory_order_relaxed);
+}
+
+void StreamQuota::UpdatePerConnectionLimitsForAllTestOnly() {
+  UpdatePerConnectionLimits();
+}
+
+}  // namespace grpc_core

data/src/core/lib/resource_quota/stream_quota.h

@@ -26,6 +26,69 @@
 #include "src/core/util/ref_counted.h"
 #include "src/core/util/ref_counted_ptr.h"
 
-namespace grpc_core {}  // namespace grpc_core
+namespace grpc_core {
+
+// Tracks the amount of streams in a resource quota.
+class StreamQuota : public RefCounted<StreamQuota> {
+ public:
+  StreamQuota() = default;
+  ~StreamQuota() override = default;
+
+  StreamQuota(const StreamQuota&) = delete;
+  StreamQuota& operator=(const StreamQuota&) = delete;
+
+  void IncrementOutstandingRequests() {
+    stats_.this_cpu().outstanding_requests.fetch_add(1,
+                                                     std::memory_order_relaxed);
+  }
+
+  void DecrementOutstandingRequests() {
+    stats_.this_cpu().outstanding_requests.fetch_sub(1,
+                                                     std::memory_order_relaxed);
+  }
+
+  void IncrementOpenChannels() {
+    limiter_.open_channels.fetch_add(1, std::memory_order_relaxed);
+  }
+
+  void DecrementOpenChannels() {
+    limiter_.open_channels.fetch_sub(1, std::memory_order_relaxed);
+  }
+
+  uint32_t GetConnectionMaxConcurrentRequests(uint32_t current_open_requests) {
+    return GetPerConnectionMaxConcurrentRequests(current_open_requests);
+  }
+
+  void SetMaxOutstandingStreams(uint32_t new_max_outstanding_streams);
+
+  void UpdatePerConnectionLimitsForAllTestOnly();
+
+ private:
+  uint32_t GetPerConnectionMaxConcurrentRequests(
+      uint32_t current_open_requests);
+
+  struct alignas(GPR_CACHELINE_SIZE) Statistics {
+    std::atomic<int64_t> outstanding_requests{0};
+  };
+  PerCpu<Statistics> stats_{PerCpuOptions()};
+
+  struct alignas(GPR_CACHELINE_SIZE) Limiter {
+    PeriodicUpdate periodic_update{Duration::Seconds(1)};
+    std::atomic<uint64_t> allowed_requests_per_channel{
+        std::numeric_limits<uint32_t>::max()};
+    std::atomic<uint64_t> target_mean_requests_per_channel{
+        std::numeric_limits<uint32_t>::max()};
+    std::atomic<uint64_t> max_outstanding_requests{
+        std::numeric_limits<uint32_t>::max()};
+    std::atomic<uint64_t> open_channels{0};
+  };
+  Limiter limiter_;
+
+  void UpdatePerConnectionLimits();
+};
+
+using StreamQuotaRefPtr = RefCountedPtr<StreamQuota>;
+
+}  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_STREAM_QUOTA_H

data/src/core/lib/resource_quota/telemetry.h

@@ -21,9 +21,9 @@ namespace grpc_core {
 
 class ResourceQuotaDomain final : public InstrumentDomain<ResourceQuotaDomain> {
  public:
+  GRPC_INSTRUMENT_DOMAIN_LABELS("grpc.resource_quota");
   using Backend = HighContentionBackend;
   static constexpr absl::string_view kName = "resource_quota";
-  static constexpr auto kLabels = Labels("grpc.resource_quota");
 
   static inline const auto kCallsDropped = RegisterCounter(
       "grpc.resource_quota.calls_dropped",

data/src/core/lib/surface/call.cc

@@ -607,3 +607,16 @@ void grpc_call_run_in_event_engine(const grpc_call* call,
       ->GetContext<grpc_event_engine::experimental::EventEngine>()
       ->Run(std::move(cb));
 }
+
+void grpc_call_run_cq_cb(const grpc_call* call,
+                         absl::AnyInvocable<void()>&& cb) {
+  if (grpc_core::IsUseCallEventEngineInCompletionQueueEnabled()) {
+    grpc_call_run_in_event_engine(
+        call, [cb = std::forward<absl::AnyInvocable<void()>>(cb)]() mutable {
+          grpc_core::ExecCtx exec_ctx;
+          cb();
+        });
+  } else {
+    cb();
+  }
+}

data/src/core/lib/surface/call_utils.h

@@ -46,6 +46,7 @@
 #include "src/core/call/metadata.h"
 #include "src/core/call/metadata_batch.h"
 #include "src/core/channelz/property_list.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/cancel_callback.h"
 #include "src/core/lib/promise/detail/promise_like.h"
@@ -76,37 +77,48 @@ class PublishToAppEncoder {
     Append(key.c_slice(), value.c_slice());
   }
 
-  // Catch anything that is not explicitly handled, and do not publish it to the
-  // application. If new metadata is added to a batch that needs to be
-  // published, it should be called out here.
+  // Publish only metadata traits that have kPublishToApp == true.
   template <typename Which>
-  void Encode(Which, const typename Which::ValueType&) {}
-
-  void Encode(UserAgentMetadata, const Slice& slice) {
-    Append(UserAgentMetadata::key(), slice);
-  }
-
-  void Encode(HostMetadata, const Slice& slice) {
-    Append(HostMetadata::key(), slice);
-  }
-
-  void Encode(GrpcPreviousRpcAttemptsMetadata, uint32_t count) {
-    Append(GrpcPreviousRpcAttemptsMetadata::key(), count);
-  }
-
-  void Encode(GrpcRetryPushbackMsMetadata, Duration count) {
-    Append(GrpcRetryPushbackMsMetadata::key(), count.millis());
-  }
-
-  void Encode(LbTokenMetadata, const Slice& slice) {
-    Append(LbTokenMetadata::key(), slice);
+  void Encode(Which, const typename Which::ValueType& value) {
+    if (IsMetadataPublishToAppTagEnabled()) {
+      if constexpr (Which::kPublishToApp) {
+        Append(Which::key(), value);
+      }
+    } else {
+      if constexpr (std::is_same<UserAgentMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<HostMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<GrpcPreviousRpcAttemptsMetadata,
+                                 Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<GrpcRetryPushbackMsMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<LbTokenMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<W3CTraceParentMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<XForwardedForMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+      if constexpr (std::is_same<XForwardedHostMetadata, Which>::value) {
+        Append(Which::key(), value);
+      }
+    }
   }
 
-  void Encode(W3CTraceParentMetadata, const Slice& slice) {
-    Append(W3CTraceParentMetadata::key(), slice);
+ private:
+  void Append(absl::string_view key, Duration value) {
+    Append(StaticSlice::FromStaticString(key).c_slice(),
+           Slice::FromInt64(value.millis()).c_slice());
   }
 
- private:
   void Append(absl::string_view key, int64_t value) {
     Append(StaticSlice::FromStaticString(key).c_slice(),
            Slice::FromInt64(value).c_slice());
@@ -168,8 +180,9 @@ class OpHandlerImpl {
                 "PromiseFactory must return a promise");
 
   OpHandlerImpl() : state_(State::kDismissed) {}
-  explicit OpHandlerImpl(SetupResult result) : state_(State::kPromiseFactory) {
-    Construct(&promise_factory_, std::move(result));
+  explicit OpHandlerImpl(SetupResult&& result)
+      : state_(State::kPromiseFactory) {
+    Construct(&promise_factory_, std::forward<SetupResult>(result));
   }
 
   ~OpHandlerImpl() {
@@ -274,8 +287,9 @@ class OpHandlerImpl {
 };
 
 template <grpc_op_type op_type, typename PromiseFactory>
-auto OpHandler(PromiseFactory setup) {
-  return OpHandlerImpl<PromiseFactory, op_type>(std::move(setup));
+auto OpHandler(PromiseFactory&& setup) {
+  return OpHandlerImpl<PromiseFactory, op_type>(
+      std::forward<PromiseFactory>(setup));
 }
 
 class BatchOpIndex {
@@ -329,7 +343,7 @@ class WaitForCqEndOp {
     grpc_completion_queue* cq;
   };
   struct Started {
-    explicit Started(Waker waker) : waker(std::move(waker)) {}
+    explicit Started(Waker&& waker) : waker(std::forward<Waker>(waker)) {}
     Waker waker;
     grpc_cq_completion completion;
     std::atomic<bool> done{false};
@@ -388,7 +402,7 @@ class WaitForCqEndOp {
 };
 
 template <typename FalliblePart, typename FinalPart>
-auto InfallibleBatch(FalliblePart fallible_part, FinalPart final_part,
+auto InfallibleBatch(FalliblePart&& fallible_part, FinalPart&& final_part,
                      bool is_notify_tag_closure, void* notify_tag,
                      grpc_completion_queue* cq) {
   // Perform fallible_part, then final_part, then wait for the
@@ -397,9 +411,9 @@ auto InfallibleBatch(FalliblePart fallible_part, FinalPart final_part,
   // There's a slight bug here in that if we cancel this promise after
   // the WaitForCqEndOp we'll double post -- but we don't currently do that.
   return OnCancelFactory(
-      [fallible_part = std::move(fallible_part),
-       final_part = std::move(final_part), is_notify_tag_closure, notify_tag,
-       cq]() mutable {
+      [fallible_part = std::forward<FalliblePart>(fallible_part),
+       final_part = std::forward<FinalPart>(final_part), is_notify_tag_closure,
+       notify_tag, cq]() mutable {
         return LogPollBatch(notify_tag,
                             Seq(std::move(fallible_part), std::move(final_part),
                                 [is_notify_tag_closure, notify_tag, cq]() {
@@ -417,15 +431,15 @@ auto InfallibleBatch(FalliblePart fallible_part, FinalPart final_part,
 }
 
 template <typename FalliblePart>
-auto FallibleBatch(FalliblePart fallible_part, bool is_notify_tag_closure,
+auto FallibleBatch(FalliblePart&& fallible_part, bool is_notify_tag_closure,
                    void* notify_tag, grpc_completion_queue* cq) {
   // Perform fallible_part, then wait for the completion queue to be done.
   // If cancelled, we'll ensure the completion queue is notified.
   // There's a slight bug here in that if we cancel this promise after
   // the WaitForCqEndOp we'll double post -- but we don't currently do that.
   return OnCancelFactory(
-      [fallible_part = std::move(fallible_part), is_notify_tag_closure,
-       notify_tag, cq]() mutable {
+      [fallible_part = std::forward<FalliblePart>(fallible_part),
+       is_notify_tag_closure, notify_tag, cq]() mutable {
         return LogPollBatch(
             notify_tag,
             Seq(std::move(fallible_part),
@@ -445,7 +459,7 @@ auto FallibleBatch(FalliblePart fallible_part, bool is_notify_tag_closure,
 template <typename F>
 class PollBatchLogger {
  public:
-  PollBatchLogger(void* tag, F f) : tag_(tag), f_(std::move(f)) {}
+  PollBatchLogger(void* tag, F&& f) : tag_(tag), f_(std::forward<F>(f)) {}
 
   auto operator()() {
     GRPC_TRACE_LOG(call, INFO) << "Poll batch " << tag_;
@@ -474,8 +488,8 @@ class PollBatchLogger {
 };
 
 template <typename F>
-PollBatchLogger<F> LogPollBatch(void* tag, F f) {
-  return PollBatchLogger<F>(tag, std::move(f));
+PollBatchLogger<F> LogPollBatch(void* tag, F&& f) {
+  return PollBatchLogger<F>(tag, std::forward<F>(f));
 }
 
 class MessageReceiver {
@@ -497,8 +511,9 @@ class MessageReceiver {
     recv_message_ = op.data.recv_message.recv_message;
     return [this, puller]() mutable {
       return Map(puller->PullMessage(),
-                 [this](typename Puller::NextMessage msg) {
-                   return FinishRecvMessage(std::move(msg));
+                 [this](typename Puller::NextMessage&& msg) {
+                   return FinishRecvMessage(
+                       std::forward<typename Puller::NextMessage>(msg));
                  });
     };
   }

data/src/core/lib/surface/channel.h

@@ -133,10 +133,7 @@ class Channel : public UnstartedCallDestination,
   }
 
   grpc_event_engine::experimental::MemoryAllocator* memory_allocator() const {
-    if (IsTrackWritesInResourceQuotaEnabled()) {
-      return memory_allocator_;
-    }
-    return nullptr;
+    return memory_allocator_;
   }
 
  protected:

data/src/core/lib/surface/completion_queue.cc

@@ -33,11 +33,11 @@
 #include <vector>
 
 #include "src/core/lib/event_engine/shim.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/surface/event_string.h"
-#include "src/core/telemetry/stats.h"
 #include "src/core/telemetry/stats_data.h"
 #include "src/core/util/atomic_utils.h"
 #include "src/core/util/debug_location.h"
@@ -908,11 +908,18 @@ static void cq_end_op_for_callback(
   }
 
   auto* functor = static_cast<grpc_completion_queue_functor*>(tag);
-  cqd->event_engine->Run(
-      [engine = cqd->event_engine, functor, ok = error.ok()]() {
-        grpc_core::ExecCtx exec_ctx;
-        (*functor->functor_run)(functor, ok);
-      });
+  if (grpc_core::IsUseCallEventEngineInCompletionQueueEnabled()) {
+    (*functor->functor_run)(functor, error.ok());
+  } else {
+    // While the experiment is rolling out, let us keep the cqd->event_engine
+    // field to prevent repeated calls to GetDefaultEventEngine() since it
+    // acquires a global lock.
+    cqd->event_engine->Run(
+        [engine = cqd->event_engine, functor, ok = error.ok()]() {
+          grpc_core::ExecCtx exec_ctx;
+          (*functor->functor_run)(functor, ok);
+        });
+  }
 }
 
 void grpc_cq_end_op(grpc_completion_queue* cq, void* tag,

data/src/core/lib/surface/validate_metadata.cc

@@ -42,6 +42,17 @@ class LegalHeaderKeyBits : public BitSet<256> {
 };
 constexpr LegalHeaderKeyBits g_legal_header_key_bits;
 
+class LegalHeaderNonBinValueBits : public BitSet<256> {
+ public:
+  constexpr LegalHeaderNonBinValueBits() {
+    for (int i = 32; i <= 126; i++) {
+      set(i);
+    }
+  }
+};
+
+constexpr LegalHeaderNonBinValueBits g_legal_header_non_bin_value_bits;
+
 ValidateMetadataResult ConformsTo(absl::string_view x,
                                   const BitSet<256>& legal_bits,
                                   ValidateMetadataResult error) {
@@ -71,6 +82,12 @@ ValidateMetadataResult ValidateHeaderKeyIsLegal(absl::string_view key) {
                     ValidateMetadataResult::kIllegalHeaderKey);
 }
 
+ValidateMetadataResult ValidateNonBinaryHeaderValueIsLegal(
+    absl::string_view value) {
+  return ConformsTo(value, g_legal_header_non_bin_value_bits,
+                    ValidateMetadataResult::kIllegalHeaderValue);
+}
+
 const char* ValidateMetadataResultToString(ValidateMetadataResult result) {
   switch (result) {
     case ValidateMetadataResult::kOk:
@@ -103,23 +120,11 @@ int grpc_header_key_is_legal(grpc_slice slice) {
   return error2int(grpc_validate_header_key_is_legal(slice));
 }
 
-namespace {
-class LegalHeaderNonBinValueBits : public grpc_core::BitSet<256> {
- public:
-  constexpr LegalHeaderNonBinValueBits() {
-    for (int i = 32; i <= 126; i++) {
-      set(i);
-    }
-  }
-};
-constexpr LegalHeaderNonBinValueBits g_legal_header_non_bin_value_bits;
-}  // namespace
-
 grpc_error_handle grpc_validate_header_nonbin_value_is_legal(
     const grpc_slice& slice) {
-  return grpc_core::UpgradeToStatus(grpc_core::ConformsTo(
-      grpc_core::StringViewFromSlice(slice), g_legal_header_non_bin_value_bits,
-      grpc_core::ValidateMetadataResult::kIllegalHeaderValue));
+  return grpc_core::UpgradeToStatus(
+      grpc_core::ValidateNonBinaryHeaderValueIsLegal(
+          grpc_core::StringViewFromSlice(slice)));
 }
 
 int grpc_header_nonbin_value_is_legal(grpc_slice slice) {

data/src/core/lib/surface/validate_metadata.h

@@ -41,9 +41,11 @@ enum class ValidateMetadataResult : uint8_t {
 
 const char* ValidateMetadataResultToString(ValidateMetadataResult result);
 
-// Returns nullopt if the key is legal, otherwise returns an error message.
 ValidateMetadataResult ValidateHeaderKeyIsLegal(absl::string_view key);
 
+ValidateMetadataResult ValidateNonBinaryHeaderValueIsLegal(
+    absl::string_view value);
+
 }  // namespace grpc_core
 
 grpc_error_handle grpc_validate_header_key_is_legal(const grpc_slice& slice);

data/src/core/lib/surface/version.cc

@@ -22,6 +22,6 @@
 #include <grpc/grpc.h>
 #include <grpc/support/port_platform.h>
 
-const char* grpc_version_string(void) { return "52.0.0"; }
+const char* grpc_version_string(void) { return "53.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gutsy"; }
+const char* grpc_g_stands_for(void) { return "glimmering"; }

data/src/core/lib/transport/promise_endpoint.cc

@@ -94,7 +94,7 @@ void PromiseEndpoint::ReadState::Complete(absl::Status status,
                 ExecCtx exec_ctx;
                 self->Complete(std::move(status), num_bytes_requested);
               },
-              &pending_buffer, std::move(read_args))) {
+              &pending_buffer, read_args)) {
         continue;
       }
       return;

data/src/core/lib/transport/promise_endpoint.h

@@ -157,7 +157,7 @@ class PromiseEndpoint {
                 ExecCtx exec_ctx;
                 read_state->Complete(std::move(status), num_bytes);
               },
-              &read_state_->pending_buffer, std::move(read_args))) {
+              &read_state_->pending_buffer, read_args)) {
         read_state_->waker = Waker();
         read_state_->pending_buffer.MoveFirstNBytesIntoSliceBuffer(
             read_state_->pending_buffer.Length(), read_state_->buffer);

data/src/core/lib/transport/transport.h

@@ -409,6 +409,11 @@ struct grpc_transport_stream_op_batch_payload {
     // This should be set for cancellations that result from malformed client
     // initial metadata.
     bool tarpit = false;
+    // Server-side only: If non-null, the transport sends this trailing
+    // metadata to the client.
+    // NOTE: This metadata bypasses subsequent filters and is sent directly
+    // to the client. Ensure it contains only fields intended for the client.
+    grpc_core::ServerMetadataHandle send_trailing_metadata = nullptr;
   } cancel_stream;
 };
 

data/src/core/load_balancing/health_check_client.cc

@@ -263,8 +263,7 @@ void HealthProducer::HealthChecker::StartHealthStreamLocked() {
       << "HealthProducer " << producer_.get() << " HealthChecker " << this
       << ": creating HealthClient for \"" << health_check_service_name_ << "\"";
   stream_client_ = MakeOrphanable<SubchannelStreamClient>(
-      producer_->connected_subchannel_, producer_->subchannel_->pollset_set(),
-      std::make_unique<HealthStreamEventHandler>(Ref()),
+      producer_->subchannel_, std::make_unique<HealthStreamEventHandler>(Ref()),
       GRPC_TRACE_FLAG_ENABLED(health_check_client) ? "HealthClient" : nullptr);
 }
 
@@ -304,10 +303,6 @@ void HealthProducer::Start(WeakRefCountedPtr<Subchannel> subchannel) {
       << "HealthProducer " << this << ": starting with subchannel "
       << subchannel.get();
   subchannel_ = std::move(subchannel);
-  {
-    MutexLock lock(&mu_);
-    connected_subchannel_ = subchannel_->connected_subchannel();
-  }
   auto connectivity_watcher =
       MakeRefCounted<ConnectivityWatcher>(WeakRefAsSubclass<HealthProducer>());
   connectivity_watcher_ = connectivity_watcher.get();
@@ -369,15 +364,6 @@ void HealthProducer::OnConnectivityStateChange(grpc_connectivity_state state,
       << ": subchannel state update: state=" << ConnectivityStateName(state)
       << " status=" << status;
   MutexLock lock(&mu_);
-  if (state == GRPC_CHANNEL_READY) {
-    connected_subchannel_ = subchannel_->connected_subchannel();
-    // If the subchannel became disconnected again before we got this
-    // notification, then just ignore the READY notification.  We should
-    // get another notification shortly indicating a different state.
-    if (connected_subchannel_ == nullptr) return;
-  } else {
-    connected_subchannel_.reset();
-  }
   state_ = state;
   status_ = status;
   for (const auto& [_, health_checker] : health_checkers_) {

data/src/core/load_balancing/health_check_client_internal.h

@@ -144,8 +144,6 @@ class HealthProducer final : public Subchannel::DataProducerInterface {
   Mutex mu_;
   std::optional<grpc_connectivity_state> state_ ABSL_GUARDED_BY(&mu_);
   absl::Status status_ ABSL_GUARDED_BY(&mu_);
-  RefCountedPtr<ConnectedSubchannel> connected_subchannel_
-      ABSL_GUARDED_BY(&mu_);
   std::map<std::string /*health_check_service_name*/,
            OrphanablePtr<HealthChecker>>
       health_checkers_ ABSL_GUARDED_BY(&mu_);