grpc 1.73.0 → 1.74.0

data/src/core/lib/transport/promise_endpoint.h

@@ -0,0 +1,307 @@
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_PROMISE_ENDPOINT_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_PROMISE_ENDPOINT_H
+
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/event_engine/slice.h>
+#include <grpc/event_engine/slice_buffer.h>
+#include <grpc/slice_buffer.h>
+#include <grpc/support/port_platform.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <atomic>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/log/check.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "src/core/lib/event_engine/extensions/chaotic_good_extension.h"
+#include "src/core/lib/event_engine/query_extensions.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/cancel_callback.h"
+#include "src/core/lib/promise/if.h"
+#include "src/core/lib/promise/map.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/slice/slice.h"
+#include "src/core/lib/slice/slice_buffer.h"
+#include "src/core/util/dump_args.h"
+#include "src/core/util/sync.h"
+
+namespace grpc_core {
+
+// Wrapper around event engine endpoint that provides a promise like API.
+class PromiseEndpoint {
+ public:
+  using WriteArgs =
+      grpc_event_engine::experimental::EventEngine::Endpoint::WriteArgs;
+
+  PromiseEndpoint(
+      std::unique_ptr<grpc_event_engine::experimental::EventEngine::Endpoint>
+          endpoint,
+      SliceBuffer already_received);
+  PromiseEndpoint() = default;
+  ~PromiseEndpoint() = default;
+  /// Prevent copying of PromiseEndpoint; moving is fine.
+  PromiseEndpoint(const PromiseEndpoint&) = delete;
+  PromiseEndpoint& operator=(const PromiseEndpoint&) = delete;
+  PromiseEndpoint(PromiseEndpoint&&) = default;
+  PromiseEndpoint& operator=(PromiseEndpoint&&) = default;
+
+  // Returns a promise that resolves to a `absl::Status` indicating the result
+  // of the write operation.
+  //
+  // Concurrent writes are not supported, which means callers should not call
+  // `Write()` before the previous write finishes. Doing that results in
+  // undefined behavior.
+  auto Write(SliceBuffer data, WriteArgs write_args) {
+    GRPC_LATENT_SEE_PARENT_SCOPE("GRPC:Write");
+    // Start write and assert previous write finishes.
+    auto prev = write_state_->state.exchange(WriteState::kWriting,
+                                             std::memory_order_relaxed);
+    CHECK(prev == WriteState::kIdle);
+    bool completed;
+    if (data.Length() == 0) {
+      completed = true;
+    } else {
+      // TODO(ladynana): Replace this with `SliceBufferCast<>` when it is
+      // available.
+      grpc_slice_buffer_swap(write_state_->buffer.c_slice_buffer(),
+                             data.c_slice_buffer());
+      // If `Write()` returns true immediately, the callback will not be called.
+      // We still need to call our callback to pick up the result.
+      write_state_->waker = GetContext<Activity>()->MakeNonOwningWaker();
+      completed = endpoint_->Write(
+          [write_state = write_state_](absl::Status status) {
+            ExecCtx exec_ctx;
+            write_state->Complete(std::move(status));
+          },
+          &write_state_->buffer, std::move(write_args));
+      if (completed) write_state_->waker = Waker();
+    }
+    return If(
+        completed,
+        [this]() {
+          return [write_state = write_state_]() {
+            auto prev = write_state->state.exchange(WriteState::kIdle,
+                                                    std::memory_order_relaxed);
+            CHECK(prev == WriteState::kWriting);
+            return absl::OkStatus();
+          };
+        },
+        GRPC_LATENT_SEE_PROMISE(
+            "DelayedWrite", ([this]() {
+              return [write_state = write_state_]() -> Poll<absl::Status> {
+                // If current write isn't finished return `Pending()`, else
+                // return write result.
+                WriteState::State expected = WriteState::kWritten;
+                if (write_state->state.compare_exchange_strong(
+                        expected, WriteState::kIdle, std::memory_order_acquire,
+                        std::memory_order_relaxed)) {
+                  // State was Written, and we changed it to Idle. We can return
+                  // the result.
+                  return std::move(write_state->result);
+                }
+                // State was not Written; since we're polling it must be
+                // Writing. Assert that and return Pending.
+                CHECK(expected == WriteState::kWriting);
+                return Pending();
+              };
+            })));
+  }
+
+  // Returns a promise that resolves to `SliceBuffer` with
+  // `num_bytes` bytes.
+  //
+  // Concurrent reads are not supported, which means callers should not call
+  // `Read()` before the previous read finishes. Doing that results in
+  // undefined behavior.
+  auto Read(size_t num_bytes) {
+    GRPC_LATENT_SEE_PARENT_SCOPE("GRPC:Read");
+    // Assert previous read finishes.
+    CHECK(!read_state_->complete.load(std::memory_order_relaxed));
+    // Should not have pending reads.
+    CHECK_EQ(read_state_->pending_buffer.Count(), 0u);
+    bool complete = true;
+    while (read_state_->buffer.Length() < num_bytes) {
+      GRPC_LATENT_SEE_INNER_SCOPE("GRPC:Read:Loop");
+      // Set read args with hinted bytes.
+      grpc_event_engine::experimental::EventEngine::Endpoint::ReadArgs
+          read_args;
+      read_args.set_read_hint_bytes(
+          static_cast<int64_t>(num_bytes - read_state_->buffer.Length()));
+      // If `Read()` returns true immediately, the callback will not be
+      // called.
+      read_state_->waker = GetContext<Activity>()->MakeNonOwningWaker();
+      if (endpoint_->Read(
+              [read_state = read_state_, num_bytes](absl::Status status) {
+                ExecCtx exec_ctx;
+                read_state->Complete(std::move(status), num_bytes);
+              },
+              &read_state_->pending_buffer, std::move(read_args))) {
+        read_state_->waker = Waker();
+        read_state_->pending_buffer.MoveFirstNBytesIntoSliceBuffer(
+            read_state_->pending_buffer.Length(), read_state_->buffer);
+        DCHECK_EQ(read_state_->pending_buffer.Count(), 0u);
+      } else {
+        complete = false;
+        break;
+      }
+    }
+    return If(
+        complete,
+        [this, num_bytes]() {
+          SliceBuffer ret;
+          grpc_slice_buffer_move_first_no_inline(
+              read_state_->buffer.c_slice_buffer(), num_bytes,
+              ret.c_slice_buffer());
+          return [ret = std::move(
+                      ret)]() mutable -> Poll<absl::StatusOr<SliceBuffer>> {
+            return std::move(ret);
+          };
+        },
+        GRPC_LATENT_SEE_PROMISE(
+            "DelayedRead", ([this, num_bytes]() {
+              return [read_state = read_state_,
+                      num_bytes]() -> Poll<absl::StatusOr<SliceBuffer>> {
+                if (!read_state->complete.load(std::memory_order_acquire)) {
+                  return Pending();
+                }
+                // If read succeeds, return `SliceBuffer` with `num_bytes`
+                // bytes.
+                if (read_state->result.ok()) {
+                  SliceBuffer ret;
+                  grpc_slice_buffer_move_first_no_inline(
+                      read_state->buffer.c_slice_buffer(), num_bytes,
+                      ret.c_slice_buffer());
+                  read_state->complete.store(false, std::memory_order_relaxed);
+                  return std::move(ret);
+                }
+                read_state->complete.store(false, std::memory_order_relaxed);
+                return std::move(read_state->result);
+              };
+            })));
+  }
+
+  // Returns a promise that resolves to `Slice` with at least
+  // `num_bytes` bytes which should be less than INT64_MAX bytes.
+  //
+  // Concurrent reads are not supported, which means callers should not call
+  // `ReadSlice()` before the previous read finishes. Doing that results in
+  // undefined behavior.
+  auto ReadSlice(size_t num_bytes) {
+    return Map(Read(num_bytes),
+               [](absl::StatusOr<SliceBuffer> buffer) -> absl::StatusOr<Slice> {
+                 if (!buffer.ok()) return buffer.status();
+                 return buffer->JoinIntoSlice();
+               });
+  }
+
+  // Returns a promise that resolves to a byte with type `uint8_t`.
+  auto ReadByte() {
+    return Map(ReadSlice(1),
+               [](absl::StatusOr<Slice> slice) -> absl::StatusOr<uint8_t> {
+                 if (!slice.ok()) return slice.status();
+                 return (*slice)[0];
+               });
+  }
+
+  // Enables RPC receive coalescing and alignment of memory holding received
+  // RPCs.
+  void EnforceRxMemoryAlignmentAndCoalescing() {
+    auto* chaotic_good_ext = grpc_event_engine::experimental::QueryExtension<
+        grpc_event_engine::experimental::ChaoticGoodExtension>(endpoint_.get());
+    if (chaotic_good_ext != nullptr) {
+      chaotic_good_ext->EnforceRxMemoryAlignment();
+      chaotic_good_ext->EnableRpcReceiveCoalescing();
+    }
+  }
+
+  const grpc_event_engine::experimental::EventEngine::ResolvedAddress&
+  GetPeerAddress() const;
+  const grpc_event_engine::experimental::EventEngine::ResolvedAddress&
+  GetLocalAddress() const;
+
+  std::shared_ptr<grpc_event_engine::experimental::EventEngine::Endpoint>
+  GetEventEngineEndpoint() const {
+    return endpoint_;
+  }
+
+ private:
+  std::shared_ptr<grpc_event_engine::experimental::EventEngine::Endpoint>
+      endpoint_;
+
+  struct ReadState : public RefCounted<ReadState> {
+    std::atomic<bool> complete{false};
+    // Read buffer used for storing successful reads given by
+    // `EventEngine::Endpoint` but not yet requested by the caller.
+    grpc_event_engine::experimental::SliceBuffer buffer;
+    // Buffer used to accept data from `EventEngine::Endpoint`.
+    // Every time after a successful read from `EventEngine::Endpoint`, the data
+    // in this buffer should be appended to `buffer`.
+    grpc_event_engine::experimental::SliceBuffer pending_buffer;
+    // Used for store the result from `EventEngine::Endpoint::Read()`.
+    absl::Status result;
+    Waker waker;
+    // Backing endpoint: we keep this on ReadState as reads will need to
+    // repeatedly read until the target size is hit, and we don't want to access
+    // the main object during this dance (indeed the main object may be
+    // deleted).
+    std::weak_ptr<grpc_event_engine::experimental::EventEngine::Endpoint>
+        endpoint;
+
+    void Complete(absl::Status status, size_t num_bytes_requested);
+  };
+
+  struct WriteState : public RefCounted<WriteState> {
+    enum State : uint8_t {
+      kIdle,     // Not writing.
+      kWriting,  // Write started, but not completed.
+      kWritten,  // Write completed.
+    };
+
+    std::atomic<State> state{kIdle};
+    // Write buffer used for `EventEngine::Endpoint::Write()` to ensure the
+    // memory behind the buffer is not lost.
+    grpc_event_engine::experimental::SliceBuffer buffer;
+    // Used for store the result from `EventEngine::Endpoint::Write()`.
+    absl::Status result;
+    Waker waker;
+
+    void Complete(absl::Status status) {
+      result = std::move(status);
+      auto w = std::move(waker);
+      auto prev = state.exchange(kWritten, std::memory_order_release);
+      // Previous state should be Writing. If we got anything else we've entered
+      // the callback path twice.
+      CHECK(prev == kWriting);
+      w.Wakeup();
+    }
+  };
+
+  RefCountedPtr<WriteState> write_state_ = MakeRefCounted<WriteState>();
+  RefCountedPtr<ReadState> read_state_ = MakeRefCounted<ReadState>();
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_PROMISE_ENDPOINT_H

data/src/core/load_balancing/child_policy_handler.cc

@@ -102,11 +102,10 @@ class ChildPolicyHandler::Helper final
     parent()->channel_control_helper()->RequestReresolution();
   }
 
-  void AddTraceEvent(TraceSeverity severity,
-                     absl::string_view message) override {
+  void AddTraceEvent(absl::string_view message) override {
     if (parent()->shutting_down_) return;
     if (!CalledByPendingChild() && !CalledByCurrentChild()) return;
-    parent()->channel_control_helper()->AddTraceEvent(severity, message);
+    parent()->channel_control_helper()->AddTraceEvent(message);
   }
 
   void set_child(LoadBalancingPolicy* child) { child_ = child; }
@@ -289,7 +288,6 @@ OrphanablePtr<LoadBalancingPolicy> ChildPolicyHandler::CreateChildPolicy(
               << lb_policy.get() << ")";
   }
   channel_control_helper()->AddTraceEvent(
-      ChannelControlHelper::TRACE_INFO,
       absl::StrCat("Created new LB policy \"", child_policy_name, "\""));
   grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(),
                                    interested_parties());

data/src/core/load_balancing/delegating_helper.h

@@ -81,9 +81,8 @@ class LoadBalancingPolicy::DelegatingChannelControlHelper
     return parent_helper()->GetStatsPluginGroup();
   }
 
-  void AddTraceEvent(TraceSeverity severity,
-                     absl::string_view message) override {
-    parent_helper()->AddTraceEvent(severity, message);
+  void AddTraceEvent(absl::string_view message) override {
+    parent_helper()->AddTraceEvent(message);
   }
 
  private:

data/src/core/load_balancing/health_check_client.cc

@@ -221,11 +221,7 @@ class HealthProducer::HealthChecker::HealthStreamEventHandler final
       LOG(ERROR) << kErrorMessage;
       auto* channelz_node =
           health_checker_->producer_->subchannel_->channelz_node();
-      if (channelz_node != nullptr) {
-        channelz_node->AddTraceEvent(
-            channelz::ChannelTrace::Error,
-            grpc_slice_from_static_string(kErrorMessage));
-      }
+      GRPC_CHANNELZ_LOG(channelz_node) << kErrorMessage;
       SetHealthStatusLocked(client, GRPC_CHANNEL_READY, kErrorMessage);
     }
   }

data/src/core/load_balancing/lb_policy.h

@@ -336,9 +336,7 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
     GetStatsPluginGroup() = 0;
 
     /// Adds a trace message associated with the channel.
-    enum TraceSeverity { TRACE_INFO, TRACE_WARNING, TRACE_ERROR };
-    virtual void AddTraceEvent(TraceSeverity severity,
-                               absl::string_view message) = 0;
+    virtual void AddTraceEvent(absl::string_view message) = 0;
   };
 
   class DelegatingChannelControlHelper;

data/src/core/load_balancing/oob_backend_metric.cc

@@ -145,11 +145,7 @@ class OrcaProducer::OrcaStreamEventHandler final
           "Orca stream returned UNIMPLEMENTED; disabling";
       LOG(ERROR) << kErrorMessage;
       auto* channelz_node = producer_->subchannel_->channelz_node();
-      if (channelz_node != nullptr) {
-        channelz_node->AddTraceEvent(
-            channelz::ChannelTrace::Error,
-            grpc_slice_from_static_string(kErrorMessage));
-      }
+      GRPC_CHANNELZ_LOG(channelz_node) << kErrorMessage;
     }
   }
 

data/src/core/load_balancing/pick_first/pick_first.cc

@@ -528,6 +528,9 @@ absl::Status PickFirst::UpdateLocked(UpdateArgs args) {
     });
     if (endpoints.empty()) {
       status = absl::UnavailableError("address list must not be empty");
+      // TODO(roth): Replace this one-off special case with a more
+      // general solution.
+      if (IsPickFirstIgnoreEmptyUpdatesEnabled()) args.addresses = status;
     } else {
       // Shuffle the list if needed.
       auto config = static_cast<PickFirstConfig*>(args.config.get());

data/src/core/load_balancing/xds/cds.cc

@@ -397,7 +397,14 @@ absl::Status CdsLb::UpdateLocked(UpdateArgs args) {
             cluster_name_, new_cluster_config->cluster->use_http_connect,
             endpoint_config.endpoints,
             child_name_state_.priority_child_numbers);
-        update_args.resolution_note = endpoint_config.resolution_note;
+        std::vector<absl::string_view> resolution_notes;
+        if (!args.resolution_note.empty()) {
+          resolution_notes.emplace_back(args.resolution_note);
+        }
+        if (!endpoint_config.resolution_note.empty()) {
+          resolution_notes.emplace_back(endpoint_config.resolution_note);
+        }
+        update_args.resolution_note = absl::StrJoin(resolution_notes, "; ");
         // Construct child policy config.
         child_policy_config_json = CreateChildPolicyConfigForLeafCluster(
             *new_cluster_config, endpoint_config, aggregate_cluster_resource);
@@ -405,6 +412,8 @@ absl::Status CdsLb::UpdateLocked(UpdateArgs args) {
       // Aggregate cluster.
       [&](const XdsConfig::ClusterConfig::AggregateConfig& aggregate_config) {
         child_name_state_.Reset();
+        // Populate resolution_note for child policy.
+        update_args.resolution_note = aggregate_config.resolution_note;
         // Construct child policy config.
         child_policy_config_json =
             CreateChildPolicyConfigForAggregateCluster(aggregate_config);

data/src/core/plugin_registry/grpc_plugin_registry_extra.cc

@@ -28,6 +28,7 @@ extern void GcpAuthenticationFilterRegister(
 extern void RegisterXdsChannelStackModifier(
     CoreConfiguration::Builder* builder);
 extern void RegisterChannelDefaultCreds(CoreConfiguration::Builder* builder);
+extern void RegisterDefaultCallCreds(CoreConfiguration::Builder* builder);
 extern void RegisterXdsResolver(CoreConfiguration::Builder* builder);
 extern void RegisterCloud2ProdResolver(CoreConfiguration::Builder* builder);
 extern void RegisterXdsClusterManagerLbPolicy(
@@ -52,6 +53,7 @@ void RegisterExtraFilters(CoreConfiguration::Builder* builder) {
   GcpAuthenticationFilterRegister(builder);
   RegisterXdsChannelStackModifier(builder);
   RegisterChannelDefaultCreds(builder);
+  RegisterDefaultCallCreds(builder);
   RegisterXdsResolver(builder);
   RegisterCloud2ProdResolver(builder);
   RegisterXdsClusterManagerLbPolicy(builder);

data/src/core/resolver/xds/xds_config.cc

@@ -41,10 +41,10 @@ XdsConfig::ClusterConfig::ClusterConfig(
 
 XdsConfig::ClusterConfig::ClusterConfig(
     std::shared_ptr<const XdsClusterResource> cluster,
-    std::vector<absl::string_view> leaf_clusters)
+    std::vector<absl::string_view> leaf_clusters, std::string resolution_note)
     : cluster(std::move(cluster)),
       children(absl::in_place_type_t<AggregateConfig>(),
-               std::move(leaf_clusters)) {}
+               std::move(leaf_clusters), std::move(resolution_note)) {}
 
 //
 // XdsConfig
@@ -81,7 +81,10 @@ std::string XdsConfig::ToString() const {
           [&](const ClusterConfig::AggregateConfig& aggregate_config) {
             parts.push_back(absl::StrCat(
                 "        leaf_clusters: [",
-                absl::StrJoin(aggregate_config.leaf_clusters, ", "), "]\n"));
+                absl::StrJoin(aggregate_config.leaf_clusters, ", "),
+                "],\n"
+                "        resolution_note: \"",
+                aggregate_config.resolution_note, "\"\n"));
           });
       parts.push_back(
           "      }\n"

data/src/core/resolver/xds/xds_config.h

@@ -66,11 +66,15 @@ struct XdsConfig : public RefCounted<XdsConfig> {
     // The list of leaf clusters for an aggregate cluster.
     struct AggregateConfig {
       std::vector<absl::string_view> leaf_clusters;
+      std::string resolution_note;
 
-      explicit AggregateConfig(std::vector<absl::string_view> leaf_clusters)
-          : leaf_clusters(std::move(leaf_clusters)) {}
+      AggregateConfig(std::vector<absl::string_view> leaf_clusters,
+                      std::string resolution_note)
+          : leaf_clusters(std::move(leaf_clusters)),
+            resolution_note(std::move(resolution_note)) {}
       bool operator==(const AggregateConfig& other) const {
-        return leaf_clusters == other.leaf_clusters;
+        return leaf_clusters == other.leaf_clusters &&
+               resolution_note == other.resolution_note;
       }
     };
     std::variant<EndpointConfig, AggregateConfig> children;
@@ -81,7 +85,8 @@ struct XdsConfig : public RefCounted<XdsConfig> {
                   std::string resolution_note);
     // Ctor for aggregate clusters.
     ClusterConfig(std::shared_ptr<const XdsClusterResource> cluster,
-                  std::vector<absl::string_view> leaf_clusters);
+                  std::vector<absl::string_view> leaf_clusters,
+                  std::string resolution_note);
 
     bool operator==(const ClusterConfig& other) const {
       return cluster == other.cluster && children == other.children;

data/src/core/resolver/xds/xds_dependency_manager.cc

@@ -656,8 +656,12 @@ bool XdsDependencyManager::PopulateClusterConfigMap(
         cluster_config_map,
     std::set<absl::string_view>* eds_resources_seen,
     std::set<absl::string_view>* dns_names_seen,
-    absl::StatusOr<std::vector<absl::string_view>>* leaf_clusters) {
-  if (depth > 0) CHECK_NE(leaf_clusters, nullptr);
+    absl::StatusOr<std::vector<absl::string_view>>* leaf_clusters,
+    std::vector<std::string>* resolution_notes) {
+  if (depth > 0) {
+    CHECK_NE(leaf_clusters, nullptr);
+    CHECK_NE(resolution_notes, nullptr);
+  }
   if (depth == kMaxXdsAggregateClusterRecursionDepth) {
     *leaf_clusters =
         absl::UnavailableError("aggregate cluster graph exceeds max depth");
@@ -687,6 +691,9 @@ bool XdsDependencyManager::PopulateClusterConfigMap(
   // If there was an error fetching the CDS resource, report the error.
   if (!state.update.ok()) {
     cluster_config = state.update.status();
+    if (resolution_notes != nullptr) {
+      resolution_notes->emplace_back(state.update.status().message());
+    }
     return true;
   }
   // If we don't have the resource yet, we can't return a config yet.
@@ -785,12 +792,13 @@ bool XdsDependencyManager::PopulateClusterConfigMap(
         // Recursively expand leaf clusters.
         absl::StatusOr<std::vector<absl::string_view>> child_leaf_clusters;
         child_leaf_clusters.emplace();
+        std::vector<std::string> child_resolution_notes;
         bool have_all_resources = true;
         for (const std::string& child_name :
              aggregate.prioritized_cluster_names) {
           have_all_resources &= PopulateClusterConfigMap(
               child_name, depth + 1, cluster_config_map, eds_resources_seen,
-              dns_names_seen, &child_leaf_clusters);
+              dns_names_seen, &child_leaf_clusters, &child_resolution_notes);
           if (!child_leaf_clusters.ok()) break;
         }
         // Note that we cannot use the cluster_config reference we
@@ -807,12 +815,18 @@ bool XdsDependencyManager::PopulateClusterConfigMap(
           }
           return true;
         }
-        // If needed, propagate leaf cluster list up the tree.
+        // If needed, propagate leaf cluster list and resolution note
+        // lists up the tree.
         if (leaf_clusters != nullptr) {
           (*leaf_clusters)
               ->insert((*leaf_clusters)->end(), child_leaf_clusters->begin(),
                        child_leaf_clusters->end());
         }
+        if (resolution_notes != nullptr) {
+          resolution_notes->insert(resolution_notes->end(),
+                                   child_resolution_notes.begin(),
+                                   child_resolution_notes.end());
+        }
         // If there are no leaf clusters, report an error for the cluster.
         if (have_all_resources && child_leaf_clusters->empty()) {
           aggregate_cluster_config = absl::UnavailableError(
@@ -825,8 +839,9 @@ bool XdsDependencyManager::PopulateClusterConfigMap(
         // at the root of the tree, because we need to make sure the list
         // of underlying cluster names stays alive so that the leaf cluster
         // list of the root aggregate cluster can point to those strings.
-        aggregate_cluster_config.emplace(std::move(cluster_resource),
-                                         std::move(*child_leaf_clusters));
+        aggregate_cluster_config.emplace(
+            std::move(cluster_resource), std::move(*child_leaf_clusters),
+            absl::StrJoin(child_resolution_notes, "; "));
         return have_all_resources;
       });
 }

data/src/core/resolver/xds/xds_dependency_manager.h

@@ -172,7 +172,8 @@ class XdsDependencyManager final : public RefCounted<XdsDependencyManager>,
           cluster_config_map,
       std::set<absl::string_view>* eds_resources_seen,
       std::set<absl::string_view>* dns_names_seen,
-      absl::StatusOr<std::vector<absl::string_view>>* leaf_clusters = nullptr);
+      absl::StatusOr<std::vector<absl::string_view>>* leaf_clusters = nullptr,
+      std::vector<std::string>* resolution_notes = nullptr);
 
   // Called when an external cluster subscription is unreffed.
   void OnClusterSubscriptionUnref(absl::string_view cluster_name,