grpc 1.62.3 → 1.63.0

data/src/core/ext/xds/xds_client.h

@@ -37,6 +37,7 @@
 #include "src/core/ext/xds/xds_api.h"
 #include "src/core/ext/xds/xds_bootstrap.h"
 #include "src/core/ext/xds/xds_client_stats.h"
+#include "src/core/ext/xds/xds_metrics.h"
 #include "src/core/ext/xds/xds_resource_type.h"
 #include "src/core/ext/xds/xds_transport.h"
 #include "src/core/lib/debug/trace.h"
@@ -60,6 +61,9 @@ extern TraceFlag grpc_xds_client_refcount_trace;
 
 class XdsClient : public DualRefCounted<XdsClient> {
  public:
+  // The authority reported for old-style (non-xdstp) resource names.
+  static constexpr absl::string_view kOldStyleAuthority = "#old";
+
   class ReadDelayHandle : public RefCounted<ReadDelayHandle> {
    public:
     static RefCountedPtr<ReadDelayHandle> NoWait() { return nullptr; }
@@ -87,6 +91,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
       std::unique_ptr<XdsBootstrap> bootstrap,
       OrphanablePtr<XdsTransportFactory> transport_factory,
       std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine,
+      std::unique_ptr<XdsMetricsReporter> metrics_reporter,
       std::string user_agent_name, std::string user_agent_version,
       Duration resource_request_timeout = Duration::Seconds(15));
   ~XdsClient() override;
@@ -99,8 +104,6 @@ class XdsClient : public DualRefCounted<XdsClient> {
     return transport_factory_.get();
   }
 
-  void Orphan() override;
-
   // Start and cancel watch for a resource.
   //
   // The XdsClient takes ownership of the watcher, but the caller may
@@ -156,6 +159,10 @@ class XdsClient : public DualRefCounted<XdsClient> {
   }
 
  protected:
+  void Orphaned() override;
+
+  Mutex* mu() ABSL_LOCK_RETURNED(&mu_) { return &mu_; }
+
   // Dumps the active xDS config to the provided
   // envoy.service.status.v3.ClientConfig message including the config status
   // (e.g., CLIENT_REQUESTED, CLIENT_ACKED, CLIENT_NACKED).
@@ -163,7 +170,22 @@ class XdsClient : public DualRefCounted<XdsClient> {
                         envoy_service_status_v3_ClientConfig* client_config)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_);
 
-  Mutex* mu() ABSL_LOCK_RETURNED(&mu_) { return &mu_; }
+  // Invokes func once for each combination of labels to report the
+  // resource count for those labels.
+  struct ResourceCountLabels {
+    absl::string_view xds_authority;
+    absl::string_view resource_type;
+    absl::string_view cache_state;
+  };
+  void ReportResourceCounts(
+      absl::FunctionRef<void(const ResourceCountLabels&, uint64_t)> func)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_);
+
+  // Invokes func once for each xDS server to report whether the
+  // connection to that server is working.
+  void ReportServerConnections(
+      absl::FunctionRef<void(absl::string_view /*xds_server*/, bool)> func)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_);
 
  private:
   friend testing::XdsClientTestPeer;
@@ -179,6 +201,8 @@ class XdsClient : public DualRefCounted<XdsClient> {
     }
   };
 
+  struct AuthorityState;
+
   struct XdsResourceName {
     std::string authority;
     XdsResourceKey key;
@@ -186,7 +210,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
 
   // Contains a channel to the xds server and all the data related to the
   // channel.  Holds a ref to the xds client object.
-  class XdsChannel : public DualRefCounted<XdsChannel> {
+  class XdsChannel final : public DualRefCounted<XdsChannel> {
    public:
     template <typename T>
     class RetryableCall;
@@ -198,8 +222,6 @@ class XdsClient : public DualRefCounted<XdsClient> {
                const XdsBootstrap::XdsServer& server);
     ~XdsChannel() override;
 
-    void Orphan() override;
-
     XdsClient* xds_client() const { return xds_client_.get(); }
     AdsCall* ads_call() const;
     LrsCall* lrs_call() const;
@@ -221,7 +243,17 @@ class XdsClient : public DualRefCounted<XdsClient> {
                            bool delay_unsubscription)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
+    absl::string_view server_uri() const { return server_.server_uri(); }
+
    private:
+    // Attempts to find a suitable Xds fallback server. Returns true if
+    // a connection to a suitable server had been established.
+    bool MaybeFallbackLocked(const std::string& authority,
+                             XdsClient::AuthorityState& authority_state)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+    void SetHealthyLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+    void Orphaned() override;
+
     void OnConnectivityFailure(absl::Status status);
 
     // Enqueues error notifications to watchers.  Caller must drain
@@ -259,7 +291,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
   };
 
   struct AuthorityState {
-    RefCountedPtr<XdsChannel> xds_channel;
+    std::vector<RefCountedPtr<XdsChannel>> xds_channels;
     std::map<const XdsResourceType*, std::map<XdsResourceKey, ResourceState>>
         resource_map;
   };
@@ -315,10 +347,10 @@ class XdsClient : public DualRefCounted<XdsClient> {
   XdsApi::ClusterLoadReportMap BuildLoadReportSnapshotLocked(
       const XdsBootstrap::XdsServer& xds_server, bool send_all_clusters,
       const std::set<std::string>& clusters) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
-
   RefCountedPtr<XdsChannel> GetOrCreateXdsChannelLocked(
       const XdsBootstrap::XdsServer& server, const char* reason)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  bool HasUncachedResources(const AuthorityState& authority_state);
 
   std::unique_ptr<XdsBootstrap> bootstrap_;
   OrphanablePtr<XdsTransportFactory> transport_factory_;
@@ -327,6 +359,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
   XdsApi api_;
   WorkSerializer work_serializer_;
   std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine_;
+  std::unique_ptr<XdsMetricsReporter> metrics_reporter_;
 
   Mutex mu_;
 

data/src/core/ext/xds/xds_client_grpc.cc

@@ -49,6 +49,7 @@
 #include "src/core/ext/xds/xds_transport.h"
 #include "src/core/ext/xds/xds_transport_grpc.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/metrics.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/event_engine/default_event_engine.h"
 #include "src/core/lib/gprpp/debug_location.h"
@@ -64,8 +65,6 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/error_utils.h"
 
-namespace grpc_core {
-
 // If gRPC is built with -DGRPC_XDS_USER_AGENT_NAME_SUFFIX="...", that string
 // will be appended to the user agent name reported to the xDS server.
 #ifdef GRPC_XDS_USER_AGENT_NAME_SUFFIX
@@ -84,21 +83,99 @@ namespace grpc_core {
 #define GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING ""
 #endif
 
+namespace grpc_core {
+
+namespace {
+
+// Metric labels.
+constexpr absl::string_view kMetricLabelXdsServer = "grpc.xds.server";
+constexpr absl::string_view kMetricLabelXdsAuthority = "grpc.xds.authority";
+constexpr absl::string_view kMetricLabelXdsResourceType =
+    "grpc.xds.resource_type";
+constexpr absl::string_view kMetricLabelXdsCacheState = "grpc.xds.cache_state";
+
+const auto kMetricResourceUpdatesValid =
+    GlobalInstrumentsRegistry::RegisterUInt64Counter(
+        "grpc.xds_client.resource_updates_valid",
+        "EXPERIMENTAL.  A counter of resources received that were considered "
+        "valid.  The counter will be incremented even for resources that "
+        "have not changed.",
+        "{resource}",
+        {kMetricLabelTarget, kMetricLabelXdsServer,
+         kMetricLabelXdsResourceType},
+        {}, false);
+
+const auto kMetricResourceUpdatesInvalid =
+    GlobalInstrumentsRegistry::RegisterUInt64Counter(
+        "grpc.xds_client.resource_updates_invalid",
+        "EXPERIMENTAL.  A counter of resources received that were considered "
+        "invalid.",
+        "{resource}",
+        {kMetricLabelTarget, kMetricLabelXdsServer,
+         kMetricLabelXdsResourceType},
+        {}, false);
+
+const auto kMetricConnected =
+    GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
+        "grpc.xds_client.connected",
+        "EXPERIMENTAL.  Whether or not the xDS client currently has a "
+        "working ADS stream to the xDS server. For a given server, this "
+        "will be set to 0 when we have a connectivity failure or when the "
+        "ADS stream fails without seeing a response message, as per gRFC "
+        "A57.  It will be set to 1 when we receive the first response on "
+        "an ADS stream.",
+        "{bool}", {kMetricLabelTarget, kMetricLabelXdsServer}, {}, false);
+
+const auto kMetricResources =
+    GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
+        "grpc.xds_client.resources", "EXPERIMENTAL.  Number of xDS resources.",
+        "{resource}",
+        {kMetricLabelTarget, kMetricLabelXdsAuthority,
+         kMetricLabelXdsResourceType, kMetricLabelXdsCacheState},
+        {}, false);
+
+}  // namespace
+
+//
+// GrpcXdsClient::MetricsReporter
+//
+
+class GrpcXdsClient::MetricsReporter final : public XdsMetricsReporter {
+ public:
+  explicit MetricsReporter(GrpcXdsClient& xds_client)
+      : xds_client_(xds_client) {}
+
+  void ReportResourceUpdates(absl::string_view xds_server,
+                             absl::string_view resource_type,
+                             uint64_t num_valid_resources,
+                             uint64_t num_invalid_resources) override {
+    xds_client_.stats_plugin_group_.AddCounter(
+        kMetricResourceUpdatesValid, num_valid_resources,
+        {xds_client_.key_, xds_server, resource_type}, {});
+    xds_client_.stats_plugin_group_.AddCounter(
+        kMetricResourceUpdatesInvalid, num_invalid_resources,
+        {xds_client_.key_, xds_server, resource_type}, {});
+  }
+
+ private:
+  GrpcXdsClient& xds_client_;
+};
+
 //
 // GrpcXdsClient
 //
 
+constexpr absl::string_view GrpcXdsClient::kServerKey;
+
 namespace {
 
 Mutex* g_mu = new Mutex;
 const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr;
-GrpcXdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr;
+// Key bytes live in clients so they outlive the entries in this map
+NoDestruct<std::map<absl::string_view, GrpcXdsClient*>> g_xds_client_map
+    ABSL_GUARDED_BY(*g_mu);
 char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr;
 
-}  // namespace
-
-namespace {
-
 absl::StatusOr<std::string> GetBootstrapContents(const char* fallback_config) {
   // First, try GRPC_XDS_BOOTSTRAP env var.
   auto path = GetEnv("GRPC_XDS_BOOTSTRAP");
@@ -136,23 +213,10 @@ absl::StatusOr<std::string> GetBootstrapContents(const char* fallback_config) {
       "not defined");
 }
 
-std::vector<RefCountedPtr<GrpcXdsClient>> GetAllXdsClients() {
-  MutexLock lock(g_mu);
-  std::vector<RefCountedPtr<GrpcXdsClient>> xds_clients;
-  if (g_xds_client != nullptr) {
-    auto xds_client =
-        g_xds_client->RefIfNonZero(DEBUG_LOCATION, "DumpAllClientConfigs");
-    if (xds_client != nullptr) {
-      xds_clients.emplace_back(xds_client.TakeAsSubclass<GrpcXdsClient>());
-    }
-  }
-  return xds_clients;
-}
-
 }  // namespace
 
 absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
-    const ChannelArgs& args, const char* reason) {
+    absl::string_view key, const ChannelArgs& args, const char* reason) {
   // If getting bootstrap from channel args, create a local XdsClient
   // instance for the channel or server instead of using the global instance.
   absl::optional<absl::string_view> bootstrap_config = args.GetString(
@@ -164,13 +228,14 @@ absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
         GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS);
     auto channel_args = ChannelArgs::FromC(xds_channel_args);
     return MakeRefCounted<GrpcXdsClient>(
-        std::move(*bootstrap), channel_args,
+        key, std::move(*bootstrap), channel_args,
         MakeOrphanable<GrpcXdsTransportFactory>(channel_args));
   }
   // Otherwise, use the global instance.
   MutexLock lock(g_mu);
-  if (g_xds_client != nullptr) {
-    auto xds_client = g_xds_client->RefIfNonZero(DEBUG_LOCATION, reason);
+  auto it = g_xds_client_map->find(key);
+  if (it != g_xds_client_map->end()) {
+    auto xds_client = it->second->RefIfNonZero(DEBUG_LOCATION, reason);
     if (xds_client != nullptr) {
       return xds_client.TakeAsSubclass<GrpcXdsClient>();
     }
@@ -188,12 +253,90 @@ absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
   // Instantiate XdsClient.
   auto channel_args = ChannelArgs::FromC(g_channel_args);
   auto xds_client = MakeRefCounted<GrpcXdsClient>(
-      std::move(*bootstrap), channel_args,
+      key, std::move(*bootstrap), channel_args,
       MakeOrphanable<GrpcXdsTransportFactory>(channel_args));
-  g_xds_client = xds_client.get();
+  g_xds_client_map->emplace(xds_client->key(), xds_client.get());
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+    gpr_log(GPR_INFO, "[xds_client %p] Created xDS client for key %s",
+            xds_client.get(), std::string(key).c_str());
+  }
   return xds_client;
 }
 
+namespace {
+
+GlobalStatsPluginRegistry::StatsPluginGroup GetStatsPluginGroupForKey(
+    absl::string_view key) {
+  if (key == GrpcXdsClient::kServerKey) {
+    return GlobalStatsPluginRegistry::GetStatsPluginsForServer(ChannelArgs{});
+  }
+  // TODO(roth): How do we set the authority here?
+  experimental::StatsPluginChannelScope scope(key, "");
+  return GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope);
+}
+
+}  // namespace
+
+GrpcXdsClient::GrpcXdsClient(
+    absl::string_view key, std::unique_ptr<GrpcXdsBootstrap> bootstrap,
+    const ChannelArgs& args,
+    OrphanablePtr<XdsTransportFactory> transport_factory)
+    : XdsClient(
+          std::move(bootstrap), std::move(transport_factory),
+          grpc_event_engine::experimental::GetDefaultEventEngine(),
+          std::make_unique<MetricsReporter>(*this),
+          absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING,
+                       GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING),
+          absl::StrCat("C-core ", grpc_version_string(),
+                       GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING,
+                       GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING),
+          std::max(Duration::Zero(),
+                   args.GetDurationFromIntMillis(
+                           GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS)
+                       .value_or(Duration::Seconds(15)))),
+      key_(key),
+      certificate_provider_store_(MakeOrphanable<CertificateProviderStore>(
+          static_cast<const GrpcXdsBootstrap&>(this->bootstrap())
+              .certificate_providers())),
+      stats_plugin_group_(GetStatsPluginGroupForKey(key_)),
+      registered_metric_callback_(stats_plugin_group_.RegisterCallback(
+          [this](CallbackMetricReporter& reporter) {
+            ReportCallbackMetrics(reporter);
+          },
+          {kMetricConnected, kMetricResources})) {}
+
+void GrpcXdsClient::Orphaned() {
+  registered_metric_callback_.reset();
+  XdsClient::Orphaned();
+  MutexLock lock(g_mu);
+  auto it = g_xds_client_map->find(key_);
+  if (it != g_xds_client_map->end() && it->second == this) {
+    g_xds_client_map->erase(it);
+  }
+}
+
+grpc_pollset_set* GrpcXdsClient::interested_parties() const {
+  return reinterpret_cast<GrpcXdsTransportFactory*>(transport_factory())
+      ->interested_parties();
+}
+
+namespace {
+
+std::vector<RefCountedPtr<GrpcXdsClient>> GetAllXdsClients() {
+  MutexLock lock(g_mu);
+  std::vector<RefCountedPtr<GrpcXdsClient>> xds_clients;
+  for (const auto& key_client : *g_xds_client_map) {
+    auto xds_client =
+        key_client.second->RefIfNonZero(DEBUG_LOCATION, "DumpAllClientConfigs");
+    if (xds_client != nullptr) {
+      xds_clients.emplace_back(xds_client.TakeAsSubclass<GrpcXdsClient>());
+    }
+  }
+  return xds_clients;
+}
+
+}  // namespace
+
 // ABSL_NO_THREAD_SAFETY_ANALYSIS because we have to manually manage locks for
 // individual XdsClients and compiler struggles with checking the validity
 grpc_slice GrpcXdsClient::DumpAllClientConfigs()
@@ -211,6 +354,8 @@ grpc_slice GrpcXdsClient::DumpAllClientConfigs()
                                                                 arena.ptr());
     xds_client->mu()->Lock();
     xds_client->DumpClientConfig(&string_pool, arena.ptr(), client_config);
+    envoy_service_status_v3_ClientConfig_set_client_scope(
+        client_config, StdStringToUpbString(xds_client->key()));
   }
   // Serialize the upb message to bytes
   size_t output_length;
@@ -222,33 +367,17 @@ grpc_slice GrpcXdsClient::DumpAllClientConfigs()
   return grpc_slice_from_cpp_string(std::string(output, output_length));
 }
 
-GrpcXdsClient::GrpcXdsClient(
-    std::unique_ptr<GrpcXdsBootstrap> bootstrap, const ChannelArgs& args,
-    OrphanablePtr<XdsTransportFactory> transport_factory)
-    : XdsClient(
-          std::move(bootstrap), std::move(transport_factory),
-          grpc_event_engine::experimental::GetDefaultEventEngine(),
-          absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING,
-                       GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING),
-          absl::StrCat("C-core ", grpc_version_string(),
-                       GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING,
-                       GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING),
-          std::max(Duration::Zero(),
-                   args.GetDurationFromIntMillis(
-                           GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS)
-                       .value_or(Duration::Seconds(15)))),
-      certificate_provider_store_(MakeOrphanable<CertificateProviderStore>(
-          static_cast<const GrpcXdsBootstrap&>(this->bootstrap())
-              .certificate_providers())) {}
-
-GrpcXdsClient::~GrpcXdsClient() {
-  MutexLock lock(g_mu);
-  if (g_xds_client == this) g_xds_client = nullptr;
-}
-
-grpc_pollset_set* GrpcXdsClient::interested_parties() const {
-  return reinterpret_cast<GrpcXdsTransportFactory*>(transport_factory())
-      ->interested_parties();
+void GrpcXdsClient::ReportCallbackMetrics(CallbackMetricReporter& reporter) {
+  MutexLock lock(mu());
+  ReportResourceCounts([&](const ResourceCountLabels& labels, uint64_t count) {
+    reporter.Report(
+        kMetricResources, count,
+        {key_, labels.xds_authority, labels.resource_type, labels.cache_state},
+        {});
+  });
+  ReportServerConnections([&](absl::string_view xds_server, bool connected) {
+    reporter.Report(kMetricConnected, connected, {key_, xds_server}, {});
+  });
 }
 
 namespace internal {
@@ -258,9 +387,9 @@ void SetXdsChannelArgsForTest(grpc_channel_args* args) {
   g_channel_args = args;
 }
 
-void UnsetGlobalXdsClientForTest() {
+void UnsetGlobalXdsClientsForTest() {
   MutexLock lock(g_mu);
-  g_xds_client = nullptr;
+  g_xds_client_map->clear();
 }
 
 void SetXdsFallbackBootstrapConfig(const char* config) {

data/src/core/ext/xds/xds_client_grpc.h

@@ -31,6 +31,7 @@
 #include "src/core/ext/xds/xds_client.h"
 #include "src/core/ext/xds/xds_transport.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/metrics.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -39,14 +40,14 @@
 
 namespace grpc_core {
 
-class GrpcXdsClient : public XdsClient {
+class GrpcXdsClient final : public XdsClient {
  public:
+  // The key to pass to GetOrCreate() for gRPC servers.
+  static constexpr absl::string_view kServerKey = "#server";
+
   // Factory function to get or create the global XdsClient instance.
   static absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GetOrCreate(
-      const ChannelArgs& args, const char* reason);
-
-  // Builds ClientStatusResponse containing all resources from all XdsClients
-  static grpc_slice DumpAllClientConfigs();
+      absl::string_view key, const ChannelArgs& args, const char* reason);
 
   // Do not instantiate directly -- use GetOrCreate() instead.
   // TODO(roth): The transport factory is injectable here to support
@@ -60,10 +61,10 @@ class GrpcXdsClient : public XdsClient {
   // work for callers that use interested_parties() but not for callers
   // that also use certificate_provider_store(), but we should consider
   // alternatives for that case as well.
-  GrpcXdsClient(std::unique_ptr<GrpcXdsBootstrap> bootstrap,
+  GrpcXdsClient(absl::string_view key,
+                std::unique_ptr<GrpcXdsBootstrap> bootstrap,
                 const ChannelArgs& args,
                 OrphanablePtr<XdsTransportFactory> transport_factory);
-  ~GrpcXdsClient() override;
 
   // Helpers for encoding the XdsClient object in channel args.
   static absl::string_view ChannelArgName() {
@@ -79,13 +80,26 @@ class GrpcXdsClient : public XdsClient {
     return *certificate_provider_store_;
   }
 
+  absl::string_view key() const { return key_; }
+
+  // Builds ClientStatusResponse containing all resources from all XdsClients
+  static grpc_slice DumpAllClientConfigs();
+
  private:
+  class MetricsReporter;
+
+  void ReportCallbackMetrics(CallbackMetricReporter& reporter);
+  void Orphaned() override;
+
+  std::string key_;
   OrphanablePtr<CertificateProviderStore> certificate_provider_store_;
+  GlobalStatsPluginRegistry::StatsPluginGroup stats_plugin_group_;
+  std::unique_ptr<RegisteredMetricCallback> registered_metric_callback_;
 };
 
 namespace internal {
 void SetXdsChannelArgsForTest(grpc_channel_args* args);
-void UnsetGlobalXdsClientForTest();
+void UnsetGlobalXdsClientsForTest();
 // Sets bootstrap config to be used when no env var is set.
 // Does not take ownership of config.
 void SetXdsFallbackBootstrapConfig(const char* config);

data/src/core/ext/xds/xds_client_stats.cc

@@ -111,7 +111,7 @@ XdsClusterLocalityStats::XdsClusterLocalityStats(
             xds_client_.get(), this, std::string(lrs_server_).c_str(),
             std::string(cluster_name_).c_str(),
             std::string(eds_service_name_).c_str(),
-            name_->AsHumanReadableString().c_str());
+            name_->human_readable_string().c_str());
   }
 }
 
@@ -122,7 +122,7 @@ XdsClusterLocalityStats::~XdsClusterLocalityStats() {
             xds_client_.get(), this, std::string(lrs_server_).c_str(),
             std::string(cluster_name_).c_str(),
             std::string(eds_service_name_).c_str(),
-            name_->AsHumanReadableString().c_str());
+            name_->human_readable_string().c_str());
   }
   xds_client_->RemoveClusterLocalityStats(lrs_server_, cluster_name_,
                                           eds_service_name_, name_, this);

data/src/core/ext/xds/xds_client_stats.h

@@ -32,6 +32,7 @@
 #include "absl/strings/string_view.h"
 
 #include "src/core/ext/xds/xds_bootstrap.h"
+#include "src/core/lib/channel/call_tracer.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/per_cpu.h"
 #include "src/core/lib/gprpp/ref_counted.h"
@@ -45,7 +46,7 @@ namespace grpc_core {
 class XdsClient;
 
 // Locality name.
-class XdsLocalityName : public RefCounted<XdsLocalityName> {
+class XdsLocalityName final : public RefCounted<XdsLocalityName> {
  public:
   struct Less {
     bool operator()(const XdsLocalityName* lhs,
@@ -63,7 +64,10 @@ class XdsLocalityName : public RefCounted<XdsLocalityName> {
   XdsLocalityName(std::string region, std::string zone, std::string sub_zone)
       : region_(std::move(region)),
         zone_(std::move(zone)),
-        sub_zone_(std::move(sub_zone)) {}
+        sub_zone_(std::move(sub_zone)),
+        human_readable_string_(
+            absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}",
+                            region_, zone_, sub_zone_)) {}
 
   bool operator==(const XdsLocalityName& other) const {
     return region_ == other.region_ && zone_ == other.zone_ &&
@@ -86,9 +90,8 @@ class XdsLocalityName : public RefCounted<XdsLocalityName> {
   const std::string& zone() const { return zone_; }
   const std::string& sub_zone() const { return sub_zone_; }
 
-  std::string AsHumanReadableString() const {
-    return absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}",
-                           region_, zone_, sub_zone_);
+  const RefCountedStringValue& human_readable_string() const {
+    return human_readable_string_;
   }
 
   // Channel args traits.
@@ -104,10 +107,11 @@ class XdsLocalityName : public RefCounted<XdsLocalityName> {
   std::string region_;
   std::string zone_;
   std::string sub_zone_;
+  RefCountedStringValue human_readable_string_;
 };
 
 // Drop stats for an xds cluster.
-class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> {
+class XdsClusterDropStats final : public RefCounted<XdsClusterDropStats> {
  public:
   // The total number of requests dropped for any reason is the sum of
   // uncategorized_drops, and dropped_requests map.
@@ -161,7 +165,8 @@ class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> {
 };
 
 // Locality stats for an xds cluster.
-class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> {
+class XdsClusterLocalityStats final
+    : public RefCounted<XdsClusterLocalityStats> {
  public:
   struct BackendMetric {
     uint64_t num_requests_finished_with_metric = 0;
@@ -223,6 +228,8 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> {
   void AddCallFinished(const std::map<absl::string_view, double>* named_metrics,
                        bool fail = false);
 
+  XdsLocalityName* locality_name() const { return name_.get(); }
+
  private:
   struct Stats {
     std::atomic<uint64_t> total_successful_requests{0};

data/src/core/ext/xds/xds_cluster.cc

@@ -714,8 +714,6 @@ absl::StatusOr<std::shared_ptr<const XdsClusterResource>> CdsResourceParse(
             StdStringToUpbString(
                 absl::string_view("com.google.csm.telemetry_labels")),
             &telemetry_labels_struct)) {
-      auto telemetry_labels =
-          std::make_shared<std::map<std::string, std::string>>();
       size_t iter = kUpb_Map_Begin;
       const google_protobuf_Struct_FieldsEntry* fields_entry;
       while ((fields_entry = google_protobuf_Struct_fields_next(
@@ -724,15 +722,17 @@ absl::StatusOr<std::shared_ptr<const XdsClusterResource>> CdsResourceParse(
         const google_protobuf_Value* value =
             google_protobuf_Struct_FieldsEntry_value(fields_entry);
         if (google_protobuf_Value_has_string_value(value)) {
-          telemetry_labels->emplace(
-              UpbStringToStdString(
-                  google_protobuf_Struct_FieldsEntry_key(fields_entry)),
-              UpbStringToStdString(google_protobuf_Value_string_value(value)));
+          if (UpbStringToAbsl(google_protobuf_Struct_FieldsEntry_key(
+                  fields_entry)) == "service_name") {
+            cds_update->service_telemetry_label = RefCountedStringValue(
+                UpbStringToAbsl(google_protobuf_Value_string_value(value)));
+          } else if (UpbStringToAbsl(google_protobuf_Struct_FieldsEntry_key(
+                         fields_entry)) == "service_namespace") {
+            cds_update->namespace_telemetry_label = RefCountedStringValue(
+                UpbStringToAbsl(google_protobuf_Value_string_value(value)));
+          }
         }
       }
-      if (!telemetry_labels->empty()) {
-        cds_update->telemetry_labels = std::move(telemetry_labels);
-      }
     }
   }
   // Return result.

data/src/core/ext/xds/xds_cluster.h

@@ -101,7 +101,8 @@ struct XdsClusterResource : public XdsResourceType::ResourceData {
 
   XdsHealthStatusSet override_host_statuses;
 
-  std::shared_ptr<std::map<std::string, std::string>> telemetry_labels;
+  RefCountedStringValue service_telemetry_label;
+  RefCountedStringValue namespace_telemetry_label;
 
   bool operator==(const XdsClusterResource& other) const {
     return type == other.type && lb_policy_config == other.lb_policy_config &&
@@ -110,7 +111,9 @@ struct XdsClusterResource : public XdsResourceType::ResourceData {
            connection_idle_timeout == other.connection_idle_timeout &&
            max_concurrent_requests == other.max_concurrent_requests &&
            outlier_detection == other.outlier_detection &&
-           override_host_statuses == other.override_host_statuses;
+           override_host_statuses == other.override_host_statuses &&
+           service_telemetry_label == other.service_telemetry_label &&
+           namespace_telemetry_label == other.namespace_telemetry_label;
   }
 
   std::string ToString() const;