grpc 1.81.1 → 1.85.0

data/src/core/load_balancing/rls/rls.cc

@@ -51,8 +51,8 @@
 #include <utility>
 #include <vector>
 
+#include "grpc/lookup/v1/rls.upb.h"
 #include "src/core/channelz/channelz.h"
-#include "src/core/client_channel/client_channel_filter.h"
 #include "src/core/config/core_configuration.h"
 #include "src/core/credentials/transport/fake/fake_credentials.h"
 #include "src/core/lib/channel/channel_args.h"
@@ -61,6 +61,8 @@
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/call.h"
@@ -76,6 +78,7 @@
 #include "src/core/resolver/resolver_registry.h"
 #include "src/core/service_config/service_config_impl.h"
 #include "src/core/telemetry/metrics.h"
+#include "src/core/telemetry/telemetry_label.h"
 #include "src/core/util/backoff.h"
 #include "src/core/util/debug_location.h"
 #include "src/core/util/dual_ref_counted.h"
@@ -95,7 +98,6 @@
 #include "src/core/util/uuid_v4.h"
 #include "src/core/util/validation_errors.h"
 #include "src/core/util/work_serializer.h"
-#include "src/proto/grpc/lookup/v1/rls.upb.h"
 #include "upb/base/string_view.h"
 #include "upb/mem/arena.hpp"
 #include "absl/base/thread_annotations.h"
@@ -146,6 +148,7 @@ const auto kMetricDefaultTargetPicks =
         "{pick}", false)
         .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
                 kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .OptionalLabels(kMetricLabelTelemetry)
         .Build();
 
 const auto kMetricTargetPicks =
@@ -158,6 +161,7 @@ const auto kMetricTargetPicks =
         "{pick}", false)
         .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
                 kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .OptionalLabels(kMetricLabelTelemetry)
         .Build();
 
 const auto kMetricFailedPicks =
@@ -167,6 +171,7 @@ const auto kMetricFailedPicks =
         "request or the RLS channel being throttled.",
         "{pick}", false)
         .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget)
+        .OptionalLabels(kMetricLabelTelemetry)
         .Build();
 
 const char kGrpc[] = "grpc";
@@ -696,7 +701,8 @@ class RlsLb final : public LoadBalancingPolicy {
   template <typename HandleType>
   void MaybeExportPickCount(HandleType handle, absl::string_view target,
                             absl::string_view lookup_service,
-                            const PickResult& pick_result);
+                            const PickResult& pick_result,
+                            absl::string_view telemetry_label);
 
   const std::string instance_uuid_;
 
@@ -866,7 +872,7 @@ void RlsLb::ChildPolicyWrapper::ChildPolicyHelper::UpdateState(
       << ", status=" << status << ", picker=" << picker.get() << ")";
   if (wrapper_->is_shutdown_) return;
   {
-    MutexLock lock(&wrapper_->lb_policy_->mu_);
+    MutexLock lock(wrapper_->lb_policy_->mu_);
     // TODO(roth): It looks like this ignores subsequent TF updates that
     // might change the status used to fail picks, which seems wrong.
     if (wrapper_->connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE &&
@@ -967,7 +973,7 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
       << "[rlslb " << lb_policy_.get() << "] picker=" << this
       << ": request keys: " << key.ToString();
   Timestamp now = Timestamp::Now();
-  MutexLock lock(&lb_policy_->mu_);
+  MutexLock lock(lb_policy_->mu_);
   if (lb_policy_->is_shutdown_) {
     return PickResult::Fail(
         absl::UnavailableError("LB policy already shut down"));
@@ -1023,14 +1029,19 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
 
 LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail(
     const char* reason, PickArgs args, absl::Status status) {
+  absl::string_view telemetry_label;
+  if (auto* label = GetContext<Arena>()->GetContext<TelemetryLabel>();
+      label != nullptr) {
+    telemetry_label = label->value;
+  }
   if (default_child_policy_ != nullptr) {
     GRPC_TRACE_LOG(rls_lb, INFO)
         << "[rlslb " << lb_policy_.get() << "] picker=" << this << ": "
         << reason << "; using default target";
     auto pick_result = default_child_policy_->Pick(args);
-    lb_policy_->MaybeExportPickCount(kMetricDefaultTargetPicks,
-                                     config_->default_target(),
-                                     config_->lookup_service(), pick_result);
+    lb_policy_->MaybeExportPickCount(
+        kMetricDefaultTargetPicks, config_->default_target(),
+        config_->lookup_service(), pick_result, telemetry_label);
     return pick_result;
   }
   GRPC_TRACE_LOG(rls_lb, INFO)
@@ -1041,7 +1052,7 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail(
   stats_plugins.AddCounter(kMetricFailedPicks, 1,
                            {lb_policy_->channel_control_helper()->GetTarget(),
                             config_->lookup_service()},
-                           {});
+                           {telemetry_label});
   return PickResult::Fail(std::move(status));
 }
 
@@ -1079,7 +1090,7 @@ void RlsLb::Cache::Entry::BackoffTimer::Orphan() {
 
 void RlsLb::Cache::Entry::BackoffTimer::OnBackoffTimerLocked() {
   {
-    MutexLock lock(&entry_->lb_policy_->mu_);
+    MutexLock lock(entry_->lb_policy_->mu_);
     GRPC_TRACE_LOG(rls_lb, INFO)
         << "[rlslb " << entry_->lb_policy_.get()
         << "] cache entry=" << entry_.get() << " "
@@ -1170,10 +1181,15 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(
       << child_policy_wrappers_.size() << ") in state "
       << ConnectivityStateName(child_policy_wrapper->connectivity_state())
       << "; delegating";
+  absl::string_view telemetry_label;
+  if (auto* label = GetContext<Arena>()->GetContext<TelemetryLabel>();
+      label != nullptr) {
+    telemetry_label = label->value;
+  }
   auto pick_result = child_policy_wrapper->Pick(args);
-  lb_policy_->MaybeExportPickCount(kMetricTargetPicks,
-                                   child_policy_wrapper->target(),
-                                   lookup_service, pick_result);
+  lb_policy_->MaybeExportPickCount(
+      kMetricTargetPicks, child_policy_wrapper->target(), lookup_service,
+      pick_result, telemetry_label);
   // Add header data.
   if (!header_data_.empty()) {
     auto* complete_pick =
@@ -1402,7 +1418,7 @@ void RlsLb::Cache::OnCleanupTimer() {
       << "[rlslb " << lb_policy_ << "] cache cleanup timer fired";
   std::vector<RefCountedPtr<ChildPolicyWrapper>>
       child_policy_wrappers_to_delete;
-  MutexLock lock(&lb_policy_->mu_);
+  MutexLock lock(lb_policy_->mu_);
   if (!cleanup_timer_handle_.has_value()) return;
   if (lb_policy_->is_shutdown_) return;
   for (auto it = map_.begin(); it != map_.end();) {
@@ -1457,7 +1473,7 @@ void RlsLb::RlsChannel::StateWatcher::OnConnectivityStateChange(
       << " StateWatcher=" << this << ": state changed to "
       << ConnectivityStateName(new_state) << " (" << status << ")";
   if (rls_channel_->is_shutdown_) return;
-  MutexLock lock(&lb_policy->mu_);
+  MutexLock lock(lb_policy->mu_);
   if (new_state == GRPC_CHANNEL_READY && was_transient_failure_) {
     was_transient_failure_ = false;
     // Reset the backoff of all cache entries, so that we don't
@@ -1524,9 +1540,15 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
       lb_policy_->channel_control_helper()->GetUnsafeChannelCredentials();
   // Use the parent channel's authority.
   auto authority = lb_policy_->channel_control_helper()->GetAuthority();
-  ChannelArgs args = ChannelArgs()
-                         .Set(GRPC_ARG_DEFAULT_AUTHORITY, authority)
-                         .Set(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL, 1);
+  ChannelArgs args;
+  const grpc_channel_args* child_args =
+      lb_policy_->channel_args_.GetPointer<grpc_channel_args>(
+          GRPC_ARG_CHILD_CHANNEL_ARGS);
+  if (child_args != nullptr) {
+    args = ChannelArgs::FromC(child_args).UnionWith(args);
+  }
+  args = args.Set(GRPC_ARG_DEFAULT_AUTHORITY, authority)
+             .Set(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL, 1);
   // Propagate fake security connector expected targets, if any.
   // (This is ugly, but it seems better than propagating all channel args
   // from the parent channel by default and then having a giant
@@ -1669,7 +1691,7 @@ void RlsLb::RlsRequest::StartCall(void* arg, grpc_error_handle /*error*/) {
 
 void RlsLb::RlsRequest::StartCallLocked() {
   {
-    MutexLock lock(&lb_policy_->mu_);
+    MutexLock lock(lb_policy_->mu_);
     if (lb_policy_->is_shutdown_) return;
   }
   Timestamp now = Timestamp::Now();
@@ -1680,7 +1702,8 @@ void RlsLb::RlsRequest::StartCallLocked() {
       /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, /*cq=*/nullptr,
       lb_policy_->interested_parties(),
       Slice::FromStaticString(kRlsRequestPath), /*authority=*/std::nullopt,
-      deadline_, /*registered_method=*/true);
+      deadline_, /*registered_method=*/true,
+      /*arena_init_function=*/std::nullopt);
   grpc_op ops[6];
   memset(ops, 0, sizeof(ops));
   grpc_op* op = ops;
@@ -1758,7 +1781,7 @@ void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
       child_policy_wrappers_to_delete;
   OrphanablePtr<ChildPolicyHandler> child_policy_to_delete;
   {
-    MutexLock lock(&lb_policy_->mu_);
+    MutexLock lock(lb_policy_->mu_);
     if (lb_policy_->is_shutdown_) return;
     rls_channel_->ReportResponseLocked(response.status.ok());
     Cache::Entry* cache_entry =
@@ -1946,7 +1969,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
       child_policy_wrappers_to_delete;
   OrphanablePtr<ChildPolicyHandler> child_policy_to_delete;
   {
-    MutexLock lock(&mu_);
+    MutexLock lock(mu_);
     // Swap out RLS channel if needed.
     if (old_config == nullptr ||
         config_->lookup_service() != old_config->lookup_service()) {
@@ -2003,7 +2026,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
     registered_metric_callback_ =
         channel_control_helper()->GetStatsPluginGroup().RegisterCallback(
             [this](CallbackMetricReporter& reporter) {
-              MutexLock lock(&mu_);
+              MutexLock lock(mu_);
               cache_.ReportMetricsLocked(reporter);
             },
             Duration::Seconds(5), kMetricCacheSize, kMetricCacheEntries);
@@ -2024,7 +2047,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
 }
 
 void RlsLb::ExitIdleLocked() {
-  MutexLock lock(&mu_);
+  MutexLock lock(mu_);
   for (auto& [_, child] : child_policy_map_) {
     child->ExitIdleLocked();
   }
@@ -2032,7 +2055,7 @@ void RlsLb::ExitIdleLocked() {
 
 void RlsLb::ResetBackoffLocked() {
   {
-    MutexLock lock(&mu_);
+    MutexLock lock(mu_);
     rls_channel_->ResetBackoff();
     cache_.ResetAllBackoff();
   }
@@ -2049,7 +2072,7 @@ void RlsLb::ShutdownLocked() {
       child_policy_wrappers_to_delete;
   OrphanablePtr<RlsChannel> rls_channel_to_delete;
   {
-    MutexLock lock(&mu_);
+    MutexLock lock(mu_);
     is_shutdown_ = true;
     config_.reset(DEBUG_LOCATION, "ShutdownLocked");
     child_policy_wrappers_to_delete = cache_.Shutdown();
@@ -2082,7 +2105,7 @@ void RlsLb::UpdatePickerLocked() {
     int num_idle = 0;
     int num_connecting = 0;
     {
-      MutexLock lock(&mu_);
+      MutexLock lock(mu_);
       if (is_shutdown_) return;
       for (auto& [_, child] : child_policy_map_) {
         grpc_connectivity_state child_state = child->connectivity_state();
@@ -2121,7 +2144,8 @@ void RlsLb::UpdatePickerLocked() {
 template <typename HandleType>
 void RlsLb::MaybeExportPickCount(HandleType handle, absl::string_view target,
                                  absl::string_view lookup_service,
-                                 const PickResult& pick_result) {
+                                 const PickResult& pick_result,
+                                 absl::string_view telemetry_label) {
   absl::string_view pick_result_string = Match(
       pick_result.result,
       [](const LoadBalancingPolicy::PickResult::Complete&) {
@@ -2135,7 +2159,7 @@ void RlsLb::MaybeExportPickCount(HandleType handle, absl::string_view target,
   stats_plugins.AddCounter(handle, 1,
                            {channel_control_helper()->GetTarget(),
                             lookup_service, target, pick_result_string},
-                           {});
+                           {telemetry_label});
 }
 
 //

data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc

@@ -14,6 +14,8 @@
 // limitations under the License.
 //
 
+#include "src/core/load_balancing/weighted_round_robin/weighted_round_robin.h"
+
 #include <grpc/event_engine/event_engine.h>
 #include <grpc/impl/connectivity_state.h>
 #include <grpc/support/port_platform.h>
@@ -32,7 +34,9 @@
 #include <variant>
 #include <vector>
 
+#include "src/core/client_channel/client_channel_service_config.h"
 #include "src/core/config/core_configuration.h"
+#include "src/core/config/experiment_env_var.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/experiments/experiments.h"
@@ -80,10 +84,6 @@ namespace {
 
 constexpr absl::string_view kWeightedRoundRobin = "weighted_round_robin";
 
-constexpr absl::string_view kMetricLabelLocality = "grpc.lb.locality";
-constexpr absl::string_view kMetricLabelBackendService =
-    "grpc.lb.backend_service";
-
 const auto kMetricRrFallback =
     GlobalInstrumentsRegistry::RegisterUInt64Counter(
         "grpc.lb.wrr.rr_fallback",
@@ -129,66 +129,128 @@ const auto kMetricEndpointWeights =
         .OptionalLabels(kMetricLabelLocality, kMetricLabelBackendService)
         .Build();
 
-// Config for WRR policy.
-class WeightedRoundRobinConfig final : public LoadBalancingPolicy::Config {
- public:
-  WeightedRoundRobinConfig() = default;
-
-  WeightedRoundRobinConfig(const WeightedRoundRobinConfig&) = delete;
-  WeightedRoundRobinConfig& operator=(const WeightedRoundRobinConfig&) = delete;
+double GetUtilization(const BackendMetricData& backend_metric_data,
+                      const std::vector<WeightedRoundRobinConfig::ParsedMetric>&
+                          parsed_custom_metrics) {
+  double utilization = 0;
+  bool custom_metric_found = false;
+  if (!parsed_custom_metrics.empty()) {
+    for (const auto& metric : parsed_custom_metrics) {
+      double value = 0;
+      switch (metric.type) {
+        case WeightedRoundRobinConfig::ParsedMetric::Type::kCpu:
+          value = backend_metric_data.cpu_utilization;
+          break;
+        case WeightedRoundRobinConfig::ParsedMetric::Type::kMem:
+          value = backend_metric_data.mem_utilization;
+          break;
+        case WeightedRoundRobinConfig::ParsedMetric::Type::kApplication:
+          value = backend_metric_data.application_utilization;
+          break;
+        case WeightedRoundRobinConfig::ParsedMetric::Type::kNamedMetric: {
+          auto it = backend_metric_data.named_metrics.find(metric.name);
+          if (it != backend_metric_data.named_metrics.end()) {
+            value = it->second;
+          }
+          break;
+        }
+        case WeightedRoundRobinConfig::ParsedMetric::Type::kUtilization: {
+          auto it = backend_metric_data.utilization.find(metric.name);
+          if (it != backend_metric_data.utilization.end()) {
+            value = it->second;
+          }
+          break;
+        }
+      }
+      if (value > 0) {
+        utilization = std::max(utilization, value);
+        custom_metric_found = true;
+      }
+    }
+  }
+  if (!custom_metric_found) {
+    utilization = backend_metric_data.application_utilization;
+  }
+  if (utilization <= 0) {
+    utilization = backend_metric_data.cpu_utilization;
+  }
+  return utilization;
+}
+}  // namespace
 
-  WeightedRoundRobinConfig(WeightedRoundRobinConfig&&) = delete;
-  WeightedRoundRobinConfig& operator=(WeightedRoundRobinConfig&&) = delete;
+absl::string_view WeightedRoundRobinConfig::name() const {
+  return kWeightedRoundRobin;
+}
 
-  absl::string_view name() const override { return kWeightedRoundRobin; }
+const JsonLoaderInterface* WeightedRoundRobinConfig::JsonLoader(
+    const JsonArgs&) {
+  static const auto* loader =
+      JsonObjectLoader<WeightedRoundRobinConfig>()
+          .OptionalField("enableOobLoadReport",
+                         &WeightedRoundRobinConfig::enable_oob_load_report_)
+          .OptionalField("oobReportingPeriod",
+                         &WeightedRoundRobinConfig::oob_reporting_period_)
+          .OptionalField("blackoutPeriod",
+                         &WeightedRoundRobinConfig::blackout_period_)
+          .OptionalField("weightUpdatePeriod",
+                         &WeightedRoundRobinConfig::weight_update_period_)
+          .OptionalField("weightExpirationPeriod",
+                         &WeightedRoundRobinConfig::weight_expiration_period_)
+          .OptionalField("errorUtilizationPenalty",
+                         &WeightedRoundRobinConfig::error_utilization_penalty_)
+          .Finish();
+  return loader;
+}
 
-  bool enable_oob_load_report() const { return enable_oob_load_report_; }
-  Duration oob_reporting_period() const { return oob_reporting_period_; }
-  Duration blackout_period() const { return blackout_period_; }
-  Duration weight_update_period() const { return weight_update_period_; }
-  Duration weight_expiration_period() const {
-    return weight_expiration_period_;
-  }
-  float error_utilization_penalty() const { return error_utilization_penalty_; }
-
-  static const JsonLoaderInterface* JsonLoader(const JsonArgs&) {
-    static const auto* loader =
-        JsonObjectLoader<WeightedRoundRobinConfig>()
-            .OptionalField("enableOobLoadReport",
-                           &WeightedRoundRobinConfig::enable_oob_load_report_)
-            .OptionalField("oobReportingPeriod",
-                           &WeightedRoundRobinConfig::oob_reporting_period_)
-            .OptionalField("blackoutPeriod",
-                           &WeightedRoundRobinConfig::blackout_period_)
-            .OptionalField("weightUpdatePeriod",
-                           &WeightedRoundRobinConfig::weight_update_period_)
-            .OptionalField("weightExpirationPeriod",
-                           &WeightedRoundRobinConfig::weight_expiration_period_)
-            .OptionalField(
-                "errorUtilizationPenalty",
-                &WeightedRoundRobinConfig::error_utilization_penalty_)
-            .Finish();
-    return loader;
-  }
-
-  void JsonPostLoad(const Json&, const JsonArgs&, ValidationErrors* errors) {
-    // Impose lower bound of 100ms on weightUpdatePeriod.
-    weight_update_period_ =
-        std::max(weight_update_period_, Duration::Milliseconds(100));
-    if (error_utilization_penalty_ < 0) {
-      ValidationErrors::ScopedField field(errors, ".errorUtilizationPenalty");
-      errors->AddError("must be non-negative");
+void WeightedRoundRobinConfig::JsonPostLoad(const Json& json,
+                                            const JsonArgs& args,
+                                            ValidationErrors* errors) {
+  // Impose lower bound of 100ms on weightUpdatePeriod.
+  weight_update_period_ =
+      std::max(weight_update_period_, Duration::Milliseconds(100));
+  if (error_utilization_penalty_ < 0) {
+    ValidationErrors::ScopedField field(errors, ".errorUtilizationPenalty");
+    errors->AddError("must be non-negative");
+  }
+  // Handle metricNamesForComputingUtilization.
+  std::optional<std::vector<std::string>>
+      metric_names_for_computing_utilization =
+          LoadJsonObjectField<std::vector<std::string>>(
+              json.object(), args, "metricNamesForComputingUtilization", errors,
+              /*required=*/false);
+  if (metric_names_for_computing_utilization.has_value()) {
+    size_t i = 0;
+    for (const auto& metric_name : *metric_names_for_computing_utilization) {
+      if (metric_name == "cpu_utilization") {
+        parsed_custom_metrics_.push_back(
+            {WeightedRoundRobinConfig::ParsedMetric::Type::kCpu, ""});
+      } else if (metric_name == "mem_utilization") {
+        parsed_custom_metrics_.push_back(
+            {WeightedRoundRobinConfig::ParsedMetric::Type::kMem, ""});
+      } else if (metric_name == "application_utilization") {
+        parsed_custom_metrics_.push_back(
+            {WeightedRoundRobinConfig::ParsedMetric::Type::kApplication, ""});
+      } else if (absl::StartsWith(metric_name, "named_metrics.")) {
+        parsed_custom_metrics_.push_back(
+            {WeightedRoundRobinConfig::ParsedMetric::Type::kNamedMetric,
+             std::string(absl::StripPrefix(metric_name, "named_metrics."))});
+      } else if (absl::StartsWith(metric_name, "utilization.")) {
+        parsed_custom_metrics_.push_back(
+            {WeightedRoundRobinConfig::ParsedMetric::Type::kUtilization,
+             std::string(absl::StripPrefix(metric_name, "utilization."))});
+      } else {
+        ValidationErrors::ScopedField field(
+            errors,
+            absl::StrCat(".metricNamesForComputingUtilization[", i, "]"));
+        errors->AddError(
+            absl::StrCat("unsupported metric name \"", metric_name, "\""));
+      }
+      ++i;
     }
   }
+}
 
- private:
-  bool enable_oob_load_report_ = false;
-  Duration oob_reporting_period_ = Duration::Seconds(10);
-  Duration blackout_period_ = Duration::Seconds(10);
-  Duration weight_update_period_ = Duration::Seconds(1);
-  Duration weight_expiration_period_ = Duration::Minutes(3);
-  float error_utilization_penalty_ = 1.0;
-};
+namespace {
 
 // WRR LB policy
 class WeightedRoundRobin final : public LoadBalancingPolicy {
@@ -252,16 +314,15 @@ class WeightedRoundRobin final : public LoadBalancingPolicy {
       class OobWatcher final : public OobBackendMetricWatcher {
        public:
         OobWatcher(RefCountedPtr<EndpointWeight> weight,
-                   float error_utilization_penalty)
-            : weight_(std::move(weight)),
-              error_utilization_penalty_(error_utilization_penalty) {}
+                   RefCountedPtr<WeightedRoundRobinConfig> config)
+            : weight_(std::move(weight)), config_(std::move(config)) {}
 
         void OnBackendMetricReport(
             const BackendMetricData& backend_metric_data) override;
 
        private:
         RefCountedPtr<EndpointWeight> weight_;
-        const float error_utilization_penalty_;
+        RefCountedPtr<WeightedRoundRobinConfig> config_;
       };
 
       RefCountedPtr<SubchannelInterface> CreateSubchannel(
@@ -341,17 +402,18 @@ class WeightedRoundRobin final : public LoadBalancingPolicy {
     class SubchannelCallTracker final : public SubchannelCallTrackerInterface {
      public:
       SubchannelCallTracker(
-          RefCountedPtr<EndpointWeight> weight, float error_utilization_penalty,
+          RefCountedPtr<EndpointWeight> weight,
+          RefCountedPtr<WeightedRoundRobinConfig> config,
           std::unique_ptr<SubchannelCallTrackerInterface> child_tracker)
           : weight_(std::move(weight)),
-            error_utilization_penalty_(error_utilization_penalty),
+            config_(std::move(config)),
             child_tracker_(std::move(child_tracker)) {}
 
       void Finish(FinishArgs args) override;
 
      private:
       RefCountedPtr<EndpointWeight> weight_;
-      const float error_utilization_penalty_;
+      RefCountedPtr<WeightedRoundRobinConfig> config_;
       std::unique_ptr<SubchannelCallTrackerInterface> child_tracker_;
     };
 
@@ -427,7 +489,7 @@ class WeightedRoundRobin final : public LoadBalancingPolicy {
 //
 
 WeightedRoundRobin::EndpointWeight::~EndpointWeight() {
-  MutexLock lock(&wrr_->endpoint_weight_map_mu_);
+  MutexLock lock(wrr_->endpoint_weight_map_mu_);
   auto it = wrr_->endpoint_weight_map_.find(key_);
   if (it != wrr_->endpoint_weight_map_.end() && it->second == this) {
     wrr_->endpoint_weight_map_.erase(it);
@@ -456,7 +518,7 @@ void WeightedRoundRobin::EndpointWeight::MaybeUpdateWeight(
   }
   Timestamp now = Timestamp::Now();
   // Grab the lock and update the data.
-  MutexLock lock(&mu_);
+  MutexLock lock(mu_);
   GRPC_TRACE_LOG(weighted_round_robin_lb, INFO)
       << "[WRR " << wrr_.get() << "] subchannel " << key_.ToString()
       << ": qps=" << qps << ", eps=" << eps << ", utilization=" << utilization
@@ -473,7 +535,7 @@ void WeightedRoundRobin::EndpointWeight::MaybeUpdateWeight(
 float WeightedRoundRobin::EndpointWeight::GetWeight(
     Timestamp now, Duration weight_expiration_period, Duration blackout_period,
     uint64_t* num_not_yet_usable, uint64_t* num_stale) {
-  MutexLock lock(&mu_);
+  MutexLock lock(mu_);
   GRPC_TRACE_LOG(weighted_round_robin_lb, INFO)
       << "[WRR " << wrr_.get() << "] subchannel " << key_.ToString()
       << ": getting weight: now=" << now.ToString()
@@ -501,7 +563,7 @@ float WeightedRoundRobin::EndpointWeight::GetWeight(
 }
 
 void WeightedRoundRobin::EndpointWeight::ResetNonEmptySince() {
-  MutexLock lock(&mu_);
+  MutexLock lock(mu_);
   non_empty_since_ = Timestamp::InfFuture();
 }
 
@@ -520,12 +582,11 @@ void WeightedRoundRobin::Picker::SubchannelCallTracker::Finish(
   if (backend_metric_data != nullptr) {
     qps = backend_metric_data->qps;
     eps = backend_metric_data->eps;
-    utilization = backend_metric_data->application_utilization;
-    if (utilization <= 0) {
-      utilization = backend_metric_data->cpu_utilization;
-    }
+    utilization =
+        GetUtilization(*backend_metric_data, config_->parsed_custom_metrics());
   }
-  weight_->MaybeUpdateWeight(qps, eps, utilization, error_utilization_penalty_);
+  weight_->MaybeUpdateWeight(qps, eps, utilization,
+                             config_->error_utilization_penalty());
 }
 
 //
@@ -551,7 +612,7 @@ WeightedRoundRobin::Picker::Picker(RefCountedPtr<WeightedRoundRobin> wrr,
       << endpoints_.size() << " subchannels";
   // Note: BuildSchedulerAndStartTimerLocked() passes out pointers to `this`,
   // so we need to ensure that we really hold timer_mu_.
-  MutexLock lock(&timer_mu_);
+  MutexLock lock(timer_mu_);
   BuildSchedulerAndStartTimerLocked();
 }
 
@@ -561,7 +622,7 @@ WeightedRoundRobin::Picker::~Picker() {
 }
 
 void WeightedRoundRobin::Picker::Orphaned() {
-  MutexLock lock(&timer_mu_);
+  MutexLock lock(timer_mu_);
   GRPC_TRACE_LOG(weighted_round_robin_lb, INFO)
       << "[WRR " << wrr_.get() << " picker " << this << "] cancelling timer";
   wrr_->channel_control_helper()->GetEventEngine()->Cancel(*timer_handle_);
@@ -583,7 +644,7 @@ WeightedRoundRobin::PickResult WeightedRoundRobin::Picker::Pick(PickArgs args) {
     if (complete != nullptr) {
       complete->subchannel_call_tracker =
           std::make_unique<SubchannelCallTracker>(
-              endpoint_info.weight, config_->error_utilization_penalty(),
+              endpoint_info.weight, config_,
               std::move(complete->subchannel_call_tracker));
     }
   }
@@ -594,7 +655,7 @@ size_t WeightedRoundRobin::Picker::PickIndex() {
   // Grab a ref to the scheduler.
   std::shared_ptr<StaticStrideScheduler> scheduler;
   {
-    MutexLock lock(&scheduler_mu_);
+    MutexLock lock(scheduler_mu_);
     scheduler = scheduler_;
   }
   // If we have a scheduler, use it to do a WRR pick.
@@ -650,7 +711,7 @@ void WeightedRoundRobin::Picker::BuildSchedulerAndStartTimerLocked() {
         {wrr_->locality_name_, wrr_->backend_service_name_});
   }
   {
-    MutexLock lock(&scheduler_mu_);
+    MutexLock lock(scheduler_mu_);
     scheduler_ = std::move(scheduler);
   }
   // Start timer.
@@ -668,7 +729,7 @@ void WeightedRoundRobin::Picker::BuildSchedulerAndStartTimerLocked() {
        work_serializer = wrr_->work_serializer()]() mutable {
         ExecCtx exec_ctx;
         {
-          MutexLock lock(&self->timer_mu_);
+          MutexLock lock(self->timer_mu_);
           if (self->timer_handle_.has_value()) {
             GRPC_TRACE_LOG(weighted_round_robin_lb, INFO)
                 << "[WRR " << self->wrr_.get() << " picker " << self.get()
@@ -799,7 +860,7 @@ RefCountedPtr<WeightedRoundRobin::EndpointWeight>
 WeightedRoundRobin::GetOrCreateWeight(
     const std::vector<grpc_resolved_address>& addresses) {
   EndpointAddressSet key(addresses);
-  MutexLock lock(&endpoint_weight_map_mu_);
+  MutexLock lock(endpoint_weight_map_mu_);
   auto it = endpoint_weight_map_.find(key);
   if (it != endpoint_weight_map_.end()) {
     auto weight = it->second->RefIfNonZero();
@@ -817,12 +878,10 @@ WeightedRoundRobin::GetOrCreateWeight(
 
 void WeightedRoundRobin::WrrEndpointList::WrrEndpoint::OobWatcher::
     OnBackendMetricReport(const BackendMetricData& backend_metric_data) {
-  double utilization = backend_metric_data.application_utilization;
-  if (utilization <= 0) {
-    utilization = backend_metric_data.cpu_utilization;
-  }
+  double utilization =
+      GetUtilization(backend_metric_data, config_->parsed_custom_metrics());
   weight_->MaybeUpdateWeight(backend_metric_data.qps, backend_metric_data.eps,
-                             utilization, error_utilization_penalty_);
+                             utilization, config_->error_utilization_penalty());
 }
 
 //
@@ -840,8 +899,7 @@ WeightedRoundRobin::WrrEndpointList::WrrEndpoint::CreateSubchannel(
   if (wrr->config_->enable_oob_load_report()) {
     subchannel->AddDataWatcher(MakeOobBackendMetricWatcher(
         wrr->config_->oob_reporting_period(),
-        std::make_unique<OobWatcher>(
-            weight_, wrr->config_->error_utilization_penalty())));
+        std::make_unique<OobWatcher>(weight_, wrr->config_)));
   }
   return subchannel;
 }