grpc 1.62.0 → 1.65.1

data/src/core/load_balancing/rls/rls.cc

@@ -20,7 +20,7 @@
 // to use. A child policy that recognizes the name as a field of its
 // configuration will take further load balancing action on the request.
 
-#include <grpc/support/port_platform.h>
+#include "src/core/load_balancing/rls/rls.h"
 
 #include <inttypes.h>
 #include <stdlib.h>
@@ -41,6 +41,8 @@
 
 #include "absl/base/thread_annotations.h"
 #include "absl/hash/hash.h"
+#include "absl/log/check.h"
+#include "absl/random/random.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
@@ -62,54 +64,110 @@
 #include <grpc/status.h>
 #include <grpc/support/json.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
+#include "src/core/channelz/channelz.h"
 #include "src/core/client_channel/client_channel_filter.h"
-#include "src/core/load_balancing/child_policy_handler.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/dual_ref_counted.h"
+#include "src/core/lib/gprpp/match.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/status_helper.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/gprpp/uuid_v4.h"
 #include "src/core/lib/gprpp/validation_errors.h"
 #include "src/core/lib/gprpp/work_serializer.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/json/json_args.h"
-#include "src/core/lib/json/json_object_loader.h"
-#include "src/core/lib/json/json_writer.h"
 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
-#include "src/core/service_config/service_config_impl.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/error_utils.h"
+#include "src/core/load_balancing/child_policy_handler.h"
 #include "src/core/load_balancing/delegating_helper.h"
 #include "src/core/load_balancing/lb_policy.h"
 #include "src/core/load_balancing/lb_policy_factory.h"
 #include "src/core/load_balancing/lb_policy_registry.h"
 #include "src/core/resolver/endpoint_addresses.h"
 #include "src/core/resolver/resolver_registry.h"
+#include "src/core/service_config/service_config_impl.h"
+#include "src/core/telemetry/metrics.h"
+#include "src/core/util/json/json.h"
+#include "src/core/util/json/json_args.h"
+#include "src/core/util/json/json_object_loader.h"
+#include "src/core/util/json/json_writer.h"
 #include "src/proto/grpc/lookup/v1/rls.upb.h"
 
-namespace grpc_core {
+using ::grpc_event_engine::experimental::EventEngine;
 
-TraceFlag grpc_lb_rls_trace(false, "rls_lb");
+namespace grpc_core {
 
 namespace {
 
-using ::grpc_event_engine::experimental::EventEngine;
+constexpr absl::string_view kMetricLabelRlsServerTarget =
+    "grpc.lb.rls.server_target";
+constexpr absl::string_view kMetricLabelRlsInstanceUuid =
+    "grpc.lb.rls.instance_uuid";
+constexpr absl::string_view kMetricRlsDataPlaneTarget =
+    "grpc.lb.rls.data_plane_target";
+constexpr absl::string_view kMetricLabelPickResult = "grpc.lb.pick_result";
+
+const auto kMetricCacheSize =
+    GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
+        "grpc.lb.rls.cache_size", "EXPERIMENTAL.  Size of the RLS cache.", "By",
+        false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricLabelRlsInstanceUuid)
+        .Build();
+
+const auto kMetricCacheEntries =
+    GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
+        "grpc.lb.rls.cache_entries",
+        "EXPERIMENTAL.  Number of entries in the RLS cache.", "{entry}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricLabelRlsInstanceUuid)
+        .Build();
+
+const auto kMetricDefaultTargetPicks =
+    GlobalInstrumentsRegistry::RegisterUInt64Counter(
+        "grpc.lb.rls.default_target_picks",
+        "EXPERIMENTAL.  Number of LB picks sent to the default target.",
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .Build();
+
+const auto kMetricTargetPicks =
+    GlobalInstrumentsRegistry::RegisterUInt64Counter(
+        "grpc.lb.rls.target_picks",
+        "EXPERIMENTAL.  Number of LB picks sent to each RLS target.  Note that "
+        "if the default target is also returned by the RLS server, RPCs sent "
+        "to that target from the cache will be counted in this metric, not "
+        "in grpc.rls.default_target_picks.",
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .Build();
+
+const auto kMetricFailedPicks =
+    GlobalInstrumentsRegistry::RegisterUInt64Counter(
+        "grpc.lb.rls.failed_picks",
+        "EXPERIMENTAL.  Number of LB picks failed due to either a failed RLS "
+        "request or the RLS channel being throttled.",
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget)
+        .Build();
 
 constexpr absl::string_view kRls = "rls_experimental";
 const char kGrpc[] = "grpc";
@@ -131,7 +189,7 @@ const Duration kCacheCleanupTimerInterval = Duration::Minutes(1);
 const int64_t kMaxCacheSizeBytes = 5 * 1024 * 1024;
 
 // Parsed RLS LB policy configuration.
-class RlsLbConfig : public LoadBalancingPolicy::Config {
+class RlsLbConfig final : public LoadBalancingPolicy::Config {
  public:
   struct KeyBuilder {
     std::map<std::string /*key*/, std::vector<std::string /*header*/>>
@@ -210,7 +268,7 @@ class RlsLbConfig : public LoadBalancingPolicy::Config {
 };
 
 // RLS LB policy.
-class RlsLb : public LoadBalancingPolicy {
+class RlsLb final : public LoadBalancingPolicy {
  public:
   explicit RlsLb(Args args);
 
@@ -266,15 +324,10 @@ class RlsLb : public LoadBalancingPolicy {
   };
 
   // Wraps a child policy for a given RLS target.
-  class ChildPolicyWrapper : public DualRefCounted<ChildPolicyWrapper> {
+  class ChildPolicyWrapper final : public DualRefCounted<ChildPolicyWrapper> {
    public:
     ChildPolicyWrapper(RefCountedPtr<RlsLb> lb_policy, std::string target);
 
-    // Note: We are forced to disable lock analysis here because
-    // Orphan() is called by OrphanablePtr<>, which cannot have lock
-    // annotations for this particular caller.
-    void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS;
-
     const std::string& target() const { return target_; }
 
     PickResult Pick(PickArgs args) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_) {
@@ -322,7 +375,7 @@ class RlsLb : public LoadBalancingPolicy {
    private:
     // ChannelControlHelper object that allows the child policy to update state
     // with the wrapper.
-    class ChildPolicyHelper : public DelegatingChannelControlHelper {
+    class ChildPolicyHelper final : public DelegatingChannelControlHelper {
      public:
       explicit ChildPolicyHelper(WeakRefCountedPtr<ChildPolicyWrapper> wrapper)
           : wrapper_(std::move(wrapper)) {}
@@ -342,6 +395,11 @@ class RlsLb : public LoadBalancingPolicy {
       WeakRefCountedPtr<ChildPolicyWrapper> wrapper_;
     };
 
+    // Note: We are forced to disable lock analysis here because
+    // Orphan() is called by Unref() which is called by RefCountedPtr<>, which
+    // cannot have lock annotations for this particular caller.
+    void Orphaned() override ABSL_NO_THREAD_SAFETY_ANALYSIS;
+
     RefCountedPtr<RlsLb> lb_policy_;
     std::string target_;
 
@@ -358,24 +416,28 @@ class RlsLb : public LoadBalancingPolicy {
 
   // A picker that uses the cache and the request map in the LB policy
   // (synchronized via a mutex) to determine how to route requests.
-  class Picker : public LoadBalancingPolicy::SubchannelPicker {
+  class Picker final : public LoadBalancingPolicy::SubchannelPicker {
    public:
     explicit Picker(RefCountedPtr<RlsLb> lb_policy);
 
     PickResult Pick(PickArgs args) override;
 
    private:
+    PickResult PickFromDefaultTargetOrFail(const char* reason, PickArgs args,
+                                           absl::Status status)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
+
     RefCountedPtr<RlsLb> lb_policy_;
     RefCountedPtr<RlsLbConfig> config_;
     RefCountedPtr<ChildPolicyWrapper> default_child_policy_;
   };
 
   // An LRU cache with adjustable size.
-  class Cache {
+  class Cache final {
    public:
     using Iterator = std::list<RequestKey>::iterator;
 
-    class Entry : public InternallyRefCounted<Entry> {
+    class Entry final : public InternallyRefCounted<Entry> {
      public:
       Entry(RefCountedPtr<RlsLb> lb_policy, const RequestKey& key);
 
@@ -447,7 +509,7 @@ class RlsLb : public LoadBalancingPolicy {
       void MarkUsed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
      private:
-      class BackoffTimer : public InternallyRefCounted<BackoffTimer> {
+      class BackoffTimer final : public InternallyRefCounted<BackoffTimer> {
        public:
         BackoffTimer(RefCountedPtr<Entry> entry, Timestamp backoff_time);
 
@@ -516,6 +578,9 @@ class RlsLb : public LoadBalancingPolicy {
     // Shutdown the cache; clean-up and orphan all the stored cache entries.
     void Shutdown() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
+    void ReportMetricsLocked(CallbackMetricReporter& reporter)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
+
    private:
     // Shared logic for starting the cleanup timer
     void StartCleanupTimer() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
@@ -543,7 +608,7 @@ class RlsLb : public LoadBalancingPolicy {
 
   // Channel for communicating with the RLS server.
   // Contains throttling logic for RLS requests.
-  class RlsChannel : public InternallyRefCounted<RlsChannel> {
+  class RlsChannel final : public InternallyRefCounted<RlsChannel> {
    public:
     explicit RlsChannel(RefCountedPtr<RlsLb> lb_policy);
 
@@ -568,12 +633,12 @@ class RlsLb : public LoadBalancingPolicy {
     // Resets the channel's backoff.
     void ResetBackoff();
 
-    grpc_channel* channel() const { return channel_; }
+    Channel* channel() const { return channel_.get(); }
 
    private:
     // Watches the state of the RLS channel. Notifies the LB policy when
     // the channel was previously in TRANSIENT_FAILURE and then becomes READY.
-    class StateWatcher : public AsyncConnectivityStateWatcherInterface {
+    class StateWatcher final : public AsyncConnectivityStateWatcherInterface {
      public:
       explicit StateWatcher(RefCountedPtr<RlsChannel> rls_channel)
           : AsyncConnectivityStateWatcherInterface(
@@ -589,7 +654,7 @@ class RlsLb : public LoadBalancingPolicy {
     };
 
     // Throttle state for RLS requests.
-    class Throttle {
+    class Throttle final {
      public:
       explicit Throttle(
           Duration window_size = kDefaultThrottleWindowSize,
@@ -620,14 +685,14 @@ class RlsLb : public LoadBalancingPolicy {
     RefCountedPtr<RlsLb> lb_policy_;
     bool is_shutdown_ = false;
 
-    grpc_channel* channel_ = nullptr;
+    RefCountedPtr<Channel> channel_;
     RefCountedPtr<channelz::ChannelNode> parent_channelz_node_;
     StateWatcher* watcher_ = nullptr;
     Throttle throttle_ ABSL_GUARDED_BY(&RlsLb::mu_);
   };
 
   // A pending RLS request.  Instances will be tracked in request_map_.
-  class RlsRequest : public InternallyRefCounted<RlsRequest> {
+  class RlsRequest final : public InternallyRefCounted<RlsRequest> {
    public:
     // Asynchronously starts a call on rls_channel for key.
     // Stores backoff_state, which will be transferred to the data cache
@@ -690,6 +755,12 @@ class RlsLb : public LoadBalancingPolicy {
   // Updates the picker in the work serializer.
   void UpdatePickerLocked() ABSL_LOCKS_EXCLUDED(&mu_);
 
+  template <typename HandleType>
+  void MaybeExportPickCount(HandleType handle, absl::string_view target,
+                            const PickResult& pick_result);
+
+  const std::string instance_uuid_;
+
   // Mutex to guard LB policy state that is accessed by the picker.
   Mutex mu_;
   bool is_shutdown_ ABSL_GUARDED_BY(mu_) = false;
@@ -712,6 +783,9 @@ class RlsLb : public LoadBalancingPolicy {
   RefCountedPtr<RlsLbConfig> config_;
   RefCountedPtr<ChildPolicyWrapper> default_child_policy_;
   std::map<std::string /*target*/, ChildPolicyWrapper*> child_policy_map_;
+
+  // Must be after mu_, so that it is destroyed before mu_.
+  std::unique_ptr<RegisteredMetricCallback> registered_metric_callback_;
 };
 
 //
@@ -721,16 +795,15 @@ class RlsLb : public LoadBalancingPolicy {
 RlsLb::ChildPolicyWrapper::ChildPolicyWrapper(RefCountedPtr<RlsLb> lb_policy,
                                               std::string target)
     : DualRefCounted<ChildPolicyWrapper>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "ChildPolicyWrapper"
-                                                     : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "ChildPolicyWrapper" : nullptr),
       lb_policy_(std::move(lb_policy)),
       target_(std::move(target)),
       picker_(MakeRefCounted<QueuePicker>(nullptr)) {
   lb_policy_->child_policy_map_.emplace(target_, this);
 }
 
-void RlsLb::ChildPolicyWrapper::Orphan() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+void RlsLb::ChildPolicyWrapper::Orphaned() {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] ChildPolicyWrapper=%p [%s]: shutdown",
             lb_policy_.get(), this, target_.c_str());
   }
@@ -788,8 +861,8 @@ void RlsLb::ChildPolicyWrapper::StartUpdate() {
   auto child_policy_config = InsertOrUpdateChildPolicyField(
       lb_policy_->config_->child_policy_config_target_field_name(), target_,
       lb_policy_->config_->child_policy_config(), &errors);
-  GPR_ASSERT(child_policy_config.has_value());
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  CHECK(child_policy_config.has_value());
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(
         GPR_INFO,
         "[rlslb %p] ChildPolicyWrapper=%p [%s]: validating update, config: %s",
@@ -801,7 +874,7 @@ void RlsLb::ChildPolicyWrapper::StartUpdate() {
           *child_policy_config);
   // Returned RLS target fails the validation.
   if (!config.ok()) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO,
               "[rlslb %p] ChildPolicyWrapper=%p [%s]: config failed to parse: "
               "%s",
@@ -829,8 +902,8 @@ absl::Status RlsLb::ChildPolicyWrapper::MaybeFinishUpdate() {
         WeakRef(DEBUG_LOCATION, "ChildPolicyHelper"));
     create_args.args = lb_policy_->channel_args_;
     child_policy_ = MakeOrphanable<ChildPolicyHandler>(std::move(create_args),
-                                                       &grpc_lb_rls_trace);
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+                                                       &rls_lb_trace);
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO,
               "[rlslb %p] ChildPolicyWrapper=%p [%s], created new child policy "
               "handler %p",
@@ -840,7 +913,7 @@ absl::Status RlsLb::ChildPolicyWrapper::MaybeFinishUpdate() {
                                      lb_policy_->interested_parties());
   }
   // Send the child the updated config.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] ChildPolicyWrapper=%p [%s], updating child policy "
             "handler %p",
@@ -860,7 +933,7 @@ absl::Status RlsLb::ChildPolicyWrapper::MaybeFinishUpdate() {
 void RlsLb::ChildPolicyWrapper::ChildPolicyHelper::UpdateState(
     grpc_connectivity_state state, const absl::Status& status,
     RefCountedPtr<SubchannelPicker> picker) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] ChildPolicyWrapper=%p [%s] ChildPolicyHelper=%p: "
             "UpdateState(state=%s, status=%s, picker=%p)",
@@ -878,7 +951,7 @@ void RlsLb::ChildPolicyWrapper::ChildPolicyHelper::UpdateState(
       return;
     }
     wrapper_->connectivity_state_ = state;
-    GPR_DEBUG_ASSERT(picker != nullptr);
+    DCHECK(picker != nullptr);
     if (picker != nullptr) {
       wrapper_->picker_ = std::move(picker);
     }
@@ -901,7 +974,7 @@ std::map<std::string, std::string> BuildKeyMap(
   if (it == key_builder_map.end()) {
     // Didn't find exact match, try method wildcard.
     last_slash_pos = path.rfind('/');
-    GPR_DEBUG_ASSERT(last_slash_pos != path.npos);
+    DCHECK(last_slash_pos != path.npos);
     if (GPR_UNLIKELY(last_slash_pos == path.npos)) return {};
     std::string service(path.substr(0, last_slash_pos + 1));
     it = key_builder_map.find(service);
@@ -935,7 +1008,7 @@ std::map<std::string, std::string> BuildKeyMap(
   if (!key_builder->service_key.empty()) {
     if (last_slash_pos == path.npos) {
       last_slash_pos = path.rfind('/');
-      GPR_DEBUG_ASSERT(last_slash_pos != path.npos);
+      DCHECK(last_slash_pos != path.npos);
       if (GPR_UNLIKELY(last_slash_pos == path.npos)) return {};
     }
     key_map[key_builder->service_key] =
@@ -945,7 +1018,7 @@ std::map<std::string, std::string> BuildKeyMap(
   if (!key_builder->method_key.empty()) {
     if (last_slash_pos == path.npos) {
       last_slash_pos = path.rfind('/');
-      GPR_DEBUG_ASSERT(last_slash_pos != path.npos);
+      DCHECK(last_slash_pos != path.npos);
       if (GPR_UNLIKELY(last_slash_pos == path.npos)) return {};
     }
     key_map[key_builder->method_key] =
@@ -968,7 +1041,7 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
       BuildKeyMap(config_->key_builder_map(), args.path,
                   lb_policy_->channel_control_helper()->GetAuthority(),
                   args.initial_metadata)};
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] picker=%p: request keys: %s",
             lb_policy_.get(), this, key.ToString().c_str());
   }
@@ -992,21 +1065,8 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
       // If there is no non-expired data in the cache, then we use the
       // default target if set, or else we fail the pick.
       if (entry == nullptr || entry->data_expiration_time() < now) {
-        if (default_child_policy_ != nullptr) {
-          if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-            gpr_log(GPR_INFO,
-                    "[rlslb %p] picker=%p: RLS call throttled; "
-                    "using default target",
-                    lb_policy_.get(), this);
-          }
-          return default_child_policy_->Pick(args);
-        }
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-          gpr_log(GPR_INFO,
-                  "[rlslb %p] picker=%p: RLS call throttled; failing pick",
-                  lb_policy_.get(), this);
-        }
-        return PickResult::Fail(
+        return PickFromDefaultTargetOrFail(
+            "RLS call throttled", args,
             absl::UnavailableError("RLS request throttled"));
       }
     }
@@ -1019,7 +1079,7 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
   if (entry != nullptr) {
     // If the entry has non-expired data, use it.
     if (entry->data_expiration_time() >= now) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+      if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
         gpr_log(GPR_INFO, "[rlslb %p] picker=%p: using cache entry %p",
                 lb_policy_.get(), this, entry);
       }
@@ -1028,32 +1088,45 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
     // If the entry is in backoff, then use the default target if set,
     // or else fail the pick.
     if (entry->backoff_time() >= now) {
-      if (default_child_policy_ != nullptr) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-          gpr_log(
-              GPR_INFO,
-              "[rlslb %p] picker=%p: RLS call in backoff; using default target",
-              lb_policy_.get(), this);
-        }
-        return default_child_policy_->Pick(args);
-      }
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO,
-                "[rlslb %p] picker=%p: RLS call in backoff; failing pick",
-                lb_policy_.get(), this);
-      }
-      return PickResult::Fail(absl::UnavailableError(
-          absl::StrCat("RLS request failed: ", entry->status().ToString())));
+      return PickFromDefaultTargetOrFail(
+          "RLS call in backoff", args,
+          absl::UnavailableError(absl::StrCat("RLS request failed: ",
+                                              entry->status().ToString())));
     }
   }
   // RLS call pending.  Queue the pick.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] picker=%p: RLS request pending; queuing pick",
             lb_policy_.get(), this);
   }
   return PickResult::Queue();
 }
 
+LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail(
+    const char* reason, PickArgs args, absl::Status status) {
+  if (default_child_policy_ != nullptr) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
+      gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; using default target",
+              lb_policy_.get(), this, reason);
+    }
+    auto pick_result = default_child_policy_->Pick(args);
+    lb_policy_->MaybeExportPickCount(kMetricDefaultTargetPicks,
+                                     config_->default_target(), pick_result);
+    return pick_result;
+  }
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
+    gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; failing pick",
+            lb_policy_.get(), this, reason);
+  }
+  auto& stats_plugins =
+      lb_policy_->channel_control_helper()->GetStatsPluginGroup();
+  stats_plugins.AddCounter(kMetricFailedPicks, 1,
+                           {lb_policy_->channel_control_helper()->GetTarget(),
+                            config_->lookup_service()},
+                           {});
+  return PickResult::Fail(std::move(status));
+}
+
 //
 // RlsLb::Cache::Entry::BackoffTimer
 //
@@ -1078,7 +1151,7 @@ void RlsLb::Cache::Entry::BackoffTimer::Orphan() {
   if (backoff_timer_task_handle_.has_value() &&
       entry_->lb_policy_->channel_control_helper()->GetEventEngine()->Cancel(
           *backoff_timer_task_handle_)) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s, backoff timer canceled",
               entry_->lb_policy_.get(), entry_.get(),
               entry_->is_shutdown_ ? "(shut down)"
@@ -1092,7 +1165,7 @@ void RlsLb::Cache::Entry::BackoffTimer::Orphan() {
 void RlsLb::Cache::Entry::BackoffTimer::OnBackoffTimerLocked() {
   {
     MutexLock lock(&entry_->lb_policy_->mu_);
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s, backoff timer fired",
               entry_->lb_policy_.get(), entry_.get(),
               entry_->is_shutdown_ ? "(shut down)"
@@ -1122,8 +1195,8 @@ std::unique_ptr<BackOff> MakeCacheEntryBackoff() {
 
 RlsLb::Cache::Entry::Entry(RefCountedPtr<RlsLb> lb_policy,
                            const RequestKey& key)
-    : InternallyRefCounted<Entry>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "CacheEntry" : nullptr),
+    : InternallyRefCounted<Entry>(GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "CacheEntry"
+                                                                  : nullptr),
       lb_policy_(std::move(lb_policy)),
       backoff_state_(MakeCacheEntryBackoff()),
       min_expiration_time_(Timestamp::Now() + kMinExpirationTime),
@@ -1131,7 +1204,7 @@ RlsLb::Cache::Entry::Entry(RefCountedPtr<RlsLb> lb_policy,
           lb_policy_->cache_.lru_list_.end(), key)) {}
 
 void RlsLb::Cache::Entry::Orphan() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s: cache entry evicted",
             lb_policy_.get(), this, lru_iterator_->ToString().c_str());
   }
@@ -1149,7 +1222,7 @@ void RlsLb::Cache::Entry::Orphan() {
 
 size_t RlsLb::Cache::Entry::Size() const {
   // lru_iterator_ is not valid once we're shut down.
-  GPR_ASSERT(!is_shutdown_);
+  CHECK(!is_shutdown_);
   return lb_policy_->cache_.EntrySizeForKey(*lru_iterator_);
 }
 
@@ -1162,7 +1235,7 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(PickArgs args) {
     if (child_policy_wrapper->connectivity_state() ==
             GRPC_CHANNEL_TRANSIENT_FAILURE &&
         i < child_policy_wrappers_.size() - 1) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+      if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
         gpr_log(GPR_INFO,
                 "[rlslb %p] cache entry=%p %s: target %s (%" PRIuPTR
                 " of %" PRIuPTR ") in state TRANSIENT_FAILURE; skipping",
@@ -1176,7 +1249,7 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(PickArgs args) {
   }
   // Child policy not in TRANSIENT_FAILURE or is the last target in
   // the list, so delegate.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] cache entry=%p %s: target %s (%" PRIuPTR " of %" PRIuPTR
             ") in state %s; delegating",
@@ -1195,7 +1268,10 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(PickArgs args) {
     strcpy(copied_header_data, header_data_.c_str());
     args.initial_metadata->Add(kRlsHeaderKey, copied_header_data);
   }
-  return child_policy_wrapper->Pick(args);
+  auto pick_result = child_policy_wrapper->Pick(args);
+  lb_policy_->MaybeExportPickCount(kMetricTargetPicks,
+                                   child_policy_wrapper->target(), pick_result);
+  return pick_result;
 }
 
 void RlsLb::Cache::Entry::ResetBackoff() {
@@ -1330,14 +1406,14 @@ RlsLb::Cache::Entry* RlsLb::Cache::FindOrInsert(const RequestKey& key) {
         lb_policy_->RefAsSubclass<RlsLb>(DEBUG_LOCATION, "CacheEntry"), key);
     map_.emplace(key, OrphanablePtr<Entry>(entry));
     size_ += entry_size;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] key=%s: cache entry added, entry=%p",
               lb_policy_, key.ToString().c_str(), entry);
     }
     return entry;
   }
   // Entry found, so use it.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] key=%s: found cache entry %p", lb_policy_,
             key.ToString().c_str(), it->second.get());
   }
@@ -1346,7 +1422,7 @@ RlsLb::Cache::Entry* RlsLb::Cache::FindOrInsert(const RequestKey& key) {
 }
 
 void RlsLb::Cache::Resize(size_t bytes) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] resizing cache to %" PRIuPTR " bytes",
             lb_policy_, bytes);
   }
@@ -1367,13 +1443,26 @@ void RlsLb::Cache::Shutdown() {
   if (cleanup_timer_handle_.has_value() &&
       lb_policy_->channel_control_helper()->GetEventEngine()->Cancel(
           *cleanup_timer_handle_)) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] cache cleanup timer canceled", lb_policy_);
     }
   }
   cleanup_timer_handle_.reset();
 }
 
+void RlsLb::Cache::ReportMetricsLocked(CallbackMetricReporter& reporter) {
+  reporter.Report(
+      kMetricCacheSize, size_,
+      {lb_policy_->channel_control_helper()->GetTarget(),
+       lb_policy_->config_->lookup_service(), lb_policy_->instance_uuid_},
+      {});
+  reporter.Report(
+      kMetricCacheEntries, map_.size(),
+      {lb_policy_->channel_control_helper()->GetTarget(),
+       lb_policy_->config_->lookup_service(), lb_policy_->instance_uuid_},
+      {});
+}
+
 void RlsLb::Cache::StartCleanupTimer() {
   cleanup_timer_handle_ =
       lb_policy_->channel_control_helper()->GetEventEngine()->RunAfter(
@@ -1392,7 +1481,7 @@ void RlsLb::Cache::StartCleanupTimer() {
 }
 
 void RlsLb::Cache::OnCleanupTimer() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] cache cleanup timer fired", lb_policy_);
   }
   MutexLock lock(&lb_policy_->mu_);
@@ -1419,16 +1508,16 @@ void RlsLb::Cache::MaybeShrinkSize(size_t bytes) {
     auto lru_it = lru_list_.begin();
     if (GPR_UNLIKELY(lru_it == lru_list_.end())) break;
     auto map_it = map_.find(*lru_it);
-    GPR_ASSERT(map_it != map_.end());
+    CHECK(map_it != map_.end());
     if (!map_it->second->CanEvict()) break;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] LRU eviction: removing entry %p %s",
               lb_policy_, map_it->second.get(), lru_it->ToString().c_str());
     }
     size_ -= map_it->second->Size();
     map_.erase(map_it);
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] LRU pass complete: desired size=%" PRIuPTR
             " size=%" PRIuPTR,
@@ -1443,7 +1532,7 @@ void RlsLb::Cache::MaybeShrinkSize(size_t bytes) {
 void RlsLb::RlsChannel::StateWatcher::OnConnectivityStateChange(
     grpc_connectivity_state new_state, const absl::Status& status) {
   auto* lb_policy = rls_channel_->lb_policy_.get();
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] RlsChannel=%p StateWatcher=%p: "
             "state changed to %s (%s)",
@@ -1508,7 +1597,7 @@ void RlsLb::RlsChannel::Throttle::RegisterResponse(bool success) {
 
 RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
     : InternallyRefCounted<RlsChannel>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "RlsChannel" : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "RlsChannel" : nullptr),
       lb_policy_(std::move(lb_policy)) {
   // Get channel creds from parent channel.
   // Note that we are using the "unsafe" channel creds here, which do
@@ -1539,17 +1628,17 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
     args = args.Set(GRPC_ARG_SERVICE_CONFIG, service_config)
                .Set(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION, 1);
   }
-  channel_ = grpc_channel_create(lb_policy_->config_->lookup_service().c_str(),
-                                 creds.get(), args.ToC().get());
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  channel_.reset(Channel::FromC(
+      grpc_channel_create(lb_policy_->config_->lookup_service().c_str(),
+                          creds.get(), args.ToC().get())));
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p: created channel %p for %s",
-            lb_policy_.get(), this, channel_,
+            lb_policy_.get(), this, channel_.get(),
             lb_policy_->config_->lookup_service().c_str());
   }
   if (channel_ != nullptr) {
     // Set up channelz linkage.
-    channelz::ChannelNode* child_channelz_node =
-        grpc_channel_get_channelz_node(channel_);
+    channelz::ChannelNode* child_channelz_node = channel_->channelz_node();
     auto parent_channelz_node =
         lb_policy_->channel_args_.GetObjectRef<channelz::ChannelNode>();
     if (child_channelz_node != nullptr && parent_channelz_node != nullptr) {
@@ -1557,39 +1646,32 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
       parent_channelz_node_ = std::move(parent_channelz_node);
     }
     // Start connectivity watch.
-    ClientChannelFilter* client_channel =
-        ClientChannelFilter::GetFromChannel(Channel::FromC(channel_));
-    GPR_ASSERT(client_channel != nullptr);
     watcher_ = new StateWatcher(Ref(DEBUG_LOCATION, "StateWatcher"));
-    client_channel->AddConnectivityWatcher(
+    channel_->AddConnectivityWatcher(
         GRPC_CHANNEL_IDLE,
         OrphanablePtr<AsyncConnectivityStateWatcherInterface>(watcher_));
   }
 }
 
 void RlsLb::RlsChannel::Orphan() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p, channel=%p: shutdown",
-            lb_policy_.get(), this, channel_);
+            lb_policy_.get(), this, channel_.get());
   }
   is_shutdown_ = true;
   if (channel_ != nullptr) {
     // Remove channelz linkage.
     if (parent_channelz_node_ != nullptr) {
-      channelz::ChannelNode* child_channelz_node =
-          grpc_channel_get_channelz_node(channel_);
-      GPR_ASSERT(child_channelz_node != nullptr);
+      channelz::ChannelNode* child_channelz_node = channel_->channelz_node();
+      CHECK_NE(child_channelz_node, nullptr);
       parent_channelz_node_->RemoveChildChannel(child_channelz_node->uuid());
     }
     // Stop connectivity watch.
     if (watcher_ != nullptr) {
-      ClientChannelFilter* client_channel =
-          ClientChannelFilter::GetFromChannel(Channel::FromC(channel_));
-      GPR_ASSERT(client_channel != nullptr);
-      client_channel->RemoveConnectivityWatcher(watcher_);
+      channel_->RemoveConnectivityWatcher(watcher_);
       watcher_ = nullptr;
     }
-    grpc_channel_destroy_internal(channel_);
+    channel_.reset();
   }
   Unref(DEBUG_LOCATION, "Orphan");
 }
@@ -1617,8 +1699,8 @@ void RlsLb::RlsChannel::ReportResponseLocked(bool response_succeeded) {
 }
 
 void RlsLb::RlsChannel::ResetBackoff() {
-  GPR_DEBUG_ASSERT(channel_ != nullptr);
-  grpc_channel_reset_connect_backoff(channel_);
+  DCHECK(channel_ != nullptr);
+  channel_->ResetConnectionBackoff();
 }
 
 //
@@ -1631,14 +1713,14 @@ RlsLb::RlsRequest::RlsRequest(RefCountedPtr<RlsLb> lb_policy, RequestKey key,
                               grpc_lookup_v1_RouteLookupRequest_Reason reason,
                               std::string stale_header_data)
     : InternallyRefCounted<RlsRequest>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "RlsRequest" : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "RlsRequest" : nullptr),
       lb_policy_(std::move(lb_policy)),
       key_(std::move(key)),
       rls_channel_(std::move(rls_channel)),
       backoff_state_(std::move(backoff_state)),
       reason_(reason),
       stale_header_data_(std::move(stale_header_data)) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO,
             "[rlslb %p] rls_request=%p: RLS request created for key %s",
             lb_policy_.get(), this, key_.ToString().c_str());
@@ -1651,11 +1733,11 @@ RlsLb::RlsRequest::RlsRequest(RefCountedPtr<RlsLb> lb_policy, RequestKey key,
       absl::OkStatus());
 }
 
-RlsLb::RlsRequest::~RlsRequest() { GPR_ASSERT(call_ == nullptr); }
+RlsLb::RlsRequest::~RlsRequest() { CHECK_EQ(call_, nullptr); }
 
 void RlsLb::RlsRequest::Orphan() {
   if (call_ != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] rls_request=%p %s: cancelling RLS call",
               lb_policy_.get(), this, key_.ToString().c_str());
     }
@@ -1683,11 +1765,11 @@ void RlsLb::RlsRequest::StartCallLocked() {
   deadline_ = now + lb_policy_->config_->lookup_service_timeout();
   grpc_metadata_array_init(&recv_initial_metadata_);
   grpc_metadata_array_init(&recv_trailing_metadata_);
-  call_ = grpc_channel_create_pollset_set_call(
-      rls_channel_->channel(), nullptr, GRPC_PROPAGATE_DEFAULTS,
+  call_ = rls_channel_->channel()->CreateCall(
+      /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, /*cq=*/nullptr,
       lb_policy_->interested_parties(),
-      grpc_slice_from_static_string(kRlsRequestPath), nullptr, deadline_,
-      nullptr);
+      Slice::FromStaticString(kRlsRequestPath), /*authority=*/absl::nullopt,
+      deadline_, /*registered_method=*/true);
   grpc_op ops[6];
   memset(ops, 0, sizeof(ops));
   grpc_op* op = ops;
@@ -1714,7 +1796,7 @@ void RlsLb::RlsRequest::StartCallLocked() {
   Ref(DEBUG_LOCATION, "OnRlsCallComplete").release();
   auto call_error = grpc_call_start_batch_and_execute(
       call_, ops, static_cast<size_t>(op - ops), &call_complete_cb_);
-  GPR_ASSERT(call_error == GRPC_CALL_OK);
+  CHECK_EQ(call_error, GRPC_CALL_OK);
 }
 
 void RlsLb::RlsRequest::OnRlsCallComplete(void* arg, grpc_error_handle error) {
@@ -1728,7 +1810,7 @@ void RlsLb::RlsRequest::OnRlsCallComplete(void* arg, grpc_error_handle error) {
 }
 
 void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     std::string status_message(StringViewFromSlice(status_details_recv_));
     gpr_log(GPR_INFO,
             "[rlslb %p] rls_request=%p %s, error=%s, status={%d, %s} RLS call "
@@ -1761,7 +1843,7 @@ void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
   grpc_call_unref(call_);
   call_ = nullptr;
   // Return result to cache.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] rls_request=%p %s: response info: %s",
             lb_policy_.get(), this, key_.ToString().c_str(),
             response.ToString().c_str());
@@ -1779,9 +1861,16 @@ void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
   // Now that we've released the lock, finish the update on any newly
   // created child policies.
   for (ChildPolicyWrapper* child : child_policies_to_finish_update) {
-    // TODO(roth): If the child reports an error with the update, we
-    // need to propagate that back to the resolver somehow.
-    (void)child->MaybeFinishUpdate();
+    // If the child policy returns a non-OK status, request re-resolution.
+    // Note that this will initially cause fixed backoff delay in the
+    // resolver instead of exponential delay.  However, once the
+    // resolver returns the initial re-resolution, we will be able to
+    // return non-OK from UpdateLocked(), which will trigger
+    // exponential backoff instead.
+    absl::Status status = child->MaybeFinishUpdate();
+    if (!status.ok()) {
+      lb_policy_->channel_control_helper()->RequestReresolution();
+    }
   }
 }
 
@@ -1852,8 +1941,28 @@ RlsLb::ResponseInfo RlsLb::RlsRequest::ParseResponseProto() {
 // RlsLb
 //
 
-RlsLb::RlsLb(Args args) : LoadBalancingPolicy(std::move(args)), cache_(this) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+std::string GenerateUUID() {
+  absl::uniform_int_distribution<uint64_t> distribution;
+  absl::BitGen bitgen;
+  uint64_t hi = distribution(bitgen);
+  uint64_t lo = distribution(bitgen);
+  return GenerateUUIDv4(hi, lo);
+}
+
+RlsLb::RlsLb(Args args)
+    : LoadBalancingPolicy(std::move(args)),
+      instance_uuid_(channel_args()
+                         .GetOwnedString(GRPC_ARG_TEST_ONLY_RLS_INSTANCE_ID)
+                         .value_or(GenerateUUID())),
+      cache_(this),
+      registered_metric_callback_(
+          channel_control_helper()->GetStatsPluginGroup().RegisterCallback(
+              [this](CallbackMetricReporter& reporter) {
+                MutexLock lock(&mu_);
+                cache_.ReportMetricsLocked(reporter);
+              },
+              Duration::Seconds(5), kMetricCacheSize, kMetricCacheEntries)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] policy created", this);
   }
 }
@@ -1880,14 +1989,14 @@ bool EndpointsEqual(
 }
 
 absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] policy updated", this);
   }
   update_in_progress_ = true;
   // Swap out config.
   RefCountedPtr<RlsLbConfig> old_config = std::move(config_);
   config_ = args.config.TakeAsSubclass<RlsLbConfig>();
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) &&
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb) &&
       (old_config == nullptr ||
        old_config->child_policy_config() != config_->child_policy_config())) {
     gpr_log(GPR_INFO, "[rlslb %p] updated child policy config: %s", this,
@@ -1915,14 +2024,14 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
   if (old_config == nullptr ||
       config_->default_target() != old_config->default_target()) {
     if (config_->default_target().empty()) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+      if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
         gpr_log(GPR_INFO, "[rlslb %p] unsetting default target", this);
       }
       default_child_policy_.reset();
     } else {
       auto it = child_policy_map_.find(config_->default_target());
       if (it == child_policy_map_.end()) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+        if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
           gpr_log(GPR_INFO, "[rlslb %p] creating new default target", this);
         }
         default_child_policy_ = MakeRefCounted<ChildPolicyWrapper>(
@@ -1930,7 +2039,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
             config_->default_target());
         created_default_child = true;
       } else {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+        if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
           gpr_log(GPR_INFO,
                   "[rlslb %p] using existing child for default target", this);
         }
@@ -1955,14 +2064,14 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
     }
     // Start update of child policies if needed.
     if (update_child_policies) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+      if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
         gpr_log(GPR_INFO, "[rlslb %p] starting child policy updates", this);
       }
       for (auto& p : child_policy_map_) {
         p.second->StartUpdate();
       }
     } else if (created_default_child) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+      if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
         gpr_log(GPR_INFO, "[rlslb %p] starting default child policy update",
                 this);
       }
@@ -1972,7 +2081,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
   // Now that we've released the lock, finish update of child policies.
   std::vector<std::string> errors;
   if (update_child_policies) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] finishing child policy updates", this);
     }
     for (auto& p : child_policy_map_) {
@@ -1983,7 +2092,7 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
       }
     }
   } else if (created_default_child) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
       gpr_log(GPR_INFO, "[rlslb %p] finishing default child policy update",
               this);
     }
@@ -2028,9 +2137,10 @@ void RlsLb::ResetBackoffLocked() {
 }
 
 void RlsLb::ShutdownLocked() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] policy shutdown", this);
   }
+  registered_metric_callback_.reset();
   MutexLock lock(&mu_);
   is_shutdown_ = true;
   config_.reset(DEBUG_LOCATION, "ShutdownLocked");
@@ -2071,7 +2181,7 @@ void RlsLb::UpdatePickerLocked() {
   // all children.  This avoids unnecessary picker churn while an update
   // is being propagated to our children.
   if (update_in_progress_) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] updating picker", this);
   }
   grpc_connectivity_state state = GRPC_CHANNEL_IDLE;
@@ -2084,7 +2194,7 @@ void RlsLb::UpdatePickerLocked() {
       if (is_shutdown_) return;
       for (auto& p : child_policy_map_) {
         grpc_connectivity_state child_state = p.second->connectivity_state();
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+        if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
           gpr_log(GPR_INFO, "[rlslb %p] target %s in state %s", this,
                   p.second->target().c_str(),
                   ConnectivityStateName(child_state));
@@ -2107,7 +2217,7 @@ void RlsLb::UpdatePickerLocked() {
       }
     }
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     gpr_log(GPR_INFO, "[rlslb %p] reporting state %s", this,
             ConnectivityStateName(state));
   }
@@ -2120,6 +2230,26 @@ void RlsLb::UpdatePickerLocked() {
       MakeRefCounted<Picker>(RefAsSubclass<RlsLb>(DEBUG_LOCATION, "Picker")));
 }
 
+template <typename HandleType>
+void RlsLb::MaybeExportPickCount(HandleType handle, absl::string_view target,
+                                 const PickResult& pick_result) {
+  absl::string_view pick_result_string = Match(
+      pick_result.result,
+      [](const LoadBalancingPolicy::PickResult::Complete&) {
+        return "complete";
+      },
+      [](const LoadBalancingPolicy::PickResult::Queue&) { return ""; },
+      [](const LoadBalancingPolicy::PickResult::Fail&) { return "fail"; },
+      [](const LoadBalancingPolicy::PickResult::Drop&) { return "drop"; });
+  if (pick_result_string.empty()) return;  // Don't report queued picks.
+  auto& stats_plugins = channel_control_helper()->GetStatsPluginGroup();
+  stats_plugins.AddCounter(
+      handle, 1,
+      {channel_control_helper()->GetTarget(), config_->lookup_service(), target,
+       pick_result_string},
+      {});
+}
+
 //
 // RlsLbFactory
 //
@@ -2451,7 +2581,7 @@ void RlsLbConfig::JsonPostLoad(const Json& json, const JsonArgs&,
   }
 }
 
-class RlsLbFactory : public LoadBalancingPolicyFactory {
+class RlsLbFactory final : public LoadBalancingPolicyFactory {
  public:
   absl::string_view name() const override { return kRls; }