grpc 1.64.3 → 1.71.0

data/src/core/load_balancing/rls/rls.cc

@@ -22,6 +22,17 @@
 
 #include "src/core/load_balancing/rls/rls.h"
 
+#include <grpc/byte_buffer.h>
+#include <grpc/byte_buffer_reader.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/channel_arg_names.h>
+#include <grpc/impl/connectivity_state.h>
+#include <grpc/impl/propagation_bits.h>
+#include <grpc/slice.h>
+#include <grpc/status.h>
+#include <grpc/support/json.h>
+#include <grpc/support/port_platform.h>
 #include <inttypes.h>
 #include <stdlib.h>
 #include <string.h>
@@ -31,6 +42,7 @@
 #include <list>
 #include <map>
 #include <memory>
+#include <optional>
 #include <random>
 #include <set>
 #include <string>
@@ -42,6 +54,7 @@
 #include "absl/base/thread_annotations.h"
 #include "absl/hash/hash.h"
 #include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/random/random.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
@@ -49,49 +62,15 @@
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
-#include "absl/types/optional.h"
-#include "upb/base/string_view.h"
-#include "upb/mem/arena.hpp"
-
-#include <grpc/byte_buffer.h>
-#include <grpc/byte_buffer_reader.h>
-#include <grpc/event_engine/event_engine.h>
-#include <grpc/grpc.h>
-#include <grpc/impl/channel_arg_names.h>
-#include <grpc/impl/connectivity_state.h>
-#include <grpc/impl/propagation_bits.h>
-#include <grpc/slice.h>
-#include <grpc/status.h>
-#include <grpc/support/json.h>
-#include <grpc/support/log.h>
-#include <grpc/support/port_platform.h>
-
 #include "src/core/channelz/channelz.h"
 #include "src/core/client_channel/client_channel_filter.h"
-#include "src/core/lib/backoff/backoff.h"
+#include "src/core/config/core_configuration.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/metrics.h"
-#include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/debug_location.h"
-#include "src/core/lib/gprpp/dual_ref_counted.h"
-#include "src/core/lib/gprpp/match.h"
-#include "src/core/lib/gprpp/orphanable.h"
-#include "src/core/lib/gprpp/ref_counted_ptr.h"
-#include "src/core/lib/gprpp/status_helper.h"
-#include "src/core/lib/gprpp/sync.h"
-#include "src/core/lib/gprpp/time.h"
-#include "src/core/lib/gprpp/uuid_v4.h"
-#include "src/core/lib/gprpp/validation_errors.h"
-#include "src/core/lib/gprpp/work_serializer.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/json/json_args.h"
-#include "src/core/lib/json/json_object_loader.h"
-#include "src/core/lib/json/json_writer.h"
 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_internal.h"
@@ -107,14 +86,32 @@
 #include "src/core/resolver/endpoint_addresses.h"
 #include "src/core/resolver/resolver_registry.h"
 #include "src/core/service_config/service_config_impl.h"
+#include "src/core/telemetry/metrics.h"
+#include "src/core/util/backoff.h"
+#include "src/core/util/debug_location.h"
+#include "src/core/util/dual_ref_counted.h"
+#include "src/core/util/json/json.h"
+#include "src/core/util/json/json_args.h"
+#include "src/core/util/json/json_object_loader.h"
+#include "src/core/util/json/json_writer.h"
+#include "src/core/util/match.h"
+#include "src/core/util/orphanable.h"
+#include "src/core/util/ref_counted_ptr.h"
+#include "src/core/util/status_helper.h"
+#include "src/core/util/sync.h"
+#include "src/core/util/time.h"
+#include "src/core/util/upb_utils.h"
+#include "src/core/util/uuid_v4.h"
+#include "src/core/util/validation_errors.h"
+#include "src/core/util/work_serializer.h"
 #include "src/proto/grpc/lookup/v1/rls.upb.h"
+#include "upb/base/string_view.h"
+#include "upb/mem/arena.hpp"
 
 using ::grpc_event_engine::experimental::EventEngine;
 
 namespace grpc_core {
 
-TraceFlag grpc_lb_rls_trace(false, "rls_lb");
-
 namespace {
 
 constexpr absl::string_view kMetricLabelRlsServerTarget =
@@ -128,26 +125,27 @@ constexpr absl::string_view kMetricLabelPickResult = "grpc.lb.pick_result";
 const auto kMetricCacheSize =
     GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
         "grpc.lb.rls.cache_size", "EXPERIMENTAL.  Size of the RLS cache.", "By",
-        {kMetricLabelTarget, kMetricLabelRlsServerTarget,
-         kMetricLabelRlsInstanceUuid},
-        {}, false);
+        false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricLabelRlsInstanceUuid)
+        .Build();
 
 const auto kMetricCacheEntries =
     GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge(
         "grpc.lb.rls.cache_entries",
-        "EXPERIMENTAL.  Number of entries in the RLS cache.", "{entry}",
-        {kMetricLabelTarget, kMetricLabelRlsServerTarget,
-         kMetricLabelRlsInstanceUuid},
-        {}, false);
+        "EXPERIMENTAL.  Number of entries in the RLS cache.", "{entry}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricLabelRlsInstanceUuid)
+        .Build();
 
 const auto kMetricDefaultTargetPicks =
     GlobalInstrumentsRegistry::RegisterUInt64Counter(
         "grpc.lb.rls.default_target_picks",
         "EXPERIMENTAL.  Number of LB picks sent to the default target.",
-        "{pick}",
-        {kMetricLabelTarget, kMetricLabelRlsServerTarget,
-         kMetricRlsDataPlaneTarget, kMetricLabelPickResult},
-        {}, false);
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .Build();
 
 const auto kMetricTargetPicks =
     GlobalInstrumentsRegistry::RegisterUInt64Counter(
@@ -156,26 +154,25 @@ const auto kMetricTargetPicks =
         "if the default target is also returned by the RLS server, RPCs sent "
         "to that target from the cache will be counted in this metric, not "
         "in grpc.rls.default_target_picks.",
-        "{pick}",
-        {kMetricLabelTarget, kMetricLabelRlsServerTarget,
-         kMetricRlsDataPlaneTarget, kMetricLabelPickResult},
-        {}, false);
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget,
+                kMetricRlsDataPlaneTarget, kMetricLabelPickResult)
+        .Build();
 
 const auto kMetricFailedPicks =
     GlobalInstrumentsRegistry::RegisterUInt64Counter(
         "grpc.lb.rls.failed_picks",
         "EXPERIMENTAL.  Number of LB picks failed due to either a failed RLS "
         "request or the RLS channel being throttled.",
-        "{pick}", {kMetricLabelTarget, kMetricLabelRlsServerTarget}, {}, false);
+        "{pick}", false)
+        .Labels(kMetricLabelTarget, kMetricLabelRlsServerTarget)
+        .Build();
 
-constexpr absl::string_view kRls = "rls_experimental";
 const char kGrpc[] = "grpc";
 const char* kRlsRequestPath = "/grpc.lookup.v1.RouteLookupService/RouteLookup";
 const char* kFakeTargetFieldValue = "fake_target_field_value";
 const char* kRlsHeaderKey = "x-google-rls-data";
 
-const Duration kDefaultLookupServiceTimeout = Duration::Seconds(10);
-const Duration kMaxMaxAge = Duration::Minutes(5);
 const Duration kMinExpirationTime = Duration::Seconds(5);
 const Duration kCacheBackoffInitial = Duration::Seconds(1);
 const double kCacheBackoffMultiplier = 1.6;
@@ -187,91 +184,12 @@ const int kDefaultThrottlePadding = 8;
 const Duration kCacheCleanupTimerInterval = Duration::Minutes(1);
 const int64_t kMaxCacheSizeBytes = 5 * 1024 * 1024;
 
-// Parsed RLS LB policy configuration.
-class RlsLbConfig final : public LoadBalancingPolicy::Config {
- public:
-  struct KeyBuilder {
-    std::map<std::string /*key*/, std::vector<std::string /*header*/>>
-        header_keys;
-    std::string host_key;
-    std::string service_key;
-    std::string method_key;
-    std::map<std::string /*key*/, std::string /*value*/> constant_keys;
-  };
-  using KeyBuilderMap = std::unordered_map<std::string /*path*/, KeyBuilder>;
-
-  struct RouteLookupConfig {
-    KeyBuilderMap key_builder_map;
-    std::string lookup_service;
-    Duration lookup_service_timeout = kDefaultLookupServiceTimeout;
-    Duration max_age = kMaxMaxAge;
-    Duration stale_age = kMaxMaxAge;
-    int64_t cache_size_bytes = 0;
-    std::string default_target;
-
-    static const JsonLoaderInterface* JsonLoader(const JsonArgs&);
-    void JsonPostLoad(const Json& json, const JsonArgs& args,
-                      ValidationErrors* errors);
-  };
-
-  RlsLbConfig() = default;
-
-  RlsLbConfig(const RlsLbConfig&) = delete;
-  RlsLbConfig& operator=(const RlsLbConfig&) = delete;
-
-  RlsLbConfig(RlsLbConfig&& other) = delete;
-  RlsLbConfig& operator=(RlsLbConfig&& other) = delete;
-
-  absl::string_view name() const override { return kRls; }
-
-  const KeyBuilderMap& key_builder_map() const {
-    return route_lookup_config_.key_builder_map;
-  }
-  const std::string& lookup_service() const {
-    return route_lookup_config_.lookup_service;
-  }
-  Duration lookup_service_timeout() const {
-    return route_lookup_config_.lookup_service_timeout;
-  }
-  Duration max_age() const { return route_lookup_config_.max_age; }
-  Duration stale_age() const { return route_lookup_config_.stale_age; }
-  int64_t cache_size_bytes() const {
-    return route_lookup_config_.cache_size_bytes;
-  }
-  const std::string& default_target() const {
-    return route_lookup_config_.default_target;
-  }
-  const std::string& rls_channel_service_config() const {
-    return rls_channel_service_config_;
-  }
-  const Json& child_policy_config() const { return child_policy_config_; }
-  const std::string& child_policy_config_target_field_name() const {
-    return child_policy_config_target_field_name_;
-  }
-  RefCountedPtr<LoadBalancingPolicy::Config>
-  default_child_policy_parsed_config() const {
-    return default_child_policy_parsed_config_;
-  }
-
-  static const JsonLoaderInterface* JsonLoader(const JsonArgs&);
-  void JsonPostLoad(const Json& json, const JsonArgs&,
-                    ValidationErrors* errors);
-
- private:
-  RouteLookupConfig route_lookup_config_;
-  std::string rls_channel_service_config_;
-  Json child_policy_config_;
-  std::string child_policy_config_target_field_name_;
-  RefCountedPtr<LoadBalancingPolicy::Config>
-      default_child_policy_parsed_config_;
-};
-
 // RLS LB policy.
 class RlsLb final : public LoadBalancingPolicy {
  public:
   explicit RlsLb(Args args);
 
-  absl::string_view name() const override { return kRls; }
+  absl::string_view name() const override { return RlsLbConfig::Name(); }
   absl::Status UpdateLocked(UpdateArgs args) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
@@ -288,17 +206,16 @@ class RlsLb final : public LoadBalancingPolicy {
     template <typename H>
     friend H AbslHashValue(H h, const RequestKey& key) {
       std::hash<std::string> string_hasher;
-      for (auto& kv : key.key_map) {
-        h = H::combine(std::move(h), string_hasher(kv.first),
-                       string_hasher(kv.second));
+      for (auto& [key, value] : key.key_map) {
+        h = H::combine(std::move(h), string_hasher(key), string_hasher(value));
       }
       return h;
     }
 
     size_t Size() const {
       size_t size = sizeof(RequestKey);
-      for (auto& kv : key_map) {
-        size += kv.first.length() + kv.second.length();
+      for (auto& [key, value] : key_map) {
+        size += key.length() + value.length();
       }
       return size;
     }
@@ -313,12 +230,12 @@ class RlsLb final : public LoadBalancingPolicy {
   struct ResponseInfo {
     absl::Status status;
     std::vector<std::string> targets;
-    std::string header_data;
+    grpc_event_engine::experimental::Slice header_data;
 
     std::string ToString() const {
       return absl::StrFormat("{status=%s, targets=[%s], header_data=\"%s\"}",
                              status.ToString(), absl::StrJoin(targets, ","),
-                             header_data);
+                             header_data.as_string_view());
     }
   };
 
@@ -351,7 +268,8 @@ class RlsLb final : public LoadBalancingPolicy {
     // is called after releasing it.
     //
     // Both methods grab the data they need from the parent object.
-    void StartUpdate() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
+    void StartUpdate(OrphanablePtr<ChildPolicyHandler>* child_policy_to_delete)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
     absl::Status MaybeFinishUpdate() ABSL_LOCKS_EXCLUDED(&RlsLb::mu_);
 
     void ExitIdleLocked() {
@@ -395,14 +313,14 @@ class RlsLb final : public LoadBalancingPolicy {
     };
 
     // Note: We are forced to disable lock analysis here because
-    // Orphan() is called by Unref() which is called by RefCountedPtr<>, which
+    // Orphaned() is called by Unref() which is called by RefCountedPtr<>, which
     // cannot have lock annotations for this particular caller.
     void Orphaned() override ABSL_NO_THREAD_SAFETY_ANALYSIS;
 
     RefCountedPtr<RlsLb> lb_policy_;
     std::string target_;
 
-    bool is_shutdown_ = false;
+    bool is_shutdown_ = false;  // Protected by WorkSerializer
 
     OrphanablePtr<ChildPolicyHandler> child_policy_;
     RefCountedPtr<LoadBalancingPolicy::Config> pending_config_;
@@ -462,7 +380,7 @@ class RlsLb final : public LoadBalancingPolicy {
           ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_) {
         return data_expiration_time_;
       }
-      const std::string& header_data() const
+      const grpc_event_engine::experimental::Slice& header_data() const
           ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_) {
         return header_data_;
       }
@@ -501,16 +419,29 @@ class RlsLb final : public LoadBalancingPolicy {
       // Returns a list of child policy wrappers on which FinishUpdate()
       // needs to be called after releasing the lock.
       std::vector<ChildPolicyWrapper*> OnRlsResponseLocked(
-          ResponseInfo response, std::unique_ptr<BackOff> backoff_state)
+          ResponseInfo response, std::unique_ptr<BackOff> backoff_state,
+          OrphanablePtr<ChildPolicyHandler>* child_policy_to_delete)
           ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
       // Moves entry to the end of the LRU list.
       void MarkUsed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
+      // Takes entries from child_policy_wrappers_ and appends them to the end
+      // of \a child_policy_wrappers.
+      void TakeChildPolicyWrappers(
+          std::vector<RefCountedPtr<ChildPolicyWrapper>>* child_policy_wrappers)
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_) {
+        child_policy_wrappers->insert(
+            child_policy_wrappers->end(),
+            std::make_move_iterator(child_policy_wrappers_.begin()),
+            std::make_move_iterator(child_policy_wrappers_.end()));
+        child_policy_wrappers_.clear();
+      }
+
      private:
       class BackoffTimer final : public InternallyRefCounted<BackoffTimer> {
        public:
-        BackoffTimer(RefCountedPtr<Entry> entry, Timestamp backoff_time);
+        BackoffTimer(RefCountedPtr<Entry> entry, Duration delay);
 
         // Note: We are forced to disable lock analysis here because
         // Orphan() is called by OrphanablePtr<>, which cannot have lock
@@ -521,7 +452,7 @@ class RlsLb final : public LoadBalancingPolicy {
         void OnBackoffTimerLocked();
 
         RefCountedPtr<Entry> entry_;
-        absl::optional<EventEngine::TaskHandle> backoff_timer_task_handle_
+        std::optional<EventEngine::TaskHandle> backoff_timer_task_handle_
             ABSL_GUARDED_BY(&RlsLb::mu_);
       };
 
@@ -541,7 +472,8 @@ class RlsLb final : public LoadBalancingPolicy {
       // RLS response states
       std::vector<RefCountedPtr<ChildPolicyWrapper>> child_policy_wrappers_
           ABSL_GUARDED_BY(&RlsLb::mu_);
-      std::string header_data_ ABSL_GUARDED_BY(&RlsLb::mu_);
+      grpc_event_engine::experimental::Slice header_data_
+          ABSL_GUARDED_BY(&RlsLb::mu_);
       Timestamp data_expiration_time_ ABSL_GUARDED_BY(&RlsLb::mu_) =
           Timestamp::InfPast();
       Timestamp stale_time_ ABSL_GUARDED_BY(&RlsLb::mu_) = Timestamp::InfPast();
@@ -563,19 +495,24 @@ class RlsLb final : public LoadBalancingPolicy {
     // the caller. Otherwise, the entry found is returned to the caller. The
     // entry returned to the user is considered recently used and its order in
     // the LRU list of the cache is updated.
-    Entry* FindOrInsert(const RequestKey& key)
+    Entry* FindOrInsert(const RequestKey& key,
+                        std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                            child_policy_wrappers_to_delete)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
     // Resizes the cache. If the new cache size is greater than the current size
     // of the cache, do nothing. Otherwise, evict the oldest entries that
     // exceed the new size limit of the cache.
-    void Resize(size_t bytes) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
+    void Resize(size_t bytes, std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                                  child_policy_wrappers_to_delete)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
     // Resets backoff of all the cache entries.
     void ResetAllBackoff() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
     // Shutdown the cache; clean-up and orphan all the stored cache entries.
-    void Shutdown() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
+    GRPC_MUST_USE_RESULT std::vector<RefCountedPtr<ChildPolicyWrapper>>
+    Shutdown() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
     void ReportMetricsLocked(CallbackMetricReporter& reporter)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
@@ -591,7 +528,9 @@ class RlsLb final : public LoadBalancingPolicy {
 
     // Evicts oversized cache elements when the current size is greater than
     // the specified limit.
-    void MaybeShrinkSize(size_t bytes)
+    void MaybeShrinkSize(size_t bytes,
+                         std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                             child_policy_wrappers_to_delete)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_);
 
     RlsLb* lb_policy_;
@@ -602,7 +541,7 @@ class RlsLb final : public LoadBalancingPolicy {
     std::list<RequestKey> lru_list_ ABSL_GUARDED_BY(&RlsLb::mu_);
     std::unordered_map<RequestKey, OrphanablePtr<Entry>, absl::Hash<RequestKey>>
         map_ ABSL_GUARDED_BY(&RlsLb::mu_);
-    absl::optional<EventEngine::TaskHandle> cleanup_timer_handle_;
+    std::optional<EventEngine::TaskHandle> cleanup_timer_handle_;
   };
 
   // Channel for communicating with the RLS server.
@@ -684,7 +623,7 @@ class RlsLb final : public LoadBalancingPolicy {
     RefCountedPtr<RlsLb> lb_policy_;
     bool is_shutdown_ = false;
 
-    OrphanablePtr<Channel> channel_;
+    RefCountedPtr<Channel> channel_;
     RefCountedPtr<channelz::ChannelNode> parent_channelz_node_;
     StateWatcher* watcher_ = nullptr;
     Throttle throttle_ ABSL_GUARDED_BY(&RlsLb::mu_);
@@ -700,7 +639,7 @@ class RlsLb final : public LoadBalancingPolicy {
                RefCountedPtr<RlsChannel> rls_channel,
                std::unique_ptr<BackOff> backoff_state,
                grpc_lookup_v1_RouteLookupRequest_Reason reason,
-               std::string stale_header_data);
+               grpc_event_engine::experimental::Slice stale_header_data);
     ~RlsRequest() override;
 
     // Shuts down the request.  If the request is still in flight, it is
@@ -728,7 +667,7 @@ class RlsLb final : public LoadBalancingPolicy {
     RefCountedPtr<RlsChannel> rls_channel_;
     std::unique_ptr<BackOff> backoff_state_;
     grpc_lookup_v1_RouteLookupRequest_Reason reason_;
-    std::string stale_header_data_;
+    grpc_event_engine::experimental::Slice stale_header_data_;
 
     // RLS call state.
     Timestamp deadline_;
@@ -754,9 +693,9 @@ class RlsLb final : public LoadBalancingPolicy {
   // Updates the picker in the work serializer.
   void UpdatePickerLocked() ABSL_LOCKS_EXCLUDED(&mu_);
 
-  void MaybeExportPickCount(
-      GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle,
-      absl::string_view target, const PickResult& pick_result);
+  template <typename HandleType>
+  void MaybeExportPickCount(HandleType handle, absl::string_view target,
+                            const PickResult& pick_result);
 
   const std::string instance_uuid_;
 
@@ -794,8 +733,7 @@ class RlsLb final : public LoadBalancingPolicy {
 RlsLb::ChildPolicyWrapper::ChildPolicyWrapper(RefCountedPtr<RlsLb> lb_policy,
                                               std::string target)
     : DualRefCounted<ChildPolicyWrapper>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "ChildPolicyWrapper"
-                                                     : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "ChildPolicyWrapper" : nullptr),
       lb_policy_(std::move(lb_policy)),
       target_(std::move(target)),
       picker_(MakeRefCounted<QueuePicker>(nullptr)) {
@@ -803,10 +741,9 @@ RlsLb::ChildPolicyWrapper::ChildPolicyWrapper(RefCountedPtr<RlsLb> lb_policy,
 }
 
 void RlsLb::ChildPolicyWrapper::Orphaned() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] ChildPolicyWrapper=%p [%s]: shutdown",
-            lb_policy_.get(), this, target_.c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] ChildPolicyWrapper=" << this
+      << " [" << target_ << "]: shutdown";
   is_shutdown_ = true;
   lb_policy_->child_policy_map_.erase(target_);
   if (child_policy_ != nullptr) {
@@ -817,13 +754,13 @@ void RlsLb::ChildPolicyWrapper::Orphaned() {
   picker_.reset();
 }
 
-absl::optional<Json> InsertOrUpdateChildPolicyField(const std::string& field,
-                                                    const std::string& value,
-                                                    const Json& config,
-                                                    ValidationErrors* errors) {
+std::optional<Json> InsertOrUpdateChildPolicyField(const std::string& field,
+                                                   const std::string& value,
+                                                   const Json& config,
+                                                   ValidationErrors* errors) {
   if (config.type() != Json::Type::kArray) {
     errors->AddError("is not an array");
-    return absl::nullopt;
+    return std::nullopt;
   }
   const size_t original_num_errors = errors->size();
   Json::Array array;
@@ -832,59 +769,50 @@ absl::optional<Json> InsertOrUpdateChildPolicyField(const std::string& field,
     ValidationErrors::ScopedField json_field(errors, absl::StrCat("[", i, "]"));
     if (child_json.type() != Json::Type::kObject) {
       errors->AddError("is not an object");
+    } else if (const Json::Object& child = child_json.object();
+               child.size() != 1) {
+      errors->AddError("child policy object contains more than one field");
     } else {
-      const Json::Object& child = child_json.object();
-      if (child.size() != 1) {
-        errors->AddError("child policy object contains more than one field");
+      const auto& [child_name, child_config_json] = *child.begin();
+      ValidationErrors::ScopedField json_field(
+          errors, absl::StrCat("[\"", child_name, "\"]"));
+      if (child_config_json.type() != Json::Type::kObject) {
+        errors->AddError("child policy config is not an object");
       } else {
-        const std::string& child_name = child.begin()->first;
-        ValidationErrors::ScopedField json_field(
-            errors, absl::StrCat("[\"", child_name, "\"]"));
-        const Json& child_config_json = child.begin()->second;
-        if (child_config_json.type() != Json::Type::kObject) {
-          errors->AddError("child policy config is not an object");
-        } else {
-          Json::Object child_config = child_config_json.object();
-          child_config[field] = Json::FromString(value);
-          array.emplace_back(Json::FromObject(
-              {{child_name, Json::FromObject(std::move(child_config))}}));
-        }
+        Json::Object child_config = child_config_json.object();
+        child_config[field] = Json::FromString(value);
+        array.emplace_back(Json::FromObject(
+            {{child_name, Json::FromObject(std::move(child_config))}}));
       }
     }
   }
-  if (errors->size() != original_num_errors) return absl::nullopt;
+  if (errors->size() != original_num_errors) return std::nullopt;
   return Json::FromArray(std::move(array));
 }
 
-void RlsLb::ChildPolicyWrapper::StartUpdate() {
+void RlsLb::ChildPolicyWrapper::StartUpdate(
+    OrphanablePtr<ChildPolicyHandler>* child_policy_to_delete) {
   ValidationErrors errors;
   auto child_policy_config = InsertOrUpdateChildPolicyField(
       lb_policy_->config_->child_policy_config_target_field_name(), target_,
       lb_policy_->config_->child_policy_config(), &errors);
   CHECK(child_policy_config.has_value());
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(
-        GPR_INFO,
-        "[rlslb %p] ChildPolicyWrapper=%p [%s]: validating update, config: %s",
-        lb_policy_.get(), this, target_.c_str(),
-        JsonDump(*child_policy_config).c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] ChildPolicyWrapper=" << this
+      << " [" << target_
+      << "]: validating update, config: " << JsonDump(*child_policy_config);
   auto config =
       CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig(
           *child_policy_config);
   // Returned RLS target fails the validation.
   if (!config.ok()) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO,
-              "[rlslb %p] ChildPolicyWrapper=%p [%s]: config failed to parse: "
-              "%s",
-              lb_policy_.get(), this, target_.c_str(),
-              config.status().ToString().c_str());
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_.get() << "] ChildPolicyWrapper=" << this
+        << " [" << target_ << "]: config failed to parse: " << config.status();
     pending_config_.reset();
     picker_ = MakeRefCounted<TransientFailurePicker>(
         absl::UnavailableError(config.status().message()));
-    child_policy_.reset();
+    *child_policy_to_delete = std::move(child_policy_);
   } else {
     pending_config_ = std::move(*config);
   }
@@ -902,23 +830,19 @@ absl::Status RlsLb::ChildPolicyWrapper::MaybeFinishUpdate() {
         WeakRef(DEBUG_LOCATION, "ChildPolicyHelper"));
     create_args.args = lb_policy_->channel_args_;
     child_policy_ = MakeOrphanable<ChildPolicyHandler>(std::move(create_args),
-                                                       &grpc_lb_rls_trace);
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO,
-              "[rlslb %p] ChildPolicyWrapper=%p [%s], created new child policy "
-              "handler %p",
-              lb_policy_.get(), this, target_.c_str(), child_policy_.get());
-    }
+                                                       &rls_lb_trace);
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_.get() << "] ChildPolicyWrapper=" << this
+        << " [" << target_ << "], created new child policy handler "
+        << child_policy_.get();
     grpc_pollset_set_add_pollset_set(child_policy_->interested_parties(),
                                      lb_policy_->interested_parties());
   }
   // Send the child the updated config.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] ChildPolicyWrapper=%p [%s], updating child policy "
-            "handler %p",
-            lb_policy_.get(), this, target_.c_str(), child_policy_.get());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] ChildPolicyWrapper=" << this
+      << " [" << target_ << "], updating child policy handler "
+      << child_policy_.get();
   UpdateArgs update_args;
   update_args.config = std::move(pending_config_);
   update_args.addresses = lb_policy_->addresses_;
@@ -933,17 +857,15 @@ absl::Status RlsLb::ChildPolicyWrapper::MaybeFinishUpdate() {
 void RlsLb::ChildPolicyWrapper::ChildPolicyHelper::UpdateState(
     grpc_connectivity_state state, const absl::Status& status,
     RefCountedPtr<SubchannelPicker> picker) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] ChildPolicyWrapper=%p [%s] ChildPolicyHelper=%p: "
-            "UpdateState(state=%s, status=%s, picker=%p)",
-            wrapper_->lb_policy_.get(), wrapper_.get(),
-            wrapper_->target_.c_str(), this, ConnectivityStateName(state),
-            status.ToString().c_str(), picker.get());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << wrapper_->lb_policy_.get()
+      << "] ChildPolicyWrapper=" << wrapper_.get() << " [" << wrapper_->target_
+      << "] ChildPolicyHelper=" << this
+      << ": UpdateState(state=" << ConnectivityStateName(state)
+      << ", status=" << status << ", picker=" << picker.get() << ")";
+  if (wrapper_->is_shutdown_) return;
   {
     MutexLock lock(&wrapper_->lb_policy_->mu_);
-    if (wrapper_->is_shutdown_) return;
     // TODO(roth): It looks like this ignores subsequent TF updates that
     // might change the status used to fail picks, which seems wrong.
     if (wrapper_->connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE &&
@@ -953,7 +875,8 @@ void RlsLb::ChildPolicyWrapper::ChildPolicyHelper::UpdateState(
     wrapper_->connectivity_state_ = state;
     DCHECK(picker != nullptr);
     if (picker != nullptr) {
-      wrapper_->picker_ = std::move(picker);
+      // We want to unref the picker after we release the lock.
+      wrapper_->picker_.swap(picker);
     }
   }
   wrapper_->lb_policy_->UpdatePickerLocked();
@@ -984,12 +907,10 @@ std::map<std::string, std::string> BuildKeyMap(
   // Construct key map using key builder.
   std::map<std::string, std::string> key_map;
   // Add header keys.
-  for (const auto& p : key_builder->header_keys) {
-    const std::string& key = p.first;
-    const std::vector<std::string>& header_names = p.second;
+  for (const auto& [key, header_names] : key_builder->header_keys) {
     for (const std::string& header_name : header_names) {
       std::string buffer;
-      absl::optional<absl::string_view> value =
+      std::optional<absl::string_view> value =
           initial_metadata->Lookup(header_name, &buffer);
       if (value.has_value()) {
         key_map[key] = std::string(*value);
@@ -1041,10 +962,9 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
       BuildKeyMap(config_->key_builder_map(), args.path,
                   lb_policy_->channel_control_helper()->GetAuthority(),
                   args.initial_metadata)};
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] picker=%p: request keys: %s",
-            lb_policy_.get(), this, key.ToString().c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] picker=" << this
+      << ": request keys: " << key.ToString();
   Timestamp now = Timestamp::Now();
   MutexLock lock(&lb_policy_->mu_);
   if (lb_policy_->is_shutdown_) {
@@ -1079,10 +999,9 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
   if (entry != nullptr) {
     // If the entry has non-expired data, use it.
     if (entry->data_expiration_time() >= now) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO, "[rlslb %p] picker=%p: using cache entry %p",
-                lb_policy_.get(), this, entry);
-      }
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << lb_policy_.get() << "] picker=" << this
+          << ": using cache entry " << entry;
       return entry->Pick(args);
     }
     // If the entry is in backoff, then use the default target if set,
@@ -1095,29 +1014,26 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) {
     }
   }
   // RLS call pending.  Queue the pick.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] picker=%p: RLS request pending; queuing pick",
-            lb_policy_.get(), this);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] picker=" << this
+      << ": RLS request pending; queuing pick";
   return PickResult::Queue();
 }
 
 LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail(
     const char* reason, PickArgs args, absl::Status status) {
   if (default_child_policy_ != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; using default target",
-              lb_policy_.get(), this, reason);
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_.get() << "] picker=" << this << ": "
+        << reason << "; using default target";
     auto pick_result = default_child_policy_->Pick(args);
     lb_policy_->MaybeExportPickCount(kMetricDefaultTargetPicks,
                                      config_->default_target(), pick_result);
     return pick_result;
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; failing pick",
-            lb_policy_.get(), this, reason);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] picker=" << this << ": " << reason
+      << "; failing pick";
   auto& stats_plugins =
       lb_policy_->channel_control_helper()->GetStatsPluginGroup();
   stats_plugins.AddCounter(kMetricFailedPicks, 1,
@@ -1132,18 +1048,15 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail(
 //
 
 RlsLb::Cache::Entry::BackoffTimer::BackoffTimer(RefCountedPtr<Entry> entry,
-                                                Timestamp backoff_time)
+                                                Duration delay)
     : entry_(std::move(entry)) {
   backoff_timer_task_handle_ =
       entry_->lb_policy_->channel_control_helper()->GetEventEngine()->RunAfter(
-          backoff_time - Timestamp::Now(),
-          [self = Ref(DEBUG_LOCATION, "BackoffTimer")]() mutable {
-            ApplicationCallbackExecCtx callback_exec_ctx;
+          delay, [self = Ref(DEBUG_LOCATION, "BackoffTimer")]() mutable {
             ExecCtx exec_ctx;
             auto self_ptr = self.get();
             self_ptr->entry_->lb_policy_->work_serializer()->Run(
-                [self = std::move(self)]() { self->OnBackoffTimerLocked(); },
-                DEBUG_LOCATION);
+                [self = std::move(self)]() { self->OnBackoffTimerLocked(); });
           });
 }
 
@@ -1151,12 +1064,12 @@ void RlsLb::Cache::Entry::BackoffTimer::Orphan() {
   if (backoff_timer_task_handle_.has_value() &&
       entry_->lb_policy_->channel_control_helper()->GetEventEngine()->Cancel(
           *backoff_timer_task_handle_)) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s, backoff timer canceled",
-              entry_->lb_policy_.get(), entry_.get(),
-              entry_->is_shutdown_ ? "(shut down)"
-                                   : entry_->lru_iterator_->ToString().c_str());
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << entry_->lb_policy_.get()
+        << "] cache entry=" << entry_.get() << " "
+        << (entry_->is_shutdown_ ? "(shut down)"
+                                 : entry_->lru_iterator_->ToString())
+        << ", backoff timer canceled";
   }
   backoff_timer_task_handle_.reset();
   Unref(DEBUG_LOCATION, "Orphan");
@@ -1165,12 +1078,12 @@ void RlsLb::Cache::Entry::BackoffTimer::Orphan() {
 void RlsLb::Cache::Entry::BackoffTimer::OnBackoffTimerLocked() {
   {
     MutexLock lock(&entry_->lb_policy_->mu_);
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s, backoff timer fired",
-              entry_->lb_policy_.get(), entry_.get(),
-              entry_->is_shutdown_ ? "(shut down)"
-                                   : entry_->lru_iterator_->ToString().c_str());
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << entry_->lb_policy_.get()
+        << "] cache entry=" << entry_.get() << " "
+        << (entry_->is_shutdown_ ? "(shut down)"
+                                 : entry_->lru_iterator_->ToString())
+        << ", backoff timer fired";
     // Skip the update if Orphaned
     if (!backoff_timer_task_handle_.has_value()) return;
     backoff_timer_task_handle_.reset();
@@ -1195,8 +1108,8 @@ std::unique_ptr<BackOff> MakeCacheEntryBackoff() {
 
 RlsLb::Cache::Entry::Entry(RefCountedPtr<RlsLb> lb_policy,
                            const RequestKey& key)
-    : InternallyRefCounted<Entry>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "CacheEntry" : nullptr),
+    : InternallyRefCounted<Entry>(GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "CacheEntry"
+                                                                  : nullptr),
       lb_policy_(std::move(lb_policy)),
       backoff_state_(MakeCacheEntryBackoff()),
       min_expiration_time_(Timestamp::Now() + kMinExpirationTime),
@@ -1204,19 +1117,19 @@ RlsLb::Cache::Entry::Entry(RefCountedPtr<RlsLb> lb_policy,
           lb_policy_->cache_.lru_list_.end(), key)) {}
 
 void RlsLb::Cache::Entry::Orphan() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] cache entry=%p %s: cache entry evicted",
-            lb_policy_.get(), this, lru_iterator_->ToString().c_str());
-  }
+  // We should be holding RlsLB::mu_.
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] cache entry=" << this << " "
+      << lru_iterator_->ToString() << ": cache entry evicted";
   is_shutdown_ = true;
   lb_policy_->cache_.lru_list_.erase(lru_iterator_);
   lru_iterator_ = lb_policy_->cache_.lru_list_.end();  // Just in case.
+  CHECK(child_policy_wrappers_.empty());
   backoff_state_.reset();
   if (backoff_timer_ != nullptr) {
     backoff_timer_.reset();
     lb_policy_->UpdatePickerAsync();
   }
-  child_policy_wrappers_.clear();
   Unref(DEBUG_LOCATION, "Orphan");
 }
 
@@ -1235,42 +1148,36 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(PickArgs args) {
     if (child_policy_wrapper->connectivity_state() ==
             GRPC_CHANNEL_TRANSIENT_FAILURE &&
         i < child_policy_wrappers_.size() - 1) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO,
-                "[rlslb %p] cache entry=%p %s: target %s (%" PRIuPTR
-                " of %" PRIuPTR ") in state TRANSIENT_FAILURE; skipping",
-                lb_policy_.get(), this, lru_iterator_->ToString().c_str(),
-                child_policy_wrapper->target().c_str(), i,
-                child_policy_wrappers_.size());
-      }
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << lb_policy_.get() << "] cache entry=" << this << " "
+          << lru_iterator_->ToString() << ": target "
+          << child_policy_wrapper->target() << " (" << i << " of "
+          << child_policy_wrappers_.size()
+          << ") in state TRANSIENT_FAILURE; skipping";
       continue;
     }
     break;
   }
   // Child policy not in TRANSIENT_FAILURE or is the last target in
   // the list, so delegate.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] cache entry=%p %s: target %s (%" PRIuPTR " of %" PRIuPTR
-            ") in state %s; delegating",
-            lb_policy_.get(), this, lru_iterator_->ToString().c_str(),
-            child_policy_wrapper->target().c_str(), i,
-            child_policy_wrappers_.size(),
-            ConnectivityStateName(child_policy_wrapper->connectivity_state()));
-  }
-  // Add header data.
-  // Note that even if the target we're using is in TRANSIENT_FAILURE,
-  // the pick might still succeed (e.g., if the child is ring_hash), so
-  // we need to pass the right header info down in all cases.
-  if (!header_data_.empty()) {
-    char* copied_header_data =
-        static_cast<char*>(args.call_state->Alloc(header_data_.length() + 1));
-    strcpy(copied_header_data, header_data_.c_str());
-    args.initial_metadata->Add(kRlsHeaderKey, copied_header_data);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] cache entry=" << this << " "
+      << lru_iterator_->ToString() << ": target "
+      << child_policy_wrapper->target() << " (" << i << " of "
+      << child_policy_wrappers_.size() << ") in state "
+      << ConnectivityStateName(child_policy_wrapper->connectivity_state())
+      << "; delegating";
   auto pick_result = child_policy_wrapper->Pick(args);
   lb_policy_->MaybeExportPickCount(kMetricTargetPicks,
                                    child_policy_wrapper->target(), pick_result);
+  // Add header data.
+  if (!header_data_.empty()) {
+    auto* complete_pick =
+        std::get_if<PickResult::Complete>(&pick_result.result);
+    if (complete_pick != nullptr) {
+      complete_pick->metadata_mutations.Set(kRlsHeaderKey, header_data_.Ref());
+    }
+  }
   return pick_result;
 }
 
@@ -1298,7 +1205,8 @@ void RlsLb::Cache::Entry::MarkUsed() {
 
 std::vector<RlsLb::ChildPolicyWrapper*>
 RlsLb::Cache::Entry::OnRlsResponseLocked(
-    ResponseInfo response, std::unique_ptr<BackOff> backoff_state) {
+    ResponseInfo response, std::unique_ptr<BackOff> backoff_state,
+    OrphanablePtr<ChildPolicyHandler>* child_policy_to_delete) {
   // Move the entry to the end of the LRU list.
   MarkUsed();
   // If the request failed, store the failed status and update the
@@ -1310,11 +1218,12 @@ RlsLb::Cache::Entry::OnRlsResponseLocked(
     } else {
       backoff_state_ = MakeCacheEntryBackoff();
     }
-    backoff_time_ = backoff_state_->NextAttemptTime();
-    Timestamp now = Timestamp::Now();
-    backoff_expiration_time_ = now + (backoff_time_ - now) * 2;
+    const Duration delay = backoff_state_->NextAttemptDelay();
+    const Timestamp now = Timestamp::Now();
+    backoff_time_ = now + delay;
+    backoff_expiration_time_ = now + delay * 2;
     backoff_timer_ = MakeOrphanable<BackoffTimer>(
-        Ref(DEBUG_LOCATION, "BackoffTimer"), backoff_time_);
+        Ref(DEBUG_LOCATION, "BackoffTimer"), delay);
     lb_policy_->UpdatePickerAsync();
     return {};
   }
@@ -1359,7 +1268,7 @@ RlsLb::Cache::Entry::OnRlsResponseLocked(
     if (it == lb_policy_->child_policy_map_.end()) {
       auto new_child = MakeRefCounted<ChildPolicyWrapper>(
           lb_policy_.Ref(DEBUG_LOCATION, "ChildPolicyWrapper"), target);
-      new_child->StartUpdate();
+      new_child->StartUpdate(child_policy_to_delete);
       child_policies_to_finish_update.push_back(new_child.get());
       new_child_policy_wrappers.emplace_back(std::move(new_child));
     } else {
@@ -1396,58 +1305,64 @@ RlsLb::Cache::Entry* RlsLb::Cache::Find(const RequestKey& key) {
   return it->second.get();
 }
 
-RlsLb::Cache::Entry* RlsLb::Cache::FindOrInsert(const RequestKey& key) {
+RlsLb::Cache::Entry* RlsLb::Cache::FindOrInsert(
+    const RequestKey& key, std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                               child_policy_wrappers_to_delete) {
   auto it = map_.find(key);
   // If not found, create new entry.
   if (it == map_.end()) {
     size_t entry_size = EntrySizeForKey(key);
-    MaybeShrinkSize(size_limit_ - std::min(size_limit_, entry_size));
+    MaybeShrinkSize(size_limit_ - std::min(size_limit_, entry_size),
+                    child_policy_wrappers_to_delete);
     Entry* entry = new Entry(
         lb_policy_->RefAsSubclass<RlsLb>(DEBUG_LOCATION, "CacheEntry"), key);
     map_.emplace(key, OrphanablePtr<Entry>(entry));
     size_ += entry_size;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] key=%s: cache entry added, entry=%p",
-              lb_policy_, key.ToString().c_str(), entry);
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_ << "] key=" << key.ToString()
+        << ": cache entry added, entry=" << entry;
     return entry;
   }
   // Entry found, so use it.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] key=%s: found cache entry %p", lb_policy_,
-            key.ToString().c_str(), it->second.get());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_ << "] key=" << key.ToString()
+      << ": found cache entry " << it->second.get();
   it->second->MarkUsed();
   return it->second.get();
 }
 
-void RlsLb::Cache::Resize(size_t bytes) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] resizing cache to %" PRIuPTR " bytes",
-            lb_policy_, bytes);
-  }
+void RlsLb::Cache::Resize(size_t bytes,
+                          std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                              child_policy_wrappers_to_delete) {
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_ << "] resizing cache to " << bytes << " bytes";
   size_limit_ = bytes;
-  MaybeShrinkSize(size_limit_);
+  MaybeShrinkSize(size_limit_, child_policy_wrappers_to_delete);
 }
 
 void RlsLb::Cache::ResetAllBackoff() {
-  for (auto& p : map_) {
-    p.second->ResetBackoff();
+  for (auto& [_, entry] : map_) {
+    entry->ResetBackoff();
   }
   lb_policy_->UpdatePickerAsync();
 }
 
-void RlsLb::Cache::Shutdown() {
+std::vector<RefCountedPtr<RlsLb::ChildPolicyWrapper>> RlsLb::Cache::Shutdown() {
+  std::vector<RefCountedPtr<ChildPolicyWrapper>>
+      child_policy_wrappers_to_delete;
+  for (auto& [_, entry] : map_) {
+    entry->TakeChildPolicyWrappers(&child_policy_wrappers_to_delete);
+  }
   map_.clear();
   lru_list_.clear();
   if (cleanup_timer_handle_.has_value() &&
       lb_policy_->channel_control_helper()->GetEventEngine()->Cancel(
           *cleanup_timer_handle_)) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] cache cleanup timer canceled", lb_policy_);
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_ << "] cache cleanup timer canceled";
   }
   cleanup_timer_handle_.reset();
+  return child_policy_wrappers_to_delete;
 }
 
 void RlsLb::Cache::ReportMetricsLocked(CallbackMetricReporter& reporter) {
@@ -1469,27 +1384,28 @@ void RlsLb::Cache::StartCleanupTimer() {
           kCacheCleanupTimerInterval,
           [this, lb_policy = lb_policy_->Ref(DEBUG_LOCATION,
                                              "CacheCleanupTimer")]() mutable {
-            ApplicationCallbackExecCtx callback_exec_ctx;
             ExecCtx exec_ctx;
             lb_policy_->work_serializer()->Run(
                 [this, lb_policy = std::move(lb_policy)]() {
                   // The lb_policy ref is held until the callback completes
                   OnCleanupTimer();
-                },
-                DEBUG_LOCATION);
+                });
           });
 }
 
 void RlsLb::Cache::OnCleanupTimer() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] cache cleanup timer fired", lb_policy_);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_ << "] cache cleanup timer fired";
+  std::vector<RefCountedPtr<ChildPolicyWrapper>>
+      child_policy_wrappers_to_delete;
   MutexLock lock(&lb_policy_->mu_);
   if (!cleanup_timer_handle_.has_value()) return;
   if (lb_policy_->is_shutdown_) return;
   for (auto it = map_.begin(); it != map_.end();) {
-    if (GPR_UNLIKELY(it->second->ShouldRemove() && it->second->CanEvict())) {
-      size_ -= it->second->Size();
+    auto& entry = it->second;
+    if (GPR_UNLIKELY(entry->ShouldRemove() && entry->CanEvict())) {
+      size_ -= entry->Size();
+      entry->TakeChildPolicyWrappers(&child_policy_wrappers_to_delete);
       it = map_.erase(it);
     } else {
       ++it;
@@ -1503,26 +1419,26 @@ size_t RlsLb::Cache::EntrySizeForKey(const RequestKey& key) {
   return (key.Size() * 2) + sizeof(Entry);
 }
 
-void RlsLb::Cache::MaybeShrinkSize(size_t bytes) {
+void RlsLb::Cache::MaybeShrinkSize(
+    size_t bytes, std::vector<RefCountedPtr<ChildPolicyWrapper>>*
+                      child_policy_wrappers_to_delete) {
   while (size_ > bytes) {
     auto lru_it = lru_list_.begin();
     if (GPR_UNLIKELY(lru_it == lru_list_.end())) break;
     auto map_it = map_.find(*lru_it);
     CHECK(map_it != map_.end());
-    if (!map_it->second->CanEvict()) break;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] LRU eviction: removing entry %p %s",
-              lb_policy_, map_it->second.get(), lru_it->ToString().c_str());
-    }
-    size_ -= map_it->second->Size();
+    auto& entry = map_it->second;
+    if (!entry->CanEvict()) break;
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_ << "] LRU eviction: removing entry "
+        << entry.get() << " " << lru_it->ToString();
+    size_ -= entry->Size();
+    entry->TakeChildPolicyWrappers(child_policy_wrappers_to_delete);
     map_.erase(map_it);
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] LRU pass complete: desired size=%" PRIuPTR
-            " size=%" PRIuPTR,
-            lb_policy_, bytes, size_);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_
+      << "] LRU pass complete: desired size=" << bytes << " size=" << size_;
 }
 
 //
@@ -1532,13 +1448,10 @@ void RlsLb::Cache::MaybeShrinkSize(size_t bytes) {
 void RlsLb::RlsChannel::StateWatcher::OnConnectivityStateChange(
     grpc_connectivity_state new_state, const absl::Status& status) {
   auto* lb_policy = rls_channel_->lb_policy_.get();
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] RlsChannel=%p StateWatcher=%p: "
-            "state changed to %s (%s)",
-            lb_policy, rls_channel_.get(), this,
-            ConnectivityStateName(new_state), status.ToString().c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy << "] RlsChannel=" << rls_channel_.get()
+      << " StateWatcher=" << this << ": state changed to "
+      << ConnectivityStateName(new_state) << " (" << status << ")";
   if (rls_channel_->is_shutdown_) return;
   MutexLock lock(&lb_policy->mu_);
   if (new_state == GRPC_CHANNEL_READY && was_transient_failure_) {
@@ -1597,7 +1510,7 @@ void RlsLb::RlsChannel::Throttle::RegisterResponse(bool success) {
 
 RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
     : InternallyRefCounted<RlsChannel>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "RlsChannel" : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "RlsChannel" : nullptr),
       lb_policy_(std::move(lb_policy)) {
   // Get channel creds from parent channel.
   // Note that we are using the "unsafe" channel creds here, which do
@@ -1614,7 +1527,7 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
   // (This is ugly, but it seems better than propagating all channel args
   // from the parent channel by default and then having a giant
   // exclude list of args to strip out, like we do in grpclb.)
-  absl::optional<absl::string_view> fake_security_expected_targets =
+  std::optional<absl::string_view> fake_security_expected_targets =
       lb_policy_->channel_args_.GetString(
           GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS);
   if (fake_security_expected_targets.has_value()) {
@@ -1631,11 +1544,10 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
   channel_.reset(Channel::FromC(
       grpc_channel_create(lb_policy_->config_->lookup_service().c_str(),
                           creds.get(), args.ToC().get())));
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p: created channel %p for %s",
-            lb_policy_.get(), this, channel_.get(),
-            lb_policy_->config_->lookup_service().c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] RlsChannel=" << this
+      << ": created channel " << channel_.get() << " for "
+      << lb_policy_->config_->lookup_service();
   if (channel_ != nullptr) {
     // Set up channelz linkage.
     channelz::ChannelNode* child_channelz_node = channel_->channelz_node();
@@ -1654,10 +1566,9 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr<RlsLb> lb_policy)
 }
 
 void RlsLb::RlsChannel::Orphan() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p, channel=%p: shutdown",
-            lb_policy_.get(), this, channel_.get());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] RlsChannel=" << this
+      << ", channel=" << channel_.get() << ": shutdown";
   is_shutdown_ = true;
   if (channel_ != nullptr) {
     // Remove channelz linkage.
@@ -1681,11 +1592,11 @@ void RlsLb::RlsChannel::StartRlsCall(const RequestKey& key,
   std::unique_ptr<BackOff> backoff_state;
   grpc_lookup_v1_RouteLookupRequest_Reason reason =
       grpc_lookup_v1_RouteLookupRequest_REASON_MISS;
-  std::string stale_header_data;
+  grpc_event_engine::experimental::Slice stale_header_data;
   if (stale_entry != nullptr) {
     backoff_state = stale_entry->TakeBackoffState();
     reason = grpc_lookup_v1_RouteLookupRequest_REASON_STALE;
-    stale_header_data = stale_entry->header_data();
+    stale_header_data = stale_entry->header_data().Ref();
   }
   lb_policy_->request_map_.emplace(
       key, MakeOrphanable<RlsRequest>(
@@ -1707,24 +1618,23 @@ void RlsLb::RlsChannel::ResetBackoff() {
 // RlsLb::RlsRequest
 //
 
-RlsLb::RlsRequest::RlsRequest(RefCountedPtr<RlsLb> lb_policy, RequestKey key,
-                              RefCountedPtr<RlsChannel> rls_channel,
-                              std::unique_ptr<BackOff> backoff_state,
-                              grpc_lookup_v1_RouteLookupRequest_Reason reason,
-                              std::string stale_header_data)
+RlsLb::RlsRequest::RlsRequest(
+    RefCountedPtr<RlsLb> lb_policy, RequestKey key,
+    RefCountedPtr<RlsChannel> rls_channel,
+    std::unique_ptr<BackOff> backoff_state,
+    grpc_lookup_v1_RouteLookupRequest_Reason reason,
+    grpc_event_engine::experimental::Slice stale_header_data)
     : InternallyRefCounted<RlsRequest>(
-          GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) ? "RlsRequest" : nullptr),
+          GRPC_TRACE_FLAG_ENABLED(rls_lb) ? "RlsRequest" : nullptr),
       lb_policy_(std::move(lb_policy)),
       key_(std::move(key)),
       rls_channel_(std::move(rls_channel)),
       backoff_state_(std::move(backoff_state)),
       reason_(reason),
       stale_header_data_(std::move(stale_header_data)) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO,
-            "[rlslb %p] rls_request=%p: RLS request created for key %s",
-            lb_policy_.get(), this, key_.ToString().c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] rls_request=" << this
+      << ": RLS request created for key " << key_.ToString();
   GRPC_CLOSURE_INIT(&call_complete_cb_, OnRlsCallComplete, this, nullptr);
   ExecCtx::Run(
       DEBUG_LOCATION,
@@ -1737,10 +1647,9 @@ RlsLb::RlsRequest::~RlsRequest() { CHECK_EQ(call_, nullptr); }
 
 void RlsLb::RlsRequest::Orphan() {
   if (call_ != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] rls_request=%p %s: cancelling RLS call",
-              lb_policy_.get(), this, key_.ToString().c_str());
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << lb_policy_.get() << "] rls_request=" << this << " "
+        << key_.ToString() << ": cancelling RLS call";
     grpc_call_cancel_internal(call_);
   }
   Unref(DEBUG_LOCATION, "Orphan");
@@ -1748,12 +1657,10 @@ void RlsLb::RlsRequest::Orphan() {
 
 void RlsLb::RlsRequest::StartCall(void* arg, grpc_error_handle /*error*/) {
   auto* request = static_cast<RlsRequest*>(arg);
-  request->lb_policy_->work_serializer()->Run(
-      [request]() {
-        request->StartCallLocked();
-        request->Unref(DEBUG_LOCATION, "StartCall");
-      },
-      DEBUG_LOCATION);
+  request->lb_policy_->work_serializer()->Run([request]() {
+    request->StartCallLocked();
+    request->Unref(DEBUG_LOCATION, "StartCall");
+  });
 }
 
 void RlsLb::RlsRequest::StartCallLocked() {
@@ -1768,7 +1675,7 @@ void RlsLb::RlsRequest::StartCallLocked() {
   call_ = rls_channel_->channel()->CreateCall(
       /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, /*cq=*/nullptr,
       lb_policy_->interested_parties(),
-      Slice::FromStaticString(kRlsRequestPath), /*authority=*/absl::nullopt,
+      Slice::FromStaticString(kRlsRequestPath), /*authority=*/std::nullopt,
       deadline_, /*registered_method=*/true);
   grpc_op ops[6];
   memset(ops, 0, sizeof(ops));
@@ -1801,23 +1708,19 @@ void RlsLb::RlsRequest::StartCallLocked() {
 
 void RlsLb::RlsRequest::OnRlsCallComplete(void* arg, grpc_error_handle error) {
   auto* request = static_cast<RlsRequest*>(arg);
-  request->lb_policy_->work_serializer()->Run(
-      [request, error]() {
-        request->OnRlsCallCompleteLocked(error);
-        request->Unref(DEBUG_LOCATION, "OnRlsCallComplete");
-      },
-      DEBUG_LOCATION);
+  request->lb_policy_->work_serializer()->Run([request, error]() {
+    request->OnRlsCallCompleteLocked(error);
+    request->Unref(DEBUG_LOCATION, "OnRlsCallComplete");
+  });
 }
 
 void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb)) {
     std::string status_message(StringViewFromSlice(status_details_recv_));
-    gpr_log(GPR_INFO,
-            "[rlslb %p] rls_request=%p %s, error=%s, status={%d, %s} RLS call "
-            "response received",
-            lb_policy_.get(), this, key_.ToString().c_str(),
-            StatusToString(error).c_str(), status_recv_,
-            status_message.c_str());
+    LOG(INFO) << "[rlslb " << lb_policy_.get() << "] rls_request=" << this
+              << " " << key_.ToString() << ", error=" << StatusToString(error)
+              << ", status={" << status_recv_ << ", " << status_message << "}"
+              << " RLS call response received";
   }
   // Parse response.
   ResponseInfo response;
@@ -1843,19 +1746,22 @@ void RlsLb::RlsRequest::OnRlsCallCompleteLocked(grpc_error_handle error) {
   grpc_call_unref(call_);
   call_ = nullptr;
   // Return result to cache.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] rls_request=%p %s: response info: %s",
-            lb_policy_.get(), this, key_.ToString().c_str(),
-            response.ToString().c_str());
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO)
+      << "[rlslb " << lb_policy_.get() << "] rls_request=" << this << " "
+      << key_.ToString() << ": response info: " << response.ToString();
   std::vector<ChildPolicyWrapper*> child_policies_to_finish_update;
+  std::vector<RefCountedPtr<ChildPolicyWrapper>>
+      child_policy_wrappers_to_delete;
+  OrphanablePtr<ChildPolicyHandler> child_policy_to_delete;
   {
     MutexLock lock(&lb_policy_->mu_);
     if (lb_policy_->is_shutdown_) return;
     rls_channel_->ReportResponseLocked(response.status.ok());
-    Cache::Entry* cache_entry = lb_policy_->cache_.FindOrInsert(key_);
+    Cache::Entry* cache_entry =
+        lb_policy_->cache_.FindOrInsert(key_, &child_policy_wrappers_to_delete);
     child_policies_to_finish_update = cache_entry->OnRlsResponseLocked(
-        std::move(response), std::move(backoff_state_));
+        std::move(response), std::move(backoff_state_),
+        &child_policy_to_delete);
     lb_policy_->request_map_.erase(key_);
   }
   // Now that we've released the lock, finish the update on any newly
@@ -1880,17 +1786,16 @@ grpc_byte_buffer* RlsLb::RlsRequest::MakeRequestProto() {
       grpc_lookup_v1_RouteLookupRequest_new(arena.ptr());
   grpc_lookup_v1_RouteLookupRequest_set_target_type(
       req, upb_StringView_FromDataAndSize(kGrpc, sizeof(kGrpc) - 1));
-  for (const auto& kv : key_.key_map) {
+  for (const auto& [key, value] : key_.key_map) {
     grpc_lookup_v1_RouteLookupRequest_key_map_set(
-        req, upb_StringView_FromDataAndSize(kv.first.data(), kv.first.size()),
-        upb_StringView_FromDataAndSize(kv.second.data(), kv.second.size()),
+        req, upb_StringView_FromDataAndSize(key.data(), key.size()),
+        upb_StringView_FromDataAndSize(value.data(), value.size()),
         arena.ptr());
   }
   grpc_lookup_v1_RouteLookupRequest_set_reason(req, reason_);
   if (!stale_header_data_.empty()) {
     grpc_lookup_v1_RouteLookupRequest_set_stale_header_data(
-        req, upb_StringView_FromDataAndSize(stale_header_data_.data(),
-                                            stale_header_data_.size()));
+        req, StdStringToUpbString(stale_header_data_.as_string_view()));
   }
   size_t len;
   char* buf =
@@ -1933,7 +1838,8 @@ RlsLb::ResponseInfo RlsLb::RlsRequest::ParseResponseProto() {
   upb_StringView header_data_strview =
       grpc_lookup_v1_RouteLookupResponse_header_data(response);
   response_info.header_data =
-      std::string(header_data_strview.data, header_data_strview.size);
+      grpc_event_engine::experimental::Slice::FromCopiedBuffer(
+          header_data_strview.data, header_data_strview.size);
   return response_info;
 }
 
@@ -1954,17 +1860,8 @@ RlsLb::RlsLb(Args args)
       instance_uuid_(channel_args()
                          .GetOwnedString(GRPC_ARG_TEST_ONLY_RLS_INSTANCE_ID)
                          .value_or(GenerateUUID())),
-      cache_(this),
-      registered_metric_callback_(
-          channel_control_helper()->GetStatsPluginGroup().RegisterCallback(
-              [this](CallbackMetricReporter& reporter) {
-                MutexLock lock(&mu_);
-                cache_.ReportMetricsLocked(reporter);
-              },
-              {kMetricCacheSize, kMetricCacheEntries})) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] policy created", this);
-  }
+      cache_(this) {
+  GRPC_TRACE_LOG(rls_lb, INFO) << "[rlslb " << this << "] policy created";
 }
 
 bool EndpointsEqual(
@@ -1989,18 +1886,16 @@ bool EndpointsEqual(
 }
 
 absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] policy updated", this);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO) << "[rlslb " << this << "] policy updated";
   update_in_progress_ = true;
   // Swap out config.
   RefCountedPtr<RlsLbConfig> old_config = std::move(config_);
   config_ = args.config.TakeAsSubclass<RlsLbConfig>();
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace) &&
+  if (GRPC_TRACE_FLAG_ENABLED(rls_lb) &&
       (old_config == nullptr ||
        old_config->child_policy_config() != config_->child_policy_config())) {
-    gpr_log(GPR_INFO, "[rlslb %p] updated child policy config: %s", this,
-            JsonDump(config_->child_policy_config()).c_str());
+    LOG(INFO) << "[rlslb " << this << "] updated child policy config: "
+              << JsonDump(config_->child_policy_config());
   }
   // Swap out addresses.
   // If the new address list is an error and we have an existing address list,
@@ -2024,31 +1919,28 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
   if (old_config == nullptr ||
       config_->default_target() != old_config->default_target()) {
     if (config_->default_target().empty()) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO, "[rlslb %p] unsetting default target", this);
-      }
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << this << "] unsetting default target";
       default_child_policy_.reset();
+    } else if (auto it = child_policy_map_.find(config_->default_target());
+               it == child_policy_map_.end()) {
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << this << "] creating new default target";
+      default_child_policy_ = MakeRefCounted<ChildPolicyWrapper>(
+          RefAsSubclass<RlsLb>(DEBUG_LOCATION, "ChildPolicyWrapper"),
+          config_->default_target());
+      created_default_child = true;
     } else {
-      auto it = child_policy_map_.find(config_->default_target());
-      if (it == child_policy_map_.end()) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-          gpr_log(GPR_INFO, "[rlslb %p] creating new default target", this);
-        }
-        default_child_policy_ = MakeRefCounted<ChildPolicyWrapper>(
-            RefAsSubclass<RlsLb>(DEBUG_LOCATION, "ChildPolicyWrapper"),
-            config_->default_target());
-        created_default_child = true;
-      } else {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-          gpr_log(GPR_INFO,
-                  "[rlslb %p] using existing child for default target", this);
-        }
-        default_child_policy_ =
-            it->second->Ref(DEBUG_LOCATION, "DefaultChildPolicy");
-      }
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << this << "] using existing child for default target";
+      default_child_policy_ =
+          it->second->Ref(DEBUG_LOCATION, "DefaultChildPolicy");
     }
   }
   // Now grab the lock to swap out the state it guards.
+  std::vector<RefCountedPtr<ChildPolicyWrapper>>
+      child_policy_wrappers_to_delete;
+  OrphanablePtr<ChildPolicyHandler> child_policy_to_delete;
   {
     MutexLock lock(&mu_);
     // Swap out RLS channel if needed.
@@ -2060,42 +1952,37 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
     // Resize cache if needed.
     if (old_config == nullptr ||
         config_->cache_size_bytes() != old_config->cache_size_bytes()) {
-      cache_.Resize(static_cast<size_t>(config_->cache_size_bytes()));
+      cache_.Resize(static_cast<size_t>(config_->cache_size_bytes()),
+                    &child_policy_wrappers_to_delete);
     }
     // Start update of child policies if needed.
     if (update_child_policies) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO, "[rlslb %p] starting child policy updates", this);
-      }
-      for (auto& p : child_policy_map_) {
-        p.second->StartUpdate();
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << this << "] starting child policy updates";
+      for (auto& [_, child] : child_policy_map_) {
+        child->StartUpdate(&child_policy_to_delete);
       }
     } else if (created_default_child) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-        gpr_log(GPR_INFO, "[rlslb %p] starting default child policy update",
-                this);
-      }
-      default_child_policy_->StartUpdate();
+      GRPC_TRACE_LOG(rls_lb, INFO)
+          << "[rlslb " << this << "] starting default child policy update";
+      default_child_policy_->StartUpdate(&child_policy_to_delete);
     }
   }
   // Now that we've released the lock, finish update of child policies.
   std::vector<std::string> errors;
   if (update_child_policies) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] finishing child policy updates", this);
-    }
-    for (auto& p : child_policy_map_) {
-      absl::Status status = p.second->MaybeFinishUpdate();
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << this << "] finishing child policy updates";
+    for (auto& [name, child] : child_policy_map_) {
+      absl::Status status = child->MaybeFinishUpdate();
       if (!status.ok()) {
         errors.emplace_back(
-            absl::StrCat("target ", p.first, ": ", status.ToString()));
+            absl::StrCat("target ", name, ": ", status.ToString()));
       }
     }
   } else if (created_default_child) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-      gpr_log(GPR_INFO, "[rlslb %p] finishing default child policy update",
-              this);
-    }
+    GRPC_TRACE_LOG(rls_lb, INFO)
+        << "[rlslb " << this << "] finishing default child policy update";
     absl::Status status = default_child_policy_->MaybeFinishUpdate();
     if (!status.ok()) {
       errors.emplace_back(absl::StrCat("target ", config_->default_target(),
@@ -2103,6 +1990,20 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
     }
   }
   update_in_progress_ = false;
+  // On the initial update only, we set the gauge metric callback.  We
+  // can't do this before the initial update, because otherwise the
+  // callback could be invoked before we've set state that we need for
+  // the label values (e.g., we'd add metrics with empty string for the
+  // RLS server name).
+  if (registered_metric_callback_ == nullptr) {
+    registered_metric_callback_ =
+        channel_control_helper()->GetStatsPluginGroup().RegisterCallback(
+            [this](CallbackMetricReporter& reporter) {
+              MutexLock lock(&mu_);
+              cache_.ReportMetricsLocked(reporter);
+            },
+            Duration::Seconds(5), kMetricCacheSize, kMetricCacheEntries);
+  }
   // In principle, we need to update the picker here only if the config
   // fields used by the picker have changed.  However, it seems fragile
   // to check individual fields, since the picker logic could change in
@@ -2120,8 +2021,8 @@ absl::Status RlsLb::UpdateLocked(UpdateArgs args) {
 
 void RlsLb::ExitIdleLocked() {
   MutexLock lock(&mu_);
-  for (auto& child_entry : child_policy_map_) {
-    child_entry.second->ExitIdleLocked();
+  for (auto& [_, child] : child_policy_map_) {
+    child->ExitIdleLocked();
   }
 }
 
@@ -2131,24 +2032,28 @@ void RlsLb::ResetBackoffLocked() {
     rls_channel_->ResetBackoff();
     cache_.ResetAllBackoff();
   }
-  for (auto& child : child_policy_map_) {
-    child.second->ResetBackoffLocked();
+  for (auto& [_, child] : child_policy_map_) {
+    child->ResetBackoffLocked();
   }
 }
 
 void RlsLb::ShutdownLocked() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] policy shutdown", this);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO) << "[rlslb " << this << "] policy shutdown";
   registered_metric_callback_.reset();
-  MutexLock lock(&mu_);
-  is_shutdown_ = true;
-  config_.reset(DEBUG_LOCATION, "ShutdownLocked");
+  RefCountedPtr<ChildPolicyWrapper> child_policy_to_delete;
+  std::vector<RefCountedPtr<ChildPolicyWrapper>>
+      child_policy_wrappers_to_delete;
+  OrphanablePtr<RlsChannel> rls_channel_to_delete;
+  {
+    MutexLock lock(&mu_);
+    is_shutdown_ = true;
+    config_.reset(DEBUG_LOCATION, "ShutdownLocked");
+    child_policy_wrappers_to_delete = cache_.Shutdown();
+    request_map_.clear();
+    rls_channel_to_delete = std::move(rls_channel_);
+    child_policy_to_delete = std::move(default_child_policy_);
+  }
   channel_args_ = ChannelArgs();
-  cache_.Shutdown();
-  request_map_.clear();
-  rls_channel_.reset();
-  default_child_policy_.reset();
 }
 
 void RlsLb::UpdatePickerAsync() {
@@ -2165,13 +2070,11 @@ void RlsLb::UpdatePickerAsync() {
 
 void RlsLb::UpdatePickerCallback(void* arg, grpc_error_handle /*error*/) {
   auto* rls_lb = static_cast<RlsLb*>(arg);
-  rls_lb->work_serializer()->Run(
-      [rls_lb]() {
-        RefCountedPtr<RlsLb> lb_policy(rls_lb);
-        lb_policy->UpdatePickerLocked();
-        lb_policy.reset(DEBUG_LOCATION, "UpdatePickerCallback");
-      },
-      DEBUG_LOCATION);
+  rls_lb->work_serializer()->Run([rls_lb]() {
+    RefCountedPtr<RlsLb> lb_policy(rls_lb);
+    lb_policy->UpdatePickerLocked();
+    lb_policy.reset(DEBUG_LOCATION, "UpdatePickerCallback");
+  });
 }
 
 void RlsLb::UpdatePickerLocked() {
@@ -2181,9 +2084,7 @@ void RlsLb::UpdatePickerLocked() {
   // all children.  This avoids unnecessary picker churn while an update
   // is being propagated to our children.
   if (update_in_progress_) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] updating picker", this);
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO) << "[rlslb " << this << "] updating picker";
   grpc_connectivity_state state = GRPC_CHANNEL_IDLE;
   if (!child_policy_map_.empty()) {
     state = GRPC_CHANNEL_TRANSIENT_FAILURE;
@@ -2192,13 +2093,11 @@ void RlsLb::UpdatePickerLocked() {
     {
       MutexLock lock(&mu_);
       if (is_shutdown_) return;
-      for (auto& p : child_policy_map_) {
-        grpc_connectivity_state child_state = p.second->connectivity_state();
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-          gpr_log(GPR_INFO, "[rlslb %p] target %s in state %s", this,
-                  p.second->target().c_str(),
-                  ConnectivityStateName(child_state));
-        }
+      for (auto& [_, child] : child_policy_map_) {
+        grpc_connectivity_state child_state = child->connectivity_state();
+        GRPC_TRACE_LOG(rls_lb, INFO)
+            << "[rlslb " << this << "] target " << child->target()
+            << " in state " << ConnectivityStateName(child_state);
         if (child_state == GRPC_CHANNEL_READY) {
           state = GRPC_CHANNEL_READY;
           break;
@@ -2217,10 +2116,8 @@ void RlsLb::UpdatePickerLocked() {
       }
     }
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) {
-    gpr_log(GPR_INFO, "[rlslb %p] reporting state %s", this,
-            ConnectivityStateName(state));
-  }
+  GRPC_TRACE_LOG(rls_lb, INFO) << "[rlslb " << this << "] reporting state "
+                               << ConnectivityStateName(state);
   absl::Status status;
   if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     status = absl::UnavailableError("no children available");
@@ -2230,9 +2127,9 @@ void RlsLb::UpdatePickerLocked() {
       MakeRefCounted<Picker>(RefAsSubclass<RlsLb>(DEBUG_LOCATION, "Picker")));
 }
 
-void RlsLb::MaybeExportPickCount(
-    GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle,
-    absl::string_view target, const PickResult& pick_result) {
+template <typename HandleType>
+void RlsLb::MaybeExportPickCount(HandleType handle, absl::string_view target,
+                                 const PickResult& pick_result) {
   absl::string_view pick_result_string = Match(
       pick_result.result,
       [](const LoadBalancingPolicy::PickResult::Complete&) {
@@ -2251,7 +2148,7 @@ void RlsLb::MaybeExportPickCount(
 }
 
 //
-// RlsLbFactory
+// RlsLbConfig
 //
 
 struct GrpcKeyBuilder {
@@ -2271,7 +2168,7 @@ struct GrpcKeyBuilder {
   struct NameMatcher {
     std::string key;
     std::vector<std::string> names;
-    absl::optional<bool> required_match;
+    std::optional<bool> required_match;
 
     static const JsonLoaderInterface* JsonLoader(const JsonArgs&) {
       static const auto* loader =
@@ -2317,9 +2214,9 @@ struct GrpcKeyBuilder {
   };
 
   struct ExtraKeys {
-    absl::optional<std::string> host_key;
-    absl::optional<std::string> service_key;
-    absl::optional<std::string> method_key;
+    std::optional<std::string> host_key;
+    std::optional<std::string> service_key;
+    std::optional<std::string> method_key;
 
     static const JsonLoaderInterface* JsonLoader(const JsonArgs&) {
       static const auto* loader =
@@ -2333,7 +2230,7 @@ struct GrpcKeyBuilder {
 
     void JsonPostLoad(const Json&, const JsonArgs&, ValidationErrors* errors) {
       auto check_field = [&](const std::string& field_name,
-                             absl::optional<std::string>* struct_field) {
+                             std::optional<std::string>* struct_field) {
         ValidationErrors::ScopedField field(errors,
                                             absl::StrCat(".", field_name));
         if (struct_field->has_value() && (*struct_field)->empty()) {
@@ -2394,9 +2291,9 @@ struct GrpcKeyBuilder {
       duplicate_key_check_func(header.key,
                                absl::StrCat(".headers[", i, "].key"));
     }
-    for (const auto& p : constant_keys) {
-      duplicate_key_check_func(
-          p.first, absl::StrCat(".constantKeys[\"", p.first, "\"]"));
+    for (const auto& [key, value] : constant_keys) {
+      duplicate_key_check_func(key,
+                               absl::StrCat(".constantKeys[\"", key, "\"]"));
     }
     if (extra_keys.host_key.has_value()) {
       duplicate_key_check_func(*extra_keys.host_key, ".extraKeys.host");
@@ -2410,6 +2307,8 @@ struct GrpcKeyBuilder {
   }
 };
 
+}  // namespace
+
 const JsonLoaderInterface* RlsLbConfig::RouteLookupConfig::JsonLoader(
     const JsonArgs&) {
   static const auto* loader =
@@ -2523,6 +2422,7 @@ void RlsLbConfig::JsonPostLoad(const Json& json, const JsonArgs&,
   // Parse routeLookupChannelServiceConfig.
   auto it = json.object().find("routeLookupChannelServiceConfig");
   if (it != json.object().end()) {
+    rls_channel_service_config_ = JsonDump(it->second);
     ValidationErrors::ScopedField field(errors,
                                         ".routeLookupChannelServiceConfig");
     // Don't need to save the result here, just need the errors (if any).
@@ -2581,9 +2481,15 @@ void RlsLbConfig::JsonPostLoad(const Json& json, const JsonArgs&,
   }
 }
 
+//
+// RlsLbFactory
+//
+
+namespace {
+
 class RlsLbFactory final : public LoadBalancingPolicyFactory {
  public:
-  absl::string_view name() const override { return kRls; }
+  absl::string_view name() const override { return RlsLbConfig::Name(); }
 
   OrphanablePtr<LoadBalancingPolicy> CreateLoadBalancingPolicy(
       LoadBalancingPolicy::Args args) const override {
@@ -2593,7 +2499,7 @@ class RlsLbFactory final : public LoadBalancingPolicyFactory {
   absl::StatusOr<RefCountedPtr<LoadBalancingPolicy::Config>>
   ParseLoadBalancingConfig(const Json& json) const override {
     return LoadFromJson<RefCountedPtr<RlsLbConfig>>(
-        json, JsonArgs(), "errors validing RLS LB policy config");
+        json, JsonArgs(), "errors validating RLS LB policy config");
   }
 };