grpc 1.71.0 → 1.75.0

data/src/core/channelz/ztrace_collector.h

@@ -0,0 +1,316 @@
+// Copyright 2025 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H
+#define GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H
+
+#include <grpc/support/time.h>
+
+#include <memory>
+#include <tuple>
+#include <vector>
+
+#include "absl/container/flat_hash_set.h"
+#include "src/core/channelz/channelz.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/util/json/json_writer.h"
+#include "src/core/util/memory_usage.h"
+#include "src/core/util/single_set_ptr.h"
+#include "src/core/util/string.h"
+#include "src/core/util/sync.h"
+#include "src/core/util/time.h"
+
+#ifdef GRPC_NO_ZTRACE
+namespace grpc_core::channelz {
+namespace ztrace_collector_detail {
+class ZTraceImpl final : public ZTrace {
+ public:
+  explicit ZTraceImpl() {}
+
+  void Run(Timestamp deadline, std::map<std::string, std::string> args,
+           std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+               event_engine,
+           absl::AnyInvocable<void(Json)> callback) override {
+    event_engine->Run([callback = std::move(callback)]() mutable {
+      callback(Json::FromBool(false));
+    });
+  }
+};
+
+class StubImpl {
+ public:
+  template <typename T>
+  void Append(const T&) {}
+
+  std::unique_ptr<ZTrace> MakeZTrace() {
+    return std::make_unique<ZTraceImpl>();
+  }
+};
+}  // namespace ztrace_collector_detail
+
+template <typename...>
+class ZTraceCollector : public ztrace_collector_detail::StubImpl {};
+}  // namespace grpc_core::channelz
+#else
+namespace grpc_core::channelz {
+namespace ztrace_collector_detail {
+
+template <typename T>
+using Collection = std::deque<std::pair<gpr_cycle_counter, T> >;
+
+template <typename T>
+void AppendResults(const Collection<T>& data, Json::Array& results) {
+  for (const auto& value : data) {
+    Json::Object object;
+    object["timestamp"] =
+        Json::FromString(gpr_format_timespec(gpr_convert_clock_type(
+            gpr_cycle_counter_to_time(value.first), GPR_CLOCK_REALTIME)));
+    value.second.RenderJson(object);
+    results.emplace_back(Json::FromObject(std::move(object)));
+  }
+}
+
+template <typename Needle, typename... Haystack>
+constexpr bool kIsElement = false;
+
+template <typename Needle, typename... Haystack>
+constexpr bool kIsElement<Needle, Needle, Haystack...> = true;
+
+template <typename Needle, typename H, typename... Haystack>
+constexpr bool kIsElement<Needle, H, Haystack...> =
+    kIsElement<Needle, Haystack...>;
+
+}  // namespace ztrace_collector_detail
+
+inline std::optional<int64_t> IntFromArgs(
+    const std::map<std::string, std::string>& args, const std::string& name) {
+  auto it = args.find(name);
+  if (it == args.end()) return std::nullopt;
+  int64_t out;
+  if (!absl::SimpleAtoi(it->second, &out)) return std::nullopt;
+  return out;
+}
+
+// Generic collector infrastructure for ztrace queries.
+// Abstracts away most of the ztrace requirements in an efficient manner,
+// allowing system authors to concentrate on emitting useful data.
+// If no trace is performed, overhead is one pointer and one relaxed atomic read
+// per trace event.
+//
+// Two kinds of objects are required:
+// 1. A `Config`
+//    - This type should be constructible with a std::map<std::string,
+//    std::string>
+//      and provides overall query configuration - the map can be used to pull
+//      predicates from the calling system.
+//    - Needs a `bool Finishes(T)` method for each Data type (see 2).
+//      This allows the config to terminate a query in the event of reaching
+//      some configured predicate.
+// 2. N `Data` types
+//    - One for each kind of data captured in the trace
+//    - Allows avoiding e.g. variant<> data types; these are inefficient
+//      in this context because they force every recorded entry to use the
+//      same number of bytes whilst pending.
+template <typename Config, typename... Data>
+class ZTraceCollector {
+ public:
+  template <typename X>
+  void Append(X producer_or_value) {
+    GRPC_TRACE_LOG(ztrace, INFO) << "ZTRACE[" << this << "]: " << [&]() {
+      Json::Object obj;
+      if constexpr (ztrace_collector_detail::kIsElement<X, Data...>) {
+        producer_or_value.RenderJson(obj);
+      } else {
+        producer_or_value().RenderJson(obj);
+      }
+      return JsonDump(Json::FromObject(std::move(obj)));
+    }();
+    if (!impl_.is_set()) return;
+    if constexpr (ztrace_collector_detail::kIsElement<X, Data...>) {
+      AppendValue(std::move(producer_or_value));
+    } else {
+      AppendValue(producer_or_value());
+    }
+  }
+
+  std::unique_ptr<ZTrace> MakeZTrace() {
+    return std::make_unique<ZTraceImpl>(impl_.GetOrCreate());
+  }
+
+ private:
+  template <typename T>
+  using Collection = ztrace_collector_detail::Collection<T>;
+
+  struct Instance : public RefCounted<Instance> {
+    Instance(std::map<std::string, std::string> args,
+             std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+                 event_engine,
+             absl::AnyInvocable<void(Json)> done)
+        : memory_cap_(IntFromArgs(args, "memory_cap").value_or(1024 * 1024)),
+          config(args),
+          event_engine(std::move(event_engine)),
+          done(std::move(done)) {}
+    using Collections = std::tuple<Collection<Data>...>;
+    struct RemoveMostRecentState {
+      void (*enact)(Instance*) = nullptr;
+      gpr_cycle_counter most_recent =
+          std::numeric_limits<gpr_cycle_counter>::max();
+    };
+    template <typename T>
+    void Append(std::pair<gpr_cycle_counter, T> value) {
+      memory_used_ += MemoryUsageOf(value.second);
+      while (memory_used_ > memory_cap_) RemoveMostRecent();
+      std::get<Collection<T> >(data).push_back(std::move(value));
+    }
+    void RemoveMostRecent() {
+      RemoveMostRecentState state;
+      (UpdateRemoveMostRecentState<Data>(&state), ...);
+      CHECK(state.enact != nullptr);
+      state.enact(this);
+      ++items_removed_;
+    }
+    template <typename T>
+    void UpdateRemoveMostRecentState(RemoveMostRecentState* state) {
+      auto& collection = std::get<Collection<T> >(data);
+      if (collection.empty()) return;
+      if (state->enact == nullptr ||
+          collection.front().first < state->most_recent) {
+        state->enact = +[](Instance* instance) {
+          auto& collection = std::get<Collection<T> >(instance->data);
+          const size_t ent_usage = MemoryUsageOf(collection.front().second);
+          CHECK_GE(instance->memory_used_, ent_usage);
+          instance->memory_used_ -= ent_usage;
+          collection.pop_front();
+        };
+        state->most_recent = collection.front().first;
+      }
+    }
+    void Finish(absl::Status status) {
+      event_engine->Run([data = std::move(data), done = std::move(done),
+                         status = std::move(status), memory_used = memory_used_,
+                         items_removed = items_removed_]() mutable {
+        Json::Array entries;
+        (ztrace_collector_detail::AppendResults(
+             std::get<Collection<Data> >(data), entries),
+         ...);
+        Json::Object result;
+        result["entries"] = Json::FromArray(entries);
+        result["status"] = Json::FromString(status.ToString());
+        result["memory_used"] =
+            Json::FromNumber(static_cast<uint64_t>(memory_used));
+        result["items_removed"] = Json::FromNumber(items_removed);
+        done(Json::FromObject(std::move(result)));
+      });
+    }
+    size_t memory_used_ = 0;
+    size_t memory_cap_ = 0;
+    uint64_t items_removed_ = 0;
+    Config config;
+    const Timestamp start_time = Timestamp::Now();
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine;
+    grpc_event_engine::experimental::EventEngine::TaskHandle task_handle{
+        grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid};
+    Collections data;
+    absl::AnyInvocable<void(Json)> done;
+  };
+  struct Impl : public RefCounted<Impl> {
+    Mutex mu;
+    absl::flat_hash_set<RefCountedPtr<Instance> > instances ABSL_GUARDED_BY(mu);
+  };
+  class ZTraceImpl final : public ZTrace {
+   public:
+    explicit ZTraceImpl(RefCountedPtr<Impl> impl) : impl_(std::move(impl)) {}
+
+    void Run(Timestamp deadline, std::map<std::string, std::string> args,
+             std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+                 event_engine,
+             absl::AnyInvocable<void(Json)> callback) override {
+      auto instance = MakeRefCounted<Instance>(std::move(args), event_engine,
+                                               std::move(callback));
+      auto impl = std::move(impl_);
+      RefCountedPtr<Instance> oldest_instance;
+      MutexLock lock(&impl->mu);
+      if (impl->instances.size() > 20) {
+        // Eject oldest running trace
+        Timestamp oldest_time = Timestamp::InfFuture();
+        for (auto& instance : impl->instances) {
+          if (instance->start_time < oldest_time) {
+            oldest_time = instance->start_time;
+            oldest_instance = instance;
+          }
+        }
+        CHECK(oldest_instance != nullptr);
+        impl->instances.erase(oldest_instance);
+        oldest_instance->Finish(
+            absl::ResourceExhaustedError("Too many concurrent ztrace queries"));
+      }
+      instance->task_handle = event_engine->RunAfter(
+          deadline - Timestamp::Now(), [instance, impl]() {
+            bool finish;
+            {
+              MutexLock lock(&impl->mu);
+              finish = impl->instances.erase(instance);
+            }
+            if (finish) instance->Finish(absl::DeadlineExceededError(""));
+          });
+      impl->instances.insert(instance);
+    }
+
+   private:
+    RefCountedPtr<Impl> impl_;
+  };
+
+  template <typename T>
+  void AppendValue(T&& data) {
+    auto value = std::pair(gpr_get_cycle_counter(), std::forward<T>(data));
+    auto* impl = impl_.Get();
+    {
+      MutexLock lock(&impl->mu);
+      switch (impl->instances.size()) {
+        case 0:
+          return;
+        case 1: {
+          auto& instances = impl->instances;
+          auto& instance = *instances.begin();
+          const bool finishes = instance->config.Finishes(value.second);
+          instance->Append(std::move(value));
+          if (finishes) {
+            instance->Finish(absl::OkStatus());
+            instances.clear();
+          }
+        } break;
+        default: {
+          std::vector<RefCountedPtr<Instance> > finished;
+          for (auto& instance : impl->instances) {
+            const bool finishes = instance->config.Finishes(value.second);
+            instance->Append(value);
+            if (finishes) {
+              finished.push_back(instance);
+            }
+          }
+          for (const auto& instance : finished) {
+            instance->Finish(absl::OkStatus());
+            impl->instances.erase(instance);
+          }
+        }
+      }
+    }
+  }
+
+  SingleSetRefCountedPtr<Impl> impl_;
+};
+}  // namespace grpc_core::channelz
+#endif  // GRPC_NO_ZTRACE
+
+#endif  // GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H

data/src/core/client_channel/backup_poller.cc

@@ -62,10 +62,15 @@ static grpc_core::Duration g_poll_interval =
 static bool g_backup_polling_disabled;
 
 void grpc_client_channel_global_init_backup_polling() {
+#ifndef GRPC_DO_NOT_INSTANTIATE_POSIX_POLLER
   // Disable backup polling if EventEngine is used everywhere.
   g_backup_polling_disabled = grpc_core::IsEventEngineClientEnabled() &&
                               grpc_core::IsEventEngineListenerEnabled() &&
                               grpc_core::IsEventEngineDnsEnabled();
+#else
+  // EventEngine polling not supported, keep using the backup poller.
+  g_backup_polling_disabled = false;
+#endif
   if (g_backup_polling_disabled) {
     return;
   }
@@ -155,11 +160,21 @@ static void g_poller_init_locked() {
   }
 }
 
+static bool g_can_poll_in_background() {
+#ifndef GRPC_DO_NOT_INSTANTIATE_POSIX_POLLER
+  return grpc_iomgr_run_in_background();
+#else
+  // No iomgr "event_engines" (not to be confused with the new EventEngine)
+  // are able to run in backgroung.
+  return false;
+#endif
+}
+
 void grpc_client_channel_start_backup_polling(
     grpc_pollset_set* interested_parties) {
   if (g_backup_polling_disabled ||
       g_poll_interval == grpc_core::Duration::Zero() ||
-      grpc_iomgr_run_in_background()) {
+      g_can_poll_in_background()) {
     return;
   }
   gpr_mu_lock(&g_poller_mu);
@@ -179,7 +194,7 @@ void grpc_client_channel_stop_backup_polling(
     grpc_pollset_set* interested_parties) {
   if (g_backup_polling_disabled ||
       g_poll_interval == grpc_core::Duration::Zero() ||
-      grpc_iomgr_run_in_background()) {
+      g_can_poll_in_background()) {
     return;
   }
   grpc_pollset_set_del_pollset(interested_parties, g_poller->pollset);

data/src/core/client_channel/client_channel.cc

@@ -45,6 +45,10 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
+#include "src/core/call/call_spine.h"
+#include "src/core/call/client_call.h"
+#include "src/core/call/metadata_batch.h"
+#include "src/core/call/status_util.h"
 #include "src/core/client_channel/client_channel_internal.h"
 #include "src/core/client_channel/client_channel_service_config.h"
 #include "src/core/client_channel/config_selector.h"
@@ -55,10 +59,10 @@
 #include "src/core/client_channel/subchannel.h"
 #include "src/core/client_channel/subchannel_interface_internal.h"
 #include "src/core/config/core_configuration.h"
+#include "src/core/credentials/transport/transport_credentials.h"
 #include "src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/status_util.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/event_engine/channel_args_endpoint_config.h"
 #include "src/core/lib/iomgr/resolved_address.h"
@@ -70,16 +74,12 @@
 #include "src/core/lib/promise/sleep.h"
 #include "src/core/lib/promise/try_seq.h"
 #include "src/core/lib/resource_quota/arena.h"
-#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
-#include "src/core/lib/surface/client_call.h"
 #include "src/core/lib/surface/completion_queue.h"
-#include "src/core/lib/transport/call_spine.h"
 #include "src/core/lib/transport/connectivity_state.h"
-#include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/load_balancing/child_policy_handler.h"
 #include "src/core/load_balancing/lb_policy.h"
 #include "src/core/load_balancing/lb_policy_registry.h"
@@ -228,21 +228,19 @@ class ClientChannel::SubchannelWrapper::WatcherWrapper
     subchannel_wrapper_.reset(DEBUG_LOCATION, "WatcherWrapper");
   }
 
-  void OnConnectivityStateChange(
-      RefCountedPtr<ConnectivityStateWatcherInterface> self,
-      grpc_connectivity_state state, const absl::Status& status) override {
+  void OnConnectivityStateChange(grpc_connectivity_state state,
+                                 const absl::Status& status) override {
     GRPC_TRACE_LOG(client_channel, INFO)
         << "client_channel=" << subchannel_wrapper_->client_channel_.get()
         << ": connectivity change for subchannel wrapper "
         << subchannel_wrapper_.get() << " subchannel "
         << subchannel_wrapper_->subchannel_.get()
         << "; hopping into work_serializer";
-    self.release();  // Held by callback.
+    auto self = RefAsSubclass<WatcherWrapper>();
     subchannel_wrapper_->client_channel_->work_serializer_->Run(
-        [this, state, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
-            *subchannel_wrapper_->client_channel_->work_serializer_) {
-          ApplyUpdateInControlPlaneWorkSerializer(state, status);
-          Unref();
+        [self, state, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
+            *self->subchannel_wrapper_->client_channel_->work_serializer_) {
+          self->ApplyUpdateInControlPlaneWorkSerializer(state, status);
         });
   }
 
@@ -324,8 +322,7 @@ ClientChannel::SubchannelWrapper::SubchannelWrapper(
       auto it =
           client_channel_->subchannel_refcount_map_.find(subchannel_.get());
       if (it == client_channel_->subchannel_refcount_map_.end()) {
-        client_channel_->channelz_node_->AddChildSubchannel(
-            subchannel_node->uuid());
+        subchannel_node->AddParent(client_channel_->channelz_node_);
         it = client_channel_->subchannel_refcount_map_
                  .emplace(subchannel_.get(), 0)
                  .first;
@@ -360,8 +357,8 @@ void ClientChannel::SubchannelWrapper::Orphaned() {
             CHECK(it != self->client_channel_->subchannel_refcount_map_.end());
             --it->second;
             if (it->second == 0) {
-              self->client_channel_->channelz_node_->RemoveChildSubchannel(
-                  subchannel_node->uuid());
+              subchannel_node->RemoveParent(
+                  self->client_channel_->channelz_node_);
               self->client_channel_->subchannel_refcount_map_.erase(it);
             }
           }
@@ -495,27 +492,16 @@ class ClientChannel::ClientChannelControlHelper
   }
 
   GlobalStatsPluginRegistry::StatsPluginGroup& GetStatsPluginGroup() override {
-    return client_channel_->stats_plugin_group_;
+    return *client_channel_->stats_plugin_group_;
   }
 
-  void AddTraceEvent(TraceSeverity severity, absl::string_view message) override
+  void AddTraceEvent(absl::string_view message) override
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(*client_channel_->work_serializer_) {
     if (client_channel_->resolver_ == nullptr) return;  // Shutting down.
-    if (client_channel_->channelz_node_ != nullptr) {
-      client_channel_->channelz_node_->AddTraceEvent(
-          ConvertSeverityEnum(severity),
-          grpc_slice_from_copied_buffer(message.data(), message.size()));
-    }
+    GRPC_CHANNELZ_LOG(client_channel_->channelz_node_) << std::string(message);
   }
 
  private:
-  static channelz::ChannelTrace::Severity ConvertSeverityEnum(
-      TraceSeverity severity) {
-    if (severity == TRACE_INFO) return channelz::ChannelTrace::Info;
-    if (severity == TRACE_WARNING) return channelz::ChannelTrace::Warning;
-    return channelz::ChannelTrace::Error;
-  }
-
   WeakRefCountedPtr<ClientChannel> client_channel_;
 };
 
@@ -530,7 +516,11 @@ RefCountedPtr<SubchannelPoolInterface> GetSubchannelPool(
   if (args.GetBool(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL).value_or(false)) {
     return MakeRefCounted<LocalSubchannelPool>();
   }
-  return GlobalSubchannelPool::instance();
+  if (IsShardGlobalConnectionPoolEnabled()) {
+    return GlobalSubchannelPool::instance();
+  } else {
+    return LegacyGlobalSubchannelPool::instance();
+  }
 }
 
 }  // namespace
@@ -587,6 +577,7 @@ absl::StatusOr<RefCountedPtr<Channel>> ClientChannel::Create(
 }
 
 namespace {
+
 std::string GetDefaultAuthorityFromChannelArgs(const ChannelArgs& channel_args,
                                                absl::string_view target) {
   std::optional<std::string> default_authority =
@@ -598,6 +589,18 @@ std::string GetDefaultAuthorityFromChannelArgs(const ChannelArgs& channel_args,
     return std::move(*default_authority);
   }
 }
+
+std::shared_ptr<GlobalStatsPluginRegistry::StatsPluginGroup>
+GetStatsPluginGroupFromChannelArgs(const ChannelArgs& channel_args,
+                                   absl::string_view target,
+                                   absl::string_view default_authority) {
+  grpc_event_engine::experimental::ChannelArgsEndpointConfig endpoint_config(
+      channel_args);
+  experimental::StatsPluginChannelScope scope(target, default_authority,
+                                              endpoint_config);
+  return GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope);
+}
+
 }  // namespace
 
 ClientChannel::ClientChannel(
@@ -606,15 +609,17 @@ ClientChannel::ClientChannel(
     ClientChannelFactory* client_channel_factory,
     CallDestinationFactory* call_destination_factory)
     : Channel(std::move(target), channel_args),
-      channel_args_(std::move(channel_args)),
+      default_authority_(
+          GetDefaultAuthorityFromChannelArgs(channel_args, this->target())),
+      stats_plugin_group_(GetStatsPluginGroupFromChannelArgs(
+          channel_args, this->target(), default_authority_)),
+      channel_args_(channel_args.SetObject(stats_plugin_group_)),
       event_engine_(channel_args_.GetObjectRef<EventEngine>()),
       uri_to_resolve_(std::move(uri_to_resolve)),
       service_config_parser_index_(
           internal::ClientChannelServiceConfigParser::ParserIndex()),
       default_service_config_(std::move(default_service_config)),
       client_channel_factory_(client_channel_factory),
-      default_authority_(
-          GetDefaultAuthorityFromChannelArgs(channel_args_, this->target())),
       channelz_node_(channel_args_.GetObject<channelz::ChannelNode>()),
       idle_timeout_(GetClientIdleTimeout(channel_args_)),
       resolver_data_for_calls_(ResolverDataForCalls{}),
@@ -634,13 +639,6 @@ ClientChannel::ClientChannel(
   } else {
     keepalive_time_ = -1;  // unset
   }
-  // Get stats plugins for channel.
-  grpc_event_engine::experimental::ChannelArgsEndpointConfig endpoint_config(
-      channel_args_);
-  experimental::StatsPluginChannelScope scope(
-      this->target(), default_authority_, endpoint_config);
-  stats_plugin_group_ =
-      GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope);
 }
 
 ClientChannel::~ClientChannel() {
@@ -693,6 +691,7 @@ class ExternalStateWatcher : public RefCounted<ExternalStateWatcher> {
                        grpc_connectivity_state last_observed_state,
                        Timestamp deadline)
       : channel_(std::move(channel)), cq_(cq), tag_(tag) {
+    grpc_cq_begin_op(cq, tag);
     MutexLock lock(&mu_);
     // Start watch.  This inherits the ref from creation.
     auto watcher =
@@ -772,17 +771,14 @@ void ClientChannel::WatchConnectivityState(grpc_connectivity_state state,
 }
 
 void ClientChannel::AddConnectivityWatcher(
-    grpc_connectivity_state,
-    OrphanablePtr<AsyncConnectivityStateWatcherInterface>) {
-  Crash("not implemented");
-  // TODO(ctiller): to make this work, need to change WorkSerializer to use
-  // absl::AnyInvocable<> instead of std::function<>
-  //  work_serializer_->Run(
-  //      [self = RefAsSubclass<ClientChannel>(), initial_state,
-  //       watcher = std::move(watcher)]()
-  //            ABSL_EXCLUSIVE_LOCKS_REQUIRED(*work_serializer_) {
-  //        self->state_tracker_.AddWatcher(initial_state, std::move(watcher));
-  //      });
+    grpc_connectivity_state initial_state,
+    OrphanablePtr<AsyncConnectivityStateWatcherInterface> watcher) {
+  auto self = RefAsSubclass<ClientChannel>();
+  work_serializer_->Run(
+      [self, initial_state, watcher = std::move(watcher)]()
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(*self->work_serializer_) mutable {
+            self->state_tracker_.AddWatcher(initial_state, std::move(watcher));
+          });
 }
 
 void ClientChannel::RemoveConnectivityWatcher(
@@ -1151,12 +1147,8 @@ void ClientChannel::OnResolverResultChangedLocked(Resolver::Result result) {
   }
   // Add channel trace event.
   if (!trace_strings.empty()) {
-    std::string message =
-        absl::StrCat("Resolution event: ", absl::StrJoin(trace_strings, ", "));
-    if (channelz_node_ != nullptr) {
-      channelz_node_->AddTraceEvent(channelz::ChannelTrace::Severity::Info,
-                                    grpc_slice_from_cpp_string(message));
-    }
+    GRPC_CHANNELZ_LOG(channelz_node_)
+        << "Resolution event: " << absl::StrJoin(trace_strings, ", ");
   }
 }
 
@@ -1272,8 +1264,7 @@ void ClientChannel::UpdateServiceConfigInDataPlaneLocked(
   ChannelArgs new_args = args.SetObject(this).SetObject(saved_service_config_);
   // Construct filter stack.
   auto new_blackboard = MakeRefCounted<Blackboard>();
-  InterceptionChainBuilder builder(new_args, blackboard_.get(),
-                                   new_blackboard.get());
+  InterceptionChainBuilder builder(new_args, new_blackboard.get());
   if (idle_timeout_ != Duration::Zero()) {
     builder.AddOnServerTrailingMetadata([this](ServerMetadata&) {
       if (idle_state_.DecreaseCallCount()) StartIdleTimer();
@@ -1282,16 +1273,18 @@ void ClientChannel::UpdateServiceConfigInDataPlaneLocked(
   CoreConfiguration::Get().channel_init().AddToInterceptionChainBuilder(
       GRPC_CLIENT_CHANNEL, builder);
   // Add filters returned by the config selector (e.g., xDS HTTP filters).
-  config_selector->AddFilters(builder);
+  config_selector->AddFilters(builder, blackboard_.get(), new_blackboard.get());
   const bool enable_retries =
       !channel_args_.WantMinimalStack() &&
       channel_args_.GetBool(GRPC_ARG_ENABLE_RETRIES).value_or(true);
   if (enable_retries) {
+    RetryInterceptor::UpdateBlackboard(*saved_service_config_,
+                                       blackboard_.get(), new_blackboard.get());
     builder.Add<RetryInterceptor>();
   }
+  blackboard_ = std::move(new_blackboard);
   // Create call destination.
   auto top_of_stack_call_destination = builder.Build(call_destination_);
-  blackboard_ = std::move(new_blackboard);
   // Send result to data plane.
   if (!top_of_stack_call_destination.ok()) {
     resolver_data_for_calls_.Set(MaybeRewriteIllegalStatusCode(
@@ -1312,11 +1305,16 @@ void ClientChannel::UpdateStateLocked(grpc_connectivity_state state,
   state_tracker_.SetState(state, status, reason);
   if (channelz_node_ != nullptr) {
     channelz_node_->SetConnectivityState(state);
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_static_string(
-            channelz::ChannelNode::GetChannelConnectivityStateChangeString(
-                state)));
+    if (!status.ok() || state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      GRPC_CHANNELZ_LOG(channelz_node_)
+          << channelz::ChannelNode::GetChannelConnectivityStateChangeString(
+                 state)
+          << " status: " << status;
+    } else {
+      GRPC_CHANNELZ_LOG(channelz_node_)
+          << channelz::ChannelNode::GetChannelConnectivityStateChangeString(
+                 state);
+    }
   }
 }