grpc 1.72.0 → 1.73.0

data/src/core/channelz/channelz_registry.h

@@ -25,8 +25,10 @@
 #include <map>
 #include <string>
 
-#include "absl/base/thread_annotations.h"
+#include "absl/container/btree_map.h"
+#include "absl/functional/function_ref.h"
 #include "src/core/channelz/channelz.h"
+#include "src/core/util/json/json_writer.h"
 #include "src/core/util/ref_counted_ptr.h"
 #include "src/core/util/sync.h"
 
@@ -40,36 +42,134 @@ class ChannelzRegistry final {
   static void Register(BaseNode* node) {
     return Default()->InternalRegister(node);
   }
-  static void Unregister(intptr_t uuid) { Default()->InternalUnregister(uuid); }
-  static RefCountedPtr<BaseNode> Get(intptr_t uuid) {
+  static void Unregister(BaseNode* node) {
+    Default()->InternalUnregister(node);
+  }
+  static WeakRefCountedPtr<BaseNode> Get(intptr_t uuid) {
     return Default()->InternalGet(uuid);
   }
+  static intptr_t NumberNode(BaseNode* node) {
+    return Default()->InternalNumberNode(node);
+  }
+
+  static WeakRefCountedPtr<SubchannelNode> GetSubchannel(intptr_t uuid) {
+    return Default()
+        ->InternalGetTyped<SubchannelNode, BaseNode::EntityType::kSubchannel>(
+            uuid);
+  }
+
+  static WeakRefCountedPtr<ChannelNode> GetChannel(intptr_t uuid) {
+    auto node = Default()->InternalGet(uuid);
+    if (node == nullptr) return nullptr;
+    if (node->type() == BaseNode::EntityType::kTopLevelChannel) {
+      return node->WeakRefAsSubclass<ChannelNode>();
+    }
+    if (node->type() == BaseNode::EntityType::kInternalChannel) {
+      return node->WeakRefAsSubclass<ChannelNode>();
+    }
+    return nullptr;
+  }
+
+  static WeakRefCountedPtr<ServerNode> GetServer(intptr_t uuid) {
+    return Default()
+        ->InternalGetTyped<ServerNode, BaseNode::EntityType::kServer>(uuid);
+  }
+
+  static WeakRefCountedPtr<SocketNode> GetSocket(intptr_t uuid) {
+    return Default()
+        ->InternalGetTyped<SocketNode, BaseNode::EntityType::kSocket>(uuid);
+  }
 
   // Returns the allocated JSON string that represents the proto
   // GetTopChannelsResponse as per channelz.proto.
-  static std::string GetTopChannels(intptr_t start_channel_id) {
-    return Default()->InternalGetTopChannels(start_channel_id);
+  static auto GetTopChannels(intptr_t start_channel_id) {
+    return Default()
+        ->InternalGetObjects<ChannelNode,
+                             BaseNode::EntityType::kTopLevelChannel>(
+            start_channel_id);
   }
 
+  static std::string GetTopChannelsJson(intptr_t start_channel_id);
+  static std::string GetServersJson(intptr_t start_server_id);
+
   // Returns the allocated JSON string that represents the proto
   // GetServersResponse as per channelz.proto.
-  static std::string GetServers(intptr_t start_server_id) {
-    return Default()->InternalGetServers(start_server_id);
+  static auto GetServers(intptr_t start_server_id) {
+    return Default()
+        ->InternalGetObjects<ServerNode, BaseNode::EntityType::kServer>(
+            start_server_id);
+  }
+
+  static std::tuple<std::vector<WeakRefCountedPtr<BaseNode>>, bool>
+  GetChildrenOfType(intptr_t start_node, const BaseNode* parent,
+                    BaseNode::EntityType type, size_t max_results) {
+    return Default()->InternalGetChildrenOfType(start_node, parent, type,
+                                                max_results);
   }
 
   // Test only helper function to dump the JSON representation to std out.
   // This can aid in debugging channelz code.
   static void LogAllEntities() { Default()->InternalLogAllEntities(); }
 
-  // Test only helper function to reset to initial state.
-  static void TestOnlyReset() {
-    auto* p = Default();
-    MutexLock lock(&p->mu_);
-    p->node_map_.clear();
-    p->uuid_generator_ = 0;
+  static std::vector<WeakRefCountedPtr<BaseNode>> GetAllEntities() {
+    return Default()->InternalGetAllEntities();
   }
 
+  // Test only helper function to reset to initial state.
+  static void TestOnlyReset();
+
  private:
+  ChannelzRegistry() { LoadConfig(); }
+
+  void LoadConfig();
+
+  // Takes a callable F: (WeakRefCountedPtr<BaseNode>) -> bool, and returns
+  // a (BaseNode*) -> bool that filters unreffed objects and returns true.
+  // The ref must be unreffed outside the NodeMapInterface iteration.
+  template <typename F>
+  static auto CollectReferences(F fn) {
+    return [fn = std::move(fn)](BaseNode* n) {
+      auto node = n->RefIfNonZero();
+      if (node == nullptr) return true;
+      return fn(std::move(node));
+    };
+  }
+
+  struct NodeList {
+    BaseNode* head = nullptr;
+    BaseNode* tail = nullptr;
+    size_t count = 0;
+    bool Holds(BaseNode* node) const;
+    void AddToHead(BaseNode* node);
+    void Remove(BaseNode* node);
+  };
+  // Nodes traverse through up to four lists, depending on
+  // whether they have a uuid (this is becoming numbered),
+  // and whether they have been orphaned or not.
+  // The lists help us find un-numbered nodes when needed for
+  // queries, and the oldest orphaned node when needed for
+  // garbage collection.
+  // Nodes are organized into shards based on their pointer
+  // address. A shard tracks the four lists of nodes
+  // independently - we strive to have no cross-talk between
+  // shards as these are very global objects.
+  struct alignas(GPR_CACHELINE_SIZE) NodeShard {
+    Mutex mu;
+    // Nursery nodes have no uuid and are not orphaned.
+    NodeList nursery ABSL_GUARDED_BY(mu);
+    // Numbered nodes have been assigned a uuid, and are not orphaned.
+    NodeList numbered ABSL_GUARDED_BY(mu);
+    // Orphaned nodes have no uuid, but have been orphaned.
+    NodeList orphaned ABSL_GUARDED_BY(mu);
+    // Finally, orphaned numbered nodes are orphaned, and have been assigned a
+    // uuid.
+    NodeList orphaned_numbered ABSL_GUARDED_BY(mu);
+    uint64_t next_orphan_index ABSL_GUARDED_BY(mu) = 1;
+    size_t TotalOrphaned() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu) {
+      return orphaned.count + orphaned_numbered.count;
+    }
+  };
+
   // Returned the singleton instance of ChannelzRegistry;
   static ChannelzRegistry* Default();
 
@@ -78,23 +178,81 @@ class ChannelzRegistry final {
 
   // globally unregisters the object that is associated to uuid. Also does
   // sanity check that an object doesn't try to unregister the wrong type.
-  void InternalUnregister(intptr_t uuid);
+  void InternalUnregister(BaseNode* node);
+
+  intptr_t InternalNumberNode(BaseNode* node);
 
   // if object with uuid has previously been registered as the correct type,
   // returns the void* associated with that uuid. Else returns nullptr.
-  RefCountedPtr<BaseNode> InternalGet(intptr_t uuid);
+  WeakRefCountedPtr<BaseNode> InternalGet(intptr_t uuid);
 
-  std::string InternalGetTopChannels(intptr_t start_channel_id);
-  std::string InternalGetServers(intptr_t start_server_id);
+  // Generic query over nodes.
+  // This function takes care of all the gnarly locking, and allows high level
+  // code to request a start node and maximum number of results (for pagination
+  // purposes).
+  // `discriminator` allows callers to choose which nodes will be returned - if
+  // it returns true, the node is included in the result.
+  // `discriminator` *MUST NOT* ref the node, nor call into ChannelzRegistry via
+  // any code path (locks are held during the call).
+  std::tuple<std::vector<WeakRefCountedPtr<BaseNode>>, bool> QueryNodes(
+      intptr_t start_node,
+      absl::FunctionRef<bool(const BaseNode*)> discriminator,
+      size_t max_results);
+
+  std::tuple<std::vector<WeakRefCountedPtr<BaseNode>>, bool>
+  InternalGetChildrenOfType(intptr_t start_node, const BaseNode* parent,
+                            BaseNode::EntityType type, size_t max_results) {
+    return QueryNodes(
+        start_node,
+        [type, parent](const BaseNode* n) {
+          return n->type() == type && n->HasParent(parent);
+        },
+        max_results);
+  }
+
+  template <typename T, BaseNode::EntityType entity_type>
+  WeakRefCountedPtr<T> InternalGetTyped(intptr_t uuid) {
+    WeakRefCountedPtr<BaseNode> node = InternalGet(uuid);
+    if (node == nullptr || node->type() != entity_type) {
+      return nullptr;
+    }
+    return node->WeakRefAsSubclass<T>();
+  }
+
+  template <typename T, BaseNode::EntityType entity_type>
+  std::tuple<std::vector<WeakRefCountedPtr<T>>, bool> InternalGetObjects(
+      intptr_t start_id) {
+    const int kPaginationLimit = 100;
+    std::vector<WeakRefCountedPtr<T>> top_level_channels;
+    const auto [nodes, end] = QueryNodes(
+        start_id,
+        [](const BaseNode* node) { return node->type() == entity_type; },
+        kPaginationLimit);
+    for (const auto& p : nodes) {
+      top_level_channels.emplace_back(p->template WeakRefAsSubclass<T>());
+    }
+    return std::tuple(std::move(top_level_channels), end);
+  }
 
   void InternalLogAllEntities();
+  std::vector<WeakRefCountedPtr<BaseNode>> InternalGetAllEntities();
 
-  // protects members
-  Mutex mu_;
-  std::map<intptr_t, BaseNode*> node_map_ ABSL_GUARDED_BY(mu_);
-  intptr_t uuid_generator_ ABSL_GUARDED_BY(mu_) = 0;
+  static constexpr size_t kNodeShards = 63;
+  size_t NodeShardIndex(BaseNode* node) {
+    return absl::HashOf(node) % kNodeShards;
+  }
+
+  int64_t uuid_generator_{1};
+  std::vector<NodeShard> node_shards_{kNodeShards};
+  Mutex index_mu_;
+  absl::btree_map<intptr_t, BaseNode*> index_ ABSL_GUARDED_BY(index_mu_);
+  size_t max_orphaned_per_shard_;
 };
 
+// `additionalInfo` section is not yet in the protobuf format, so we
+// provide a utility to strip it for compatibility.
+std::string StripAdditionalInfoFromJson(absl::string_view json);
+
 }  // namespace channelz
 }  // namespace grpc_core
 

data/src/core/channelz/ztrace_collector.h

@@ -0,0 +1,315 @@
+// Copyright 2025 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H
+#define GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H
+
+#include <grpc/support/time.h>
+
+#include <memory>
+#include <tuple>
+#include <vector>
+
+#include "absl/container/flat_hash_set.h"
+#include "src/core/channelz/channelz.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/util/json/json_writer.h"
+#include "src/core/util/single_set_ptr.h"
+#include "src/core/util/string.h"
+#include "src/core/util/sync.h"
+#include "src/core/util/time.h"
+
+#ifdef GRPC_NO_ZTRACE
+namespace grpc_core::channelz {
+namespace ztrace_collector_detail {
+class ZTraceImpl final : public ZTrace {
+ public:
+  explicit ZTraceImpl() {}
+
+  void Run(Timestamp deadline, std::map<std::string, std::string> args,
+           std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+               event_engine,
+           absl::AnyInvocable<void(Json)> callback) override {
+    event_engine->Run([callback = std::move(callback)]() mutable {
+      callback(Json::FromBool(false));
+    });
+  }
+};
+
+class StubImpl {
+ public:
+  template <typename T>
+  void Append(const T&) {}
+
+  std::unique_ptr<ZTrace> MakeZTrace() {
+    return std::make_unique<ZTraceImpl>();
+  }
+};
+}  // namespace ztrace_collector_detail
+
+template <typename...>
+class ZTraceCollector : public ztrace_collector_detail::StubImpl {};
+}  // namespace grpc_core::channelz
+#else
+namespace grpc_core::channelz {
+namespace ztrace_collector_detail {
+
+template <typename T>
+using Collection = std::deque<std::pair<gpr_cycle_counter, T> >;
+
+template <typename T>
+void AppendResults(const Collection<T>& data, Json::Array& results) {
+  for (const auto& value : data) {
+    Json::Object object;
+    object["timestamp"] =
+        Json::FromString(gpr_format_timespec(gpr_convert_clock_type(
+            gpr_cycle_counter_to_time(value.first), GPR_CLOCK_REALTIME)));
+    value.second.RenderJson(object);
+    results.emplace_back(Json::FromObject(std::move(object)));
+  }
+}
+
+template <typename Needle, typename... Haystack>
+constexpr bool kIsElement = false;
+
+template <typename Needle, typename... Haystack>
+constexpr bool kIsElement<Needle, Needle, Haystack...> = true;
+
+template <typename Needle, typename H, typename... Haystack>
+constexpr bool kIsElement<Needle, H, Haystack...> =
+    kIsElement<Needle, Haystack...>;
+
+}  // namespace ztrace_collector_detail
+
+inline std::optional<int64_t> IntFromArgs(
+    const std::map<std::string, std::string>& args, const std::string& name) {
+  auto it = args.find(name);
+  if (it == args.end()) return std::nullopt;
+  int64_t out;
+  if (!absl::SimpleAtoi(it->second, &out)) return std::nullopt;
+  return out;
+}
+
+// Generic collector infrastructure for ztrace queries.
+// Abstracts away most of the ztrace requirements in an efficient manner,
+// allowing system authors to concentrate on emitting useful data.
+// If no trace is performed, overhead is one pointer and one relaxed atomic read
+// per trace event.
+//
+// Two kinds of objects are required:
+// 1. A `Config`
+//    - This type should be constructible with a std::map<std::string,
+//    std::string>
+//      and provides overall query configuration - the map can be used to pull
+//      predicates from the calling system.
+//    - Needs a `bool Finishes(T)` method for each Data type (see 2).
+//      This allows the config to terminate a query in the event of reaching
+//      some configured predicate.
+// 2. N `Data` types
+//    - One for each kind of data captured in the trace
+//    - Allows avoiding e.g. variant<> data types; these are inefficient
+//      in this context because they force every recorded entry to use the
+//      same number of bytes whilst pending.
+template <typename Config, typename... Data>
+class ZTraceCollector {
+ public:
+  template <typename X>
+  void Append(X producer_or_value) {
+    GRPC_TRACE_LOG(ztrace, INFO) << "ZTRACE[" << this << "]: " << [&]() {
+      Json::Object obj;
+      if constexpr (ztrace_collector_detail::kIsElement<X, Data...>) {
+        producer_or_value.RenderJson(obj);
+      } else {
+        producer_or_value().RenderJson(obj);
+      }
+      return JsonDump(Json::FromObject(std::move(obj)));
+    }();
+    if (!impl_.is_set()) return;
+    if constexpr (ztrace_collector_detail::kIsElement<X, Data...>) {
+      AppendValue(std::move(producer_or_value));
+    } else {
+      AppendValue(producer_or_value());
+    }
+  }
+
+  std::unique_ptr<ZTrace> MakeZTrace() {
+    return std::make_unique<ZTraceImpl>(impl_.GetOrCreate());
+  }
+
+ private:
+  template <typename T>
+  using Collection = ztrace_collector_detail::Collection<T>;
+
+  struct Instance : public RefCounted<Instance> {
+    Instance(std::map<std::string, std::string> args,
+             std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+                 event_engine,
+             absl::AnyInvocable<void(Json)> done)
+        : memory_cap_(IntFromArgs(args, "memory_cap").value_or(1024 * 1024)),
+          config(args),
+          event_engine(std::move(event_engine)),
+          done(std::move(done)) {}
+    using Collections = std::tuple<Collection<Data>...>;
+    struct RemoveMostRecentState {
+      void (*enact)(Instance*) = nullptr;
+      gpr_cycle_counter most_recent =
+          std::numeric_limits<gpr_cycle_counter>::max();
+    };
+    template <typename T>
+    void Append(std::pair<gpr_cycle_counter, T> value) {
+      memory_used_ += value.second.MemoryUsage();
+      while (memory_used_ > memory_cap_) RemoveMostRecent();
+      std::get<Collection<T> >(data).push_back(std::move(value));
+    }
+    void RemoveMostRecent() {
+      RemoveMostRecentState state;
+      (UpdateRemoveMostRecentState<Data>(&state), ...);
+      CHECK(state.enact != nullptr);
+      state.enact(this);
+      ++items_removed_;
+    }
+    template <typename T>
+    void UpdateRemoveMostRecentState(RemoveMostRecentState* state) {
+      auto& collection = std::get<Collection<T> >(data);
+      if (collection.empty()) return;
+      if (state->enact == nullptr ||
+          collection.front().first < state->most_recent) {
+        state->enact = +[](Instance* instance) {
+          auto& collection = std::get<Collection<T> >(instance->data);
+          const size_t ent_usage = collection.front().second.MemoryUsage();
+          CHECK_GE(instance->memory_used_, ent_usage);
+          instance->memory_used_ -= ent_usage;
+          collection.pop_front();
+        };
+        state->most_recent = collection.front().first;
+      }
+    }
+    void Finish(absl::Status status) {
+      event_engine->Run([data = std::move(data), done = std::move(done),
+                         status = std::move(status), memory_used = memory_used_,
+                         items_removed = items_removed_]() mutable {
+        Json::Array entries;
+        (ztrace_collector_detail::AppendResults(
+             std::get<Collection<Data> >(data), entries),
+         ...);
+        Json::Object result;
+        result["entries"] = Json::FromArray(entries);
+        result["status"] = Json::FromString(status.ToString());
+        result["memory_used"] =
+            Json::FromNumber(static_cast<uint64_t>(memory_used));
+        result["items_removed"] = Json::FromNumber(items_removed);
+        done(Json::FromObject(std::move(result)));
+      });
+    }
+    size_t memory_used_ = 0;
+    size_t memory_cap_ = 0;
+    uint64_t items_removed_ = 0;
+    Config config;
+    const Timestamp start_time = Timestamp::Now();
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine;
+    grpc_event_engine::experimental::EventEngine::TaskHandle task_handle{
+        grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid};
+    Collections data;
+    absl::AnyInvocable<void(Json)> done;
+  };
+  struct Impl : public RefCounted<Impl> {
+    Mutex mu;
+    absl::flat_hash_set<RefCountedPtr<Instance> > instances ABSL_GUARDED_BY(mu);
+  };
+  class ZTraceImpl final : public ZTrace {
+   public:
+    explicit ZTraceImpl(RefCountedPtr<Impl> impl) : impl_(std::move(impl)) {}
+
+    void Run(Timestamp deadline, std::map<std::string, std::string> args,
+             std::shared_ptr<grpc_event_engine::experimental::EventEngine>
+                 event_engine,
+             absl::AnyInvocable<void(Json)> callback) override {
+      auto instance = MakeRefCounted<Instance>(std::move(args), event_engine,
+                                               std::move(callback));
+      auto impl = std::move(impl_);
+      RefCountedPtr<Instance> oldest_instance;
+      MutexLock lock(&impl->mu);
+      if (impl->instances.size() > 20) {
+        // Eject oldest running trace
+        Timestamp oldest_time = Timestamp::InfFuture();
+        for (auto& instance : impl->instances) {
+          if (instance->start_time < oldest_time) {
+            oldest_time = instance->start_time;
+            oldest_instance = instance;
+          }
+        }
+        CHECK(oldest_instance != nullptr);
+        impl->instances.erase(oldest_instance);
+        oldest_instance->Finish(
+            absl::ResourceExhaustedError("Too many concurrent ztrace queries"));
+      }
+      instance->task_handle = event_engine->RunAfter(
+          deadline - Timestamp::Now(), [instance, impl]() {
+            bool finish;
+            {
+              MutexLock lock(&impl->mu);
+              finish = impl->instances.erase(instance);
+            }
+            if (finish) instance->Finish(absl::DeadlineExceededError(""));
+          });
+      impl->instances.insert(instance);
+    }
+
+   private:
+    RefCountedPtr<Impl> impl_;
+  };
+
+  template <typename T>
+  void AppendValue(T&& data) {
+    auto value = std::pair(gpr_get_cycle_counter(), std::forward<T>(data));
+    auto* impl = impl_.Get();
+    {
+      MutexLock lock(&impl->mu);
+      switch (impl->instances.size()) {
+        case 0:
+          return;
+        case 1: {
+          auto& instances = impl->instances;
+          auto& instance = *instances.begin();
+          const bool finishes = instance->config.Finishes(value.second);
+          instance->Append(std::move(value));
+          if (finishes) {
+            instance->Finish(absl::OkStatus());
+            instances.clear();
+          }
+        } break;
+        default: {
+          std::vector<RefCountedPtr<Instance> > finished;
+          for (auto& instance : impl->instances) {
+            const bool finishes = instance->config.Finishes(value.second);
+            instance->Append(value);
+            if (finishes) {
+              finished.push_back(instance);
+            }
+          }
+          for (const auto& instance : finished) {
+            instance->Finish(absl::OkStatus());
+            impl->instances.erase(instance);
+          }
+        }
+      }
+    }
+  }
+
+  SingleSetRefCountedPtr<Impl> impl_;
+};
+}  // namespace grpc_core::channelz
+#endif  // GRPC_NO_ZTRACE
+
+#endif  // GRPC_SRC_CORE_CHANNELZ_ZTRACE_COLLECTOR_H

data/src/core/client_channel/client_channel.cc

@@ -228,21 +228,19 @@ class ClientChannel::SubchannelWrapper::WatcherWrapper
     subchannel_wrapper_.reset(DEBUG_LOCATION, "WatcherWrapper");
   }
 
-  void OnConnectivityStateChange(
-      RefCountedPtr<ConnectivityStateWatcherInterface> self,
-      grpc_connectivity_state state, const absl::Status& status) override {
+  void OnConnectivityStateChange(grpc_connectivity_state state,
+                                 const absl::Status& status) override {
     GRPC_TRACE_LOG(client_channel, INFO)
         << "client_channel=" << subchannel_wrapper_->client_channel_.get()
         << ": connectivity change for subchannel wrapper "
         << subchannel_wrapper_.get() << " subchannel "
         << subchannel_wrapper_->subchannel_.get()
         << "; hopping into work_serializer";
-    self.release();  // Held by callback.
+    auto self = RefAsSubclass<WatcherWrapper>();
     subchannel_wrapper_->client_channel_->work_serializer_->Run(
-        [this, state, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
-            *subchannel_wrapper_->client_channel_->work_serializer_) {
-          ApplyUpdateInControlPlaneWorkSerializer(state, status);
-          Unref();
+        [self, state, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
+            *self->subchannel_wrapper_->client_channel_->work_serializer_) {
+          self->ApplyUpdateInControlPlaneWorkSerializer(state, status);
         });
   }
 
@@ -324,8 +322,7 @@ ClientChannel::SubchannelWrapper::SubchannelWrapper(
       auto it =
           client_channel_->subchannel_refcount_map_.find(subchannel_.get());
       if (it == client_channel_->subchannel_refcount_map_.end()) {
-        client_channel_->channelz_node_->AddChildSubchannel(
-            subchannel_node->uuid());
+        subchannel_node->AddParent(client_channel_->channelz_node_);
         it = client_channel_->subchannel_refcount_map_
                  .emplace(subchannel_.get(), 0)
                  .first;
@@ -360,8 +357,8 @@ void ClientChannel::SubchannelWrapper::Orphaned() {
             CHECK(it != self->client_channel_->subchannel_refcount_map_.end());
             --it->second;
             if (it->second == 0) {
-              self->client_channel_->channelz_node_->RemoveChildSubchannel(
-                  subchannel_node->uuid());
+              subchannel_node->RemoveParent(
+                  self->client_channel_->channelz_node_);
               self->client_channel_->subchannel_refcount_map_.erase(it);
             }
           }
@@ -530,7 +527,11 @@ RefCountedPtr<SubchannelPoolInterface> GetSubchannelPool(
   if (args.GetBool(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL).value_or(false)) {
     return MakeRefCounted<LocalSubchannelPool>();
   }
-  return GlobalSubchannelPool::instance();
+  if (IsShardGlobalConnectionPoolEnabled()) {
+    return GlobalSubchannelPool::instance();
+  } else {
+    return LegacyGlobalSubchannelPool::instance();
+  }
 }
 
 }  // namespace
@@ -701,6 +702,7 @@ class ExternalStateWatcher : public RefCounted<ExternalStateWatcher> {
                        grpc_connectivity_state last_observed_state,
                        Timestamp deadline)
       : channel_(std::move(channel)), cq_(cq), tag_(tag) {
+    grpc_cq_begin_op(cq, tag);
     MutexLock lock(&mu_);
     // Start watch.  This inherits the ref from creation.
     auto watcher =
@@ -780,17 +782,14 @@ void ClientChannel::WatchConnectivityState(grpc_connectivity_state state,
 }
 
 void ClientChannel::AddConnectivityWatcher(
-    grpc_connectivity_state,
-    OrphanablePtr<AsyncConnectivityStateWatcherInterface>) {
-  Crash("not implemented");
-  // TODO(ctiller): to make this work, need to change WorkSerializer to use
-  // absl::AnyInvocable<> instead of std::function<>
-  //  work_serializer_->Run(
-  //      [self = RefAsSubclass<ClientChannel>(), initial_state,
-  //       watcher = std::move(watcher)]()
-  //            ABSL_EXCLUSIVE_LOCKS_REQUIRED(*work_serializer_) {
-  //        self->state_tracker_.AddWatcher(initial_state, std::move(watcher));
-  //      });
+    grpc_connectivity_state initial_state,
+    OrphanablePtr<AsyncConnectivityStateWatcherInterface> watcher) {
+  auto self = RefAsSubclass<ClientChannel>();
+  work_serializer_->Run(
+      [self, initial_state, watcher = std::move(watcher)]()
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(*self->work_serializer_) mutable {
+            self->state_tracker_.AddWatcher(initial_state, std::move(watcher));
+          });
 }
 
 void ClientChannel::RemoveConnectivityWatcher(
@@ -1320,11 +1319,13 @@ void ClientChannel::UpdateStateLocked(grpc_connectivity_state state,
   state_tracker_.SetState(state, status, reason);
   if (channelz_node_ != nullptr) {
     channelz_node_->SetConnectivityState(state);
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_static_string(
-            channelz::ChannelNode::GetChannelConnectivityStateChangeString(
-                state)));
+    std::string trace =
+        channelz::ChannelNode::GetChannelConnectivityStateChangeString(state);
+    if (!status.ok() || state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      absl::StrAppend(&trace, " status:", status.ToString());
+    }
+    channelz_node_->AddTraceEvent(channelz::ChannelTrace::Severity::Info,
+                                  grpc_slice_from_cpp_string(std::move(trace)));
   }
 }