grpc 1.72.0 → 1.73.0

data/src/core/channelz/channelz_registry.cc

@@ -24,154 +24,356 @@
 #include <grpc/support/string_util.h>
 
 #include <algorithm>
+#include <atomic>
 #include <cstdint>
 #include <cstring>
+#include <memory>
+#include <optional>
+#include <string>
 #include <utility>
 #include <vector>
 
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "src/core/channelz/channelz.h"
+#include "src/core/config/config_vars.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/util/json/json.h"
+#include "src/core/util/json/json_reader.h"
 #include "src/core/util/json/json_writer.h"
+#include "src/core/util/shared_bit_gen.h"
 #include "src/core/util/sync.h"
 
 namespace grpc_core {
 namespace channelz {
+
 namespace {
+template <typename T>
+std::string RenderArray(std::tuple<T, bool> values_and_end,
+                        const std::string& key) {
+  auto& [values, end] = values_and_end;
+  Json::Object object;
+  if (!values.empty()) {
+    // Create list of channels.
+    Json::Array array;
+    for (size_t i = 0; i < values.size(); ++i) {
+      array.emplace_back(values[i]->RenderJson());
+    }
+    object[key] = Json::FromArray(std::move(array));
+  }
+  if (end) {
+    object["end"] = Json::FromBool(true);
+  }
+  return JsonDump(Json::FromObject(std::move(object)));
+}
+
+Json RemoveAdditionalInfo(const Json& json) {
+  switch (json.type()) {
+    case Json::Type::kArray: {
+      Json::Array out;
+      for (const auto& node : json.array()) {
+        out.emplace_back(RemoveAdditionalInfo(node));
+      }
+      return Json::FromArray(std::move(out));
+    } break;
+    case Json::Type::kObject: {
+      Json::Object out;
+      for (const auto& [key, value] : json.object()) {
+        if (key == "additionalInfo") continue;
+        out[key] = RemoveAdditionalInfo(value);
+      }
+      return Json::FromObject(std::move(out));
+    } break;
+    default:
+      return json;
+  }
+}
 
-const int kPaginationLimit = 100;
+// TODO(ctiller): Temporary hack to remove fields that are objectionable to the
+// protobuf parser (because we've not published them in protobuf yet).
+char* ApplyHacks(const std::string& json_str) {
+  return gpr_strdup(StripAdditionalInfoFromJson(json_str).c_str());
+}
+}  // namespace
 
-}  // anonymous namespace
+std::string StripAdditionalInfoFromJson(absl::string_view json_str) {
+  auto json = JsonParse(json_str);
+  if (!json.ok()) return gpr_strdup(std::string(json_str).c_str());
+  return JsonDump(RemoveAdditionalInfo(*json));
+}
 
 ChannelzRegistry* ChannelzRegistry::Default() {
   static ChannelzRegistry* singleton = new ChannelzRegistry();
   return singleton;
 }
 
-void ChannelzRegistry::InternalRegister(BaseNode* node) {
-  MutexLock lock(&mu_);
-  node->uuid_ = ++uuid_generator_;
-  node_map_[node->uuid_] = node;
+std::vector<WeakRefCountedPtr<BaseNode>>
+ChannelzRegistry::InternalGetAllEntities() {
+  return std::get<0>(QueryNodes(
+      0, [](const BaseNode*) { return true; },
+      std::numeric_limits<size_t>::max()));
 }
 
-void ChannelzRegistry::InternalUnregister(intptr_t uuid) {
-  CHECK_GE(uuid, 1);
-  MutexLock lock(&mu_);
-  CHECK(uuid <= uuid_generator_);
-  node_map_.erase(uuid);
+void ChannelzRegistry::InternalLogAllEntities() {
+  for (const auto& p : InternalGetAllEntities()) {
+    std::string json = p->RenderJsonString();
+    LOG(INFO) << json;
+  }
 }
 
-RefCountedPtr<BaseNode> ChannelzRegistry::InternalGet(intptr_t uuid) {
-  MutexLock lock(&mu_);
-  if (uuid < 1 || uuid > uuid_generator_) {
-    return nullptr;
-  }
-  auto it = node_map_.find(uuid);
-  if (it == node_map_.end()) return nullptr;
-  // Found node.  Return only if its refcount is not zero (i.e., when we
-  // know that there is no other thread about to destroy it).
-  BaseNode* node = it->second;
-  return node->RefIfNonZero();
+std::string ChannelzRegistry::GetTopChannelsJson(intptr_t start_channel_id) {
+  return RenderArray(GetTopChannels(start_channel_id), "channel");
 }
 
-std::string ChannelzRegistry::InternalGetTopChannels(
-    intptr_t start_channel_id) {
-  std::vector<RefCountedPtr<BaseNode>> top_level_channels;
-  RefCountedPtr<BaseNode> node_after_pagination_limit;
-  {
-    MutexLock lock(&mu_);
-    for (auto it = node_map_.lower_bound(start_channel_id);
-         it != node_map_.end(); ++it) {
-      BaseNode* node = it->second;
-      RefCountedPtr<BaseNode> node_ref;
-      if (node->type() == BaseNode::EntityType::kTopLevelChannel &&
-          (node_ref = node->RefIfNonZero()) != nullptr) {
-        // Check if we are over pagination limit to determine if we need to set
-        // the "end" element. If we don't go through this block, we know that
-        // when the loop terminates, we have <= to kPaginationLimit.
-        // Note that because we have already increased this node's
-        // refcount, we need to decrease it, but we can't unref while
-        // holding the lock, because this may lead to a deadlock.
-        if (top_level_channels.size() == kPaginationLimit) {
-          node_after_pagination_limit = std::move(node_ref);
-          break;
-        }
-        top_level_channels.emplace_back(std::move(node_ref));
-      }
+std::string ChannelzRegistry::GetServersJson(intptr_t start_server_id) {
+  return RenderArray(GetServers(start_server_id), "server");
+}
+
+void ChannelzRegistry::InternalRegister(BaseNode* node) {
+  DCHECK_EQ(node->uuid_, -1);
+  const size_t node_shard_index = NodeShardIndex(node);
+  NodeShard& node_shard = node_shards_[node_shard_index];
+  MutexLock lock(&node_shard.mu);
+  node_shard.nursery.AddToHead(node);
+}
+
+void ChannelzRegistry::InternalUnregister(BaseNode* node) {
+  const size_t node_shard_index = NodeShardIndex(node);
+  NodeShard& node_shard = node_shards_[node_shard_index];
+  node_shard.mu.Lock();
+  CHECK_EQ(node->orphaned_index_, 0u);
+  intptr_t uuid = node->uuid_.load(std::memory_order_relaxed);
+  NodeList& remove_list = uuid == -1 ? node_shard.nursery : node_shard.numbered;
+  remove_list.Remove(node);
+  if (max_orphaned_per_shard_ == 0) {
+    // We are not tracking orphaned nodes... remove from the index
+    // if necessary, then exit out.
+    node_shard.mu.Unlock();
+    if (uuid != -1) {
+      MutexLock lock(&index_mu_);
+      index_.erase(uuid);
     }
+    return;
   }
-  Json::Object object;
-  if (!top_level_channels.empty()) {
-    // Create list of channels.
-    Json::Array array;
-    for (size_t i = 0; i < top_level_channels.size(); ++i) {
-      array.emplace_back(top_level_channels[i]->RenderJson());
-    }
-    object["channel"] = Json::FromArray(std::move(array));
+  NodeList& add_list =
+      uuid != -1 ? node_shard.orphaned_numbered : node_shard.orphaned;
+  // Ref counting: once a node becomes orphaned we add a single weak ref to it.
+  // We hold that ref until it gets garbage collected later.
+  node->WeakRef().release();
+  node->orphaned_index_ = node_shard.next_orphan_index;
+  CHECK_GT(node->orphaned_index_, 0u);
+  ++node_shard.next_orphan_index;
+  add_list.AddToHead(node);
+  if (node_shard.TotalOrphaned() <= max_orphaned_per_shard_) {
+    // Below recycling thresholds: just exit out
+    node_shard.mu.Unlock();
+    return;
   }
-  if (node_after_pagination_limit == nullptr) {
-    object["end"] = Json::FromBool(true);
+  CHECK_EQ(node_shard.TotalOrphaned(), max_orphaned_per_shard_ + 1);
+  NodeList* gc_list;
+  // choose the oldest node to evict, regardless of numbered or not
+  if (node_shard.orphaned.tail == nullptr) {
+    CHECK_NE(node_shard.orphaned_numbered.tail, nullptr);
+    gc_list = &node_shard.orphaned_numbered;
+  } else if (node_shard.orphaned_numbered.tail == nullptr) {
+    gc_list = &node_shard.orphaned;
+  } else if (node_shard.orphaned.tail->orphaned_index_ <
+             node_shard.orphaned_numbered.tail->orphaned_index_) {
+    gc_list = &node_shard.orphaned;
+  } else {
+    gc_list = &node_shard.orphaned_numbered;
+  }
+  auto* n = gc_list->tail;
+  CHECK_GT(n->orphaned_index_, 0u);
+  gc_list->Remove(n);
+  // Note: we capture the reference to n previously added here, and release
+  // it when this smart pointer is destroyed, outside of any locks.
+  WeakRefCountedPtr<BaseNode> gcd_node(n);
+  node_shard.mu.Unlock();
+  if (gc_list == &node_shard.orphaned_numbered) {
+    MutexLock lock(&index_mu_);
+    intptr_t uuid = n->uuid_.load(std::memory_order_relaxed);
+    index_.erase(uuid);
   }
-  return JsonDump(Json::FromObject(std::move(object)));
 }
 
-std::string ChannelzRegistry::InternalGetServers(intptr_t start_server_id) {
-  std::vector<RefCountedPtr<BaseNode>> servers;
-  RefCountedPtr<BaseNode> node_after_pagination_limit;
-  {
-    MutexLock lock(&mu_);
-    for (auto it = node_map_.lower_bound(start_server_id);
-         it != node_map_.end(); ++it) {
-      BaseNode* node = it->second;
-      RefCountedPtr<BaseNode> node_ref;
-      if (node->type() == BaseNode::EntityType::kServer &&
-          (node_ref = node->RefIfNonZero()) != nullptr) {
-        // Check if we are over pagination limit to determine if we need to set
-        // the "end" element. If we don't go through this block, we know that
-        // when the loop terminates, we have <= to kPaginationLimit.
-        // Note that because we have already increased this node's
-        // refcount, we need to decrease it, but we can't unref while
-        // holding the lock, because this may lead to a deadlock.
-        if (servers.size() == kPaginationLimit) {
-          node_after_pagination_limit = std::move(node_ref);
-          break;
+void ChannelzRegistry::LoadConfig() {
+  const auto max_orphaned = ConfigVars::Get().ChannelzMaxOrphanedNodes();
+  if (max_orphaned == 0) {
+    max_orphaned_per_shard_ = 0;
+  } else {
+    max_orphaned_per_shard_ = std::max<int>(max_orphaned / kNodeShards, 1);
+  }
+}
+
+std::tuple<std::vector<WeakRefCountedPtr<BaseNode>>, bool>
+ChannelzRegistry::QueryNodes(
+    intptr_t start_node, absl::FunctionRef<bool(const BaseNode*)> discriminator,
+    size_t max_results) {
+  // Mitigate drain hotspotting by randomizing the drain order each query.
+  std::vector<size_t> nursery_visitation_order;
+  for (size_t i = 0; i < kNodeShards; ++i) {
+    nursery_visitation_order.push_back(i);
+  }
+  absl::c_shuffle(nursery_visitation_order, SharedBitGen());
+  // In the iteration below, even once we have max_results nodes, we need
+  // to find the next node in order to know if we've hit the end.  If we get
+  // through the loop without returning, then we return end=true.  But if we
+  // find a node to add after we already have max_results nodes, then we
+  // return with end=false before exiting the loop.  However, in the latter
+  // case, we will have already increased the ref count of the next node,
+  // so we need to unref it, but we can't do that while holding the lock.
+  // So instead, we store it in node_after_end, which will be unreffed
+  // after releasing the lock.
+  WeakRefCountedPtr<BaseNode> node_after_end;
+  std::vector<WeakRefCountedPtr<BaseNode>> result;
+  MutexLock index_lock(&index_mu_);
+  for (auto it = index_.lower_bound(start_node); it != index_.end(); ++it) {
+    BaseNode* node = it->second;
+    if (!discriminator(node)) continue;
+    auto node_ref = node->WeakRefIfNonZero();
+    if (node_ref == nullptr) continue;
+    if (result.size() == max_results) {
+      node_after_end = std::move(node_ref);
+      return std::tuple(std::move(result), false);
+    }
+    result.emplace_back(std::move(node_ref));
+  }
+  for (auto nursery_index : nursery_visitation_order) {
+    NodeShard& node_shard = node_shards_[nursery_index];
+    MutexLock shard_lock(&node_shard.mu);
+    for (auto [nursery, numbered] :
+         {std::pair(&node_shard.nursery, &node_shard.numbered),
+          std::pair(&node_shard.orphaned, &node_shard.orphaned_numbered)}) {
+      if (nursery->head == nullptr) continue;
+      BaseNode* n = nursery->head;
+      while (n != nullptr) {
+        if (!discriminator(n)) {
+          n = n->next_;
+          continue;
+        }
+        auto node_ref = n->WeakRefIfNonZero();
+        if (node_ref == nullptr) {
+          n = n->next_;
+          continue;
         }
-        servers.emplace_back(std::move(node_ref));
+        BaseNode* next = n->next_;
+        nursery->Remove(n);
+        numbered->AddToHead(n);
+        n->uuid_ = uuid_generator_;
+        ++uuid_generator_;
+        index_.emplace(n->uuid_, n);
+        if (n->uuid_ >= start_node) {
+          if (result.size() == max_results) {
+            node_after_end = std::move(node_ref);
+            return std::tuple(std::move(result), false);
+          }
+          result.emplace_back(std::move(node_ref));
+        }
+        n = next;
       }
     }
   }
-  Json::Object object;
-  if (!servers.empty()) {
-    // Create list of servers.
-    Json::Array array;
-    for (size_t i = 0; i < servers.size(); ++i) {
-      array.emplace_back(servers[i]->RenderJson());
-    }
-    object["server"] = Json::FromArray(std::move(array));
+  CHECK(node_after_end == nullptr);
+  return std::tuple(std::move(result), true);
+}
+
+WeakRefCountedPtr<BaseNode> ChannelzRegistry::InternalGet(intptr_t uuid) {
+  MutexLock index_lock(&index_mu_);
+  auto it = index_.find(uuid);
+  if (it == index_.end()) return nullptr;
+  BaseNode* node = it->second;
+  return node->WeakRefIfNonZero();
+}
+
+intptr_t ChannelzRegistry::InternalNumberNode(BaseNode* node) {
+  // node must be strongly owned still
+  node->AssertStronglyOwned();
+  const size_t node_shard_index = NodeShardIndex(node);
+  NodeShard& node_shard = node_shards_[node_shard_index];
+  MutexLock index_lock(&index_mu_);
+  MutexLock lock(&node_shard.mu);
+  intptr_t uuid = node->uuid_.load(std::memory_order_relaxed);
+  if (uuid != -1) return uuid;
+  uuid = uuid_generator_;
+  ++uuid_generator_;
+  node->uuid_ = uuid;
+  if (node->orphaned_index_ > 0) {
+    node_shard.orphaned.Remove(node);
+    node_shard.orphaned_numbered.AddToHead(node);
+  } else {
+    node_shard.nursery.Remove(node);
+    node_shard.numbered.AddToHead(node);
   }
-  if (node_after_pagination_limit == nullptr) {
-    object["end"] = Json::FromBool(true);
+  index_.emplace(uuid, node);
+  return uuid;
+}
+
+bool ChannelzRegistry::NodeList::Holds(BaseNode* node) const {
+  BaseNode* n = head;
+  while (n != nullptr) {
+    if (n == node) return true;
+    n = n->next_;
   }
-  return JsonDump(Json::FromObject(std::move(object)));
+  return false;
 }
 
-void ChannelzRegistry::InternalLogAllEntities() {
-  std::vector<RefCountedPtr<BaseNode>> nodes;
-  {
-    MutexLock lock(&mu_);
-    for (auto& p : node_map_) {
-      RefCountedPtr<BaseNode> node = p.second->RefIfNonZero();
-      if (node != nullptr) {
-        nodes.emplace_back(std::move(node));
-      }
+void ChannelzRegistry::NodeList::AddToHead(BaseNode* node) {
+  DCHECK(!Holds(node));
+  ++count;
+  if (head != nullptr) head->prev_ = node;
+  node->next_ = head;
+  node->prev_ = nullptr;
+  head = node;
+  if (tail == nullptr) tail = node;
+  DCHECK(Holds(node));
+}
+
+void ChannelzRegistry::NodeList::Remove(BaseNode* node) {
+  DCHECK(Holds(node));
+  DCHECK_GT(count, 0u);
+  --count;
+  if (node->prev_ == nullptr) {
+    head = node->next_;
+    if (head == nullptr) {
+      DCHECK_EQ(count, 0u);
+      tail = nullptr;
+      DCHECK(!Holds(node));
+      return;
     }
+  } else {
+    node->prev_->next_ = node->next_;
   }
-  for (size_t i = 0; i < nodes.size(); ++i) {
-    std::string json = nodes[i]->RenderJsonString();
-    LOG(INFO) << json;
+  if (node->next_ == nullptr) {
+    tail = node->prev_;
+  } else {
+    node->next_->prev_ = node->prev_;
+  }
+  DCHECK(!Holds(node));
+}
+void ChannelzRegistry::TestOnlyReset() {
+  auto* p = Default();
+  p->uuid_generator_ = 1;
+  p->LoadConfig();
+  std::vector<WeakRefCountedPtr<BaseNode>> free_nodes;
+  for (size_t i = 0; i < kNodeShards; i++) {
+    MutexLock lock(&p->node_shards_[i].mu);
+    CHECK(p->node_shards_[i].nursery.head == nullptr);
+    CHECK(p->node_shards_[i].numbered.head == nullptr);
+    while (p->node_shards_[i].orphaned.head != nullptr) {
+      free_nodes.emplace_back(p->node_shards_[i].orphaned.head);
+      p->node_shards_[i].orphaned.Remove(p->node_shards_[i].orphaned.head);
+    }
+    while (p->node_shards_[i].orphaned_numbered.head != nullptr) {
+      free_nodes.emplace_back(p->node_shards_[i].orphaned_numbered.head);
+      p->node_shards_[i].orphaned_numbered.Remove(
+          p->node_shards_[i].orphaned_numbered.head);
+    }
   }
+  std::vector<NodeShard> replace_node_shards(kNodeShards);
+  replace_node_shards.swap(p->node_shards_);
+  MutexLock lock(&p->index_mu_);
+  p->index_.clear();
 }
 
 }  // namespace channelz
@@ -179,21 +381,22 @@ void ChannelzRegistry::InternalLogAllEntities() {
 
 char* grpc_channelz_get_top_channels(intptr_t start_channel_id) {
   grpc_core::ExecCtx exec_ctx;
-  return gpr_strdup(
-      grpc_core::channelz::ChannelzRegistry::GetTopChannels(start_channel_id)
+  return grpc_core::channelz::ApplyHacks(
+      grpc_core::channelz::ChannelzRegistry::GetTopChannelsJson(
+          start_channel_id)
           .c_str());
 }
 
 char* grpc_channelz_get_servers(intptr_t start_server_id) {
   grpc_core::ExecCtx exec_ctx;
-  return gpr_strdup(
-      grpc_core::channelz::ChannelzRegistry::GetServers(start_server_id)
+  return grpc_core::channelz::ApplyHacks(
+      grpc_core::channelz::ChannelzRegistry::GetServersJson(start_server_id)
           .c_str());
 }
 
 char* grpc_channelz_get_server(intptr_t server_id) {
   grpc_core::ExecCtx exec_ctx;
-  grpc_core::RefCountedPtr<grpc_core::channelz::BaseNode> server_node =
+  grpc_core::WeakRefCountedPtr<grpc_core::channelz::BaseNode> server_node =
       grpc_core::channelz::ChannelzRegistry::Get(server_id);
   if (server_node == nullptr ||
       server_node->type() !=
@@ -203,7 +406,7 @@ char* grpc_channelz_get_server(intptr_t server_id) {
   grpc_core::Json json = grpc_core::Json::FromObject({
       {"server", server_node->RenderJson()},
   });
-  return gpr_strdup(grpc_core::JsonDump(json).c_str());
+  return grpc_core::channelz::ApplyHacks(grpc_core::JsonDump(json).c_str());
 }
 
 char* grpc_channelz_get_server_sockets(intptr_t server_id,
@@ -211,7 +414,7 @@ char* grpc_channelz_get_server_sockets(intptr_t server_id,
                                        intptr_t max_results) {
   grpc_core::ExecCtx exec_ctx;
   // Validate inputs before handing them of to the renderer.
-  grpc_core::RefCountedPtr<grpc_core::channelz::BaseNode> base_node =
+  grpc_core::WeakRefCountedPtr<grpc_core::channelz::BaseNode> base_node =
       grpc_core::channelz::ChannelzRegistry::Get(server_id);
   if (base_node == nullptr ||
       base_node->type() != grpc_core::channelz::BaseNode::EntityType::kServer ||
@@ -222,13 +425,13 @@ char* grpc_channelz_get_server_sockets(intptr_t server_id,
   // actually a server node.
   grpc_core::channelz::ServerNode* server_node =
       static_cast<grpc_core::channelz::ServerNode*>(base_node.get());
-  return gpr_strdup(
+  return grpc_core::channelz::ApplyHacks(
       server_node->RenderServerSockets(start_socket_id, max_results).c_str());
 }
 
 char* grpc_channelz_get_channel(intptr_t channel_id) {
   grpc_core::ExecCtx exec_ctx;
-  grpc_core::RefCountedPtr<grpc_core::channelz::BaseNode> channel_node =
+  grpc_core::WeakRefCountedPtr<grpc_core::channelz::BaseNode> channel_node =
       grpc_core::channelz::ChannelzRegistry::Get(channel_id);
   if (channel_node == nullptr ||
       (channel_node->type() !=
@@ -240,12 +443,12 @@ char* grpc_channelz_get_channel(intptr_t channel_id) {
   grpc_core::Json json = grpc_core::Json::FromObject({
       {"channel", channel_node->RenderJson()},
   });
-  return gpr_strdup(grpc_core::JsonDump(json).c_str());
+  return grpc_core::channelz::ApplyHacks(grpc_core::JsonDump(json).c_str());
 }
 
 char* grpc_channelz_get_subchannel(intptr_t subchannel_id) {
   grpc_core::ExecCtx exec_ctx;
-  grpc_core::RefCountedPtr<grpc_core::channelz::BaseNode> subchannel_node =
+  grpc_core::WeakRefCountedPtr<grpc_core::channelz::BaseNode> subchannel_node =
       grpc_core::channelz::ChannelzRegistry::Get(subchannel_id);
   if (subchannel_node == nullptr ||
       subchannel_node->type() !=
@@ -255,20 +458,22 @@ char* grpc_channelz_get_subchannel(intptr_t subchannel_id) {
   grpc_core::Json json = grpc_core::Json::FromObject({
       {"subchannel", subchannel_node->RenderJson()},
   });
-  return gpr_strdup(grpc_core::JsonDump(json).c_str());
+  return grpc_core::channelz::ApplyHacks(grpc_core::JsonDump(json).c_str());
 }
 
 char* grpc_channelz_get_socket(intptr_t socket_id) {
   grpc_core::ExecCtx exec_ctx;
-  grpc_core::RefCountedPtr<grpc_core::channelz::BaseNode> socket_node =
+  grpc_core::WeakRefCountedPtr<grpc_core::channelz::BaseNode> socket_node =
       grpc_core::channelz::ChannelzRegistry::Get(socket_id);
   if (socket_node == nullptr ||
-      socket_node->type() !=
-          grpc_core::channelz::BaseNode::EntityType::kSocket) {
+      (socket_node->type() !=
+           grpc_core::channelz::BaseNode::EntityType::kSocket &&
+       socket_node->type() !=
+           grpc_core::channelz::BaseNode::EntityType::kListenSocket)) {
     return nullptr;
   }
   grpc_core::Json json = grpc_core::Json::FromObject({
       {"socket", socket_node->RenderJson()},
   });
-  return gpr_strdup(grpc_core::JsonDump(json).c_str());
+  return grpc_core::channelz::ApplyHacks(grpc_core::JsonDump(json).c_str());
 }