grpc 1.71.0 → 1.73.0

data/src/core/client_channel/subchannel.cc

@@ -35,6 +35,7 @@
 #include "absl/strings/cord.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/string_view.h"
+#include "src/core/call/interception_chain.h"
 #include "src/core/channelz/channel_trace.h"
 #include "src/core/channelz/channelz.h"
 #include "src/core/client_channel/client_channel_internal.h"
@@ -57,7 +58,6 @@
 #include "src/core/lib/surface/init_internally.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/error_utils.h"
-#include "src/core/lib/transport/interception_chain.h"
 #include "src/core/lib/transport/transport.h"
 #include "src/core/telemetry/stats.h"
 #include "src/core/telemetry/stats_data.h"
@@ -244,13 +244,12 @@ SubchannelCall::SubchannelCall(Args args, grpc_error_handle* error)
       deadline_(args.deadline) {
   grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(this);
   const grpc_call_element_args call_args = {
-      callstk,              // call_stack
-      nullptr,              // server_transport_data
-      args.path.c_slice(),  // path
-      args.start_time,      // start_time
-      args.deadline,        // deadline
-      args.arena,           // arena
-      args.call_combiner    // call_combiner
+      callstk,            // call_stack
+      nullptr,            // server_transport_data
+      args.start_time,    // start_time
+      args.deadline,      // deadline
+      args.arena,         // arena
+      args.call_combiner  // call_combiner
   };
   *error = grpc_call_stack_init(connected_subchannel_->channel_stack(), 1,
                                 SubchannelCall::Destroy, this, &call_args);
@@ -453,10 +452,8 @@ void Subchannel::ConnectivityStateWatcherList::RemoveWatcherLocked(
 void Subchannel::ConnectivityStateWatcherList::NotifyLocked(
     grpc_connectivity_state state, const absl::Status& status) {
   for (const auto& watcher : watchers_) {
-    subchannel_->work_serializer_.Run([watcher = watcher->Ref(), state,
-                                       status]() mutable {
-      auto* watcher_ptr = watcher.get();
-      watcher_ptr->OnConnectivityStateChange(std::move(watcher), state, status);
+    subchannel_->work_serializer_.Run([watcher, state, status]() {
+      watcher->OnConnectivityStateChange(state, status);
     });
   }
 }
@@ -552,6 +549,8 @@ Subchannel::Subchannel(SubchannelKey key,
     channelz_node_->AddTraceEvent(
         channelz::ChannelTrace::Severity::Info,
         grpc_slice_from_static_string("subchannel created"));
+    channelz_node_->SetChannelArgs(args_);
+    args_ = args_.SetObject<channelz::BaseNode>(channelz_node_);
   }
 }
 
@@ -613,10 +612,8 @@ void Subchannel::WatchConnectivityState(
     grpc_pollset_set_add_pollset_set(pollset_set_, interested_parties);
   }
   work_serializer_.Run(
-      [watcher = watcher->Ref(), state = state_, status = status_]() mutable {
-        auto* watcher_ptr = watcher.get();
-        watcher_ptr->OnConnectivityStateChange(std::move(watcher), state,
-                                               status);
+      [watcher, state = state_, status = status_]() {
+        watcher->OnConnectivityStateChange(state, status);
       },
       DEBUG_LOCATION);
   watcher_list_.AddWatcherLocked(std::move(watcher));
@@ -793,7 +790,7 @@ void Subchannel::OnConnectingFinishedLocked(grpc_error_handle error) {
 }
 
 bool Subchannel::PublishTransportLocked() {
-  auto socket_node = std::move(connecting_result_.socket_node);
+  auto socket_node = connecting_result_.transport->GetSocketNode();
   if (connecting_result_.transport->filter_stack_transport() != nullptr) {
     // Construct channel stack.
     // Builder takes ownership of transport.

data/src/core/client_channel/subchannel.h

@@ -29,6 +29,7 @@
 #include "absl/base/thread_annotations.h"
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
+#include "src/core/call/metadata_batch.h"
 #include "src/core/client_channel/connector.h"
 #include "src/core/client_channel/subchannel_pool_interface.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
@@ -44,7 +45,6 @@
 #include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/transport/connectivity_state.h"
-#include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/transport.h"
 #include "src/core/util/backoff.h"
 #include "src/core/util/debug_location.h"
@@ -95,7 +95,6 @@ class SubchannelCall final {
   struct Args {
     RefCountedPtr<ConnectedSubchannel> connected_subchannel;
     grpc_polling_entity* pollent;
-    Slice path;
     gpr_cycle_counter start_time;
     Timestamp deadline;
     Arena* arena;
@@ -169,14 +168,8 @@ class Subchannel final : public DualRefCounted<Subchannel> {
     // Invoked whenever the subchannel's connectivity state changes.
     // There will be only one invocation of this method on a given watcher
     // instance at any given time.
-    // A ref to the watcher is passed in here so that the implementation
-    // can unref it in the appropriate synchronization context (e.g.,
-    // inside a WorkSerializer).
-    // TODO(roth): Figure out a cleaner way to guarantee that the ref is
-    // released in the right context.
-    virtual void OnConnectivityStateChange(
-        RefCountedPtr<ConnectivityStateWatcherInterface> self,
-        grpc_connectivity_state state, const absl::Status& status) = 0;
+    virtual void OnConnectivityStateChange(grpc_connectivity_state state,
+                                           const absl::Status& status) = 0;
 
     virtual grpc_pollset_set* interested_parties() = 0;
   };

data/src/core/client_channel/subchannel_pool_interface.cc

@@ -38,13 +38,12 @@ SubchannelKey::SubchannelKey(const grpc_resolved_address& address,
                              const ChannelArgs& args)
     : address_(address), args_(args) {}
 
-bool SubchannelKey::operator<(const SubchannelKey& other) const {
-  if (address_.len < other.address_.len) return true;
-  if (address_.len > other.address_.len) return false;
+int SubchannelKey::Compare(const SubchannelKey& other) const {
+  if (address_.len < other.address_.len) return -1;
+  if (address_.len > other.address_.len) return 1;
   int r = memcmp(address_.addr, other.address_.addr, address_.len);
-  if (r < 0) return true;
-  if (r > 0) return false;
-  return args_ < other.args();
+  if (r != 0) return r;
+  return QsortCompare(args_, other.args_);
 }
 
 std::string SubchannelKey::ToString() const {

data/src/core/client_channel/subchannel_pool_interface.h

@@ -45,7 +45,17 @@ class SubchannelKey final {
   SubchannelKey(SubchannelKey&& other) noexcept = default;
   SubchannelKey& operator=(SubchannelKey&& other) noexcept = default;
 
-  bool operator<(const SubchannelKey& other) const;
+  bool operator<(const SubchannelKey& other) const {
+    return Compare(other) < 0;
+  }
+  bool operator>(const SubchannelKey& other) const {
+    return Compare(other) > 0;
+  }
+  bool operator==(const SubchannelKey& other) const {
+    return Compare(other) == 0;
+  }
+
+  int Compare(const SubchannelKey& other) const;
 
   const grpc_resolved_address& address() const { return address_; }
   const ChannelArgs& args() const { return args_; }

data/src/core/client_channel/subchannel_stream_client.cc

@@ -130,7 +130,7 @@ void SubchannelStreamClient::StartRetryTimerLocked() {
   const Duration timeout = retry_backoff_.NextAttemptDelay();
   if (GPR_UNLIKELY(tracer_ != nullptr)) {
     LOG(INFO) << tracer_ << " " << this
-              << ": SubchannelStreamClient health check call lost...";
+              << ": SubchannelStreamClient call lost...";
     if (timeout > Duration::Zero()) {
       LOG(INFO) << tracer_ << " " << this << ": ... will retry in "
                 << timeout.millis() << "ms.";
@@ -139,10 +139,10 @@ void SubchannelStreamClient::StartRetryTimerLocked() {
     }
   }
   retry_timer_handle_ = event_engine_->RunAfter(
-      timeout, [self = Ref(DEBUG_LOCATION, "health_retry_timer")]() mutable {
+      timeout, [self = Ref(DEBUG_LOCATION, "retry_timer")]() mutable {
         ExecCtx exec_ctx;
         self->OnRetryTimer();
-        self.reset(DEBUG_LOCATION, "health_retry_timer");
+        self.reset(DEBUG_LOCATION, "retry_timer");
       });
 }
 
@@ -152,7 +152,7 @@ void SubchannelStreamClient::OnRetryTimer() {
       call_state_ == nullptr) {
     if (GPR_UNLIKELY(tracer_ != nullptr)) {
       LOG(INFO) << tracer_ << " " << this
-                << ": SubchannelStreamClient restarting health check call";
+                << ": SubchannelStreamClient restarting call";
     }
     StartCallLocked();
   }
@@ -164,9 +164,9 @@ void SubchannelStreamClient::OnRetryTimer() {
 //
 
 SubchannelStreamClient::CallState::CallState(
-    RefCountedPtr<SubchannelStreamClient> health_check_client,
+    RefCountedPtr<SubchannelStreamClient> subchannel_stream_client,
     grpc_pollset_set* interested_parties)
-    : subchannel_stream_client_(std::move(health_check_client)),
+    : subchannel_stream_client_(std::move(subchannel_stream_client)),
       pollent_(grpc_polling_entity_create_from_pollset_set(interested_parties)),
       arena_(subchannel_stream_client_->call_allocator_->MakeArena()) {}
 
@@ -192,7 +192,6 @@ void SubchannelStreamClient::CallState::StartCallLocked() {
   SubchannelCall::Args args = {
       subchannel_stream_client_->connected_subchannel_,
       &pollent_,
-      Slice::FromStaticString("/grpc.health.v1.Health/Watch"),
       gpr_get_cycle_counter(),  // start_time
       Timestamp::InfFuture(),   // deadline
       arena_.get(),
@@ -299,7 +298,7 @@ void SubchannelStreamClient::CallState::AfterCallStackDestruction(
 void SubchannelStreamClient::CallState::OnCancelComplete(
     void* arg, grpc_error_handle /*error*/) {
   auto* self = static_cast<SubchannelStreamClient::CallState*>(arg);
-  GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "health_cancel");
+  GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "cancel_batch");
   self->call_->Unref(DEBUG_LOCATION, "cancel");
 }
 
@@ -322,7 +321,7 @@ void SubchannelStreamClient::CallState::Cancel() {
     GRPC_CALL_COMBINER_START(
         &call_combiner_,
         GRPC_CLOSURE_CREATE(StartCancel, this, grpc_schedule_on_exec_ctx),
-        absl::OkStatus(), "health_cancel");
+        absl::OkStatus(), "cancel_batch");
   }
 }
 
@@ -406,18 +405,22 @@ void SubchannelStreamClient::CallState::RecvTrailingMetadataReady(
     LOG(INFO) << self->subchannel_stream_client_->tracer_ << " "
               << self->subchannel_stream_client_.get()
               << ": SubchannelStreamClient CallState " << self
-              << ": health watch failed with status " << status;
+              << ": call failed with status " << status;
   }
   // Clean up.
   self->recv_trailing_metadata_.Clear();
   // Report call end.
+  // Note: We hold a ref to the SubchannelStreamClient here to ensure
+  // that it lives long enough for us to release the mutex, since the
+  // call to CallEndedLocked() may release the last ref.
+  auto subchannel_stream_client = self->subchannel_stream_client_->Ref();
   MutexLock lock(&self->subchannel_stream_client_->mu_);
   if (self->subchannel_stream_client_->event_handler_ != nullptr) {
     self->subchannel_stream_client_->event_handler_
         ->RecvTrailingMetadataReadyLocked(self->subchannel_stream_client_.get(),
                                           status);
   }
-  // For status UNIMPLEMENTED, give up and assume always healthy.
+  // For status UNIMPLEMENTED, give up.
   self->CallEndedLocked(/*retry=*/status != GRPC_STATUS_UNIMPLEMENTED);
 }
 

data/src/core/client_channel/subchannel_stream_client.h

@@ -30,6 +30,7 @@
 #include "absl/base/thread_annotations.h"
 #include "absl/status/status.h"
 #include "absl/strings/string_view.h"
+#include "src/core/call/metadata_batch.h"
 #include "src/core/client_channel/subchannel.h"
 #include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/closure.h"
@@ -40,7 +41,6 @@
 #include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_buffer.h"
-#include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/transport.h"
 #include "src/core/util/backoff.h"
 #include "src/core/util/orphanable.h"
@@ -202,7 +202,7 @@ class SubchannelStreamClient final
   Mutex mu_;
   std::unique_ptr<CallEventHandler> event_handler_ ABSL_GUARDED_BY(mu_);
 
-  // The data associated with the current health check call.  It holds a ref
+  // The data associated with the current call.  It holds a ref
   // to this SubchannelStreamClient object.
   OrphanablePtr<CallState> call_state_ ABSL_GUARDED_BY(mu_);
 

data/src/core/config/config_vars.cc

@@ -76,6 +76,11 @@ ABSL_FLAG(absl::optional<bool>, grpc_cpp_experimental_disable_reflection, {},
           "EXPERIMENTAL. Only respected when there is a dependency on "
           ":grpc++_reflection. If true, no reflection server will be "
           "automatically added.");
+ABSL_FLAG(
+    absl::optional<int32_t>, grpc_channelz_max_orphaned_nodes, {},
+    "EXPERIMENTAL: If non-zero, extend the lifetime of channelz nodes past the "
+    "underlying object lifetime, up to this many nodes. The value may be "
+    "adjusted slightly to account for implementation limits.");
 
 namespace grpc_core {
 
@@ -84,6 +89,10 @@ ConfigVars::ConfigVars(const Overrides& overrides)
           LoadConfig(FLAGS_grpc_client_channel_backup_poll_interval_ms,
                      "GRPC_CLIENT_CHANNEL_BACKUP_POLL_INTERVAL_MS",
                      overrides.client_channel_backup_poll_interval_ms, 5000)),
+      channelz_max_orphaned_nodes_(
+          LoadConfig(FLAGS_grpc_channelz_max_orphaned_nodes,
+                     "GRPC_CHANNELZ_MAX_ORPHANED_NODES",
+                     overrides.channelz_max_orphaned_nodes, 0)),
       enable_fork_support_(LoadConfig(
           FLAGS_grpc_enable_fork_support, "GRPC_ENABLE_FORK_SUPPORT",
           overrides.enable_fork_support, GRPC_ENABLE_FORK_SUPPORT_DEFAULT)),
@@ -146,7 +155,8 @@ std::string ConfigVars::ToString() const {
       ", not_use_system_ssl_roots: ", NotUseSystemSslRoots() ? "true" : "false",
       ", ssl_cipher_suites: ", "\"", absl::CEscape(SslCipherSuites()), "\"",
       ", cpp_experimental_disable_reflection: ",
-      CppExperimentalDisableReflection() ? "true" : "false");
+      CppExperimentalDisableReflection() ? "true" : "false",
+      ", channelz_max_orphaned_nodes: ", ChannelzMaxOrphanedNodes());
 }
 
 }  // namespace grpc_core

data/src/core/config/config_vars.h

@@ -35,6 +35,7 @@ class GPR_DLL ConfigVars {
  public:
   struct Overrides {
     absl::optional<int32_t> client_channel_backup_poll_interval_ms;
+    absl::optional<int32_t> channelz_max_orphaned_nodes;
     absl::optional<bool> enable_fork_support;
     absl::optional<bool> abort_on_leaks;
     absl::optional<bool> not_use_system_ssl_roots;
@@ -104,12 +105,19 @@ class GPR_DLL ConfigVars {
   bool CppExperimentalDisableReflection() const {
     return cpp_experimental_disable_reflection_;
   }
+  // EXPERIMENTAL: If non-zero, extend the lifetime of channelz nodes past the
+  // underlying object lifetime, up to this many nodes. The value may be
+  // adjusted slightly to account for implementation limits.
+  int32_t ChannelzMaxOrphanedNodes() const {
+    return channelz_max_orphaned_nodes_;
+  }
 
  private:
   explicit ConfigVars(const Overrides& overrides);
   static const ConfigVars& Load();
   static std::atomic<ConfigVars*> config_vars_;
   int32_t client_channel_backup_poll_interval_ms_;
+  int32_t channelz_max_orphaned_nodes_;
   bool enable_fork_support_;
   bool abort_on_leaks_;
   bool not_use_system_ssl_roots_;

data/src/core/config/core_configuration.cc

@@ -25,8 +25,10 @@
 namespace grpc_core {
 
 std::atomic<CoreConfiguration*> CoreConfiguration::config_{nullptr};
-std::atomic<CoreConfiguration::RegisteredBuilder*> CoreConfiguration::builders_{
-    nullptr};
+std::atomic<CoreConfiguration::RegisteredBuilder*>
+    CoreConfiguration::builders_[static_cast<size_t>(BuilderScope::kCount)]{
+        nullptr, nullptr};
+std::atomic<bool> CoreConfiguration::has_config_ever_been_produced_{false};
 void (*CoreConfiguration::default_builder_)(CoreConfiguration::Builder*);
 
 CoreConfiguration::Builder::Builder() = default;
@@ -46,18 +48,31 @@ CoreConfiguration::CoreConfiguration(Builder* builder)
       lb_policy_registry_(builder->lb_policy_registry_.Build()),
       proxy_mapper_registry_(builder->proxy_mapper_registry_.Build()),
       certificate_provider_registry_(
-          builder->certificate_provider_registry_.Build()) {}
+          builder->certificate_provider_registry_.Build()),
+      endpoint_transport_registry_(
+          builder->endpoint_transport_registry_.Build()) {}
 
 void CoreConfiguration::RegisterBuilder(
-    absl::AnyInvocable<void(Builder*)> builder) {
+    BuilderScope scope, absl::AnyInvocable<void(Builder*)> builder,
+    SourceLocation whence) {
   CHECK(config_.load(std::memory_order_relaxed) == nullptr)
       << "CoreConfiguration was already instantiated before builder "
          "registration was completed";
+  if (scope == BuilderScope::kPersistent) {
+    CHECK(!has_config_ever_been_produced_.load(std::memory_order_relaxed))
+        << "Persistent builders cannot be registered after the first "
+           "CoreConfiguration has been produced";
+  }
+  CHECK_NE(scope, BuilderScope::kCount);
+  auto& head = builders_[static_cast<size_t>(scope)];
   RegisteredBuilder* n = new RegisteredBuilder();
+  VLOG(4) << "Registering " << scope << " builder from " << whence.file() << ":"
+          << whence.line();
   n->builder = std::move(builder);
-  n->next = builders_.load(std::memory_order_relaxed);
-  while (!builders_.compare_exchange_weak(n->next, n, std::memory_order_acq_rel,
-                                          std::memory_order_relaxed)) {
+  n->whence = whence;
+  n->next = head.load(std::memory_order_relaxed);
+  while (!head.compare_exchange_weak(n->next, n, std::memory_order_acq_rel,
+                                     std::memory_order_relaxed)) {
   }
   CHECK(config_.load(std::memory_order_relaxed) == nullptr)
       << "CoreConfiguration was already instantiated before builder "
@@ -65,6 +80,7 @@ void CoreConfiguration::RegisterBuilder(
 }
 
 const CoreConfiguration& CoreConfiguration::BuildNewAndMaybeSet() {
+  has_config_ever_been_produced_.store(true, std::memory_order_relaxed);
   // Construct builder, pass it up to code that knows about build configuration
   Builder builder;
   // The linked list of builders stores things in reverse registration order.
@@ -72,13 +88,21 @@ const CoreConfiguration& CoreConfiguration::BuildNewAndMaybeSet() {
   // actually need to run things in forward registration order, so we iterate
   // once over the linked list to build a vector of builders, and then iterate
   // over said vector in reverse to actually run the builders.
+  // Note that we also iterate scopes in reverse order here too, so that when
+  // we run the builders in the reverse generated order we'll actually run
+  // persistent builders before ephemeral ones.
   std::vector<RegisteredBuilder*> registered_builders;
-  for (RegisteredBuilder* b = builders_.load(std::memory_order_acquire);
-       b != nullptr; b = b->next) {
-    registered_builders.push_back(b);
+  for (auto scope : {BuilderScope::kEphemeral, BuilderScope::kPersistent}) {
+    for (RegisteredBuilder* b = builders_[static_cast<size_t>(scope)].load(
+             std::memory_order_acquire);
+         b != nullptr; b = b->next) {
+      registered_builders.push_back(b);
+    }
   }
   for (auto it = registered_builders.rbegin(); it != registered_builders.rend();
        ++it) {
+    VLOG(4) << "Running builder from " << (*it)->whence.file() << ":"
+            << (*it)->whence.line();
     (*it)->builder(&builder);
   }
   // Finally, call the built in configuration builder.
@@ -100,12 +124,27 @@ const CoreConfiguration& CoreConfiguration::BuildNewAndMaybeSet() {
 void CoreConfiguration::Reset() {
   delete config_.exchange(nullptr, std::memory_order_acquire);
   RegisteredBuilder* builder =
-      builders_.exchange(nullptr, std::memory_order_acquire);
+      builders_[static_cast<size_t>(BuilderScope::kEphemeral)].exchange(
+          nullptr, std::memory_order_acquire);
+  while (builder != nullptr) {
+    RegisteredBuilder* next = builder->next;
+    delete builder;
+    builder = next;
+  }
+}
+
+void CoreConfiguration::
+    ResetEverythingIncludingPersistentBuildersAbsolutelyNotRecommended() {
+  has_config_ever_been_produced_.store(false, std::memory_order_relaxed);
+  RegisteredBuilder* builder =
+      builders_[static_cast<size_t>(BuilderScope::kPersistent)].exchange(
+          nullptr, std::memory_order_acquire);
   while (builder != nullptr) {
     RegisteredBuilder* next = builder->next;
     delete builder;
     builder = next;
   }
+  Reset();
 }
 
 }  // namespace grpc_core

data/src/core/config/core_configuration.h

@@ -16,20 +16,23 @@
 #define GRPC_SRC_CORE_CONFIG_CORE_CONFIGURATION_H
 
 #include <grpc/support/port_platform.h>
+#include <sys/stat.h>
 
 #include <atomic>
 
 #include "absl/functional/any_invocable.h"
 #include "absl/log/check.h"
+#include "src/core/credentials/transport/channel_creds_registry.h"
+#include "src/core/credentials/transport/tls/certificate_provider_registry.h"
 #include "src/core/handshaker/handshaker_registry.h"
 #include "src/core/handshaker/proxy_mapper_registry.h"
 #include "src/core/lib/channel/channel_args_preconditioning.h"
-#include "src/core/lib/security/certificate_provider/certificate_provider_registry.h"
-#include "src/core/lib/security/credentials/channel_creds_registry.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/load_balancing/lb_policy_registry.h"
 #include "src/core/resolver/resolver_registry.h"
 #include "src/core/service_config/service_config_parser.h"
+#include "src/core/transport/endpoint_transport.h"
+#include "src/core/util/debug_location.h"
 
 namespace grpc_core {
 
@@ -40,6 +43,27 @@ class GRPC_DLL CoreConfiguration {
   CoreConfiguration(const CoreConfiguration&) = delete;
   CoreConfiguration& operator=(const CoreConfiguration&) = delete;
 
+  // BulderScope is used to indicate whether a builder is persistent - these
+  // are builders that are used every time the configuration is built, or
+  // ephemeral - each time the configuration is built these are thrown away.
+  //
+  // Considerations for choosing persistent vs ephemeral:
+  // - For testing we want ephemeral builders, so the next test can throw away
+  //   configuration.
+  // - For adapting gRPC to different environments we typically want persistent
+  //   builders.
+  //   - However, if the adaption should run only once per process, then
+  //     ephemeral is better.
+  //
+  // Builders are instantiated in scope order - persistent first, ephemeral
+  // second.
+  enum class BuilderScope {
+    kPersistent,
+    kEphemeral,
+    // Must be last, do not use as a scope.
+    kCount,
+  };
+
   // Builder is passed to plugins, etc... at initialization time to collect
   // their configuration and assemble the published CoreConfiguration.
   class Builder {
@@ -78,6 +102,10 @@ class GRPC_DLL CoreConfiguration {
       return &certificate_provider_registry_;
     }
 
+    EndpointTransportRegistry::Builder* endpoint_transport_registry() {
+      return &endpoint_transport_registry_;
+    }
+
    private:
     friend class CoreConfiguration;
 
@@ -90,6 +118,7 @@ class GRPC_DLL CoreConfiguration {
     LoadBalancingPolicyRegistry::Builder lb_policy_registry_;
     ProxyMapperRegistry::Builder proxy_mapper_registry_;
     CertificateProviderRegistry::Builder certificate_provider_registry_;
+    EndpointTransportRegistry::Builder endpoint_transport_registry_;
 
     Builder();
     CoreConfiguration* Build();
@@ -99,6 +128,7 @@ class GRPC_DLL CoreConfiguration {
   struct RegisteredBuilder {
     absl::AnyInvocable<void(Builder*)> builder;
     RegisteredBuilder* next;
+    SourceLocation whence;
   };
 
   // Temporarily replaces core configuration with what is built from the
@@ -121,8 +151,10 @@ class GRPC_DLL CoreConfiguration {
       // Backup current core configuration and replace/reset.
       config_restore_ =
           CoreConfiguration::config_.exchange(p, std::memory_order_acquire);
-      builders_restore_ = CoreConfiguration::builders_.exchange(
-          nullptr, std::memory_order_acquire);
+      builders_restore_ =
+          CoreConfiguration::builders_[static_cast<size_t>(
+                                           BuilderScope::kEphemeral)]
+              .exchange(nullptr, std::memory_order_acquire);
     }
 
     ~WithSubstituteBuilder() {
@@ -130,8 +162,10 @@ class GRPC_DLL CoreConfiguration {
       Reset();
       CHECK(CoreConfiguration::config_.exchange(
                 config_restore_, std::memory_order_acquire) == nullptr);
-      CHECK(CoreConfiguration::builders_.exchange(
-                builders_restore_, std::memory_order_acquire) == nullptr);
+      CHECK(CoreConfiguration::builders_[static_cast<size_t>(
+                                             BuilderScope::kEphemeral)]
+                .exchange(builders_restore_, std::memory_order_acquire) ==
+            nullptr);
     }
 
    private:
@@ -153,13 +187,34 @@ class GRPC_DLL CoreConfiguration {
   // Attach a registration function globally.
   // Each registration function is called *in addition to*
   // BuildCoreConfiguration for the default core configuration.
-  static void RegisterBuilder(absl::AnyInvocable<void(Builder*)> builder);
+  static void RegisterBuilder(BuilderScope scope,
+                              absl::AnyInvocable<void(Builder*)> builder,
+                              SourceLocation whence);
+
+  static void RegisterPersistentBuilder(
+      absl::AnyInvocable<void(Builder*)> builder, SourceLocation whence = {}) {
+    RegisterBuilder(BuilderScope::kPersistent, std::move(builder), whence);
+  }
+
+  static void RegisterEphemeralBuilder(
+      absl::AnyInvocable<void(Builder*)> builder, SourceLocation whence = {}) {
+    RegisterBuilder(BuilderScope::kEphemeral, std::move(builder), whence);
+  }
 
   // Drop the core configuration. Users must ensure no other threads are
   // accessing the configuration.
-  // Clears any dynamically registered builders.
+  // Clears any dynamically registered ephemeral builders.
   static void Reset();
 
+  // Reset, but also reset persistent builders. This is not recommended, but
+  // is useful for tests that assume exactly the default open source
+  // configuration when running in other environments.
+  //
+  // TODO(ctiller, roth, yashkt): Remove the need for this method, and then
+  // move the legacy plugin registration mechanism to be a persistent builder.
+  static void
+  ResetEverythingIncludingPersistentBuildersAbsolutelyNotRecommended();
+
   // Helper for tests: Reset the configuration, build a special one, run some
   // code, and then reset the configuration again.
   // Templatized to be sure no codegen in normal builds.
@@ -206,6 +261,10 @@ class GRPC_DLL CoreConfiguration {
     return certificate_provider_registry_;
   }
 
+  const EndpointTransportRegistry& endpoint_transport_registry() const {
+    return endpoint_transport_registry_;
+  }
+
   static void SetDefaultBuilder(void (*builder)(CoreConfiguration::Builder*)) {
     default_builder_ = builder;
   }
@@ -219,8 +278,13 @@ class GRPC_DLL CoreConfiguration {
 
   // The configuration
   static std::atomic<CoreConfiguration*> config_;
+  // Has a configuration *ever* been produced - we verify this is false for
+  // persistent builders so that we can prove consistency build to build for
+  // these.
+  static std::atomic<bool> has_config_ever_been_produced_;
   // Extra registered builders
-  static std::atomic<RegisteredBuilder*> builders_;
+  static std::atomic<RegisteredBuilder*>
+      builders_[static_cast<size_t>(BuilderScope::kCount)];
   // Default builder
   static void (*default_builder_)(CoreConfiguration::Builder*);
 
@@ -233,8 +297,26 @@ class GRPC_DLL CoreConfiguration {
   LoadBalancingPolicyRegistry lb_policy_registry_;
   ProxyMapperRegistry proxy_mapper_registry_;
   CertificateProviderRegistry certificate_provider_registry_;
+  EndpointTransportRegistry endpoint_transport_registry_;
 };
 
+template <typename Sink>
+void AbslStringify(Sink& sink, CoreConfiguration::BuilderScope scope) {
+  switch (scope) {
+    case CoreConfiguration::BuilderScope::kPersistent:
+      sink.Append("Persistent");
+      break;
+    case CoreConfiguration::BuilderScope::kEphemeral:
+      sink.Append("Ephemeral");
+      break;
+    case CoreConfiguration::BuilderScope::kCount:
+      sink.Append("Count(");
+      sink.Append(std::to_string(static_cast<size_t>(scope)));
+      sink.Append(")");
+      break;
+  }
+}
+
 extern void BuildCoreConfiguration(CoreConfiguration::Builder* builder);
 
 }  // namespace grpc_core