grpc 1.72.0 → 1.78.1

data/src/core/client_channel/subchannel.cc

@@ -29,12 +29,6 @@
 #include <optional>
 #include <utility>
 
-#include "absl/log/check.h"
-#include "absl/log/log.h"
-#include "absl/status/statusor.h"
-#include "absl/strings/cord.h"
-#include "absl/strings/str_cat.h"
-#include "absl/strings/string_view.h"
 #include "src/core/call/interception_chain.h"
 #include "src/core/channelz/channel_trace.h"
 #include "src/core/channelz/channelz.h"
@@ -64,12 +58,18 @@
 #include "src/core/util/alloc.h"
 #include "src/core/util/backoff.h"
 #include "src/core/util/debug_location.h"
+#include "src/core/util/grpc_check.h"
 #include "src/core/util/orphanable.h"
 #include "src/core/util/ref_counted.h"
 #include "src/core/util/ref_counted_ptr.h"
 #include "src/core/util/status_helper.h"
 #include "src/core/util/sync.h"
 #include "src/core/util/useful.h"
+#include "absl/log/log.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/cord.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 
 // Backoff parameters.
 #define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1
@@ -117,7 +117,7 @@ class LegacyConnectedSubchannel : public ConnectedSubchannel {
     channel_stack_.reset(DEBUG_LOCATION, "ConnectedSubchannel");
   }
 
-  channelz::SubchannelNode* channelz_node() const {
+  channelz::SubchannelNode* channelz_node() const override {
     return channelz_node_.get();
   }
 
@@ -220,6 +220,8 @@ class NewConnectedSubchannel : public ConnectedSubchannel {
     Crash("legacy ping method called in call v3 impl");
   }
 
+  channelz::SubchannelNode* channelz_node() const override { return nullptr; }
+
  private:
   RefCountedPtr<UnstartedCallDestination> call_destination_;
   RefCountedPtr<TransportCallDestination> transport_;
@@ -280,8 +282,8 @@ grpc_call_stack* SubchannelCall::GetCallStack() {
 }
 
 void SubchannelCall::SetAfterCallStackDestroy(grpc_closure* closure) {
-  CHECK_EQ(after_call_stack_destroy_, nullptr);
-  CHECK_NE(closure, nullptr);
+  GRPC_CHECK_EQ(after_call_stack_destroy_, nullptr);
+  GRPC_CHECK_NE(closure, nullptr);
   after_call_stack_destroy_ = closure;
 }
 
@@ -332,7 +334,7 @@ void SubchannelCall::MaybeInterceptRecvTrailingMetadata(
   GRPC_CLOSURE_INIT(&recv_trailing_metadata_ready_, RecvTrailingMetadataReady,
                     this, grpc_schedule_on_exec_ctx);
   // save some state needed for the interception callback.
-  CHECK_EQ(recv_trailing_metadata_, nullptr);
+  GRPC_CHECK_EQ(recv_trailing_metadata_, nullptr);
   recv_trailing_metadata_ =
       batch->payload->recv_trailing_metadata.recv_trailing_metadata;
   original_recv_trailing_metadata_ =
@@ -358,12 +360,12 @@ void GetCallStatus(grpc_status_code* status, Timestamp deadline,
 void SubchannelCall::RecvTrailingMetadataReady(void* arg,
                                                grpc_error_handle error) {
   SubchannelCall* call = static_cast<SubchannelCall*>(arg);
-  CHECK_NE(call->recv_trailing_metadata_, nullptr);
+  GRPC_CHECK_NE(call->recv_trailing_metadata_, nullptr);
   grpc_status_code status = GRPC_STATUS_OK;
   GetCallStatus(&status, call->deadline_, call->recv_trailing_metadata_, error);
   channelz::SubchannelNode* channelz_node =
       call->connected_subchannel_->channelz_node();
-  CHECK_NE(channelz_node, nullptr);
+  GRPC_CHECK_NE(channelz_node, nullptr);
   if (status == GRPC_STATUS_OK) {
     channelz_node->RecordCallSucceeded();
   } else {
@@ -410,23 +412,31 @@ class Subchannel::ConnectedSubchannelStateWatcher final
       // we will see TRANSIENT_FAILURE followed by SHUTDOWN, but if not, we
       // will see only SHUTDOWN.  Either way, we react to the first one we
       // see, ignoring anything that happens after that.
-      if (c->connected_subchannel_ == nullptr) return;
       if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE ||
           new_state == GRPC_CHANNEL_SHUTDOWN) {
+        RefCountedPtr<ConnectedSubchannel> connected_subchannel =
+            std::move(c->connected_subchannel_);
+        if (connected_subchannel == nullptr) return;
         GRPC_TRACE_LOG(subchannel, INFO)
             << "subchannel " << c << " " << c->key_.ToString()
-            << ": Connected subchannel " << c->connected_subchannel_.get()
+            << ": Connected subchannel " << connected_subchannel.get()
             << " reports " << ConnectivityStateName(new_state) << ": "
             << status;
-        c->connected_subchannel_.reset();
-        if (c->channelz_node() != nullptr) {
-          c->channelz_node()->SetChildSocket(nullptr);
-        }
-        // Even though we're reporting IDLE instead of TRANSIENT_FAILURE here,
-        // pass along the status from the transport, since it may have
-        // keepalive info attached to it that the channel needs.
-        // TODO(roth): Consider whether there's a cleaner way to do this.
-        c->SetConnectivityStateLocked(GRPC_CHANNEL_IDLE, status);
+        // If the subchannel was created from an endpoint, then we report
+        // TRANSIENT_FAILURE here instead of IDLE. The subchannel will never
+        // leave TRANSIENT_FAILURE state, because there is no way for us to
+        // establish a new connection.
+        //
+        // Otherwise, we report IDLE here. Note that even though we're not
+        // reporting TRANSIENT_FAILURE, we pass along the status from the
+        // transport, since it may have keepalive info attached to it that the
+        // channel needs.
+        // TODO(roth): Consider whether there's a cleaner way to propagate the
+        // keepalive info.
+        c->SetConnectivityStateLocked(c->created_from_endpoint_
+                                          ? GRPC_CHANNEL_TRANSIENT_FAILURE
+                                          : GRPC_CHANNEL_IDLE,
+                                      status);
         c->backoff_.Reset();
       }
     }
@@ -435,6 +445,68 @@ class Subchannel::ConnectedSubchannelStateWatcher final
   WeakRefCountedPtr<Subchannel> subchannel_;
 };
 
+//
+// Subchannel::ConnectionStateWatcher
+//
+
+class Subchannel::ConnectionStateWatcher final
+    : public Transport::StateWatcher {
+ public:
+  explicit ConnectionStateWatcher(WeakRefCountedPtr<Subchannel> subchannel)
+      : subchannel_(std::move(subchannel)) {}
+
+  ~ConnectionStateWatcher() override {
+    subchannel_.reset(DEBUG_LOCATION, "state_watcher");
+  }
+
+  void OnDisconnect(absl::Status status,
+                    DisconnectInfo disconnect_info) override {
+    MutexLock lock(&subchannel_->mu_);
+    // Handle keepalive update.
+    if (disconnect_info.keepalive_time.has_value()) {
+      subchannel_->ThrottleKeepaliveTimeLocked(*disconnect_info.keepalive_time);
+      subchannel_->watcher_list_.NotifyOnKeepaliveUpdateLocked(
+          *disconnect_info.keepalive_time);
+    }
+    // We shouldn't ever see OnDisconnect() more than once for a given
+    // connection, but we'll be defensive just in case: if the connected
+    // subchannel has already been cleared, then this becomes a no-op.
+    RefCountedPtr<ConnectedSubchannel> connected_subchannel =
+        std::move(subchannel_->connected_subchannel_);
+    if (connected_subchannel == nullptr) return;
+    GRPC_TRACE_LOG(subchannel, INFO)
+        << "subchannel " << subchannel_.get() << " "
+        << subchannel_->key_.ToString() << ": connected subchannel "
+        << connected_subchannel.get() << " reports disconnection: " << status;
+    // If the subchannel was created from an endpoint, then we report
+    // TRANSIENT_FAILURE here instead of IDLE. The subchannel will never
+    // leave TRANSIENT_FAILURE state, because there is no way for us to
+    // establish a new connection.  Otherwise, we report IDLE here.
+    if (subchannel_->created_from_endpoint_) {
+      subchannel_->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                              status);
+    } else {
+      subchannel_->SetConnectivityStateLocked(GRPC_CHANNEL_IDLE,
+                                              absl::OkStatus());
+    }
+    subchannel_->backoff_.Reset();
+  }
+
+  void OnPeerMaxConcurrentStreamsUpdate(
+      uint32_t /*max_concurrent_streams*/,
+      std::unique_ptr<MaxConcurrentStreamsUpdateDoneHandle> /*on_done*/)
+      override {
+    // TODO(roth): Implement this as part of adding connection scaling.
+  }
+
+  grpc_pollset_set* interested_parties() const override {
+    return subchannel_->pollset_set_;
+  }
+
+ private:
+  WeakRefCountedPtr<Subchannel> subchannel_;
+};
+
 //
 // Subchannel::ConnectivityStateWatcherList
 //
@@ -452,14 +524,33 @@ void Subchannel::ConnectivityStateWatcherList::RemoveWatcherLocked(
 void Subchannel::ConnectivityStateWatcherList::NotifyLocked(
     grpc_connectivity_state state, const absl::Status& status) {
   for (const auto& watcher : watchers_) {
-    subchannel_->work_serializer_.Run([watcher = watcher->Ref(), state,
-                                       status]() mutable {
-      auto* watcher_ptr = watcher.get();
-      watcher_ptr->OnConnectivityStateChange(std::move(watcher), state, status);
+    subchannel_->work_serializer_.Run([watcher, state, status]() {
+      watcher->OnConnectivityStateChange(state, status);
+    });
+  }
+}
+
+void Subchannel::ConnectivityStateWatcherList::NotifyOnKeepaliveUpdateLocked(
+    Duration new_keepalive_time) {
+  for (const auto& watcher : watchers_) {
+    subchannel_->work_serializer_.Run([watcher, new_keepalive_time]() {
+      watcher->OnKeepaliveUpdate(new_keepalive_time);
     });
   }
 }
 
+uint32_t
+Subchannel::ConnectivityStateWatcherList::GetMaxConnectionsPerSubchannel()
+    const {
+  uint32_t max_connections_per_subchannel = 1;
+  for (const auto& watcher : watchers_) {
+    max_connections_per_subchannel =
+        std::max(max_connections_per_subchannel,
+                 watcher->max_connections_per_subchannel());
+  }
+  return max_connections_per_subchannel;
+}
+
 //
 // Subchannel
 //
@@ -511,6 +602,7 @@ Subchannel::Subchannel(SubchannelKey key,
                                      ? "Subchannel"
                                      : nullptr),
       key_(std::move(key)),
+      created_from_endpoint_(args.Contains(GRPC_ARG_SUBCHANNEL_ENDPOINT)),
       args_(args),
       pollset_set_(grpc_pollset_set_create()),
       connector_(std::move(connector)),
@@ -548,17 +640,15 @@ Subchannel::Subchannel(SubchannelKey key,
         grpc_sockaddr_to_uri(&key_.address())
             .value_or("<unknown address type>"),
         channel_tracer_max_memory);
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_static_string("subchannel created"));
+    GRPC_CHANNELZ_LOG(channelz_node_) << "subchannel created";
+    channelz_node_->SetChannelArgs(args_);
+    args_ = args_.SetObject<channelz::BaseNode>(channelz_node_);
   }
 }
 
 Subchannel::~Subchannel() {
   if (channelz_node_ != nullptr) {
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_static_string("Subchannel destroyed"));
+    GRPC_CHANNELZ_LOG(channelz_node_) << "Subchannel destroyed";
     channelz_node_->UpdateConnectivityState(GRPC_CHANNEL_SHUTDOWN);
   }
   connector_.reset();
@@ -572,12 +662,21 @@ RefCountedPtr<Subchannel> Subchannel::Create(
     const grpc_resolved_address& address, const ChannelArgs& args) {
   SubchannelKey key(address, args);
   auto* subchannel_pool = args.GetObject<SubchannelPoolInterface>();
-  CHECK_NE(subchannel_pool, nullptr);
+  GRPC_CHECK_NE(subchannel_pool, nullptr);
   RefCountedPtr<Subchannel> c = subchannel_pool->FindSubchannel(key);
   if (c != nullptr) {
     return c;
   }
   c = MakeRefCounted<Subchannel>(std::move(key), std::move(connector), args);
+  if (c->created_from_endpoint_) {
+    // We don't interact with the subchannel pool in this case.
+    // Instead, we unconditionally return the newly created subchannel.
+    // Before returning, we explicitly trigger a connection attempt
+    // by calling RequestConnection(), which sets the subchannel’s
+    // connectivity state to CONNECTING.
+    c->RequestConnection();
+    return c;
+  }
   // Try to register the subchannel before setting the subchannel pool.
   // Otherwise, in case of a registration race, unreffing c in
   // RegisterSubchannel() will cause c to be tried to be unregistered, while
@@ -588,15 +687,19 @@ RefCountedPtr<Subchannel> Subchannel::Create(
   return registered;
 }
 
-void Subchannel::ThrottleKeepaliveTime(int new_keepalive_time) {
+void Subchannel::ThrottleKeepaliveTime(Duration new_keepalive_time) {
   MutexLock lock(&mu_);
+  ThrottleKeepaliveTimeLocked(new_keepalive_time);
+}
+
+void Subchannel::ThrottleKeepaliveTimeLocked(Duration new_keepalive_time) {
   // Only update the value if the new keepalive time is larger.
   if (new_keepalive_time > keepalive_time_) {
     keepalive_time_ = new_keepalive_time;
     GRPC_TRACE_LOG(subchannel, INFO)
         << "subchannel " << this << " " << key_.ToString()
         << ": throttling keepalive time to " << new_keepalive_time;
-    args_ = args_.Set(GRPC_ARG_KEEPALIVE_TIME_MS, new_keepalive_time);
+    args_ = args_.Set(GRPC_ARG_KEEPALIVE_TIME_MS, new_keepalive_time.millis());
   }
 }
 
@@ -612,10 +715,8 @@ void Subchannel::WatchConnectivityState(
     grpc_pollset_set_add_pollset_set(pollset_set_, interested_parties);
   }
   work_serializer_.Run(
-      [watcher = watcher->Ref(), state = state_, status = status_]() mutable {
-        auto* watcher_ptr = watcher.get();
-        watcher_ptr->OnConnectivityStateChange(std::move(watcher), state,
-                                               status);
+      [watcher, state = state_, status = status_]() {
+        watcher->OnConnectivityStateChange(state, status);
       },
       DEBUG_LOCATION);
   watcher_list_.AddWatcherLocked(std::move(watcher));
@@ -661,7 +762,7 @@ void Subchannel::Orphaned() {
     subchannel_pool_.reset();
   }
   MutexLock lock(&mu_);
-  CHECK(!shutdown_);
+  GRPC_CHECK(!shutdown_);
   shutdown_ = true;
   connector_.reset();
   connected_subchannel_.reset();
@@ -704,12 +805,15 @@ void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state,
   }
   if (channelz_node_ != nullptr) {
     channelz_node_->UpdateConnectivityState(state);
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_cpp_string(absl::StrCat(
-            "Subchannel connectivity state changed to ",
-            ConnectivityStateName(state),
-            status.ok() ? "" : absl::StrCat(": ", status_.ToString()))));
+    if (status.ok()) {
+      GRPC_CHANNELZ_LOG(channelz_node_)
+          << "Subchannel connectivity state changed to "
+          << ConnectivityStateName(state);
+    } else {
+      GRPC_CHANNELZ_LOG(channelz_node_)
+          << "Subchannel connectivity state changed to "
+          << ConnectivityStateName(state) << ": " << status;
+    }
   }
   // Notify watchers.
   watcher_list_.NotifyLocked(state, status_);
@@ -769,10 +873,15 @@ void Subchannel::OnConnectingFinishedLocked(grpc_error_handle error) {
         next_attempt_time_ - Timestamp::Now();
     GRPC_TRACE_LOG(subchannel, INFO)
         << "subchannel " << this << " " << key_.ToString()
-        << ": connect failed (" << StatusToString(error)
-        << "), backing off for " << time_until_next_attempt.millis() << " ms";
+        << ": connect failed (" << StatusToString(error) << ")"
+        << (created_from_endpoint_
+                ? ", no retry will be attempted (created from endpoint); "
+                  "remaining in TRANSIENT_FAILURE"
+                : ", backing off for " +
+                      std::to_string(time_until_next_attempt.millis()) + " ms");
     SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
                                grpc_error_to_absl_status(error));
+    if (created_from_endpoint_) return;
     retry_timer_handle_ = event_engine_->RunAfter(
         time_until_next_attempt,
         [self = WeakRef(DEBUG_LOCATION, "RetryTimer")]() mutable {
@@ -792,7 +901,12 @@ void Subchannel::OnConnectingFinishedLocked(grpc_error_handle error) {
 }
 
 bool Subchannel::PublishTransportLocked() {
-  auto socket_node = std::move(connecting_result_.socket_node);
+  auto socket_node = connecting_result_.transport->GetSocketNode();
+  if (IsTransportStateWatcherEnabled()) {
+    connecting_result_.transport->StartWatch(
+        MakeRefCounted<ConnectionStateWatcher>(
+            WeakRef(DEBUG_LOCATION, "state_watcher")));
+  }
   if (connecting_result_.transport->filter_stack_transport() != nullptr) {
     // Construct channel stack.
     // Builder takes ownership of transport.
@@ -858,12 +972,15 @@ bool Subchannel::PublishTransportLocked() {
       << "subchannel " << this << " " << key_.ToString()
       << ": new connected subchannel at " << connected_subchannel_.get();
   if (channelz_node_ != nullptr) {
-    channelz_node_->SetChildSocket(std::move(socket_node));
+    if (socket_node != nullptr) {
+      socket_node->AddParent(channelz_node_.get());
+    }
+  }
+  if (!IsTransportStateWatcherEnabled()) {
+    connected_subchannel_->StartWatch(
+        pollset_set_, MakeOrphanable<ConnectedSubchannelStateWatcher>(
+                          WeakRef(DEBUG_LOCATION, "state_watcher")));
   }
-  // Start watching connected subchannel.
-  connected_subchannel_->StartWatch(
-      pollset_set_, MakeOrphanable<ConnectedSubchannelStateWatcher>(
-                        WeakRef(DEBUG_LOCATION, "state_watcher")));
   // Report initial state.
   SetConnectivityStateLocked(GRPC_CHANNEL_READY, absl::Status());
   return true;
@@ -889,6 +1006,8 @@ ChannelArgs Subchannel::MakeSubchannelArgs(
       // uniqueness.
       .Remove(GRPC_ARG_HEALTH_CHECK_SERVICE_NAME)
       .Remove(GRPC_ARG_INHIBIT_HEALTH_CHECKING)
+      .Remove(GRPC_ARG_MAX_CONNECTIONS_PER_SUBCHANNEL)
+      .Remove(GRPC_ARG_MAX_CONNECTIONS_PER_SUBCHANNEL_CAP)
       .Remove(GRPC_ARG_CHANNELZ_CHANNEL_NODE)
       // Remove all keys with the no-subchannel prefix.
       .RemoveAllKeysWithPrefix(GRPC_ARG_NO_SUBCHANNEL_PREFIX);

data/src/core/client_channel/subchannel.h

@@ -26,9 +26,6 @@
 #include <map>
 #include <memory>
 
-#include "absl/base/thread_annotations.h"
-#include "absl/container/flat_hash_set.h"
-#include "absl/status/status.h"
 #include "src/core/call/metadata_batch.h"
 #include "src/core/client_channel/connector.h"
 #include "src/core/client_channel/subchannel_pool_interface.h"
@@ -57,6 +54,14 @@
 #include "src/core/util/time_precise.h"
 #include "src/core/util/unique_type_name.h"
 #include "src/core/util/work_serializer.h"
+#include "absl/base/thread_annotations.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/status/status.h"
+
+/** This arg is intended for internal use only, primarily
+ *  for passing endpoint information during subchannel creation or connection.
+ */
+#define GRPC_ARG_SUBCHANNEL_ENDPOINT "grpc.internal.subchannel_endpoint"
 
 namespace grpc_core {
 
@@ -66,6 +71,7 @@ class ConnectedSubchannel : public RefCounted<ConnectedSubchannel> {
  public:
   const ChannelArgs& args() const { return args_; }
 
+  // TODO(roth): Remove this when transport_state_watcher experiment is removed.
   virtual void StartWatch(
       grpc_pollset_set* interested_parties,
       OrphanablePtr<ConnectivityStateWatcherInterface> watcher) = 0;
@@ -80,6 +86,8 @@ class ConnectedSubchannel : public RefCounted<ConnectedSubchannel> {
   virtual size_t GetInitialCallSizeEstimate() const = 0;
   virtual void Ping(grpc_closure* on_initiate, grpc_closure* on_ack) = 0;
 
+  virtual channelz::SubchannelNode* channelz_node() const = 0;
+
  protected:
   explicit ConnectedSubchannel(const ChannelArgs& args);
 
@@ -160,22 +168,19 @@ class SubchannelCall final {
 // (SubchannelWrapper) that "converts" between the two.
 class Subchannel final : public DualRefCounted<Subchannel> {
  public:
-  // TODO(roth): Once we remove pollset_set, consider whether this can
-  // just use the normal AsyncConnectivityStateWatcherInterface API.
   class ConnectivityStateWatcherInterface
       : public RefCounted<ConnectivityStateWatcherInterface> {
    public:
     // Invoked whenever the subchannel's connectivity state changes.
     // There will be only one invocation of this method on a given watcher
     // instance at any given time.
-    // A ref to the watcher is passed in here so that the implementation
-    // can unref it in the appropriate synchronization context (e.g.,
-    // inside a WorkSerializer).
-    // TODO(roth): Figure out a cleaner way to guarantee that the ref is
-    // released in the right context.
-    virtual void OnConnectivityStateChange(
-        RefCountedPtr<ConnectivityStateWatcherInterface> self,
-        grpc_connectivity_state state, const absl::Status& status) = 0;
+    virtual void OnConnectivityStateChange(grpc_connectivity_state state,
+                                           const absl::Status& status) = 0;
+
+    // Invoked to report updated keepalive time.
+    virtual void OnKeepaliveUpdate(Duration keepalive_time) = 0;
+
+    virtual uint32_t max_connections_per_subchannel() const = 0;
 
     virtual grpc_pollset_set* interested_parties() = 0;
   };
@@ -207,7 +212,8 @@ class Subchannel final : public DualRefCounted<Subchannel> {
   // Throttles keepalive time to \a new_keepalive_time iff \a new_keepalive_time
   // is larger than the subchannel's current keepalive time. The updated value
   // will have an affect when the subchannel creates a new ConnectedSubchannel.
-  void ThrottleKeepaliveTime(int new_keepalive_time) ABSL_LOCKS_EXCLUDED(mu_);
+  void ThrottleKeepaliveTime(Duration new_keepalive_time)
+      ABSL_LOCKS_EXCLUDED(mu_);
 
   grpc_pollset_set* pollset_set() const { return pollset_set_; }
 
@@ -301,10 +307,15 @@ class Subchannel final : public DualRefCounted<Subchannel> {
     void NotifyLocked(grpc_connectivity_state state,
                       const absl::Status& status);
 
+    // Notifies all watchers about a keepalive update.
+    void NotifyOnKeepaliveUpdateLocked(Duration new_keepalive_time);
+
     void Clear() { watchers_.clear(); }
 
     bool empty() const { return watchers_.empty(); }
 
+    uint32_t GetMaxConnectionsPerSubchannel() const;
+
    private:
     Subchannel* subchannel_;
     absl::flat_hash_set<RefCountedPtr<ConnectivityStateWatcherInterface>,
@@ -313,13 +324,19 @@ class Subchannel final : public DualRefCounted<Subchannel> {
         watchers_;
   };
 
+  // TODO(roth): Remove this when transport_state_watcher experiment is removed.
   class ConnectedSubchannelStateWatcher;
 
+  class ConnectionStateWatcher;
+
   // Sets the subchannel's connectivity state to \a state.
   void SetConnectivityStateLocked(grpc_connectivity_state state,
                                   const absl::Status& status)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
+  void ThrottleKeepaliveTimeLocked(Duration new_keepalive_time)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
   // Methods for connection.
   void OnRetryTimer() ABSL_LOCKS_EXCLUDED(mu_);
   void OnRetryTimerLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
@@ -334,6 +351,9 @@ class Subchannel final : public DualRefCounted<Subchannel> {
   RefCountedPtr<SubchannelPoolInterface> subchannel_pool_;
   // Subchannel key that identifies this subchannel in the subchannel pool.
   const SubchannelKey key_;
+  // boolean value that identifies this subchannel is created from event engine
+  // endpoint.
+  const bool created_from_endpoint_;
   // Actual address to connect to.  May be different than the address in
   // key_ if overridden by proxy mapper.
   grpc_resolved_address address_for_connect_;
@@ -379,8 +399,8 @@ class Subchannel final : public DualRefCounted<Subchannel> {
   grpc_event_engine::experimental::EventEngine::TaskHandle retry_timer_handle_
       ABSL_GUARDED_BY(mu_);
 
-  // Keepalive time period (-1 for unset)
-  int keepalive_time_ ABSL_GUARDED_BY(mu_) = -1;
+  // Keepalive time period
+  Duration keepalive_time_ ABSL_GUARDED_BY(mu_);
 
   // Data producer map.
   std::map<UniqueTypeName, DataProducerInterface*> data_producer_map_

data/src/core/client_channel/subchannel_pool_interface.cc

@@ -21,11 +21,11 @@
 #include <grpc/support/port_platform.h>
 #include <string.h>
 
+#include "src/core/lib/address_utils/sockaddr_utils.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
-#include "src/core/lib/address_utils/sockaddr_utils.h"
-#include "src/core/lib/channel/channel_args.h"
 
 // The subchannel pool to reuse subchannels.
 #define GRPC_ARG_SUBCHANNEL_POOL "grpc.internal.subchannel_pool"
@@ -38,13 +38,12 @@ SubchannelKey::SubchannelKey(const grpc_resolved_address& address,
                              const ChannelArgs& args)
     : address_(address), args_(args) {}
 
-bool SubchannelKey::operator<(const SubchannelKey& other) const {
-  if (address_.len < other.address_.len) return true;
-  if (address_.len > other.address_.len) return false;
+int SubchannelKey::Compare(const SubchannelKey& other) const {
+  if (address_.len < other.address_.len) return -1;
+  if (address_.len > other.address_.len) return 1;
   int r = memcmp(address_.addr, other.address_.addr, address_.len);
-  if (r < 0) return true;
-  if (r > 0) return false;
-  return args_ < other.args();
+  if (r != 0) return r;
+  return QsortCompare(args_, other.args_);
 }
 
 std::string SubchannelKey::ToString() const {

data/src/core/client_channel/subchannel_pool_interface.h

@@ -23,13 +23,13 @@
 
 #include <string>
 
-#include "absl/strings/string_view.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/resolved_address.h"
 #include "src/core/util/ref_counted.h"
 #include "src/core/util/ref_counted_ptr.h"
 #include "src/core/util/useful.h"
+#include "absl/strings/string_view.h"
 
 namespace grpc_core {
 
@@ -45,7 +45,17 @@ class SubchannelKey final {
   SubchannelKey(SubchannelKey&& other) noexcept = default;
   SubchannelKey& operator=(SubchannelKey&& other) noexcept = default;
 
-  bool operator<(const SubchannelKey& other) const;
+  bool operator<(const SubchannelKey& other) const {
+    return Compare(other) < 0;
+  }
+  bool operator>(const SubchannelKey& other) const {
+    return Compare(other) > 0;
+  }
+  bool operator==(const SubchannelKey& other) const {
+    return Compare(other) == 0;
+  }
+
+  int Compare(const SubchannelKey& other) const;
 
   const grpc_resolved_address& address() const { return address_; }
   const ChannelArgs& args() const { return args_; }

data/src/core/client_channel/subchannel_stream_client.cc

@@ -23,17 +23,17 @@
 
 #include <utility>
 
-#include "absl/log/check.h"
-#include "absl/log/log.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/resource_quota/resource_quota.h"
 #include "src/core/lib/transport/error_utils.h"
 #include "src/core/util/debug_location.h"
+#include "src/core/util/grpc_check.h"
 #include "src/core/util/status_helper.h"
 #include "src/core/util/sync.h"
 #include "src/core/util/time.h"
 #include "src/core/util/time_precise.h"
+#include "absl/log/log.h"
 
 #define SUBCHANNEL_STREAM_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define SUBCHANNEL_STREAM_RECONNECT_BACKOFF_MULTIPLIER 1.6
@@ -110,7 +110,7 @@ void SubchannelStreamClient::StartCall() {
 
 void SubchannelStreamClient::StartCallLocked() {
   if (event_handler_ == nullptr) return;
-  CHECK(call_state_ == nullptr);
+  GRPC_CHECK(call_state_ == nullptr);
   if (event_handler_ != nullptr) {
     event_handler_->OnCallStartLocked(this);
   }
@@ -222,7 +222,7 @@ void SubchannelStreamClient::CallState::StartCallLocked() {
   send_initial_metadata_.Set(
       HttpPathMetadata(),
       subchannel_stream_client_->event_handler_->GetPathLocked());
-  CHECK(error.ok());
+  GRPC_CHECK(error.ok());
   payload_.send_initial_metadata.send_initial_metadata =
       &send_initial_metadata_;
   batch_.send_initial_metadata = true;
@@ -432,7 +432,7 @@ void SubchannelStreamClient::CallState::CallEndedLocked(bool retry) {
   if (this == subchannel_stream_client_->call_state_.get()) {
     subchannel_stream_client_->call_state_.reset();
     if (retry) {
-      CHECK(subchannel_stream_client_->event_handler_ != nullptr);
+      GRPC_CHECK(subchannel_stream_client_->event_handler_ != nullptr);
       if (seen_response_.load(std::memory_order_acquire)) {
         // If the call fails after we've gotten a successful response, reset
         // the backoff and restart the call immediately.