grpc 1.43.1 → 1.48.0

data/src/core/ext/filters/client_channel/subchannel.cc

@@ -23,35 +23,46 @@
 
 #include <algorithm>
 #include <cstring>
+#include <memory>
+#include <new>
+#include <type_traits>
+#include <utility>
 
-#include "absl/strings/str_format.h"
+#include "absl/status/statusor.h"
 
+#include <grpc/grpc.h>
+#include <grpc/slice.h>
+#include <grpc/status.h>
 #include <grpc/support/alloc.h>
-#include <grpc/support/string_util.h>
+#include <grpc/support/log.h>
 
-#include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/health/health_check_client.h"
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel_pool_interface.h"
-#include "src/core/lib/address_utils/parse_address.h"
+#include "src/core/ext/filters/client_channel/subchannel_stream_client.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/channel/channel_stack_builder_impl.h"
+#include "src/core/lib/channel/channel_trace.h"
+#include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/event_engine_factory.h"
 #include "src/core/lib/gpr/alloc.h"
 #include "src/core/lib/gprpp/debug_location.h"
-#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/surface/channel_stack_type.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/error_utils.h"
-#include "src/core/lib/transport/status_metadata.h"
-#include "src/core/lib/uri/uri_parser.h"
 
 // Strong and weak refs.
 #define INTERNAL_REF_BITS 16
@@ -73,6 +84,7 @@
                     GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(SubchannelCall)))
 
 namespace grpc_core {
+using ::grpc_event_engine::experimental::GetDefaultEventEngine;
 
 TraceFlag grpc_trace_subchannel(false, "subchannel");
 DebugOnlyTraceFlag grpc_trace_subchannel_refcount(false, "subchannel_refcount");
@@ -141,18 +153,18 @@ SubchannelCall::SubchannelCall(Args args, grpc_error_handle* error)
       deadline_(args.deadline) {
   grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(this);
   const grpc_call_element_args call_args = {
-      callstk,           /* call_stack */
-      nullptr,           /* server_transport_data */
-      args.context,      /* context */
-      args.path,         /* path */
-      args.start_time,   /* start_time */
-      args.deadline,     /* deadline */
-      args.arena,        /* arena */
-      args.call_combiner /* call_combiner */
+      callstk,             /* call_stack */
+      nullptr,             /* server_transport_data */
+      args.context,        /* context */
+      args.path.c_slice(), /* path */
+      args.start_time,     /* start_time */
+      args.deadline,       /* deadline */
+      args.arena,          /* arena */
+      args.call_combiner   /* call_combiner */
   };
   *error = grpc_call_stack_init(connected_subchannel_->channel_stack(), 1,
                                 SubchannelCall::Destroy, this, &call_args);
-  if (GPR_UNLIKELY(*error != GRPC_ERROR_NONE)) {
+  if (GPR_UNLIKELY(!GRPC_ERROR_IS_NONE(*error))) {
     gpr_log(GPR_ERROR, "error: %s", grpc_error_std_string(*error).c_str());
     return;
   }
@@ -247,17 +259,12 @@ void SubchannelCall::MaybeInterceptRecvTrailingMetadata(
 namespace {
 
 // Sets *status based on the rest of the parameters.
-void GetCallStatus(grpc_status_code* status, grpc_millis deadline,
+void GetCallStatus(grpc_status_code* status, Timestamp deadline,
                    grpc_metadata_batch* md_batch, grpc_error_handle error) {
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     grpc_error_get_status(error, deadline, status, nullptr, nullptr, nullptr);
   } else {
-    if (md_batch->legacy_index()->named.grpc_status != nullptr) {
-      *status = grpc_get_status_code_from_metadata(
-          md_batch->legacy_index()->named.grpc_status->md);
-    } else {
-      *status = GRPC_STATUS_UNKNOWN;
-    }
+    *status = md_batch->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN);
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -312,43 +319,33 @@ class Subchannel::ConnectedSubchannelStateWatcher
                                  const absl::Status& status) override {
     Subchannel* c = subchannel_.get();
     MutexLock lock(&c->mu_);
-    switch (new_state) {
-      case GRPC_CHANNEL_TRANSIENT_FAILURE:
-      case GRPC_CHANNEL_SHUTDOWN: {
-        if (!c->disconnected_ && c->connected_subchannel_ != nullptr) {
-          if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_subchannel)) {
-            gpr_log(GPR_INFO,
-                    "subchannel %p %s: Connected subchannel %p has gone into "
-                    "%s. Attempting to reconnect.",
-                    c, c->key_.ToString().c_str(),
-                    c->connected_subchannel_.get(),
-                    ConnectivityStateName(new_state));
-          }
-          c->connected_subchannel_.reset();
-          if (c->channelz_node() != nullptr) {
-            c->channelz_node()->SetChildSocket(nullptr);
-          }
-          // We need to construct our own status if the underlying state was
-          // shutdown since the accompanying status will be StatusCode::OK
-          // otherwise.
-          c->SetConnectivityStateLocked(
-              GRPC_CHANNEL_TRANSIENT_FAILURE,
-              new_state == GRPC_CHANNEL_SHUTDOWN
-                  ? absl::Status(absl::StatusCode::kUnavailable,
-                                 "Subchannel has disconnected.")
-                  : status);
-          c->backoff_begun_ = false;
-          c->backoff_.Reset();
-        }
-        break;
+    // If we're either shutting down or have already seen this connection
+    // failure (i.e., c->connected_subchannel_ is null), do nothing.
+    //
+    // The transport reports TRANSIENT_FAILURE upon GOAWAY but SHUTDOWN
+    // upon connection close.  So if the server gracefully shuts down,
+    // we will see TRANSIENT_FAILURE followed by SHUTDOWN, but if not, we
+    // will see only SHUTDOWN.  Either way, we react to the first one we
+    // see, ignoring anything that happens after that.
+    if (c->connected_subchannel_ == nullptr) return;
+    if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE ||
+        new_state == GRPC_CHANNEL_SHUTDOWN) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_subchannel)) {
+        gpr_log(GPR_INFO,
+                "subchannel %p %s: Connected subchannel %p reports %s: %s", c,
+                c->key_.ToString().c_str(), c->connected_subchannel_.get(),
+                ConnectivityStateName(new_state), status.ToString().c_str());
       }
-      default: {
-        // In principle, this should never happen.  We should not get
-        // a callback for READY, because that was the state we started
-        // this watch from.  And a connected subchannel should never go
-        // from READY to CONNECTING or IDLE.
-        c->SetConnectivityStateLocked(new_state, status);
+      c->connected_subchannel_.reset();
+      if (c->channelz_node() != nullptr) {
+        c->channelz_node()->SetChildSocket(nullptr);
       }
+      // Even though we're reporting IDLE instead of TRANSIENT_FAILURE here,
+      // pass along the status from the transport, since it may have
+      // keepalive info attached to it that the channel needs.
+      // TODO(roth): Consider whether there's a cleaner way to do this.
+      c->SetConnectivityStateLocked(GRPC_CHANNEL_IDLE, status);
+      c->backoff_.Reset();
     }
   }
 
@@ -434,11 +431,8 @@ class Subchannel::HealthWatcherMap::HealthWatcher
   grpc_connectivity_state state() const { return state_; }
 
   void AddWatcherLocked(
-      grpc_connectivity_state initial_state,
       RefCountedPtr<Subchannel::ConnectivityStateWatcherInterface> watcher) {
-    if (state_ != initial_state) {
-      new AsyncWatcherNotifierLocked(watcher, state_, status_);
-    }
+    new AsyncWatcherNotifierLocked(watcher, state_, status_);
     watcher_list_.AddWatcherLocked(std::move(watcher));
   }
 
@@ -492,14 +486,14 @@ class Subchannel::HealthWatcherMap::HealthWatcher
   void StartHealthCheckingLocked()
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(subchannel_->mu_) {
     GPR_ASSERT(health_check_client_ == nullptr);
-    health_check_client_ = MakeOrphanable<HealthCheckClient>(
+    health_check_client_ = MakeHealthCheckClient(
         health_check_service_name_, subchannel_->connected_subchannel_,
         subchannel_->pollset_set_, subchannel_->channelz_node_, Ref());
   }
 
   WeakRefCountedPtr<Subchannel> subchannel_;
   std::string health_check_service_name_;
-  OrphanablePtr<HealthCheckClient> health_check_client_;
+  OrphanablePtr<SubchannelStreamClient> health_check_client_;
   grpc_connectivity_state state_;
   absl::Status status_;
   ConnectivityStateWatcherList watcher_list_;
@@ -511,7 +505,6 @@ class Subchannel::HealthWatcherMap::HealthWatcher
 
 void Subchannel::HealthWatcherMap::AddWatcherLocked(
     WeakRefCountedPtr<Subchannel> subchannel,
-    grpc_connectivity_state initial_state,
     const std::string& health_check_service_name,
     RefCountedPtr<ConnectivityStateWatcherInterface> watcher) {
   // If the health check service name is not already present in the map,
@@ -527,7 +520,7 @@ void Subchannel::HealthWatcherMap::AddWatcherLocked(
     health_watcher = it->second.get();
   }
   // Add the watcher to the entry.
-  health_watcher->AddWatcherLocked(initial_state, std::move(watcher));
+  health_watcher->AddWatcherLocked(std::move(watcher));
 }
 
 void Subchannel::HealthWatcherMap::RemoveWatcherLocked(
@@ -567,77 +560,84 @@ Subchannel::HealthWatcherMap::CheckConnectivityStateLocked(
 
 void Subchannel::HealthWatcherMap::ShutdownLocked() { map_.clear(); }
 
+//
+// Subchannel::ConnectivityStateWatcherInterface
+//
+
+void Subchannel::ConnectivityStateWatcherInterface::PushConnectivityStateChange(
+    ConnectivityStateChange state_change) {
+  MutexLock lock(&mu_);
+  connectivity_state_queue_.push_back(std::move(state_change));
+}
+
+Subchannel::ConnectivityStateWatcherInterface::ConnectivityStateChange
+Subchannel::ConnectivityStateWatcherInterface::PopConnectivityStateChange() {
+  MutexLock lock(&mu_);
+  GPR_ASSERT(!connectivity_state_queue_.empty());
+  ConnectivityStateChange state_change = connectivity_state_queue_.front();
+  connectivity_state_queue_.pop_front();
+  return state_change;
+}
+
 //
 // Subchannel
 //
 
 namespace {
 
-BackOff::Options ParseArgsForBackoffValues(
-    const grpc_channel_args* args, grpc_millis* min_connect_timeout_ms) {
-  grpc_millis initial_backoff_ms =
-      GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS * 1000;
-  *min_connect_timeout_ms =
-      GRPC_SUBCHANNEL_RECONNECT_MIN_TIMEOUT_SECONDS * 1000;
-  grpc_millis max_backoff_ms =
-      GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000;
+BackOff::Options ParseArgsForBackoffValues(const grpc_channel_args* args,
+                                           Duration* min_connect_timeout) {
+  Duration initial_backoff =
+      Duration::Seconds(GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS);
+  *min_connect_timeout =
+      Duration::Seconds(GRPC_SUBCHANNEL_RECONNECT_MIN_TIMEOUT_SECONDS);
+  Duration max_backoff =
+      Duration::Seconds(GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS);
   bool fixed_reconnect_backoff = false;
   if (args != nullptr) {
     for (size_t i = 0; i < args->num_args; i++) {
       if (0 == strcmp(args->args[i].key,
                       "grpc.testing.fixed_reconnect_backoff_ms")) {
         fixed_reconnect_backoff = true;
-        initial_backoff_ms = *min_connect_timeout_ms = max_backoff_ms =
-            grpc_channel_arg_get_integer(
+        initial_backoff = *min_connect_timeout = max_backoff =
+            Duration::Milliseconds(grpc_channel_arg_get_integer(
                 &args->args[i],
-                {static_cast<int>(initial_backoff_ms), 100, INT_MAX});
+                {static_cast<int>(initial_backoff.millis()), 100, INT_MAX}));
       } else if (0 ==
                  strcmp(args->args[i].key, GRPC_ARG_MIN_RECONNECT_BACKOFF_MS)) {
         fixed_reconnect_backoff = false;
-        *min_connect_timeout_ms = grpc_channel_arg_get_integer(
-            &args->args[i],
-            {static_cast<int>(*min_connect_timeout_ms), 100, INT_MAX});
+        *min_connect_timeout =
+            Duration::Milliseconds(grpc_channel_arg_get_integer(
+                &args->args[i],
+                {static_cast<int>(min_connect_timeout->millis()), 100,
+                 INT_MAX}));
       } else if (0 ==
                  strcmp(args->args[i].key, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS)) {
         fixed_reconnect_backoff = false;
-        max_backoff_ms = grpc_channel_arg_get_integer(
-            &args->args[i], {static_cast<int>(max_backoff_ms), 100, INT_MAX});
+        max_backoff = Duration::Milliseconds(grpc_channel_arg_get_integer(
+            &args->args[i],
+            {static_cast<int>(max_backoff.millis()), 100, INT_MAX}));
       } else if (0 == strcmp(args->args[i].key,
                              GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS)) {
         fixed_reconnect_backoff = false;
-        initial_backoff_ms = grpc_channel_arg_get_integer(
+        initial_backoff = Duration::Milliseconds(grpc_channel_arg_get_integer(
             &args->args[i],
-            {static_cast<int>(initial_backoff_ms), 100, INT_MAX});
+            {static_cast<int>(initial_backoff.millis()), 100, INT_MAX}));
       }
     }
   }
   return BackOff::Options()
-      .set_initial_backoff(initial_backoff_ms)
+      .set_initial_backoff(initial_backoff)
       .set_multiplier(fixed_reconnect_backoff
                           ? 1.0
                           : GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER)
       .set_jitter(fixed_reconnect_backoff ? 0.0
                                           : GRPC_SUBCHANNEL_RECONNECT_JITTER)
-      .set_max_backoff(max_backoff_ms);
+      .set_max_backoff(max_backoff);
 }
 
 }  // namespace
 
-void Subchannel::ConnectivityStateWatcherInterface::PushConnectivityStateChange(
-    ConnectivityStateChange state_change) {
-  MutexLock lock(&mu_);
-  connectivity_state_queue_.push_back(std::move(state_change));
-}
-
-Subchannel::ConnectivityStateWatcherInterface::ConnectivityStateChange
-Subchannel::ConnectivityStateWatcherInterface::PopConnectivityStateChange() {
-  MutexLock lock(&mu_);
-  GPR_ASSERT(!connectivity_state_queue_.empty());
-  ConnectivityStateChange state_change = connectivity_state_queue_.front();
-  connectivity_state_queue_.pop_front();
-  return state_change;
-}
-
 Subchannel::Subchannel(SubchannelKey key,
                        OrphanablePtr<SubchannelConnector> connector,
                        const grpc_channel_args* args)
@@ -647,7 +647,16 @@ Subchannel::Subchannel(SubchannelKey key,
       key_(std::move(key)),
       pollset_set_(grpc_pollset_set_create()),
       connector_(std::move(connector)),
-      backoff_(ParseArgsForBackoffValues(args, &min_connect_timeout_ms_)) {
+      backoff_(ParseArgsForBackoffValues(args, &min_connect_timeout_)) {
+  // A grpc_init is added here to ensure that grpc_shutdown does not happen
+  // until the subchannel is destroyed. Subchannels can persist longer than
+  // channels because they maybe reused/shared among multiple channels. As a
+  // result the subchannel destruction happens asynchronously to channel
+  // destruction. If the last channel destruction triggers a grpc_shutdown
+  // before the last subchannel destruction, then there maybe race conditions
+  // triggering segmentation faults. To prevent this issue, we call a grpc_init
+  // here and a grpc_shutdown in the subchannel destructor.
+  grpc_init();
   GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED();
   GRPC_CLOSURE_INIT(&on_connecting_finished_, OnConnectingFinished, this,
                     grpc_schedule_on_exec_ctx);
@@ -677,7 +686,9 @@ Subchannel::Subchannel(SubchannelKey key,
             {GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT, 0,
              INT_MAX}));
     channelz_node_ = MakeRefCounted<channelz::SubchannelNode>(
-        grpc_sockaddr_to_uri(&key_.address()), channel_tracer_max_memory);
+        grpc_sockaddr_to_uri(&key_.address())
+            .value_or("<unknown address type>"),
+        channel_tracer_max_memory);
     channelz_node_->AddTraceEvent(
         channelz::ChannelTrace::Severity::Info,
         grpc_slice_from_static_string("subchannel created"));
@@ -694,6 +705,8 @@ Subchannel::~Subchannel() {
   grpc_channel_args_destroy(args_);
   connector_.reset();
   grpc_pollset_set_destroy(pollset_set_);
+  // grpc_shutdown is called here because grpc_init is called in the ctor.
+  grpc_shutdown();
 }
 
 RefCountedPtr<Subchannel> Subchannel::Create(
@@ -741,18 +754,7 @@ channelz::SubchannelNode* Subchannel::channelz_node() {
   return channelz_node_.get();
 }
 
-grpc_connectivity_state Subchannel::CheckConnectivityState(
-    const absl::optional<std::string>& health_check_service_name) {
-  MutexLock lock(&mu_);
-  if (health_check_service_name.has_value()) {
-    return health_watcher_map_.CheckConnectivityStateLocked(
-        this, *health_check_service_name);
-  }
-  return state_;
-}
-
 void Subchannel::WatchConnectivityState(
-    grpc_connectivity_state initial_state,
     const absl::optional<std::string>& health_check_service_name,
     RefCountedPtr<ConnectivityStateWatcherInterface> watcher) {
   MutexLock lock(&mu_);
@@ -761,14 +763,12 @@ void Subchannel::WatchConnectivityState(
     grpc_pollset_set_add_pollset_set(pollset_set_, interested_parties);
   }
   if (!health_check_service_name.has_value()) {
-    if (state_ != initial_state) {
-      new AsyncWatcherNotifierLocked(watcher, state_, status_);
-    }
+    new AsyncWatcherNotifierLocked(watcher, state_, status_);
     watcher_list_.AddWatcherLocked(std::move(watcher));
   } else {
     health_watcher_map_.AddWatcherLocked(
-        WeakRef(DEBUG_LOCATION, "health_watcher"), initial_state,
-        *health_check_service_name, std::move(watcher));
+        WeakRef(DEBUG_LOCATION, "health_watcher"), *health_check_service_name,
+        std::move(watcher));
   }
 }
 
@@ -788,20 +788,25 @@ void Subchannel::CancelConnectivityStateWatch(
   }
 }
 
-void Subchannel::AttemptToConnect() {
+void Subchannel::RequestConnection() {
   MutexLock lock(&mu_);
-  MaybeStartConnectingLocked();
+  if (state_ == GRPC_CHANNEL_IDLE) {
+    StartConnectingLocked();
+  }
 }
 
 void Subchannel::ResetBackoff() {
+  // Hold a ref to ensure cancellation and subsequent deletion of the closure
+  // does not eliminate the last ref and destroy the Subchannel before the
+  // method returns.
+  auto self = WeakRef(DEBUG_LOCATION, "ResetBackoff");
   MutexLock lock(&mu_);
   backoff_.Reset();
-  if (have_retry_alarm_) {
-    retry_immediately_ = true;
-    grpc_timer_cancel(&retry_alarm_);
-  } else {
-    backoff_begun_ = false;
-    MaybeStartConnectingLocked();
+  if (state_ == GRPC_CHANNEL_TRANSIENT_FAILURE &&
+      GetDefaultEventEngine()->Cancel(retry_timer_handle_)) {
+    OnRetryTimerLocked();
+  } else if (state_ == GRPC_CHANNEL_CONNECTING) {
+    next_attempt_time_ = ExecCtx::Get()->Now();
   }
 }
 
@@ -813,13 +818,36 @@ void Subchannel::Orphan() {
     subchannel_pool_.reset();
   }
   MutexLock lock(&mu_);
-  GPR_ASSERT(!disconnected_);
-  disconnected_ = true;
+  GPR_ASSERT(!shutdown_);
+  shutdown_ = true;
   connector_.reset();
   connected_subchannel_.reset();
   health_watcher_map_.ShutdownLocked();
 }
 
+void Subchannel::AddDataProducer(DataProducerInterface* data_producer) {
+  MutexLock lock(&mu_);
+  auto& entry = data_producer_map_[data_producer->type()];
+  GPR_ASSERT(entry == nullptr);
+  entry = data_producer;
+}
+
+void Subchannel::RemoveDataProducer(DataProducerInterface* data_producer) {
+  MutexLock lock(&mu_);
+  auto it = data_producer_map_.find(data_producer->type());
+  GPR_ASSERT(it != data_producer_map_.end());
+  GPR_ASSERT(it->second == data_producer);
+  data_producer_map_.erase(it);
+}
+
+Subchannel::DataProducerInterface* Subchannel::GetDataProducer(
+    UniqueTypeName type) {
+  MutexLock lock(&mu_);
+  auto it = data_producer_map_.find(type);
+  if (it == data_producer_map_.end()) return nullptr;
+  return it->second;
+}
+
 namespace {
 
 // Returns a string indicating the subchannel's connectivity state change to
@@ -861,78 +889,31 @@ void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state,
   health_watcher_map_.NotifyLocked(state, status);
 }
 
-void Subchannel::MaybeStartConnectingLocked() {
-  if (disconnected_) {
-    // Don't try to connect if we're already disconnected.
-    return;
-  }
-  if (connecting_) {
-    // Already connecting: don't restart.
-    return;
-  }
-  if (connected_subchannel_ != nullptr) {
-    // Already connected: don't restart.
-    return;
-  }
-  connecting_ = true;
-  WeakRef(DEBUG_LOCATION, "connecting")
-      .release();  // ref held by pending connect
-  if (!backoff_begun_) {
-    backoff_begun_ = true;
-    ContinueConnectingLocked();
-  } else {
-    GPR_ASSERT(!have_retry_alarm_);
-    have_retry_alarm_ = true;
-    const grpc_millis time_til_next =
-        next_attempt_deadline_ - ExecCtx::Get()->Now();
-    if (time_til_next <= 0) {
-      gpr_log(GPR_INFO, "subchannel %p %s: Retry immediately", this,
-              key_.ToString().c_str());
-    } else {
-      gpr_log(GPR_INFO, "subchannel %p %s: Retry in %" PRId64 " milliseconds",
-              this, key_.ToString().c_str(), time_til_next);
-    }
-    GRPC_CLOSURE_INIT(&on_retry_alarm_, OnRetryAlarm, this,
-                      grpc_schedule_on_exec_ctx);
-    grpc_timer_init(&retry_alarm_, next_attempt_deadline_, &on_retry_alarm_);
-  }
+void Subchannel::OnRetryTimer() {
+  MutexLock lock(&mu_);
+  OnRetryTimerLocked();
 }
 
-void Subchannel::OnRetryAlarm(void* arg, grpc_error_handle error) {
-  WeakRefCountedPtr<Subchannel> c(static_cast<Subchannel*>(arg));
-  MutexLock lock(&c->mu_);
-  c->have_retry_alarm_ = false;
-  if (c->disconnected_) {
-    error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected",
-                                                             &error, 1);
-  } else if (c->retry_immediately_) {
-    c->retry_immediately_ = false;
-    error = GRPC_ERROR_NONE;
-  } else {
-    (void)GRPC_ERROR_REF(error);
-  }
-  if (error == GRPC_ERROR_NONE) {
-    gpr_log(GPR_INFO,
-            "subchannel %p %s: failed to connect to channel, retrying", c.get(),
-            c->key_.ToString().c_str());
-    c->ContinueConnectingLocked();
-    // Still connecting, keep ref around. Note that this stolen ref won't
-    // be dropped without first acquiring c->mu_.
-    c.release();
-  }
-  GRPC_ERROR_UNREF(error);
+void Subchannel::OnRetryTimerLocked() {
+  if (shutdown_) return;
+  gpr_log(GPR_INFO, "subchannel %p %s: backoff delay elapsed, reporting IDLE",
+          this, key_.ToString().c_str());
+  SetConnectivityStateLocked(GRPC_CHANNEL_IDLE, absl::OkStatus());
 }
 
-void Subchannel::ContinueConnectingLocked() {
+void Subchannel::StartConnectingLocked() {
+  // Set next attempt time.
+  const Timestamp min_deadline = min_connect_timeout_ + ExecCtx::Get()->Now();
+  next_attempt_time_ = backoff_.NextAttemptTime();
+  // Report CONNECTING.
+  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING, absl::OkStatus());
+  // Start connection attempt.
   SubchannelConnector::Args args;
   args.address = &address_for_connect_;
   args.interested_parties = pollset_set_;
-  const grpc_millis min_deadline =
-      min_connect_timeout_ms_ + ExecCtx::Get()->Now();
-  next_attempt_deadline_ = backoff_.NextAttemptTime();
-  args.deadline = std::max(next_attempt_deadline_, min_deadline);
+  args.deadline = std::max(next_attempt_time_, min_deadline);
   args.channel_args = args_;
-  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING, absl::Status());
+  WeakRef(DEBUG_LOCATION, "Connect").release();  // Ref held by callback.
   connector_->Connect(args, &connecting_result_, &on_connecting_finished_);
 }
 
@@ -942,48 +923,63 @@ void Subchannel::OnConnectingFinished(void* arg, grpc_error_handle error) {
       c->connecting_result_.channel_args;
   {
     MutexLock lock(&c->mu_);
-    c->connecting_ = false;
-    if (c->connecting_result_.transport != nullptr &&
-        c->PublishTransportLocked()) {
-      // Do nothing, transport was published.
-    } else if (!c->disconnected_) {
-      gpr_log(GPR_INFO, "subchannel %p %s: connect failed: %s", c.get(),
-              c->key_.ToString().c_str(), grpc_error_std_string(error).c_str());
-      c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                    grpc_error_to_absl_status(error));
-    }
+    c->OnConnectingFinishedLocked(GRPC_ERROR_REF(error));
   }
   grpc_channel_args_destroy(delete_channel_args);
-  c.reset(DEBUG_LOCATION, "connecting");
+  c.reset(DEBUG_LOCATION, "Connect");
 }
 
-namespace {
-
-void ConnectionDestroy(void* arg, grpc_error_handle /*error*/) {
-  grpc_channel_stack* stk = static_cast<grpc_channel_stack*>(arg);
-  grpc_channel_stack_destroy(stk);
-  gpr_free(stk);
+void Subchannel::OnConnectingFinishedLocked(grpc_error_handle error) {
+  if (shutdown_) {
+    (void)GRPC_ERROR_UNREF(error);
+    return;
+  }
+  // If we didn't get a transport or we fail to publish it, report
+  // TRANSIENT_FAILURE and start the retry timer.
+  // Note that if the connection attempt took longer than the backoff
+  // time, then the timer will fire immediately, and we will quickly
+  // transition back to IDLE.
+  if (connecting_result_.transport == nullptr || !PublishTransportLocked()) {
+    const Duration time_until_next_attempt =
+        next_attempt_time_ - ExecCtx::Get()->Now();
+    gpr_log(GPR_INFO,
+            "subchannel %p %s: connect failed (%s), backing off for %" PRId64
+            " ms",
+            this, key_.ToString().c_str(), grpc_error_std_string(error).c_str(),
+            time_until_next_attempt.millis());
+    SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
+                               grpc_error_to_absl_status(error));
+    retry_timer_handle_ = GetDefaultEventEngine()->RunAfter(
+        time_until_next_attempt,
+        [self = WeakRef(DEBUG_LOCATION, "RetryTimer")]() mutable {
+          {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            self->OnRetryTimer();
+            // Subchannel deletion might require an active ExecCtx. So if
+            // self.reset() is not called here, the WeakRefCountedPtr destructor
+            // may run after the ExecCtx declared in the callback is destroyed.
+            // Since subchannel may get destroyed when the WeakRefCountedPtr
+            // destructor runs, it may not have an active ExecCtx - thus leading
+            // to crashes.
+            self.reset();
+          }
+        });
+  }
+  (void)GRPC_ERROR_UNREF(error);
 }
 
-}  // namespace
-
 bool Subchannel::PublishTransportLocked() {
   // Construct channel stack.
-  grpc_channel_stack_builder* builder = grpc_channel_stack_builder_create();
-  grpc_channel_stack_builder_set_channel_arguments(
-      builder, connecting_result_.channel_args);
-  grpc_channel_stack_builder_set_transport(builder,
-                                           connecting_result_.transport);
-  if (!CoreConfiguration::Get().channel_init().CreateStack(
-          builder, GRPC_CLIENT_SUBCHANNEL)) {
-    grpc_channel_stack_builder_destroy(builder);
+  ChannelStackBuilderImpl builder("subchannel", GRPC_CLIENT_SUBCHANNEL);
+  builder.SetChannelArgs(ChannelArgs::FromC(connecting_result_.channel_args))
+      .SetTransport(connecting_result_.transport);
+  if (!CoreConfiguration::Get().channel_init().CreateStack(&builder)) {
     return false;
   }
-  grpc_channel_stack* stk;
-  grpc_error_handle error = grpc_channel_stack_builder_finish(
-      builder, 0, 1, ConnectionDestroy, nullptr,
-      reinterpret_cast<void**>(&stk));
-  if (error != GRPC_ERROR_NONE) {
+  absl::StatusOr<RefCountedPtr<grpc_channel_stack>> stk = builder.Build();
+  if (!stk.ok()) {
+    auto error = absl_status_to_grpc_error(stk.status());
     grpc_transport_destroy(connecting_result_.transport);
     gpr_log(GPR_ERROR,
             "subchannel %p %s: error initializing subchannel stack: %s", this,
@@ -994,14 +990,10 @@ bool Subchannel::PublishTransportLocked() {
   RefCountedPtr<channelz::SocketNode> socket =
       std::move(connecting_result_.socket_node);
   connecting_result_.Reset();
-  if (disconnected_) {
-    grpc_channel_stack_destroy(stk);
-    gpr_free(stk);
-    return false;
-  }
+  if (shutdown_) return false;
   // Publish.
   connected_subchannel_.reset(
-      new ConnectedSubchannel(stk, args_, channelz_node_));
+      new ConnectedSubchannel(stk->release(), args_, channelz_node_));
   if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_subchannel)) {
     gpr_log(GPR_INFO, "subchannel %p %s: new connected subchannel at %p", this,
             key_.ToString().c_str(), connected_subchannel_.get());