grpc 1.21.0 → 1.32.0

data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -38,9 +38,9 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
-#include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/error_utils.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 namespace grpc_core {
@@ -63,8 +63,6 @@ class RoundRobin : public LoadBalancingPolicy {
 
   void UpdateLocked(UpdateArgs args) override;
   void ResetBackoffLocked() override;
-  void FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
-                                channelz::ChildRefsList* ignored) override;
 
  private:
   ~RoundRobin();
@@ -83,22 +81,29 @@ class RoundRobin : public LoadBalancingPolicy {
     RoundRobinSubchannelData(
         SubchannelList<RoundRobinSubchannelList, RoundRobinSubchannelData>*
             subchannel_list,
-        const ServerAddress& address, Subchannel* subchannel,
-        grpc_combiner* combiner)
-        : SubchannelData(subchannel_list, address, subchannel, combiner) {}
+        const ServerAddress& address,
+        RefCountedPtr<SubchannelInterface> subchannel)
+        : SubchannelData(subchannel_list, address, std::move(subchannel)) {}
 
     grpc_connectivity_state connectivity_state() const {
       return last_connectivity_state_;
     }
 
+    bool seen_failure_since_ready() const { return seen_failure_since_ready_; }
+
+    // Performs connectivity state updates that need to be done both when we
+    // first start watching and when a watcher notification is received.
     void UpdateConnectivityStateLocked(
         grpc_connectivity_state connectivity_state);
 
    private:
+    // Performs connectivity state updates that need to be done only
+    // after we have started watching.
     void ProcessConnectivityChangeLocked(
         grpc_connectivity_state connectivity_state) override;
 
     grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_IDLE;
+    bool seen_failure_since_ready_ = false;
   };
 
   // A list of subchannels.
@@ -108,9 +113,8 @@ class RoundRobin : public LoadBalancingPolicy {
    public:
     RoundRobinSubchannelList(RoundRobin* policy, TraceFlag* tracer,
                              const ServerAddressList& addresses,
-                             grpc_combiner* combiner,
                              const grpc_channel_args& args)
-        : SubchannelList(policy, tracer, addresses, combiner,
+        : SubchannelList(policy, tracer, addresses,
                          policy->channel_control_helper(), args) {
       // Need to maintain a ref to the LB policy as long as we maintain
       // any references to subchannels, since the subchannels'
@@ -150,32 +154,18 @@ class RoundRobin : public LoadBalancingPolicy {
    public:
     Picker(RoundRobin* parent, RoundRobinSubchannelList* subchannel_list);
 
-    PickResult Pick(PickArgs* pick, grpc_error** error) override;
+    PickResult Pick(PickArgs args) override;
 
    private:
     // Using pointer value only, no ref held -- do not dereference!
     RoundRobin* parent_;
 
     size_t last_picked_index_;
-    InlinedVector<RefCountedPtr<ConnectedSubchannel>, 10> subchannels_;
-  };
-
-  // Helper class to ensure that any function that modifies the child refs
-  // data structures will update the channelz snapshot data structures before
-  // returning.
-  class AutoChildRefsUpdater {
-   public:
-    explicit AutoChildRefsUpdater(RoundRobin* rr) : rr_(rr) {}
-    ~AutoChildRefsUpdater() { rr_->UpdateChildRefsLocked(); }
-
-   private:
-    RoundRobin* rr_;
+    absl::InlinedVector<RefCountedPtr<SubchannelInterface>, 10> subchannels_;
   };
 
   void ShutdownLocked() override;
 
-  void UpdateChildRefsLocked();
-
   /** list of subchannels */
   OrphanablePtr<RoundRobinSubchannelList> subchannel_list_;
   /** Latest version of the subchannel list.
@@ -186,11 +176,6 @@ class RoundRobin : public LoadBalancingPolicy {
   OrphanablePtr<RoundRobinSubchannelList> latest_pending_subchannel_list_;
   /** are we shutting down? */
   bool shutdown_ = false;
-  /// Lock and data used to capture snapshots of this channel's child
-  /// channels and subchannels. This data is consumed by channelz.
-  Mutex child_refs_mu_;
-  channelz::ChildRefsList child_subchannels_;
-  channelz::ChildRefsList child_channels_;
 };
 
 //
@@ -201,10 +186,9 @@ RoundRobin::Picker::Picker(RoundRobin* parent,
                            RoundRobinSubchannelList* subchannel_list)
     : parent_(parent) {
   for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
-    auto* connected_subchannel =
-        subchannel_list->subchannel(i)->connected_subchannel();
-    if (connected_subchannel != nullptr) {
-      subchannels_.push_back(connected_subchannel->Ref());
+    RoundRobinSubchannelData* sd = subchannel_list->subchannel(i);
+    if (sd->connectivity_state() == GRPC_CHANNEL_READY) {
+      subchannels_.push_back(sd->subchannel()->Ref());
     }
   }
   // For discussion on why we generate a random starting index for
@@ -221,18 +205,18 @@ RoundRobin::Picker::Picker(RoundRobin* parent,
   }
 }
 
-RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs* pick,
-                                                grpc_error** error) {
+RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs /*args*/) {
   last_picked_index_ = (last_picked_index_ + 1) % subchannels_.size();
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO,
-            "[RR %p picker %p] returning index %" PRIuPTR
-            ", connected_subchannel=%p",
+            "[RR %p picker %p] returning index %" PRIuPTR ", subchannel=%p",
             parent_, this, last_picked_index_,
             subchannels_[last_picked_index_].get());
   }
-  pick->connected_subchannel = subchannels_[last_picked_index_];
-  return PICK_COMPLETE;
+  PickResult result;
+  result.type = PickResult::PICK_COMPLETE;
+  result.subchannel = subchannels_[last_picked_index_];
+  return result;
 }
 
 //
@@ -254,7 +238,6 @@ RoundRobin::~RoundRobin() {
 }
 
 void RoundRobin::ShutdownLocked() {
-  AutoChildRefsUpdater guard(this);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] Shutting down", this);
   }
@@ -270,40 +253,6 @@ void RoundRobin::ResetBackoffLocked() {
   }
 }
 
-void RoundRobin::FillChildRefsForChannelz(
-    channelz::ChildRefsList* child_subchannels_to_fill,
-    channelz::ChildRefsList* ignored) {
-  MutexLock lock(&child_refs_mu_);
-  for (size_t i = 0; i < child_subchannels_.size(); ++i) {
-    // TODO(ncteisen): implement a de dup loop that is not O(n^2). Might
-    // have to implement lightweight set. For now, we don't care about
-    // performance when channelz requests are made.
-    bool found = false;
-    for (size_t j = 0; j < child_subchannels_to_fill->size(); ++j) {
-      if ((*child_subchannels_to_fill)[j] == child_subchannels_[i]) {
-        found = true;
-        break;
-      }
-    }
-    if (!found) {
-      child_subchannels_to_fill->push_back(child_subchannels_[i]);
-    }
-  }
-}
-
-void RoundRobin::UpdateChildRefsLocked() {
-  channelz::ChildRefsList cs;
-  if (subchannel_list_ != nullptr) {
-    subchannel_list_->PopulateChildRefsList(&cs);
-  }
-  if (latest_pending_subchannel_list_ != nullptr) {
-    latest_pending_subchannel_list_->PopulateChildRefsList(&cs);
-  }
-  // atomically update the data that channelz will actually be looking at.
-  MutexLock lock(&child_refs_mu_);
-  child_subchannels_ = std::move(cs);
-}
-
 void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
   if (num_subchannels() == 0) return;
   // Check current state of each subchannel synchronously, since any
@@ -320,6 +269,7 @@ void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
   for (size_t i = 0; i < num_subchannels(); i++) {
     if (subchannel(i)->subchannel() != nullptr) {
       subchannel(i)->StartConnectivityWatchLocked();
+      subchannel(i)->subchannel()->AttemptToConnect();
     }
   }
   // Now set the LB policy's state based on the subchannels' states.
@@ -373,12 +323,12 @@ void RoundRobin::RoundRobinSubchannelList::
   if (num_ready_ > 0) {
     /* 1) READY */
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_READY, UniquePtr<SubchannelPicker>(New<Picker>(p, this)));
+        GRPC_CHANNEL_READY, absl::Status(), absl::make_unique<Picker>(p, this));
   } else if (num_connecting_ > 0) {
     /* 2) CONNECTING */
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_CONNECTING,
-        UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
+        GRPC_CHANNEL_CONNECTING, absl::Status(),
+        absl::make_unique<QueuePicker>(p->Ref(DEBUG_LOCATION, "QueuePicker")));
   } else if (num_transient_failure_ == num_subchannels()) {
     /* 3) TRANSIENT_FAILURE */
     grpc_error* error =
@@ -386,15 +336,14 @@ void RoundRobin::RoundRobinSubchannelList::
                                "connections to all backends failing"),
                            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE,
-        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
+        absl::make_unique<TransientFailurePicker>(error));
   }
 }
 
 void RoundRobin::RoundRobinSubchannelList::
     UpdateRoundRobinStateFromSubchannelStateCountsLocked() {
   RoundRobin* p = static_cast<RoundRobin*>(policy());
-  AutoChildRefsUpdater guard(p);
   if (num_ready_ > 0) {
     if (p->subchannel_list_.get() != this) {
       // Promote this list to p->subchannel_list_.
@@ -431,11 +380,28 @@ void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
         "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s",
         p, subchannel(), subchannel_list(), Index(),
         subchannel_list()->num_subchannels(),
-        grpc_connectivity_state_name(last_connectivity_state_),
-        grpc_connectivity_state_name(connectivity_state));
+        ConnectivityStateName(last_connectivity_state_),
+        ConnectivityStateName(connectivity_state));
   }
-  subchannel_list()->UpdateStateCountersLocked(last_connectivity_state_,
-                                               connectivity_state);
+  // Decide what state to report for aggregation purposes.
+  // If we haven't seen a failure since the last time we were in state
+  // READY, then we report the state change as-is.  However, once we do see
+  // a failure, we report TRANSIENT_FAILURE and do not report any subsequent
+  // state changes until we go back into state READY.
+  if (!seen_failure_since_ready_) {
+    if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      seen_failure_since_ready_ = true;
+    }
+    subchannel_list()->UpdateStateCountersLocked(last_connectivity_state_,
+                                                 connectivity_state);
+  } else {
+    if (connectivity_state == GRPC_CHANNEL_READY) {
+      seen_failure_since_ready_ = false;
+      subchannel_list()->UpdateStateCountersLocked(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, connectivity_state);
+    }
+  }
+  // Record last seen connectivity state.
   last_connectivity_state_ = connectivity_state;
 }
 
@@ -448,6 +414,7 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
   // Otherwise, if the subchannel was already in state TRANSIENT_FAILURE
   // when the subchannel list was created, we'd wind up in a constant
   // loop of re-resolution.
+  // Also attempt to reconnect.
   if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
       gpr_log(GPR_INFO,
@@ -456,9 +423,8 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
               p, subchannel());
     }
     p->channel_control_helper()->RequestReresolution();
+    subchannel()->AttemptToConnect();
   }
-  // Renew connectivity watch.
-  RenewConnectivityWatchLocked();
   // Update state counters.
   UpdateConnectivityStateLocked(connectivity_state);
   // Update overall state and renew notification.
@@ -466,7 +432,6 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
 }
 
 void RoundRobin::UpdateLocked(UpdateArgs args) {
-  AutoChildRefsUpdater guard(this);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
             this, args.addresses.size());
@@ -480,7 +445,7 @@ void RoundRobin::UpdateLocked(UpdateArgs args) {
     }
   }
   latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
-      this, &grpc_lb_round_robin_trace, args.addresses, combiner(), *args.args);
+      this, &grpc_lb_round_robin_trace, args.addresses, *args.args);
   if (latest_pending_subchannel_list_->num_subchannels() == 0) {
     // If the new list is empty, immediately promote the new list to the
     // current list and transition to TRANSIENT_FAILURE.
@@ -488,8 +453,8 @@ void RoundRobin::UpdateLocked(UpdateArgs args) {
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
                            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE,
-        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_to_absl_status(error),
+        absl::make_unique<TransientFailurePicker>(error));
     subchannel_list_ = std::move(latest_pending_subchannel_list_);
   } else if (subchannel_list_ == nullptr) {
     // If there is no current list, immediately promote the new list to
@@ -503,7 +468,7 @@ void RoundRobin::UpdateLocked(UpdateArgs args) {
   }
 }
 
-class ParsedRoundRobinConfig : public ParsedLoadBalancingConfig {
+class RoundRobinConfig : public LoadBalancingPolicy::Config {
  public:
   const char* name() const override { return kRoundRobin; }
 };
@@ -516,18 +481,14 @@ class RoundRobinFactory : public LoadBalancingPolicyFactory {
  public:
   OrphanablePtr<LoadBalancingPolicy> CreateLoadBalancingPolicy(
       LoadBalancingPolicy::Args args) const override {
-    return OrphanablePtr<LoadBalancingPolicy>(New<RoundRobin>(std::move(args)));
+    return MakeOrphanable<RoundRobin>(std::move(args));
   }
 
   const char* name() const override { return kRoundRobin; }
 
-  RefCountedPtr<ParsedLoadBalancingConfig> ParseLoadBalancingConfig(
-      const grpc_json* json, grpc_error** error) const override {
-    if (json != nullptr) {
-      GPR_DEBUG_ASSERT(strcmp(json->key, name()) == 0);
-    }
-    return RefCountedPtr<ParsedLoadBalancingConfig>(
-        New<ParsedRoundRobinConfig>());
+  RefCountedPtr<LoadBalancingPolicy::Config> ParseLoadBalancingConfig(
+      const Json& /*json*/, grpc_error** /*error*/) const override {
+    return MakeRefCounted<RoundRobinConfig>();
   }
 };
 
@@ -538,8 +499,7 @@ class RoundRobinFactory : public LoadBalancingPolicyFactory {
 void grpc_lb_policy_round_robin_init() {
   grpc_core::LoadBalancingPolicyRegistry::Builder::
       RegisterLoadBalancingPolicyFactory(
-          grpc_core::UniquePtr<grpc_core::LoadBalancingPolicyFactory>(
-              grpc_core::New<grpc_core::RoundRobinFactory>()));
+          absl::make_unique<grpc_core::RoundRobinFactory>());
 }
 
 void grpc_lb_policy_round_robin_shutdown() {}

data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h

@@ -25,18 +25,20 @@
 
 #include <grpc/support/alloc.h>
 
+#include "absl/container/inlined_vector.h"
+
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/server_address.h"
+// TODO(roth): Should not need the include of subchannel.h here, since
+// that implementation should be hidden from the LB policy API.
 #include "src/core/ext/filters/client_channel/subchannel.h"
+#include "src/core/ext/filters/client_channel/subchannel_interface.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/abstract.h"
-#include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/closure.h"
-#include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
@@ -61,8 +63,7 @@ class MySubchannelList
 };
 
 */
-// All methods with a Locked() suffix must be called from within the
-// client_channel combiner.
+// All methods will be called from within the client_channel work serializer.
 
 namespace grpc_core {
 
@@ -88,25 +89,16 @@ class SubchannelData {
   }
 
   // Returns a pointer to the subchannel.
-  Subchannel* subchannel() const { return subchannel_; }
-
-  // Returns the connected subchannel.  Will be null if the subchannel
-  // is not connected.
-  ConnectedSubchannel* connected_subchannel() const {
-    return connected_subchannel_.get();
-  }
+  SubchannelInterface* subchannel() const { return subchannel_.get(); }
 
   // Synchronously checks the subchannel's connectivity state.
   // Must not be called while there is a connectivity notification
-  // pending (i.e., between calling StartConnectivityWatchLocked() or
-  // RenewConnectivityWatchLocked() and the resulting invocation of
-  // ProcessConnectivityChangeLocked()).
+  // pending (i.e., between calling StartConnectivityWatchLocked() and
+  // calling CancelConnectivityWatchLocked()).
   grpc_connectivity_state CheckConnectivityStateLocked() {
-    GPR_ASSERT(!connectivity_notification_pending_);
-    pending_connectivity_state_unsafe_ = subchannel()->CheckConnectivity(
-        subchannel_list_->inhibit_health_checking());
-    UpdateConnectedSubchannelLocked();
-    return pending_connectivity_state_unsafe_;
+    GPR_ASSERT(pending_watcher_ == nullptr);
+    connectivity_state_ = subchannel_->CheckConnectivityState();
+    return connectivity_state_;
   }
 
   // Resets the connection backoff.
@@ -115,78 +107,73 @@ class SubchannelData {
   void ResetBackoffLocked();
 
   // Starts watching the connectivity state of the subchannel.
-  // ProcessConnectivityChangeLocked() will be called when the
+  // ProcessConnectivityChangeLocked() will be called whenever the
   // connectivity state changes.
   void StartConnectivityWatchLocked();
 
-  // Renews watching the connectivity state of the subchannel.
-  void RenewConnectivityWatchLocked();
-
-  // Stops watching the connectivity state of the subchannel.
-  void StopConnectivityWatchLocked();
-
   // Cancels watching the connectivity state of the subchannel.
-  // Must be called only while there is a connectivity notification
-  // pending (i.e., between calling StartConnectivityWatchLocked() or
-  // RenewConnectivityWatchLocked() and the resulting invocation of
-  // ProcessConnectivityChangeLocked()).
-  // From within ProcessConnectivityChangeLocked(), use
-  // StopConnectivityWatchLocked() instead.
   void CancelConnectivityWatchLocked(const char* reason);
 
   // Cancels any pending connectivity watch and unrefs the subchannel.
   void ShutdownLocked();
 
-  GRPC_ABSTRACT_BASE_CLASS
-
  protected:
   SubchannelData(
       SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
-      const ServerAddress& address, Subchannel* subchannel,
-      grpc_combiner* combiner);
+      const ServerAddress& address,
+      RefCountedPtr<SubchannelInterface> subchannel);
 
   virtual ~SubchannelData();
 
-  // After StartConnectivityWatchLocked() or RenewConnectivityWatchLocked()
-  // is called, this method will be invoked when the subchannel's connectivity
-  // state changes.
-  // Implementations must invoke either RenewConnectivityWatchLocked() or
-  // StopConnectivityWatchLocked() before returning.
+  // After StartConnectivityWatchLocked() is called, this method will be
+  // invoked whenever the subchannel's connectivity state changes.
+  // To stop watching, use CancelConnectivityWatchLocked().
   virtual void ProcessConnectivityChangeLocked(
-      grpc_connectivity_state connectivity_state) GRPC_ABSTRACT;
-
-  // Unrefs the subchannel.
-  void UnrefSubchannelLocked(const char* reason);
+      grpc_connectivity_state connectivity_state) = 0;
 
  private:
-  // Updates connected_subchannel_ based on pending_connectivity_state_unsafe_.
-  // Returns true if the connectivity state should be reported.
-  bool UpdateConnectedSubchannelLocked();
+  // Watcher for subchannel connectivity state.
+  class Watcher
+      : public SubchannelInterface::ConnectivityStateWatcherInterface {
+   public:
+    Watcher(
+        SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data,
+        RefCountedPtr<SubchannelListType> subchannel_list)
+        : subchannel_data_(subchannel_data),
+          subchannel_list_(std::move(subchannel_list)) {}
+
+    ~Watcher() { subchannel_list_.reset(DEBUG_LOCATION, "Watcher dtor"); }
+
+    void OnConnectivityStateChange(grpc_connectivity_state new_state) override;
+
+    grpc_pollset_set* interested_parties() override {
+      return subchannel_list_->policy()->interested_parties();
+    }
 
-  static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+   private:
+    SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data_;
+    RefCountedPtr<SubchannelListType> subchannel_list_;
+  };
+
+  // Unrefs the subchannel.
+  void UnrefSubchannelLocked(const char* reason);
 
   // Backpointer to owning subchannel list.  Not owned.
   SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list_;
-
-  // The subchannel and connected subchannel.
-  Subchannel* subchannel_;
-  RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
-
-  // Notification that connectivity has changed on subchannel.
-  grpc_closure connectivity_changed_closure_;
-  // Is a connectivity notification pending?
-  bool connectivity_notification_pending_ = false;
-  // Connectivity state to be updated by
-  // grpc_subchannel_notify_on_state_change(), not guarded by
-  // the combiner.
-  grpc_connectivity_state pending_connectivity_state_unsafe_;
+  // The subchannel.
+  RefCountedPtr<SubchannelInterface> subchannel_;
+  // Will be non-null when the subchannel's state is being watched.
+  SubchannelInterface::ConnectivityStateWatcherInterface* pending_watcher_ =
+      nullptr;
+  // Data updated by the watcher.
+  grpc_connectivity_state connectivity_state_;
 };
 
 // A list of subchannels.
 template <typename SubchannelListType, typename SubchannelDataType>
 class SubchannelList : public InternallyRefCounted<SubchannelListType> {
  public:
-  typedef InlinedVector<SubchannelDataType, 10> SubchannelVector;
+  typedef absl::InlinedVector<SubchannelDataType, 10> SubchannelVector;
 
   // The number of subchannels in the list.
   size_t num_subchannels() const { return subchannels_.size(); }
@@ -197,50 +184,29 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
   // Returns true if the subchannel list is shutting down.
   bool shutting_down() const { return shutting_down_; }
 
-  // Populates refs_list with the uuids of this SubchannelLists's subchannels.
-  void PopulateChildRefsList(channelz::ChildRefsList* refs_list) {
-    for (size_t i = 0; i < subchannels_.size(); ++i) {
-      if (subchannels_[i].subchannel() != nullptr) {
-        grpc_core::channelz::SubchannelNode* subchannel_node =
-            subchannels_[i].subchannel()->channelz_node();
-        if (subchannel_node != nullptr) {
-          refs_list->push_back(subchannel_node->uuid());
-        }
-      }
-    }
-  }
-
   // Accessors.
   LoadBalancingPolicy* policy() const { return policy_; }
   TraceFlag* tracer() const { return tracer_; }
-  bool inhibit_health_checking() const { return inhibit_health_checking_; }
 
   // Resets connection backoff of all subchannels.
   // TODO(roth): We will probably need to rethink this as part of moving
   // the backoff code out of subchannels and into LB policies.
   void ResetBackoffLocked();
 
-  // Note: Caller must ensure that this is invoked inside of the combiner.
   void Orphan() override {
     ShutdownLocked();
     InternallyRefCounted<SubchannelListType>::Unref(DEBUG_LOCATION, "shutdown");
   }
 
-  GRPC_ABSTRACT_BASE_CLASS
-
  protected:
   SubchannelList(LoadBalancingPolicy* policy, TraceFlag* tracer,
-                 const ServerAddressList& addresses, grpc_combiner* combiner,
+                 const ServerAddressList& addresses,
                  LoadBalancingPolicy::ChannelControlHelper* helper,
                  const grpc_channel_args& args);
 
   virtual ~SubchannelList();
 
  private:
-  // So New() can call our private ctor.
-  template <typename T, typename... Args>
-  friend T* New(Args&&... args);
-
   // For accessing Ref() and Unref().
   friend class SubchannelData<SubchannelListType, SubchannelDataType>;
 
@@ -251,10 +217,6 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 
   TraceFlag* tracer_;
 
-  bool inhibit_health_checking_;
-
-  grpc_combiner* combiner_;
-
   // The list of subchannels.
   SubchannelVector subchannels_;
 
@@ -268,6 +230,33 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 // implementation -- no user-servicable parts below
 //
 
+//
+// SubchannelData::Watcher
+//
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::Watcher::
+    OnConnectivityStateChange(grpc_connectivity_state new_state) {
+  if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): connectivity changed: state=%s, "
+            "shutting_down=%d, pending_watcher=%p",
+            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
+            subchannel_list_.get(), subchannel_data_->Index(),
+            subchannel_list_->num_subchannels(),
+            subchannel_data_->subchannel_.get(),
+            ConnectivityStateName(new_state), subchannel_list_->shutting_down(),
+            subchannel_data_->pending_watcher_);
+  }
+  if (!subchannel_list_->shutting_down() &&
+      subchannel_data_->pending_watcher_ != nullptr) {
+    subchannel_data_->connectivity_state_ = new_state;
+    // Call the subclass's ProcessConnectivityChangeLocked() method.
+    subchannel_data_->ProcessConnectivityChangeLocked(new_state);
+  }
+}
+
 //
 // SubchannelData
 //
@@ -275,23 +264,17 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelData<SubchannelListType, SubchannelDataType>::SubchannelData(
     SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
-    const ServerAddress& address, Subchannel* subchannel,
-    grpc_combiner* combiner)
+    const ServerAddress& /*address*/,
+    RefCountedPtr<SubchannelInterface> subchannel)
     : subchannel_list_(subchannel_list),
-      subchannel_(subchannel),
+      subchannel_(std::move(subchannel)),
       // We assume that the current state is IDLE.  If not, we'll get a
       // callback telling us that.
-      pending_connectivity_state_unsafe_(GRPC_CHANNEL_IDLE) {
-  GRPC_CLOSURE_INIT(
-      &connectivity_changed_closure_,
-      (&SubchannelData<SubchannelListType,
-                       SubchannelDataType>::OnConnectivityChangedLocked),
-      this, grpc_combiner_scheduler(combiner));
-}
+      connectivity_state_(GRPC_CHANNEL_IDLE) {}
 
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelData<SubchannelListType, SubchannelDataType>::~SubchannelData() {
-  UnrefSubchannelLocked("subchannel_data_destroy");
+  GPR_ASSERT(subchannel_ == nullptr);
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
@@ -301,14 +284,12 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::
     if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
       gpr_log(GPR_INFO,
               "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-              " (subchannel %p): unreffing subchannel",
+              " (subchannel %p): unreffing subchannel (%s)",
               subchannel_list_->tracer()->name(), subchannel_list_->policy(),
               subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-              subchannel_);
+              subchannel_.get(), reason);
     }
-    GRPC_SUBCHANNEL_UNREF(subchannel_, reason);
-    subchannel_ = nullptr;
-    connected_subchannel_.reset();
+    subchannel_.reset();
   }
 }
 
@@ -326,56 +307,18 @@ void SubchannelData<SubchannelListType,
   if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
     gpr_log(GPR_INFO,
             "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): starting watch: requesting connectivity change "
-            "notification (from %s)",
+            " (subchannel %p): starting watch (from %s)",
             subchannel_list_->tracer()->name(), subchannel_list_->policy(),
             subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_,
-            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
+            subchannel_.get(), ConnectivityStateName(connectivity_state_));
   }
-  GPR_ASSERT(!connectivity_notification_pending_);
-  connectivity_notification_pending_ = true;
-  subchannel_list()->Ref(DEBUG_LOCATION, "connectivity_watch").release();
-  subchannel_->NotifyOnStateChange(
-      subchannel_list_->policy()->interested_parties(),
-      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_,
-      subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType,
-                    SubchannelDataType>::RenewConnectivityWatchLocked() {
-  if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
-    gpr_log(GPR_INFO,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): renewing watch: requesting connectivity change "
-            "notification (from %s)",
-            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_,
-            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
-  }
-  GPR_ASSERT(connectivity_notification_pending_);
-  subchannel_->NotifyOnStateChange(
-      subchannel_list_->policy()->interested_parties(),
-      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_,
-      subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType,
-                    SubchannelDataType>::StopConnectivityWatchLocked() {
-  if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
-    gpr_log(GPR_INFO,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): stopping connectivity watch",
-            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_);
-  }
-  GPR_ASSERT(connectivity_notification_pending_);
-  connectivity_notification_pending_ = false;
-  subchannel_list()->Unref(DEBUG_LOCATION, "connectivity_watch");
+  GPR_ASSERT(pending_watcher_ == nullptr);
+  pending_watcher_ =
+      new Watcher(this, subchannel_list()->Ref(DEBUG_LOCATION, "Watcher"));
+  subchannel_->WatchConnectivityState(
+      connectivity_state_,
+      std::unique_ptr<SubchannelInterface::ConnectivityStateWatcherInterface>(
+          pending_watcher_));
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
@@ -387,93 +330,18 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::
             " (subchannel %p): canceling connectivity watch (%s)",
             subchannel_list_->tracer()->name(), subchannel_list_->policy(),
             subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_, reason);
+            subchannel_.get(), reason);
   }
-  GPR_ASSERT(connectivity_notification_pending_);
-  subchannel_->NotifyOnStateChange(nullptr, nullptr,
-                                   &connectivity_changed_closure_,
-                                   subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-bool SubchannelData<SubchannelListType,
-                    SubchannelDataType>::UpdateConnectedSubchannelLocked() {
-  // If the subchannel is READY, take a ref to the connected subchannel.
-  if (pending_connectivity_state_unsafe_ == GRPC_CHANNEL_READY) {
-    connected_subchannel_ = subchannel_->connected_subchannel();
-    // If the subchannel became disconnected between the time that READY
-    // was reported and the time we got here (e.g., between when a
-    // notification callback is scheduled and when it was actually run in
-    // the combiner), then the connected subchannel may have disappeared out
-    // from under us.  In that case, we don't actually want to consider the
-    // subchannel to be in state READY.  Instead, we use IDLE as the
-    // basis for any future connectivity watch; this is the one state that
-    // the subchannel will never transition back into, so this ensures
-    // that we will get a notification for the next state, even if that state
-    // is READY again (e.g., if the subchannel has transitioned back to
-    // READY before the next watch gets requested).
-    if (connected_subchannel_ == nullptr) {
-      if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
-        gpr_log(GPR_INFO,
-                "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-                " (subchannel %p): state is READY but connected subchannel is "
-                "null; moving to state IDLE",
-                subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-                subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-                subchannel_);
-      }
-      pending_connectivity_state_unsafe_ = GRPC_CHANNEL_IDLE;
-      return false;
-    }
-  } else {
-    // For any state other than READY, unref the connected subchannel.
-    connected_subchannel_.reset();
+  if (pending_watcher_ != nullptr) {
+    subchannel_->CancelConnectivityStateWatch(pending_watcher_);
+    pending_watcher_ = nullptr;
   }
-  return true;
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType, SubchannelDataType>::
-    OnConnectivityChangedLocked(void* arg, grpc_error* error) {
-  SubchannelData* sd = static_cast<SubchannelData*>(arg);
-  if (GRPC_TRACE_FLAG_ENABLED(*sd->subchannel_list_->tracer())) {
-    gpr_log(
-        GPR_INFO,
-        "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-        " (subchannel %p): connectivity changed: state=%s, error=%s, "
-        "shutting_down=%d",
-        sd->subchannel_list_->tracer()->name(), sd->subchannel_list_->policy(),
-        sd->subchannel_list_, sd->Index(),
-        sd->subchannel_list_->num_subchannels(), sd->subchannel_,
-        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe_),
-        grpc_error_string(error), sd->subchannel_list_->shutting_down());
-  }
-  // If shutting down, unref subchannel and stop watching.
-  if (sd->subchannel_list_->shutting_down() || error == GRPC_ERROR_CANCELLED) {
-    sd->UnrefSubchannelLocked("connectivity_shutdown");
-    sd->StopConnectivityWatchLocked();
-    return;
-  }
-  // Get or release ref to connected subchannel.
-  if (!sd->UpdateConnectedSubchannelLocked()) {
-    // We don't want to report this connectivity state, so renew the watch.
-    sd->RenewConnectivityWatchLocked();
-    return;
-  }
-  // Call the subclass's ProcessConnectivityChangeLocked() method.
-  sd->ProcessConnectivityChangeLocked(sd->pending_connectivity_state_unsafe_);
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelData<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
-  // If there's a pending notification for this subchannel, cancel it;
-  // the callback is responsible for unreffing the subchannel.
-  // Otherwise, unref the subchannel directly.
-  if (connectivity_notification_pending_) {
-    CancelConnectivityWatchLocked("shutdown");
-  } else if (subchannel_ != nullptr) {
-    UnrefSubchannelLocked("shutdown");
-  }
+  if (pending_watcher_ != nullptr) CancelConnectivityWatchLocked("shutdown");
+  UnrefSubchannelLocked("shutdown");
 }
 
 //
@@ -483,13 +351,12 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
     LoadBalancingPolicy* policy, TraceFlag* tracer,
-    const ServerAddressList& addresses, grpc_combiner* combiner,
+    const ServerAddressList& addresses,
     LoadBalancingPolicy::ChannelControlHelper* helper,
     const grpc_channel_args& args)
     : InternallyRefCounted<SubchannelListType>(tracer),
       policy_(policy),
-      tracer_(tracer),
-      combiner_(GRPC_COMBINER_REF(combiner, "subchannel_list")) {
+      tracer_(tracer) {
   if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO,
             "[%s %p] Creating subchannel list %p for %" PRIuPTR " subchannels",
@@ -498,22 +365,13 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
   subchannels_.reserve(addresses.size());
   // We need to remove the LB addresses in order to be able to compare the
   // subchannel keys of subchannels from a different batch of addresses.
-  // We also remove the inhibit-health-checking arg, since we are
-  // handling that here.
-  inhibit_health_checking_ = grpc_channel_arg_get_bool(
-      grpc_channel_args_find(&args, GRPC_ARG_INHIBIT_HEALTH_CHECKING), false);
+  // We remove the service config, since it will be passed into the
+  // subchannel via call context.
   static const char* keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
-                                         GRPC_ARG_INHIBIT_HEALTH_CHECKING};
+                                         GRPC_ARG_SERVICE_CONFIG};
   // Create a subchannel for each address.
   for (size_t i = 0; i < addresses.size(); i++) {
-    // TODO(roth): we should ideally hide this from the LB policy code. In
-    // principle, if we're dealing with this special case in the client_channel
-    // code for selecting grpclb, then we should also strip out these addresses
-    // there if we're not using grpclb.
-    if (addresses[i].IsBalancer()) {
-      continue;
-    }
-    InlinedVector<grpc_arg, 3> args_to_add;
+    absl::InlinedVector<grpc_arg, 3> args_to_add;
     const size_t subchannel_address_arg_index = args_to_add.size();
     args_to_add.emplace_back(
         Subchannel::CreateSubchannelAddressArg(&addresses[i].address()));
@@ -526,30 +384,29 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
         &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove),
         args_to_add.data(), args_to_add.size());
     gpr_free(args_to_add[subchannel_address_arg_index].value.string);
-    Subchannel* subchannel = helper->CreateSubchannel(*new_args);
+    RefCountedPtr<SubchannelInterface> subchannel =
+        helper->CreateSubchannel(*new_args);
     grpc_channel_args_destroy(new_args);
     if (subchannel == nullptr) {
       // Subchannel could not be created.
       if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
-        char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
         gpr_log(GPR_INFO,
                 "[%s %p] could not create subchannel for address uri %s, "
                 "ignoring",
-                tracer_->name(), policy_, address_uri);
-        gpr_free(address_uri);
+                tracer_->name(), policy_,
+                grpc_sockaddr_to_uri(&addresses[i].address()).c_str());
       }
       continue;
     }
     if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
-      char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
       gpr_log(GPR_INFO,
               "[%s %p] subchannel list %p index %" PRIuPTR
               ": Created subchannel %p for address uri %s",
-              tracer_->name(), policy_, this, subchannels_.size(), subchannel,
-              address_uri);
-      gpr_free(address_uri);
+              tracer_->name(), policy_, this, subchannels_.size(),
+              subchannel.get(),
+              grpc_sockaddr_to_uri(&addresses[i].address()).c_str());
     }
-    subchannels_.emplace_back(this, addresses[i], subchannel, combiner);
+    subchannels_.emplace_back(this, addresses[i], std::move(subchannel));
   }
 }
 
@@ -559,7 +416,6 @@ SubchannelList<SubchannelListType, SubchannelDataType>::~SubchannelList() {
     gpr_log(GPR_INFO, "[%s %p] Destroying subchannel_list %p", tracer_->name(),
             policy_, this);
   }
-  GRPC_COMBINER_UNREF(combiner_, "subchannel_list");
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>