grpc 1.43.1 → 1.48.0

data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc

@@ -18,23 +18,47 @@
 
 #include <inttypes.h>
 #include <limits.h>
-
+#include <stddef.h>
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
-#include "absl/strings/str_format.h"
+#include "absl/strings/string_view.h"
 
-#include <grpc/grpc.h>
+#include <grpc/impl/codegen/connectivity_state.h>
+#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/support/log.h>
 
 #include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h"
 #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h"
 #include "src/core/ext/filters/client_channel/lb_policy_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
+#include "src/core/ext/filters/client_channel/subchannel_interface.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/work_serializer.h"
-#include "src/core/lib/transport/error_utils.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/resolver/server_address.h"
+#include "src/core/lib/transport/connectivity_state.h"
 
 namespace grpc_core {
 
@@ -47,11 +71,11 @@ constexpr char kPriority[] = "priority_experimental";
 // How long we keep a child around for after it is no longer being used
 // (either because it has been removed from the config or because we
 // have switched to a higher-priority child).
-constexpr int kChildRetentionIntervalMs = 15 * 60 * 1000;
+constexpr Duration kChildRetentionInterval = Duration::Minutes(15);
 
 // Default for how long we wait for a newly created child to get connected
 // before starting to attempt the next priority.  Overridable via channel arg.
-constexpr int kDefaultChildFailoverTimeoutMs = 10000;
+constexpr Duration kDefaultChildFailoverTimeout = Duration::Seconds(10);
 
 // Config for priority LB policy.
 class PriorityLbConfig : public LoadBalancingPolicy::Config {
@@ -104,15 +128,12 @@ class PriorityLb : public LoadBalancingPolicy {
                       bool ignore_reresolution_requests);
     void ExitIdleLocked();
     void ResetBackoffLocked();
-    void DeactivateLocked();
+    void MaybeDeactivateLocked();
     void MaybeReactivateLocked();
-    void MaybeCancelFailoverTimerLocked();
 
     void Orphan() override;
 
-    std::unique_ptr<SubchannelPicker> GetPicker() {
-      return absl::make_unique<RefCountedPickerWrapper>(picker_wrapper_);
-    }
+    std::unique_ptr<SubchannelPicker> GetPicker();
 
     grpc_connectivity_state connectivity_state() const {
       return connectivity_state_;
@@ -122,9 +143,7 @@ class PriorityLb : public LoadBalancingPolicy {
       return connectivity_status_;
     }
 
-    bool failover_timer_callback_pending() const {
-      return failover_timer_callback_pending_;
-    }
+    bool FailoverTimerPending() const { return failover_timer_ != nullptr; }
 
    private:
     // A simple wrapper for ref-counting a picker from the child policy.
@@ -170,6 +189,38 @@ class PriorityLb : public LoadBalancingPolicy {
       RefCountedPtr<ChildPriority> priority_;
     };
 
+    class DeactivationTimer : public InternallyRefCounted<DeactivationTimer> {
+     public:
+      explicit DeactivationTimer(RefCountedPtr<ChildPriority> child_priority);
+
+      void Orphan() override;
+
+     private:
+      static void OnTimer(void* arg, grpc_error_handle error);
+      void OnTimerLocked(grpc_error_handle);
+
+      RefCountedPtr<ChildPriority> child_priority_;
+      grpc_timer timer_;
+      grpc_closure on_timer_;
+      bool timer_pending_ = true;
+    };
+
+    class FailoverTimer : public InternallyRefCounted<FailoverTimer> {
+     public:
+      explicit FailoverTimer(RefCountedPtr<ChildPriority> child_priority);
+
+      void Orphan() override;
+
+     private:
+      static void OnTimer(void* arg, grpc_error_handle error);
+      void OnTimerLocked(grpc_error_handle);
+
+      RefCountedPtr<ChildPriority> child_priority_;
+      grpc_timer timer_;
+      grpc_closure on_timer_;
+      bool timer_pending_ = true;
+    };
+
     // Methods for dealing with the child policy.
     OrphanablePtr<LoadBalancingPolicy> CreateChildPolicyLocked(
         const grpc_channel_args* args);
@@ -178,13 +229,6 @@ class PriorityLb : public LoadBalancingPolicy {
         grpc_connectivity_state state, const absl::Status& status,
         std::unique_ptr<SubchannelPicker> picker);
 
-    void StartFailoverTimerLocked();
-
-    static void OnFailoverTimer(void* arg, grpc_error_handle error);
-    void OnFailoverTimerLocked(grpc_error_handle error);
-    static void OnDeactivationTimer(void* arg, grpc_error_handle error);
-    void OnDeactivationTimerLocked(grpc_error_handle error);
-
     RefCountedPtr<PriorityLb> priority_policy_;
     const std::string name_;
     bool ignore_reresolution_requests_ = false;
@@ -195,40 +239,60 @@ class PriorityLb : public LoadBalancingPolicy {
     absl::Status connectivity_status_;
     RefCountedPtr<RefCountedPicker> picker_wrapper_;
 
-    // States for delayed removal.
-    grpc_timer deactivation_timer_;
-    grpc_closure on_deactivation_timer_;
-    bool deactivation_timer_callback_pending_ = false;
+    bool seen_ready_or_idle_since_transient_failure_ = true;
 
-    // States of failover.
-    grpc_timer failover_timer_;
-    grpc_closure on_failover_timer_;
-    bool failover_timer_callback_pending_ = false;
+    OrphanablePtr<DeactivationTimer> deactivation_timer_;
+    OrphanablePtr<FailoverTimer> failover_timer_;
   };
 
   ~PriorityLb() override;
 
   void ShutdownLocked() override;
 
-  // Returns UINT32_MAX if child is not in current priority list.
+  // Returns the priority of the specified child name, or UINT32_MAX if
+  // the child is not in the current priority list.
   uint32_t GetChildPriorityLocked(const std::string& child_name) const;
 
+  // Called when a child's connectivity state has changed.
+  // May propagate the update to the channel or trigger choosing a new
+  // priority.
   void HandleChildConnectivityStateChangeLocked(ChildPriority* child);
-  void DeleteChild(ChildPriority* child);
 
-  void TryNextPriorityLocked(bool report_connecting);
-  void SelectPriorityLocked(uint32_t priority);
+  // Deletes a child.  Called when the child's deactivation timer fires.
+  void DeleteChild(ChildPriority* child);
 
-  const int child_failover_timeout_ms_;
+  // Iterates through the list of priorities to choose one:
+  // - If the child for a priority doesn't exist, creates it.
+  // - If a child's failover timer is pending, selects that priority
+  //   while we wait for the child to attempt to connect.
+  // - If the child is connected, selects that priority.
+  // - Otherwise, continues on to the next child.
+  // Delegates to the last child if none of the children are connecting.
+  // Reports TRANSIENT_FAILURE if the priority list is empty.
+  //
+  // This method is idempotent; it should yield the same result every
+  // time as a function of the state of the children.
+  void ChoosePriorityLocked();
+
+  // Sets the specified priority as the current priority.
+  // Deactivates any children at lower priorities.
+  // Returns the child's picker to the channel.
+  void SetCurrentPriorityLocked(uint32_t priority);
+
+  const Duration child_failover_timeout_;
 
   // Current channel args and config from the resolver.
   const grpc_channel_args* args_ = nullptr;
   RefCountedPtr<PriorityLbConfig> config_;
-  HierarchicalAddressMap addresses_;
+  absl::StatusOr<HierarchicalAddressMap> addresses_;
 
   // Internal state.
   bool shutting_down_ = false;
 
+  bool update_in_progress_ = false;
+
+  // All children that currently exist.
+  // Some of these children may be in deactivated state.
   std::map<std::string, OrphanablePtr<ChildPriority>> children_;
   // The priority that is being used.
   uint32_t current_priority_ = UINT32_MAX;
@@ -244,9 +308,11 @@ class PriorityLb : public LoadBalancingPolicy {
 
 PriorityLb::PriorityLb(Args args)
     : LoadBalancingPolicy(std::move(args)),
-      child_failover_timeout_ms_(grpc_channel_args_find_integer(
-          args.args, GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS,
-          {kDefaultChildFailoverTimeoutMs, 0, INT_MAX})) {
+      child_failover_timeout_(
+          Duration::Milliseconds(grpc_channel_args_find_integer(
+              args.args, GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS,
+              {static_cast<int>(kDefaultChildFailoverTimeout.millis()), 0,
+               INT_MAX}))) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] created", this);
   }
@@ -290,12 +356,11 @@ void PriorityLb::UpdateLocked(UpdateArgs args) {
   // Save current child.
   if (current_priority_ != UINT32_MAX) {
     const std::string& child_name = config_->priorities()[current_priority_];
-    current_child_from_before_update_ = children_[child_name].get();
-    // Unset current_priority_, since it was an index into the old
-    // config's priority list and may no longer be valid.  It will be
-    // reset later by TryNextPriorityLocked(), but we unset it here in
-    // case updating any of our children triggers a state update.
-    current_priority_ = UINT32_MAX;
+    auto* child = children_[child_name].get();
+    GPR_ASSERT(child != nullptr);
+    if (child->connectivity_state() == GRPC_CHANNEL_READY) {
+      current_child_from_before_update_ = children_[child_name].get();
+    }
   }
   // Update config.
   config_ = std::move(args.config);
@@ -306,21 +371,23 @@ void PriorityLb::UpdateLocked(UpdateArgs args) {
   // Update addresses.
   addresses_ = MakeHierarchicalAddressMap(args.addresses);
   // Check all existing children against the new config.
+  update_in_progress_ = true;
   for (const auto& p : children_) {
     const std::string& child_name = p.first;
     auto& child = p.second;
     auto config_it = config_->children().find(child_name);
     if (config_it == config_->children().end()) {
       // Existing child not found in new config.  Deactivate it.
-      child->DeactivateLocked();
+      child->MaybeDeactivateLocked();
     } else {
       // Existing child found in new config.  Update it.
       child->UpdateLocked(config_it->second.config,
                           config_it->second.ignore_reresolution_requests);
     }
   }
+  update_in_progress_ = false;
   // Try to get connected.
-  TryNextPriorityLocked(/*report_connecting=*/children_.empty());
+  ChoosePriorityLocked();
 }
 
 uint32_t PriorityLb::GetChildPriorityLocked(
@@ -334,6 +401,12 @@ uint32_t PriorityLb::GetChildPriorityLocked(
 
 void PriorityLb::HandleChildConnectivityStateChangeLocked(
     ChildPriority* child) {
+  // If we're in the process of propagating an update from our parent to
+  // our children, ignore any updates that come from the children.  We
+  // will instead choose a new priority once the update has been seen by
+  // all children.  This ensures that we don't incorrectly do the wrong
+  // thing while state is inconsistent.
+  if (update_in_progress_) return;
   // Special case for the child that was the current child before the
   // most recent update.
   if (child == current_child_from_before_update_) {
@@ -353,11 +426,11 @@ void PriorityLb::HandleChildConnectivityStateChangeLocked(
     } else {
       // If it's no longer READY or IDLE, we should stop using it.
       // We already started trying other priorities as a result of the
-      // update, but calling TryNextPriorityLocked() ensures that we will
+      // update, but calling ChoosePriorityLocked() ensures that we will
       // properly select between CONNECTING and TRANSIENT_FAILURE as the
       // new state to report to our parent.
       current_child_from_before_update_ = nullptr;
-      TryNextPriorityLocked(/*report_connecting=*/true);
+      ChoosePriorityLocked();
     }
     return;
   }
@@ -369,52 +442,37 @@ void PriorityLb::HandleChildConnectivityStateChangeLocked(
             "priority %u",
             this, child_priority, child->name().c_str(), current_priority_);
   }
-  // Ignore priorities not in the current config.
-  if (child_priority == UINT32_MAX) return;
-  // Ignore lower-than-current priorities.
-  if (child_priority > current_priority_) return;
-  // If a child reports TRANSIENT_FAILURE, start trying the next priority.
-  // Note that even if this is for a higher-than-current priority, we
-  // may still need to create some children between this priority and
-  // the current one (e.g., if we got an update that inserted new
-  // priorities ahead of the current one).
-  if (child->connectivity_state() == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    TryNextPriorityLocked(
-        /*report_connecting=*/child_priority == current_priority_);
-    return;
-  }
-  // The update is for a higher-than-current priority (or for any
-  // priority if we don't have any current priority).
-  if (child_priority < current_priority_) {
-    // If the child reports READY or IDLE, switch to that priority.
-    // Otherwise, ignore the update.
-    if (child->connectivity_state() == GRPC_CHANNEL_READY ||
-        child->connectivity_state() == GRPC_CHANNEL_IDLE) {
-      SelectPriorityLocked(child_priority);
-    }
-    return;
-  }
-  // The current priority has returned a new picker, so pass it up to
-  // our parent.
-  channel_control_helper()->UpdateState(child->connectivity_state(),
-                                        child->connectivity_status(),
-                                        child->GetPicker());
+  // Unconditionally call ChoosePriorityLocked().  It should do the
+  // right thing based on the state of all children.
+  ChoosePriorityLocked();
 }
 
 void PriorityLb::DeleteChild(ChildPriority* child) {
   // If this was the current child from before the most recent update,
   // stop using it.  We already started trying other priorities as a
-  // result of the update, but calling TryNextPriorityLocked() ensures that
+  // result of the update, but calling ChoosePriorityLocked() ensures that
   // we will properly select between CONNECTING and TRANSIENT_FAILURE as the
   // new state to report to our parent.
   if (current_child_from_before_update_ == child) {
     current_child_from_before_update_ = nullptr;
-    TryNextPriorityLocked(/*report_connecting=*/true);
+    ChoosePriorityLocked();
   }
   children_.erase(child->name());
 }
 
-void PriorityLb::TryNextPriorityLocked(bool report_connecting) {
+void PriorityLb::ChoosePriorityLocked() {
+  // If priority list is empty, report TF.
+  if (config_->priorities().empty()) {
+    current_child_from_before_update_ = nullptr;
+    absl::Status status =
+        absl::UnavailableError("priority policy has empty priority list");
+    channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        absl::make_unique<TransientFailurePicker>(status));
+    return;
+  }
+  // Iterate through priorities, searching for one in READY or IDLE,
+  // creating new children as needed.
   current_priority_ = UINT32_MAX;
   for (uint32_t priority = 0; priority < config_->priorities().size();
        ++priority) {
@@ -426,11 +484,18 @@ void PriorityLb::TryNextPriorityLocked(bool report_connecting) {
     }
     auto& child = children_[child_name];
     if (child == nullptr) {
-      if (report_connecting) {
+      // If we're not still using an old child from before the last
+      // update, report CONNECTING here.
+      // This is probably not strictly necessary, since the child should
+      // immediately report CONNECTING and cause us to report that state
+      // anyway, but we do this just in case the child fails to report
+      // state before UpdateLocked() returns.
+      if (current_child_from_before_update_ != nullptr) {
         channel_control_helper()->UpdateState(
             GRPC_CHANNEL_CONNECTING, absl::Status(),
             absl::make_unique<QueuePicker>(Ref(DEBUG_LOCATION, "QueuePicker")));
       }
+      current_priority_ = priority;
       child = MakeOrphanable<ChildPriority>(
           Ref(DEBUG_LOCATION, "ChildPriority"), child_name);
       auto child_config = config_->children().find(child_name);
@@ -444,42 +509,78 @@ void PriorityLb::TryNextPriorityLocked(bool report_connecting) {
     // If the child is in state READY or IDLE, switch to it.
     if (child->connectivity_state() == GRPC_CHANNEL_READY ||
         child->connectivity_state() == GRPC_CHANNEL_IDLE) {
-      SelectPriorityLocked(priority);
+      SetCurrentPriorityLocked(priority);
       return;
     }
     // Child is not READY or IDLE.
     // If its failover timer is still pending, give it time to fire.
-    if (child->failover_timer_callback_pending()) {
+    if (child->FailoverTimerPending()) {
       if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
         gpr_log(GPR_INFO,
                 "[priority_lb %p] priority %u, child %s: child still "
                 "attempting to connect, will wait",
                 this, priority, child_name.c_str());
       }
-      if (report_connecting) {
-        channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_CONNECTING, absl::Status(),
-            absl::make_unique<QueuePicker>(Ref(DEBUG_LOCATION, "QueuePicker")));
+      current_priority_ = priority;
+      // If we're not still using an old child from before the last
+      // update, report CONNECTING here.
+      if (current_child_from_before_update_ != nullptr) {
+        channel_control_helper()->UpdateState(child->connectivity_state(),
+                                              child->connectivity_status(),
+                                              child->GetPicker());
       }
       return;
     }
     // Child has been failing for a while.  Move on to the next priority.
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO,
+              "[priority_lb %p] skipping priority %u, child %s: state=%s, "
+              "failover timer not pending",
+              this, priority, child_name.c_str(),
+              ConnectivityStateName(child->connectivity_state()));
+    }
   }
-  // If there are no more priorities to try, report TRANSIENT_FAILURE.
+  // If we didn't find any priority to try, pick the first one in state
+  // CONNECTING.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO,
-            "[priority_lb %p] no priority reachable, putting channel in "
-            "TRANSIENT_FAILURE",
+            "[priority_lb %p] no priority reachable, checking for CONNECTING "
+            "priority to delegate to",
             this);
   }
-  current_child_from_before_update_ = nullptr;
-  absl::Status status = absl::UnavailableError("no ready priority");
-  channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_TRANSIENT_FAILURE, status,
-      absl::make_unique<TransientFailurePicker>(status));
+  for (uint32_t priority = 0; priority < config_->priorities().size();
+       ++priority) {
+    // If the child for the priority does not exist yet, create it.
+    const std::string& child_name = config_->priorities()[priority];
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO, "[priority_lb %p] trying priority %u, child %s", this,
+              priority, child_name.c_str());
+    }
+    auto& child = children_[child_name];
+    GPR_ASSERT(child != nullptr);
+    if (child->connectivity_state() == GRPC_CHANNEL_CONNECTING) {
+      channel_control_helper()->UpdateState(child->connectivity_state(),
+                                            child->connectivity_status(),
+                                            child->GetPicker());
+      return;
+    }
+  }
+  // Did not find any child in CONNECTING, delegate to last child.
+  const std::string& child_name = config_->priorities().back();
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+    gpr_log(GPR_INFO,
+            "[priority_lb %p] no priority in CONNECTING, delegating to "
+            "lowest priority child %s",
+            this, child_name.c_str());
+  }
+  auto& child = children_[child_name];
+  GPR_ASSERT(child != nullptr);
+  channel_control_helper()->UpdateState(child->connectivity_state(),
+                                        child->connectivity_status(),
+                                        child->GetPicker());
 }
 
-void PriorityLb::SelectPriorityLocked(uint32_t priority) {
+void PriorityLb::SetCurrentPriorityLocked(uint32_t priority) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] selected priority %u, child %s", this,
             priority, config_->priorities()[priority].c_str());
@@ -490,7 +591,7 @@ void PriorityLb::SelectPriorityLocked(uint32_t priority) {
   for (uint32_t p = priority + 1; p < config_->priorities().size(); ++p) {
     const std::string& child_name = config_->priorities()[p];
     auto it = children_.find(child_name);
-    if (it != children_.end()) it->second->DeactivateLocked();
+    if (it != children_.end()) it->second->MaybeDeactivateLocked();
   }
   // Update picker.
   auto& child = children_[config_->priorities()[priority]];
@@ -499,6 +600,132 @@ void PriorityLb::SelectPriorityLocked(uint32_t priority) {
                                         child->GetPicker());
 }
 
+//
+// PriorityLb::ChildPriority::DeactivationTimer
+//
+
+PriorityLb::ChildPriority::DeactivationTimer::DeactivationTimer(
+    RefCountedPtr<PriorityLb::ChildPriority> child_priority)
+    : child_priority_(std::move(child_priority)) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+    gpr_log(GPR_INFO,
+            "[priority_lb %p] child %s (%p): deactivating -- will remove in "
+            "%" PRId64 "ms",
+            child_priority_->priority_policy_.get(),
+            child_priority_->name_.c_str(), child_priority_.get(),
+            kChildRetentionInterval.millis());
+  }
+  GRPC_CLOSURE_INIT(&on_timer_, OnTimer, this, nullptr);
+  Ref(DEBUG_LOCATION, "Timer").release();
+  grpc_timer_init(&timer_, ExecCtx::Get()->Now() + kChildRetentionInterval,
+                  &on_timer_);
+}
+
+void PriorityLb::ChildPriority::DeactivationTimer::Orphan() {
+  if (timer_pending_) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): reactivating",
+              child_priority_->priority_policy_.get(),
+              child_priority_->name_.c_str(), child_priority_.get());
+    }
+    timer_pending_ = false;
+    grpc_timer_cancel(&timer_);
+  }
+  Unref();
+}
+
+void PriorityLb::ChildPriority::DeactivationTimer::OnTimer(
+    void* arg, grpc_error_handle error) {
+  auto* self = static_cast<DeactivationTimer*>(arg);
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
+  self->child_priority_->priority_policy_->work_serializer()->Run(
+      [self, error]() { self->OnTimerLocked(error); }, DEBUG_LOCATION);
+}
+
+void PriorityLb::ChildPriority::DeactivationTimer::OnTimerLocked(
+    grpc_error_handle error) {
+  if (GRPC_ERROR_IS_NONE(error) && timer_pending_) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO,
+              "[priority_lb %p] child %s (%p): deactivation timer fired, "
+              "deleting child",
+              child_priority_->priority_policy_.get(),
+              child_priority_->name_.c_str(), child_priority_.get());
+    }
+    timer_pending_ = false;
+    child_priority_->priority_policy_->DeleteChild(child_priority_.get());
+  }
+  Unref(DEBUG_LOCATION, "Timer");
+  GRPC_ERROR_UNREF(error);
+}
+
+//
+// PriorityLb::ChildPriority::FailoverTimer
+//
+
+PriorityLb::ChildPriority::FailoverTimer::FailoverTimer(
+    RefCountedPtr<PriorityLb::ChildPriority> child_priority)
+    : child_priority_(std::move(child_priority)) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+    gpr_log(
+        GPR_INFO,
+        "[priority_lb %p] child %s (%p): starting failover timer for %" PRId64
+        "ms",
+        child_priority_->priority_policy_.get(), child_priority_->name_.c_str(),
+        child_priority_.get(),
+        child_priority_->priority_policy_->child_failover_timeout_.millis());
+  }
+  GRPC_CLOSURE_INIT(&on_timer_, OnTimer, this, nullptr);
+  Ref(DEBUG_LOCATION, "Timer").release();
+  grpc_timer_init(
+      &timer_,
+      ExecCtx::Get()->Now() +
+          child_priority_->priority_policy_->child_failover_timeout_,
+      &on_timer_);
+}
+
+void PriorityLb::ChildPriority::FailoverTimer::Orphan() {
+  if (timer_pending_) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO,
+              "[priority_lb %p] child %s (%p): cancelling failover timer",
+              child_priority_->priority_policy_.get(),
+              child_priority_->name_.c_str(), child_priority_.get());
+    }
+    timer_pending_ = false;
+    grpc_timer_cancel(&timer_);
+  }
+  Unref();
+}
+
+void PriorityLb::ChildPriority::FailoverTimer::OnTimer(
+    void* arg, grpc_error_handle error) {
+  auto* self = static_cast<FailoverTimer*>(arg);
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
+  self->child_priority_->priority_policy_->work_serializer()->Run(
+      [self, error]() { self->OnTimerLocked(error); }, DEBUG_LOCATION);
+}
+
+void PriorityLb::ChildPriority::FailoverTimer::OnTimerLocked(
+    grpc_error_handle error) {
+  if (GRPC_ERROR_IS_NONE(error) && timer_pending_) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO,
+              "[priority_lb %p] child %s (%p): failover timer fired, "
+              "reporting TRANSIENT_FAILURE",
+              child_priority_->priority_policy_.get(),
+              child_priority_->name_.c_str(), child_priority_.get());
+    }
+    timer_pending_ = false;
+    child_priority_->OnConnectivityStateUpdateLocked(
+        GRPC_CHANNEL_TRANSIENT_FAILURE,
+        absl::Status(absl::StatusCode::kUnavailable, "failover timer fired"),
+        nullptr);
+  }
+  Unref(DEBUG_LOCATION, "Timer");
+  GRPC_ERROR_UNREF(error);
+}
+
 //
 // PriorityLb::ChildPriority
 //
@@ -510,12 +737,8 @@ PriorityLb::ChildPriority::ChildPriority(
     gpr_log(GPR_INFO, "[priority_lb %p] creating child %s (%p)",
             priority_policy_.get(), name_.c_str(), this);
   }
-  GRPC_CLOSURE_INIT(&on_failover_timer_, OnFailoverTimer, this,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&on_deactivation_timer_, OnDeactivationTimer, this,
-                    grpc_schedule_on_exec_ctx);
   // Start the failover timer.
-  StartFailoverTimerLocked();
+  failover_timer_ = MakeOrphanable<FailoverTimer>(Ref());
 }
 
 void PriorityLb::ChildPriority::Orphan() {
@@ -523,10 +746,8 @@ void PriorityLb::ChildPriority::Orphan() {
     gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): orphaned",
             priority_policy_.get(), name_.c_str(), this);
   }
-  MaybeCancelFailoverTimerLocked();
-  if (deactivation_timer_callback_pending_) {
-    grpc_timer_cancel(&deactivation_timer_);
-  }
+  failover_timer_.reset();
+  deactivation_timer_.reset();
   // Remove the child policy's interested_parties pollset_set from the
   // xDS policy.
   grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(),
@@ -535,12 +756,18 @@ void PriorityLb::ChildPriority::Orphan() {
   // Drop our ref to the child's picker, in case it's holding a ref to
   // the child.
   picker_wrapper_.reset();
-  if (deactivation_timer_callback_pending_) {
-    grpc_timer_cancel(&deactivation_timer_);
-  }
   Unref(DEBUG_LOCATION, "ChildPriority+Orphan");
 }
 
+std::unique_ptr<LoadBalancingPolicy::SubchannelPicker>
+PriorityLb::ChildPriority::GetPicker() {
+  if (picker_wrapper_ == nullptr) {
+    return absl::make_unique<QueuePicker>(
+        priority_policy_->Ref(DEBUG_LOCATION, "QueuePicker"));
+  }
+  return absl::make_unique<RefCountedPickerWrapper>(picker_wrapper_);
+}
+
 void PriorityLb::ChildPriority::UpdateLocked(
     RefCountedPtr<LoadBalancingPolicy::Config> config,
     bool ignore_reresolution_requests) {
@@ -557,7 +784,11 @@ void PriorityLb::ChildPriority::UpdateLocked(
   // Construct update args.
   UpdateArgs update_args;
   update_args.config = std::move(config);
-  update_args.addresses = priority_policy_->addresses_[name_];
+  if (priority_policy_->addresses_.ok()) {
+    update_args.addresses = (*priority_policy_->addresses_)[name_];
+  } else {
+    update_args.addresses = priority_policy_->addresses_.status();
+  }
   update_args.args = grpc_channel_args_copy(priority_policy_->args_);
   // Update the policy.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
@@ -594,10 +825,6 @@ PriorityLb::ChildPriority::CreateChildPolicyLocked(
 }
 
 void PriorityLb::ChildPriority::ExitIdleLocked() {
-  if (connectivity_state_ == GRPC_CHANNEL_IDLE &&
-      !failover_timer_callback_pending_) {
-    StartFailoverTimerLocked();
-  }
   child_policy_->ExitIdleLocked();
 }
 
@@ -619,122 +846,35 @@ void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked(
   connectivity_state_ = state;
   connectivity_status_ = status;
   picker_wrapper_ = MakeRefCounted<RefCountedPicker>(std::move(picker));
-  // If READY or TRANSIENT_FAILURE, cancel failover timer.
-  if (state == GRPC_CHANNEL_READY || state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    MaybeCancelFailoverTimerLocked();
+  // If we transition to state CONNECTING and we've not seen
+  // TRANSIENT_FAILURE more recently than READY or IDLE, start failover
+  // timer if not already pending.
+  // In any other state, update seen_ready_or_idle_since_transient_failure_
+  // and cancel failover timer.
+  if (state == GRPC_CHANNEL_CONNECTING) {
+    if (seen_ready_or_idle_since_transient_failure_ &&
+        failover_timer_ == nullptr) {
+      failover_timer_ = MakeOrphanable<FailoverTimer>(Ref());
+    }
+  } else if (state == GRPC_CHANNEL_READY || state == GRPC_CHANNEL_IDLE) {
+    seen_ready_or_idle_since_transient_failure_ = true;
+    failover_timer_.reset();
+  } else if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    seen_ready_or_idle_since_transient_failure_ = false;
+    failover_timer_.reset();
   }
   // Notify the parent policy.
   priority_policy_->HandleChildConnectivityStateChangeLocked(this);
 }
 
-void PriorityLb::ChildPriority::StartFailoverTimerLocked() {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-    gpr_log(GPR_INFO,
-            "[priority_lb %p] child %s (%p): starting failover timer for %d ms",
-            priority_policy_.get(), name_.c_str(), this,
-            priority_policy_->child_failover_timeout_ms_);
+void PriorityLb::ChildPriority::MaybeDeactivateLocked() {
+  if (deactivation_timer_ == nullptr) {
+    deactivation_timer_ = MakeOrphanable<DeactivationTimer>(Ref());
   }
-  Ref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked").release();
-  grpc_timer_init(
-      &failover_timer_,
-      ExecCtx::Get()->Now() + priority_policy_->child_failover_timeout_ms_,
-      &on_failover_timer_);
-  failover_timer_callback_pending_ = true;
-}
-
-void PriorityLb::ChildPriority::MaybeCancelFailoverTimerLocked() {
-  if (failover_timer_callback_pending_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-      gpr_log(GPR_INFO,
-              "[priority_lb %p] child %s (%p): cancelling failover timer",
-              priority_policy_.get(), name_.c_str(), this);
-    }
-    grpc_timer_cancel(&failover_timer_);
-    failover_timer_callback_pending_ = false;
-  }
-}
-
-void PriorityLb::ChildPriority::OnFailoverTimer(void* arg,
-                                                grpc_error_handle error) {
-  ChildPriority* self = static_cast<ChildPriority*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  self->priority_policy_->work_serializer()->Run(
-      [self, error]() { self->OnFailoverTimerLocked(error); }, DEBUG_LOCATION);
-}
-
-void PriorityLb::ChildPriority::OnFailoverTimerLocked(grpc_error_handle error) {
-  if (error == GRPC_ERROR_NONE && failover_timer_callback_pending_ &&
-      !priority_policy_->shutting_down_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-      gpr_log(GPR_INFO,
-              "[priority_lb %p] child %s (%p): failover timer fired, "
-              "reporting TRANSIENT_FAILURE",
-              priority_policy_.get(), name_.c_str(), this);
-    }
-    failover_timer_callback_pending_ = false;
-    OnConnectivityStateUpdateLocked(
-        GRPC_CHANNEL_TRANSIENT_FAILURE,
-        absl::Status(absl::StatusCode::kUnavailable, "failover timer fired"),
-        nullptr);
-  }
-  Unref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked");
-  GRPC_ERROR_UNREF(error);
-}
-
-void PriorityLb::ChildPriority::DeactivateLocked() {
-  // If already deactivated, don't do it again.
-  if (deactivation_timer_callback_pending_) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-    gpr_log(GPR_INFO,
-            "[priority_lb %p] child %s (%p): deactivating -- will remove in %d "
-            "ms.",
-            priority_policy_.get(), name_.c_str(), this,
-            kChildRetentionIntervalMs);
-  }
-  MaybeCancelFailoverTimerLocked();
-  // Start a timer to delete the child.
-  Ref(DEBUG_LOCATION, "ChildPriority+timer").release();
-  grpc_timer_init(&deactivation_timer_,
-                  ExecCtx::Get()->Now() + kChildRetentionIntervalMs,
-                  &on_deactivation_timer_);
-  deactivation_timer_callback_pending_ = true;
 }
 
 void PriorityLb::ChildPriority::MaybeReactivateLocked() {
-  if (deactivation_timer_callback_pending_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-      gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): reactivating",
-              priority_policy_.get(), name_.c_str(), this);
-    }
-    deactivation_timer_callback_pending_ = false;
-    grpc_timer_cancel(&deactivation_timer_);
-  }
-}
-
-void PriorityLb::ChildPriority::OnDeactivationTimer(void* arg,
-                                                    grpc_error_handle error) {
-  ChildPriority* self = static_cast<ChildPriority*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  self->priority_policy_->work_serializer()->Run(
-      [self, error]() { self->OnDeactivationTimerLocked(error); },
-      DEBUG_LOCATION);
-}
-
-void PriorityLb::ChildPriority::OnDeactivationTimerLocked(
-    grpc_error_handle error) {
-  if (error == GRPC_ERROR_NONE && deactivation_timer_callback_pending_ &&
-      !priority_policy_->shutting_down_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-      gpr_log(GPR_INFO,
-              "[priority_lb %p] child %s (%p): deactivation timer fired, "
-              "deleting child",
-              priority_policy_.get(), name_.c_str(), this);
-    }
-    deactivation_timer_callback_pending_ = false;
-    priority_policy_->DeleteChild(this);
-  }
-  Unref(DEBUG_LOCATION, "ChildPriority+timer");
-  GRPC_ERROR_UNREF(error);
+  deactivation_timer_.reset();
 }
 
 //
@@ -791,7 +931,7 @@ class PriorityLbFactory : public LoadBalancingPolicyFactory {
 
   RefCountedPtr<LoadBalancingPolicy::Config> ParseLoadBalancingConfig(
       const Json& json, grpc_error_handle* error) const override {
-    GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE);
+    GPR_DEBUG_ASSERT(error != nullptr && GRPC_ERROR_IS_NONE(*error));
     if (json.type() == Json::Type::JSON_NULL) {
       // priority was mentioned as a policy in the deprecated
       // loadBalancingPolicy field or in the client API.
@@ -846,7 +986,7 @@ class PriorityLbFactory : public LoadBalancingPolicyFactory {
               }
             }
             if (config == nullptr) {
-              GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE);
+              GPR_DEBUG_ASSERT(!GRPC_ERROR_IS_NONE(parse_error));
               error_list.push_back(
                   GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(
                       absl::StrCat("field:children key:", child_name).c_str(),