grpc 1.47.0 → 1.53.0

data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc

@@ -14,51 +14,51 @@
 // limitations under the License.
 //
 
-// IWYU pragma: no_include <ext/alloc_traits.h>
-
 #include <grpc/support/port_platform.h>
 
 #include <inttypes.h>
-#include <limits.h>
-#include <stddef.h>
 
 #include <algorithm>
 #include <map>
 #include <memory>
+#include <set>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
-#include "absl/memory/memory.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 
-#include <grpc/impl/codegen/connectivity_state.h>
-#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/connectivity_state.h>
 #include <grpc/support/log.h>
 
-#include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h"
 #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h"
-#include "src/core/ext/filters/client_channel/lb_policy_factory.h"
-#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
-#include "src/core/ext/filters/client_channel/subchannel_interface.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/orphanable.h"
-#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/time.h"
-#include "src/core/lib/iomgr/closure.h"
-#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/gprpp/validation_errors.h"
+#include "src/core/lib/gprpp/work_serializer.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/iomgr/work_serializer.h"
 #include "src/core/lib/json/json.h"
+#include "src/core/lib/json/json_args.h"
+#include "src/core/lib/json/json_object_loader.h"
+#include "src/core/lib/load_balancing/lb_policy.h"
+#include "src/core/lib/load_balancing/lb_policy_factory.h"
+#include "src/core/lib/load_balancing/lb_policy_registry.h"
+#include "src/core/lib/load_balancing/subchannel_interface.h"
 #include "src/core/lib/resolver/server_address.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
@@ -68,7 +68,9 @@ TraceFlag grpc_lb_priority_trace(false, "priority_lb");
 
 namespace {
 
-constexpr char kPriority[] = "priority_experimental";
+using ::grpc_event_engine::experimental::EventEngine;
+
+constexpr absl::string_view kPriority = "priority_experimental";
 
 // How long we keep a child around for after it is no longer being used
 // (either because it has been removed from the config or because we
@@ -85,22 +87,34 @@ class PriorityLbConfig : public LoadBalancingPolicy::Config {
   struct PriorityLbChild {
     RefCountedPtr<LoadBalancingPolicy::Config> config;
     bool ignore_reresolution_requests = false;
+
+    static const JsonLoaderInterface* JsonLoader(const JsonArgs&);
+    void JsonPostLoad(const Json& json, const JsonArgs&,
+                      ValidationErrors* errors);
   };
 
-  PriorityLbConfig(std::map<std::string, PriorityLbChild> children,
-                   std::vector<std::string> priorities)
-      : children_(std::move(children)), priorities_(std::move(priorities)) {}
+  PriorityLbConfig() = default;
+
+  PriorityLbConfig(const PriorityLbConfig&) = delete;
+  PriorityLbConfig& operator=(const PriorityLbConfig&) = delete;
 
-  const char* name() const override { return kPriority; }
+  PriorityLbConfig(PriorityLbConfig&& other) = delete;
+  PriorityLbConfig& operator=(PriorityLbConfig&& other) = delete;
+
+  absl::string_view name() const override { return kPriority; }
 
   const std::map<std::string, PriorityLbChild>& children() const {
     return children_;
   }
   const std::vector<std::string>& priorities() const { return priorities_; }
 
+  static const JsonLoaderInterface* JsonLoader(const JsonArgs&);
+  void JsonPostLoad(const Json& json, const JsonArgs&,
+                    ValidationErrors* errors);
+
  private:
-  const std::map<std::string, PriorityLbChild> children_;
-  const std::vector<std::string> priorities_;
+  std::map<std::string, PriorityLbChild> children_;
+  std::vector<std::string> priorities_;
 };
 
 // priority LB policy.
@@ -108,9 +122,9 @@ class PriorityLb : public LoadBalancingPolicy {
  public:
   explicit PriorityLb(Args args);
 
-  const char* name() const override { return kPriority; }
+  absl::string_view name() const override { return kPriority; }
 
-  void UpdateLocked(UpdateArgs args) override;
+  absl::Status UpdateLocked(UpdateArgs args) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
 
@@ -126,8 +140,8 @@ class PriorityLb : public LoadBalancingPolicy {
 
     const std::string& name() const { return name_; }
 
-    void UpdateLocked(RefCountedPtr<LoadBalancingPolicy::Config> config,
-                      bool ignore_reresolution_requests);
+    absl::Status UpdateLocked(RefCountedPtr<LoadBalancingPolicy::Config> config,
+                              bool ignore_reresolution_requests);
     void ExitIdleLocked();
     void ResetBackoffLocked();
     void MaybeDeactivateLocked();
@@ -135,9 +149,7 @@ class PriorityLb : public LoadBalancingPolicy {
 
     void Orphan() override;
 
-    std::unique_ptr<SubchannelPicker> GetPicker() {
-      return absl::make_unique<RefCountedPickerWrapper>(picker_wrapper_);
-    }
+    RefCountedPtr<SubchannelPicker> GetPicker();
 
     grpc_connectivity_state connectivity_state() const {
       return connectivity_state_;
@@ -150,28 +162,6 @@ class PriorityLb : public LoadBalancingPolicy {
     bool FailoverTimerPending() const { return failover_timer_ != nullptr; }
 
    private:
-    // A simple wrapper for ref-counting a picker from the child policy.
-    class RefCountedPicker : public RefCounted<RefCountedPicker> {
-     public:
-      explicit RefCountedPicker(std::unique_ptr<SubchannelPicker> picker)
-          : picker_(std::move(picker)) {}
-      PickResult Pick(PickArgs args) { return picker_->Pick(args); }
-
-     private:
-      std::unique_ptr<SubchannelPicker> picker_;
-    };
-
-    // A non-ref-counted wrapper for RefCountedPicker.
-    class RefCountedPickerWrapper : public SubchannelPicker {
-     public:
-      explicit RefCountedPickerWrapper(RefCountedPtr<RefCountedPicker> picker)
-          : picker_(std::move(picker)) {}
-      PickResult Pick(PickArgs args) override { return picker_->Pick(args); }
-
-     private:
-      RefCountedPtr<RefCountedPicker> picker_;
-    };
-
     class Helper : public ChannelControlHelper {
      public:
       explicit Helper(RefCountedPtr<ChildPriority> priority)
@@ -180,12 +170,13 @@ class PriorityLb : public LoadBalancingPolicy {
       ~Helper() override { priority_.reset(DEBUG_LOCATION, "Helper"); }
 
       RefCountedPtr<SubchannelInterface> CreateSubchannel(
-          ServerAddress address, const grpc_channel_args& args) override;
+          ServerAddress address, const ChannelArgs& args) override;
       void UpdateState(grpc_connectivity_state state,
                        const absl::Status& status,
-                       std::unique_ptr<SubchannelPicker> picker) override;
+                       RefCountedPtr<SubchannelPicker> picker) override;
       void RequestReresolution() override;
       absl::string_view GetAuthority() override;
+      EventEngine* GetEventEngine() override;
       void AddTraceEvent(TraceSeverity severity,
                          absl::string_view message) override;
 
@@ -200,13 +191,10 @@ class PriorityLb : public LoadBalancingPolicy {
       void Orphan() override;
 
      private:
-      static void OnTimer(void* arg, grpc_error_handle error);
-      void OnTimerLocked(grpc_error_handle);
+      void OnTimerLocked();
 
       RefCountedPtr<ChildPriority> child_priority_;
-      grpc_timer timer_;
-      grpc_closure on_timer_;
-      bool timer_pending_ = true;
+      absl::optional<EventEngine::TaskHandle> timer_handle_;
     };
 
     class FailoverTimer : public InternallyRefCounted<FailoverTimer> {
@@ -216,22 +204,19 @@ class PriorityLb : public LoadBalancingPolicy {
       void Orphan() override;
 
      private:
-      static void OnTimer(void* arg, grpc_error_handle error);
-      void OnTimerLocked(grpc_error_handle);
+      void OnTimerLocked();
 
       RefCountedPtr<ChildPriority> child_priority_;
-      grpc_timer timer_;
-      grpc_closure on_timer_;
-      bool timer_pending_ = true;
+      absl::optional<EventEngine::TaskHandle> timer_handle_;
     };
 
     // Methods for dealing with the child policy.
     OrphanablePtr<LoadBalancingPolicy> CreateChildPolicyLocked(
-        const grpc_channel_args* args);
+        const ChannelArgs& args);
 
     void OnConnectivityStateUpdateLocked(
         grpc_connectivity_state state, const absl::Status& status,
-        std::unique_ptr<SubchannelPicker> picker);
+        RefCountedPtr<SubchannelPicker> picker);
 
     RefCountedPtr<PriorityLb> priority_policy_;
     const std::string name_;
@@ -241,7 +226,7 @@ class PriorityLb : public LoadBalancingPolicy {
 
     grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_CONNECTING;
     absl::Status connectivity_status_;
-    RefCountedPtr<RefCountedPicker> picker_wrapper_;
+    RefCountedPtr<SubchannelPicker> picker_;
 
     bool seen_ready_or_idle_since_transient_failure_ = true;
 
@@ -257,40 +242,36 @@ class PriorityLb : public LoadBalancingPolicy {
   // the child is not in the current priority list.
   uint32_t GetChildPriorityLocked(const std::string& child_name) const;
 
-  // Called when a child's connectivity state has changed.
-  // May propagate the update to the channel or trigger choosing a new
-  // priority.
-  void HandleChildConnectivityStateChangeLocked(ChildPriority* child);
-
   // Deletes a child.  Called when the child's deactivation timer fires.
   void DeleteChild(ChildPriority* child);
 
   // Iterates through the list of priorities to choose one:
   // - If the child for a priority doesn't exist, creates it.
-  // - If a child's failover timer is pending, returns without selecting
-  //   a priority while we wait for the child to attempt to connect.  In
-  //   this case, if report_connecting is true, reports CONNECTING state to
-  //   the channel.
-  // - If the child is connected, it will be used as the current priority.
+  // - If a child's failover timer is pending, selects that priority
+  //   while we wait for the child to attempt to connect.
+  // - If the child is connected, selects that priority.
   // - Otherwise, continues on to the next child.
-  // Reports TRANSIENT_FAILURE to the channel if all children are not
-  // connected.
+  // Delegates to the last child if none of the children are connecting.
+  // Reports TRANSIENT_FAILURE if the priority list is empty.
   //
   // This method is idempotent; it should yield the same result every
   // time as a function of the state of the children.
-  void ChoosePriorityLocked(bool report_connecting);
+  void ChoosePriorityLocked();
 
   // Sets the specified priority as the current priority.
-  // Deactivates any children at lower priorities.
+  // Optionally deactivates any children at lower priorities.
   // Returns the child's picker to the channel.
-  void SetCurrentPriorityLocked(uint32_t priority);
+  void SetCurrentPriorityLocked(int32_t priority,
+                                bool deactivate_lower_priorities,
+                                const char* reason);
 
   const Duration child_failover_timeout_;
 
   // Current channel args and config from the resolver.
-  const grpc_channel_args* args_ = nullptr;
+  ChannelArgs args_;
   RefCountedPtr<PriorityLbConfig> config_;
   absl::StatusOr<HierarchicalAddressMap> addresses_;
+  std::string resolution_note_;
 
   // Internal state.
   bool shutting_down_ = false;
@@ -302,10 +283,6 @@ class PriorityLb : public LoadBalancingPolicy {
   std::map<std::string, OrphanablePtr<ChildPriority>> children_;
   // The priority that is being used.
   uint32_t current_priority_ = UINT32_MAX;
-  // Points to the current child from before the most recent update.
-  // We will continue to use this child until we decide which of the new
-  // children to use.
-  ChildPriority* current_child_from_before_update_ = nullptr;
 };
 
 //
@@ -314,11 +291,11 @@ class PriorityLb : public LoadBalancingPolicy {
 
 PriorityLb::PriorityLb(Args args)
     : LoadBalancingPolicy(std::move(args)),
-      child_failover_timeout_(
-          Duration::Milliseconds(grpc_channel_args_find_integer(
-              args.args, GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS,
-              {static_cast<int>(kDefaultChildFailoverTimeout.millis()), 0,
-               INT_MAX}))) {
+      child_failover_timeout_(std::max(
+          Duration::Zero(),
+          args.args
+              .GetDurationFromIntMillis(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS)
+              .value_or(kDefaultChildFailoverTimeout))) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] created", this);
   }
@@ -328,7 +305,6 @@ PriorityLb::~PriorityLb() {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] destroying priority LB policy", this);
   }
-  grpc_channel_args_destroy(args_);
 }
 
 void PriorityLb::ShutdownLocked() {
@@ -355,30 +331,20 @@ void PriorityLb::ResetBackoffLocked() {
   for (const auto& p : children_) p.second->ResetBackoffLocked();
 }
 
-void PriorityLb::UpdateLocked(UpdateArgs args) {
+absl::Status PriorityLb::UpdateLocked(UpdateArgs args) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] received update", this);
   }
-  // Save current child.
-  if (current_priority_ != UINT32_MAX) {
-    const std::string& child_name = config_->priorities()[current_priority_];
-    current_child_from_before_update_ = children_[child_name].get();
-    // Unset current_priority_, since it was an index into the old
-    // config's priority list and may no longer be valid.  It will be
-    // reset later by ChoosePriorityLocked(), but we unset it here in
-    // case updating any of our children triggers a state update.
-    current_priority_ = UINT32_MAX;
-  }
   // Update config.
   config_ = std::move(args.config);
   // Update args.
-  grpc_channel_args_destroy(args_);
-  args_ = args.args;
-  args.args = nullptr;
+  args_ = std::move(args.args);
   // Update addresses.
   addresses_ = MakeHierarchicalAddressMap(args.addresses);
+  resolution_note_ = std::move(args.resolution_note);
   // Check all existing children against the new config.
   update_in_progress_ = true;
+  std::vector<std::string> errors;
   for (const auto& p : children_) {
     const std::string& child_name = p.first;
     auto& child = p.second;
@@ -388,13 +354,24 @@ void PriorityLb::UpdateLocked(UpdateArgs args) {
       child->MaybeDeactivateLocked();
     } else {
       // Existing child found in new config.  Update it.
-      child->UpdateLocked(config_it->second.config,
-                          config_it->second.ignore_reresolution_requests);
+      absl::Status status =
+          child->UpdateLocked(config_it->second.config,
+                              config_it->second.ignore_reresolution_requests);
+      if (!status.ok()) {
+        errors.emplace_back(
+            absl::StrCat("child ", child_name, ": ", status.ToString()));
+      }
     }
   }
   update_in_progress_ = false;
   // Try to get connected.
-  ChoosePriorityLocked(/*report_connecting=*/children_.empty());
+  ChoosePriorityLocked();
+  // Return status.
+  if (!errors.empty()) {
+    return absl::UnavailableError(absl::StrCat(
+        "errors from children: [", absl::StrJoin(errors, "; "), "]"));
+  }
+  return absl::OkStatus();
 }
 
 uint32_t PriorityLb::GetChildPriorityLocked(
@@ -406,69 +383,22 @@ uint32_t PriorityLb::GetChildPriorityLocked(
   return UINT32_MAX;
 }
 
-void PriorityLb::HandleChildConnectivityStateChangeLocked(
-    ChildPriority* child) {
-  // If we're in the process of propagating an update from our parent to
-  // our children, ignore any updates that come from the children.  We
-  // will instead choose a new priority once the update has been seen by
-  // all children.  This ensures that we don't incorrectly do the wrong
-  // thing while state is inconsistent.
-  if (update_in_progress_) return;
-  // Special case for the child that was the current child before the
-  // most recent update.
-  if (child == current_child_from_before_update_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-      gpr_log(GPR_INFO,
-              "[priority_lb %p] state update for current child from before "
-              "config update",
-              this);
-    }
-    if (child->connectivity_state() == GRPC_CHANNEL_READY ||
-        child->connectivity_state() == GRPC_CHANNEL_IDLE) {
-      // If it's still READY or IDLE, we stick with this child, so pass
-      // the new picker up to our parent.
-      channel_control_helper()->UpdateState(child->connectivity_state(),
-                                            child->connectivity_status(),
-                                            child->GetPicker());
-    } else {
-      // If it's no longer READY or IDLE, we should stop using it.
-      // We already started trying other priorities as a result of the
-      // update, but calling ChoosePriorityLocked() ensures that we will
-      // properly select between CONNECTING and TRANSIENT_FAILURE as the
-      // new state to report to our parent.
-      current_child_from_before_update_ = nullptr;
-      ChoosePriorityLocked(/*report_connecting=*/true);
-    }
-    return;
-  }
-  // Otherwise, find the child's priority.
-  uint32_t child_priority = GetChildPriorityLocked(child->name());
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-    gpr_log(GPR_INFO,
-            "[priority_lb %p] state update for priority %u, child %s, current "
-            "priority %u",
-            this, child_priority, child->name().c_str(), current_priority_);
-  }
-  // Unconditionally call ChoosePriorityLocked().  It should do the
-  // right thing based on the state of all children.
-  ChoosePriorityLocked(
-      /*report_connecting=*/child_priority == current_priority_);
-}
-
 void PriorityLb::DeleteChild(ChildPriority* child) {
-  // If this was the current child from before the most recent update,
-  // stop using it.  We already started trying other priorities as a
-  // result of the update, but calling ChoosePriorityLocked() ensures that
-  // we will properly select between CONNECTING and TRANSIENT_FAILURE as the
-  // new state to report to our parent.
-  if (current_child_from_before_update_ == child) {
-    current_child_from_before_update_ = nullptr;
-    ChoosePriorityLocked(/*report_connecting=*/true);
-  }
   children_.erase(child->name());
 }
 
-void PriorityLb::ChoosePriorityLocked(bool report_connecting) {
+void PriorityLb::ChoosePriorityLocked() {
+  // If priority list is empty, report TF.
+  if (config_->priorities().empty()) {
+    absl::Status status =
+        absl::UnavailableError("priority policy has empty priority list");
+    channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
+        MakeRefCounted<TransientFailurePicker>(status));
+    return;
+  }
+  // Iterate through priorities, searching for one in READY or IDLE,
+  // creating new children as needed.
   current_priority_ = UINT32_MAX;
   for (uint32_t priority = 0; priority < config_->priorities().size();
        ++priority) {
@@ -479,42 +409,33 @@ void PriorityLb::ChoosePriorityLocked(bool report_connecting) {
               priority, child_name.c_str());
     }
     auto& child = children_[child_name];
+    // Create child if needed.
     if (child == nullptr) {
-      if (report_connecting) {
-        channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_CONNECTING, absl::Status(),
-            absl::make_unique<QueuePicker>(Ref(DEBUG_LOCATION, "QueuePicker")));
-      }
       child = MakeOrphanable<ChildPriority>(
           Ref(DEBUG_LOCATION, "ChildPriority"), child_name);
       auto child_config = config_->children().find(child_name);
       GPR_DEBUG_ASSERT(child_config != config_->children().end());
-      child->UpdateLocked(child_config->second.config,
-                          child_config->second.ignore_reresolution_requests);
-      return;
+      // TODO(roth): If the child reports a non-OK status with the
+      // update, we need to propagate that back to the resolver somehow.
+      (void)child->UpdateLocked(
+          child_config->second.config,
+          child_config->second.ignore_reresolution_requests);
+    } else {
+      // The child already exists.  Reactivate if needed.
+      child->MaybeReactivateLocked();
     }
-    // The child already exists.
-    child->MaybeReactivateLocked();
-    // If the child is in state READY or IDLE, switch to it.
+    // Select this child if it is in states READY or IDLE.
     if (child->connectivity_state() == GRPC_CHANNEL_READY ||
         child->connectivity_state() == GRPC_CHANNEL_IDLE) {
-      SetCurrentPriorityLocked(priority);
+      SetCurrentPriorityLocked(
+          priority, /*deactivate_lower_priorities=*/true,
+          ConnectivityStateName(child->connectivity_state()));
       return;
     }
-    // Child is not READY or IDLE.
-    // If its failover timer is still pending, give it time to fire.
+    // Select this child if its failover timer is pending.
     if (child->FailoverTimerPending()) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-        gpr_log(GPR_INFO,
-                "[priority_lb %p] priority %u, child %s: child still "
-                "attempting to connect, will wait",
-                this, priority, child_name.c_str());
-      }
-      if (report_connecting) {
-        channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_CONNECTING, absl::Status(),
-            absl::make_unique<QueuePicker>(Ref(DEBUG_LOCATION, "QueuePicker")));
-      }
+      SetCurrentPriorityLocked(priority, /*deactivate_lower_priorities=*/false,
+                               "failover timer pending");
       return;
     }
     // Child has been failing for a while.  Move on to the next priority.
@@ -526,35 +447,56 @@ void PriorityLb::ChoosePriorityLocked(bool report_connecting) {
               ConnectivityStateName(child->connectivity_state()));
     }
   }
-  // If there are no more priorities to try, report TRANSIENT_FAILURE.
+  // If we didn't find any priority to try, pick the first one in state
+  // CONNECTING.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO,
-            "[priority_lb %p] no priority reachable, putting channel in "
-            "TRANSIENT_FAILURE",
+            "[priority_lb %p] no priority reachable, checking for CONNECTING "
+            "priority to delegate to",
             this);
   }
-  current_child_from_before_update_ = nullptr;
-  absl::Status status = absl::UnavailableError("no ready priority");
-  channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_TRANSIENT_FAILURE, status,
-      absl::make_unique<TransientFailurePicker>(status));
+  for (uint32_t priority = 0; priority < config_->priorities().size();
+       ++priority) {
+    // If the child for the priority does not exist yet, create it.
+    const std::string& child_name = config_->priorities()[priority];
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
+      gpr_log(GPR_INFO, "[priority_lb %p] trying priority %u, child %s", this,
+              priority, child_name.c_str());
+    }
+    auto& child = children_[child_name];
+    GPR_ASSERT(child != nullptr);
+    if (child->connectivity_state() == GRPC_CHANNEL_CONNECTING) {
+      SetCurrentPriorityLocked(priority, /*deactivate_lower_priorities=*/false,
+                               "CONNECTING (pass 2)");
+      return;
+    }
+  }
+  // Did not find any child in CONNECTING, delegate to last child.
+  SetCurrentPriorityLocked(config_->priorities().size() - 1,
+                           /*deactivate_lower_priorities=*/false,
+                           "no usable children");
 }
 
-void PriorityLb::SetCurrentPriorityLocked(uint32_t priority) {
+void PriorityLb::SetCurrentPriorityLocked(int32_t priority,
+                                          bool deactivate_lower_priorities,
+                                          const char* reason) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
-    gpr_log(GPR_INFO, "[priority_lb %p] selected priority %u, child %s", this,
-            priority, config_->priorities()[priority].c_str());
+    gpr_log(GPR_INFO,
+            "[priority_lb %p] selecting priority %u, child %s (%s, "
+            "deactivate_lower_priorities=%d)",
+            this, priority, config_->priorities()[priority].c_str(), reason,
+            deactivate_lower_priorities);
   }
   current_priority_ = priority;
-  current_child_from_before_update_ = nullptr;
-  // Deactivate lower priorities.
-  for (uint32_t p = priority + 1; p < config_->priorities().size(); ++p) {
-    const std::string& child_name = config_->priorities()[p];
-    auto it = children_.find(child_name);
-    if (it != children_.end()) it->second->MaybeDeactivateLocked();
-  }
-  // Update picker.
+  if (deactivate_lower_priorities) {
+    for (uint32_t p = priority + 1; p < config_->priorities().size(); ++p) {
+      const std::string& child_name = config_->priorities()[p];
+      auto it = children_.find(child_name);
+      if (it != children_.end()) it->second->MaybeDeactivateLocked();
+    }
+  }
   auto& child = children_[config_->priorities()[priority]];
+  GPR_ASSERT(child != nullptr);
   channel_control_helper()->UpdateState(child->connectivity_state(),
                                         child->connectivity_status(),
                                         child->GetPicker());
@@ -575,36 +517,38 @@ PriorityLb::ChildPriority::DeactivationTimer::DeactivationTimer(
             child_priority_->name_.c_str(), child_priority_.get(),
             kChildRetentionInterval.millis());
   }
-  GRPC_CLOSURE_INIT(&on_timer_, OnTimer, this, nullptr);
-  Ref(DEBUG_LOCATION, "Timer").release();
-  grpc_timer_init(&timer_, ExecCtx::Get()->Now() + kChildRetentionInterval,
-                  &on_timer_);
+  timer_handle_ =
+      child_priority_->priority_policy_->channel_control_helper()
+          ->GetEventEngine()
+          ->RunAfter(kChildRetentionInterval, [self = Ref(DEBUG_LOCATION,
+                                                          "Timer")]() mutable {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            auto self_ptr = self.get();
+            self_ptr->child_priority_->priority_policy_->work_serializer()->Run(
+                [self = std::move(self)]() { self->OnTimerLocked(); },
+                DEBUG_LOCATION);
+          });
 }
 
 void PriorityLb::ChildPriority::DeactivationTimer::Orphan() {
-  if (timer_pending_) {
+  if (timer_handle_.has_value()) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
       gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): reactivating",
               child_priority_->priority_policy_.get(),
               child_priority_->name_.c_str(), child_priority_.get());
     }
-    timer_pending_ = false;
-    grpc_timer_cancel(&timer_);
+    child_priority_->priority_policy_->channel_control_helper()
+        ->GetEventEngine()
+        ->Cancel(*timer_handle_);
+    timer_handle_.reset();
   }
   Unref();
 }
 
-void PriorityLb::ChildPriority::DeactivationTimer::OnTimer(
-    void* arg, grpc_error_handle error) {
-  auto* self = static_cast<DeactivationTimer*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  self->child_priority_->priority_policy_->work_serializer()->Run(
-      [self, error]() { self->OnTimerLocked(error); }, DEBUG_LOCATION);
-}
-
-void PriorityLb::ChildPriority::DeactivationTimer::OnTimerLocked(
-    grpc_error_handle error) {
-  if (error == GRPC_ERROR_NONE && timer_pending_) {
+void PriorityLb::ChildPriority::DeactivationTimer::OnTimerLocked() {
+  if (timer_handle_.has_value()) {
+    timer_handle_.reset();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
       gpr_log(GPR_INFO,
               "[priority_lb %p] child %s (%p): deactivation timer fired, "
@@ -612,11 +556,8 @@ void PriorityLb::ChildPriority::DeactivationTimer::OnTimerLocked(
               child_priority_->priority_policy_.get(),
               child_priority_->name_.c_str(), child_priority_.get());
     }
-    timer_pending_ = false;
     child_priority_->priority_policy_->DeleteChild(child_priority_.get());
   }
-  Unref(DEBUG_LOCATION, "Timer");
-  GRPC_ERROR_UNREF(error);
 }
 
 //
@@ -635,40 +576,40 @@ PriorityLb::ChildPriority::FailoverTimer::FailoverTimer(
         child_priority_.get(),
         child_priority_->priority_policy_->child_failover_timeout_.millis());
   }
-  GRPC_CLOSURE_INIT(&on_timer_, OnTimer, this, nullptr);
-  Ref(DEBUG_LOCATION, "Timer").release();
-  grpc_timer_init(
-      &timer_,
-      ExecCtx::Get()->Now() +
-          child_priority_->priority_policy_->child_failover_timeout_,
-      &on_timer_);
+  timer_handle_ =
+      child_priority_->priority_policy_->channel_control_helper()
+          ->GetEventEngine()
+          ->RunAfter(
+              child_priority_->priority_policy_->child_failover_timeout_,
+              [self = Ref(DEBUG_LOCATION, "Timer")]() mutable {
+                ApplicationCallbackExecCtx callback_exec_ctx;
+                ExecCtx exec_ctx;
+                auto self_ptr = self.get();
+                self_ptr->child_priority_->priority_policy_->work_serializer()
+                    ->Run([self = std::move(self)]() { self->OnTimerLocked(); },
+                          DEBUG_LOCATION);
+              });
 }
 
 void PriorityLb::ChildPriority::FailoverTimer::Orphan() {
-  if (timer_pending_) {
+  if (timer_handle_.has_value()) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
       gpr_log(GPR_INFO,
               "[priority_lb %p] child %s (%p): cancelling failover timer",
               child_priority_->priority_policy_.get(),
               child_priority_->name_.c_str(), child_priority_.get());
     }
-    timer_pending_ = false;
-    grpc_timer_cancel(&timer_);
+    child_priority_->priority_policy_->channel_control_helper()
+        ->GetEventEngine()
+        ->Cancel(*timer_handle_);
+    timer_handle_.reset();
   }
   Unref();
 }
 
-void PriorityLb::ChildPriority::FailoverTimer::OnTimer(
-    void* arg, grpc_error_handle error) {
-  auto* self = static_cast<FailoverTimer*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  self->child_priority_->priority_policy_->work_serializer()->Run(
-      [self, error]() { self->OnTimerLocked(error); }, DEBUG_LOCATION);
-}
-
-void PriorityLb::ChildPriority::FailoverTimer::OnTimerLocked(
-    grpc_error_handle error) {
-  if (error == GRPC_ERROR_NONE && timer_pending_) {
+void PriorityLb::ChildPriority::FailoverTimer::OnTimerLocked() {
+  if (timer_handle_.has_value()) {
+    timer_handle_.reset();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
       gpr_log(GPR_INFO,
               "[priority_lb %p] child %s (%p): failover timer fired, "
@@ -676,14 +617,11 @@ void PriorityLb::ChildPriority::FailoverTimer::OnTimerLocked(
               child_priority_->priority_policy_.get(),
               child_priority_->name_.c_str(), child_priority_.get());
     }
-    timer_pending_ = false;
     child_priority_->OnConnectivityStateUpdateLocked(
         GRPC_CHANNEL_TRANSIENT_FAILURE,
         absl::Status(absl::StatusCode::kUnavailable, "failover timer fired"),
         nullptr);
   }
-  Unref(DEBUG_LOCATION, "Timer");
-  GRPC_ERROR_UNREF(error);
 }
 
 //
@@ -715,14 +653,23 @@ void PriorityLb::ChildPriority::Orphan() {
   child_policy_.reset();
   // Drop our ref to the child's picker, in case it's holding a ref to
   // the child.
-  picker_wrapper_.reset();
+  picker_.reset();
   Unref(DEBUG_LOCATION, "ChildPriority+Orphan");
 }
 
-void PriorityLb::ChildPriority::UpdateLocked(
+RefCountedPtr<LoadBalancingPolicy::SubchannelPicker>
+PriorityLb::ChildPriority::GetPicker() {
+  if (picker_ == nullptr) {
+    return MakeRefCounted<QueuePicker>(
+        priority_policy_->Ref(DEBUG_LOCATION, "QueuePicker"));
+  }
+  return picker_;
+}
+
+absl::Status PriorityLb::ChildPriority::UpdateLocked(
     RefCountedPtr<LoadBalancingPolicy::Config> config,
     bool ignore_reresolution_requests) {
-  if (priority_policy_->shutting_down_) return;
+  if (priority_policy_->shutting_down_) return absl::OkStatus();
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): start update",
             priority_policy_.get(), name_.c_str(), this);
@@ -740,24 +687,24 @@ void PriorityLb::ChildPriority::UpdateLocked(
   } else {
     update_args.addresses = priority_policy_->addresses_.status();
   }
-  update_args.args = grpc_channel_args_copy(priority_policy_->args_);
+  update_args.resolution_note = priority_policy_->resolution_note_;
+  update_args.args = priority_policy_->args_;
   // Update the policy.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO,
             "[priority_lb %p] child %s (%p): updating child policy handler %p",
             priority_policy_.get(), name_.c_str(), this, child_policy_.get());
   }
-  child_policy_->UpdateLocked(std::move(update_args));
+  return child_policy_->UpdateLocked(std::move(update_args));
 }
 
 OrphanablePtr<LoadBalancingPolicy>
-PriorityLb::ChildPriority::CreateChildPolicyLocked(
-    const grpc_channel_args* args) {
+PriorityLb::ChildPriority::CreateChildPolicyLocked(const ChannelArgs& args) {
   LoadBalancingPolicy::Args lb_policy_args;
   lb_policy_args.work_serializer = priority_policy_->work_serializer();
   lb_policy_args.args = args;
   lb_policy_args.channel_control_helper =
-      absl::make_unique<Helper>(this->Ref(DEBUG_LOCATION, "Helper"));
+      std::make_unique<Helper>(this->Ref(DEBUG_LOCATION, "Helper"));
   OrphanablePtr<LoadBalancingPolicy> lb_policy =
       MakeOrphanable<ChildPolicyHandler>(std::move(lb_policy_args),
                                          &grpc_lb_priority_trace);
@@ -785,7 +732,7 @@ void PriorityLb::ChildPriority::ResetBackoffLocked() {
 
 void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked(
     grpc_connectivity_state state, const absl::Status& status,
-    std::unique_ptr<SubchannelPicker> picker) {
+    RefCountedPtr<SubchannelPicker> picker) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) {
     gpr_log(GPR_INFO,
             "[priority_lb %p] child %s (%p): state update: %s (%s) picker %p",
@@ -796,7 +743,12 @@ void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked(
   // Store the state and picker.
   connectivity_state_ = state;
   connectivity_status_ = status;
-  picker_wrapper_ = MakeRefCounted<RefCountedPicker>(std::move(picker));
+  // When the failover timer fires, this method will be called with picker
+  // set to null, because we want to consider the child to be in
+  // TRANSIENT_FAILURE, but we have no new picker to report.  In that case,
+  // just keep using the old picker, in case we wind up delegating to this
+  // child when all priorities are failing.
+  if (picker != nullptr) picker_ = std::move(picker);
   // If we transition to state CONNECTING and we've not seen
   // TRANSIENT_FAILURE more recently than READY or IDLE, start failover
   // timer if not already pending.
@@ -814,8 +766,17 @@ void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked(
     seen_ready_or_idle_since_transient_failure_ = false;
     failover_timer_.reset();
   }
-  // Notify the parent policy.
-  priority_policy_->HandleChildConnectivityStateChangeLocked(this);
+  // Call the LB policy's ChoosePriorityLocked() to choose a priority to
+  // use based on the updated state of this child.
+  //
+  // Note that if we're in the process of propagating an update from our
+  // parent to our children, we skip this, because we don't want to
+  // choose a new priority based on inconsistent state.  Instead, the
+  // policy will choose a new priority once the update has been seen by
+  // all children.
+  if (!priority_policy_->update_in_progress_) {
+    priority_policy_->ChoosePriorityLocked();
+  }
 }
 
 void PriorityLb::ChildPriority::MaybeDeactivateLocked() {
@@ -833,8 +794,8 @@ void PriorityLb::ChildPriority::MaybeReactivateLocked() {
 //
 
 RefCountedPtr<SubchannelInterface>
-PriorityLb::ChildPriority::Helper::CreateSubchannel(
-    ServerAddress address, const grpc_channel_args& args) {
+PriorityLb::ChildPriority::Helper::CreateSubchannel(ServerAddress address,
+                                                    const ChannelArgs& args) {
   if (priority_->priority_policy_->shutting_down_) return nullptr;
   return priority_->priority_policy_->channel_control_helper()
       ->CreateSubchannel(std::move(address), args);
@@ -842,7 +803,7 @@ PriorityLb::ChildPriority::Helper::CreateSubchannel(
 
 void PriorityLb::ChildPriority::Helper::UpdateState(
     grpc_connectivity_state state, const absl::Status& status,
-    std::unique_ptr<SubchannelPicker> picker) {
+    RefCountedPtr<SubchannelPicker> picker) {
   if (priority_->priority_policy_->shutting_down_) return;
   // Notify the priority.
   priority_->OnConnectivityStateUpdateLocked(state, status, std::move(picker));
@@ -860,6 +821,11 @@ absl::string_view PriorityLb::ChildPriority::Helper::GetAuthority() {
   return priority_->priority_policy_->channel_control_helper()->GetAuthority();
 }
 
+EventEngine* PriorityLb::ChildPriority::Helper::GetEventEngine() {
+  return priority_->priority_policy_->channel_control_helper()
+      ->GetEventEngine();
+}
+
 void PriorityLb::ChildPriority::Helper::AddTraceEvent(
     TraceSeverity severity, absl::string_view message) {
   if (priority_->priority_policy_->shutting_down_) return;
@@ -871,6 +837,61 @@ void PriorityLb::ChildPriority::Helper::AddTraceEvent(
 // factory
 //
 
+const JsonLoaderInterface* PriorityLbConfig::PriorityLbChild::JsonLoader(
+    const JsonArgs&) {
+  static const auto* loader =
+      JsonObjectLoader<PriorityLbChild>()
+          // Note: The "config" field requires custom parsing, so it's
+          // handled in JsonPostLoad() instead of here.
+          .OptionalField("ignore_reresolution_requests",
+                         &PriorityLbChild::ignore_reresolution_requests)
+          .Finish();
+  return loader;
+}
+
+void PriorityLbConfig::PriorityLbChild::JsonPostLoad(const Json& json,
+                                                     const JsonArgs&,
+                                                     ValidationErrors* errors) {
+  ValidationErrors::ScopedField field(errors, ".config");
+  auto it = json.object_value().find("config");
+  if (it == json.object_value().end()) {
+    errors->AddError("field not present");
+    return;
+  }
+  auto lb_config =
+      CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig(
+          it->second);
+  if (!lb_config.ok()) {
+    errors->AddError(lb_config.status().message());
+    return;
+  }
+  config = std::move(*lb_config);
+}
+
+const JsonLoaderInterface* PriorityLbConfig::JsonLoader(const JsonArgs&) {
+  static const auto* loader =
+      JsonObjectLoader<PriorityLbConfig>()
+          .Field("children", &PriorityLbConfig::children_)
+          .Field("priorities", &PriorityLbConfig::priorities_)
+          .Finish();
+  return loader;
+}
+
+void PriorityLbConfig::JsonPostLoad(const Json& /*json*/, const JsonArgs&,
+                                    ValidationErrors* errors) {
+  std::set<std::string> unknown_priorities;
+  for (const std::string& priority : priorities_) {
+    if (children_.find(priority) == children_.end()) {
+      unknown_priorities.insert(priority);
+    }
+  }
+  if (!unknown_priorities.empty()) {
+    errors->AddError(absl::StrCat("unknown priorit(ies): [",
+                                  absl::StrJoin(unknown_priorities, ", "),
+                                  "]"));
+  }
+}
+
 class PriorityLbFactory : public LoadBalancingPolicyFactory {
  public:
   OrphanablePtr<LoadBalancingPolicy> CreateLoadBalancingPolicy(
@@ -878,132 +899,28 @@ class PriorityLbFactory : public LoadBalancingPolicyFactory {
     return MakeOrphanable<PriorityLb>(std::move(args));
   }
 
-  const char* name() const override { return kPriority; }
+  absl::string_view name() const override { return kPriority; }
 
-  RefCountedPtr<LoadBalancingPolicy::Config> ParseLoadBalancingConfig(
-      const Json& json, grpc_error_handle* error) const override {
-    GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE);
+  absl::StatusOr<RefCountedPtr<LoadBalancingPolicy::Config>>
+  ParseLoadBalancingConfig(const Json& json) const override {
     if (json.type() == Json::Type::JSON_NULL) {
       // priority was mentioned as a policy in the deprecated
       // loadBalancingPolicy field or in the client API.
-      *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+      return absl::InvalidArgumentError(
           "field:loadBalancingPolicy error:priority policy requires "
           "configuration. Please use loadBalancingConfig field of service "
           "config instead.");
-      return nullptr;
-    }
-    std::vector<grpc_error_handle> error_list;
-    // Children.
-    std::map<std::string, PriorityLbConfig::PriorityLbChild> children;
-    auto it = json.object_value().find("children");
-    if (it == json.object_value().end()) {
-      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:children error:required field missing"));
-    } else if (it->second.type() != Json::Type::OBJECT) {
-      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:children error:type should be object"));
-    } else {
-      const Json::Object& object = it->second.object_value();
-      for (const auto& p : object) {
-        const std::string& child_name = p.first;
-        const Json& element = p.second;
-        if (element.type() != Json::Type::OBJECT) {
-          error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(
-              absl::StrCat("field:children key:", child_name,
-                           " error:should be type object")));
-        } else {
-          auto it2 = element.object_value().find("config");
-          if (it2 == element.object_value().end()) {
-            error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(
-                absl::StrCat("field:children key:", child_name,
-                             " error:missing 'config' field")));
-          } else {
-            grpc_error_handle parse_error = GRPC_ERROR_NONE;
-            auto config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(
-                it2->second, &parse_error);
-            bool ignore_resolution_requests = false;
-            // If present, ignore_reresolution_requests must be of type
-            // boolean.
-            auto it3 =
-                element.object_value().find("ignore_reresolution_requests");
-            if (it3 != element.object_value().end()) {
-              if (it3->second.type() == Json::Type::JSON_TRUE) {
-                ignore_resolution_requests = true;
-              } else if (it3->second.type() != Json::Type::JSON_FALSE) {
-                error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(
-                    absl::StrCat("field:children key:", child_name,
-                                 " field:ignore_reresolution_requests:should "
-                                 "be type boolean")));
-              }
-            }
-            if (config == nullptr) {
-              GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE);
-              error_list.push_back(
-                  GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(
-                      absl::StrCat("field:children key:", child_name).c_str(),
-                      &parse_error, 1));
-              GRPC_ERROR_UNREF(parse_error);
-            }
-            children[child_name].config = std::move(config);
-            children[child_name].ignore_reresolution_requests =
-                ignore_resolution_requests;
-          }
-        }
-      }
-    }
-    // Priorities.
-    std::vector<std::string> priorities;
-    it = json.object_value().find("priorities");
-    if (it == json.object_value().end()) {
-      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:priorities error:required field missing"));
-    } else if (it->second.type() != Json::Type::ARRAY) {
-      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "field:priorities error:type should be array"));
-    } else {
-      const Json::Array& array = it->second.array_value();
-      for (size_t i = 0; i < array.size(); ++i) {
-        const Json& element = array[i];
-        if (element.type() != Json::Type::STRING) {
-          error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
-              "field:priorities element:", i, " error:should be type string")));
-        } else if (children.find(element.string_value()) == children.end()) {
-          error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
-              "field:priorities element:", i, " error:unknown child '",
-              element.string_value(), "'")));
-        } else {
-          priorities.emplace_back(element.string_value());
-        }
-      }
-      if (priorities.size() != children.size()) {
-        error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
-            "field:priorities error:priorities size (", priorities.size(),
-            ") != children size (", children.size(), ")")));
-      }
-    }
-    if (error_list.empty()) {
-      return MakeRefCounted<PriorityLbConfig>(std::move(children),
-                                              std::move(priorities));
-    } else {
-      *error = GRPC_ERROR_CREATE_FROM_VECTOR(
-          "priority_experimental LB policy config", &error_list);
-      return nullptr;
     }
+    return LoadRefCountedFromJson<PriorityLbConfig>(
+        json, JsonArgs(), "errors validating priority LB policy config");
   }
 };
 
 }  // namespace
 
-}  // namespace grpc_core
-
-//
-// Plugin registration
-//
-
-void grpc_lb_policy_priority_init() {
-  grpc_core::LoadBalancingPolicyRegistry::Builder::
-      RegisterLoadBalancingPolicyFactory(
-          absl::make_unique<grpc_core::PriorityLbFactory>());
+void RegisterPriorityLbPolicy(CoreConfiguration::Builder* builder) {
+  builder->lb_policy_registry()->RegisterLoadBalancingPolicyFactory(
+      std::make_unique<PriorityLbFactory>());
 }
 
-void grpc_lb_policy_priority_shutdown() {}
+}  // namespace grpc_core