grpc 1.46.3 → 1.53.1

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -53,23 +53,43 @@
 
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h"
 
+#include <grpc/event_engine/event_engine.h>
+
+// IWYU pragma: no_include <sys/socket.h>
+
 #include <inttypes.h>
-#include <limits.h>
 #include <string.h>
 
+#include <algorithm>
+#include <initializer_list>
+#include <map>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
 #include "absl/container/inlined_vector.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
+#include "absl/strings/string_view.h"
 #include "absl/strings/strip.h"
+#include "absl/types/optional.h"
+#include "absl/types/variant.h"
 #include "upb/upb.hpp"
 
+#include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>
 #include <grpc/grpc.h>
-#include <grpc/grpc_security.h>
+#include <grpc/impl/connectivity_state.h>
+#include <grpc/impl/propagation_bits.h>
+#include <grpc/slice.h>
+#include <grpc/status.h>
 #include <grpc/support/alloc.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/time.h>
+#include <grpc/support/log.h>
 
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h"
@@ -77,30 +97,51 @@
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h"
-#include "src/core/ext/filters/client_channel/lb_policy_factory.h"
-#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
-#include "src/core/lib/address_utils/parse_address.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/string.h"
-#include "src/core/lib/gprpp/manual_constructor.h"
-#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/crash.h"
+#include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/status_helper.h"
+#include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/gprpp/validation_errors.h"
+#include "src/core/lib/gprpp/work_serializer.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/iomgr/resolved_address.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/socket_utils.h"
-#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/json/json_args.h"
+#include "src/core/lib/json/json_object_loader.h"
+#include "src/core/lib/load_balancing/lb_policy.h"
+#include "src/core/lib/load_balancing/lb_policy_factory.h"
+#include "src/core/lib/load_balancing/lb_policy_registry.h"
+#include "src/core/lib/load_balancing/subchannel_interface.h"
+#include "src/core/lib/resolver/resolver.h"
 #include "src/core/lib/resolver/server_address.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/lib/uri/uri_parser.h"
 
 #define GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6
@@ -117,15 +158,55 @@ const char kGrpcLbAddressAttributeKey[] = "grpclb";
 
 namespace {
 
-constexpr char kGrpclb[] = "grpclb";
+using ::grpc_event_engine::experimental::EventEngine;
+
+constexpr absl::string_view kGrpclb = "grpclb";
 
 class GrpcLbConfig : public LoadBalancingPolicy::Config {
  public:
-  GrpcLbConfig(RefCountedPtr<LoadBalancingPolicy::Config> child_policy,
-               std::string service_name)
-      : child_policy_(std::move(child_policy)),
-        service_name_(std::move(service_name)) {}
-  const char* name() const override { return kGrpclb; }
+  GrpcLbConfig() = default;
+
+  GrpcLbConfig(const GrpcLbConfig&) = delete;
+  GrpcLbConfig& operator=(const GrpcLbConfig&) = delete;
+
+  GrpcLbConfig(GrpcLbConfig&& other) = delete;
+  GrpcLbConfig& operator=(GrpcLbConfig&& other) = delete;
+
+  static const JsonLoaderInterface* JsonLoader(const JsonArgs&) {
+    static const auto* loader =
+        JsonObjectLoader<GrpcLbConfig>()
+            // Note: "childPolicy" field requires custom parsing, so
+            // it's handled in JsonPostLoad() instead.
+            .OptionalField("serviceName", &GrpcLbConfig::service_name_)
+            .Finish();
+    return loader;
+  }
+
+  void JsonPostLoad(const Json& json, const JsonArgs&,
+                    ValidationErrors* errors) {
+    ValidationErrors::ScopedField field(errors, ".childPolicy");
+    Json child_policy_config_json_tmp;
+    const Json* child_policy_config_json;
+    auto it = json.object_value().find("childPolicy");
+    if (it == json.object_value().end()) {
+      child_policy_config_json_tmp = Json::Array{Json::Object{
+          {"round_robin", Json::Object()},
+      }};
+      child_policy_config_json = &child_policy_config_json_tmp;
+    } else {
+      child_policy_config_json = &it->second;
+    }
+    auto child_policy_config =
+        CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig(
+            *child_policy_config_json);
+    if (!child_policy_config.ok()) {
+      errors->AddError(child_policy_config.status().message());
+      return;
+    }
+    child_policy_ = std::move(*child_policy_config);
+  }
+
+  absl::string_view name() const override { return kGrpclb; }
 
   RefCountedPtr<LoadBalancingPolicy::Config> child_policy() const {
     return child_policy_;
@@ -142,9 +223,9 @@ class GrpcLb : public LoadBalancingPolicy {
  public:
   explicit GrpcLb(Args args);
 
-  const char* name() const override { return kGrpclb; }
+  absl::string_view name() const override { return kGrpclb; }
 
-  void UpdateLocked(UpdateArgs args) override;
+  absl::Status UpdateLocked(UpdateArgs args) override;
   void ResetBackoffLocked() override;
 
  private:
@@ -174,13 +255,14 @@ class GrpcLb : public LoadBalancingPolicy {
     void ScheduleNextClientLoadReportLocked();
     void SendClientLoadReportLocked();
 
-    static void MaybeSendClientLoadReport(void* arg, grpc_error_handle error);
+    // EventEngine callbacks
+    void MaybeSendClientLoadReportLocked();
+
     static void ClientLoadReportDone(void* arg, grpc_error_handle error);
     static void OnInitialRequestSent(void* arg, grpc_error_handle error);
     static void OnBalancerMessageReceived(void* arg, grpc_error_handle error);
     static void OnBalancerStatusReceived(void* arg, grpc_error_handle error);
 
-    void MaybeSendClientLoadReportLocked(grpc_error_handle error);
     void ClientLoadReportDoneLocked(grpc_error_handle error);
     void OnInitialRequestSentLocked();
     void OnBalancerMessageReceivedLocked();
@@ -215,13 +297,11 @@ class GrpcLb : public LoadBalancingPolicy {
     // Created after the first serverlist is received.
     RefCountedPtr<GrpcLbClientStats> client_stats_;
     Duration client_stats_report_interval_;
-    grpc_timer client_load_report_timer_;
-    bool client_load_report_timer_callback_pending_ = false;
+    absl::optional<EventEngine::TaskHandle> client_load_report_handle_;
     bool last_client_load_report_counters_were_zero_ = false;
     bool client_load_report_is_due_ = false;
-    // The closure used for either the load report timer or the callback for
-    // completion of sending the load report.
-    grpc_closure client_load_report_closure_;
+    // The closure used for the completion of sending the load report.
+    grpc_closure client_load_report_done_closure_;
   };
 
   class SubchannelWrapper : public DelegatingSubchannel {
@@ -258,8 +338,8 @@ class GrpcLb : public LoadBalancingPolicy {
           client_stats_(std::move(client_stats)) {}
 
     std::unique_ptr<AttributeInterface> Copy() const override {
-      return absl::make_unique<TokenAndClientStatsAttribute>(lb_token_,
-                                                             client_stats_);
+      return std::make_unique<TokenAndClientStatsAttribute>(lb_token_,
+                                                            client_stats_);
     }
 
     int Cmp(const AttributeInterface* other_base) const override {
@@ -327,7 +407,7 @@ class GrpcLb : public LoadBalancingPolicy {
   class Picker : public SubchannelPicker {
    public:
     Picker(RefCountedPtr<Serverlist> serverlist,
-           std::unique_ptr<SubchannelPicker> child_picker,
+           RefCountedPtr<SubchannelPicker> child_picker,
            RefCountedPtr<GrpcLbClientStats> client_stats)
         : serverlist_(std::move(serverlist)),
           child_picker_(std::move(child_picker)),
@@ -336,10 +416,43 @@ class GrpcLb : public LoadBalancingPolicy {
     PickResult Pick(PickArgs args) override;
 
    private:
+    // A subchannel call tracker that unrefs the GrpcLbClientStats object
+    // in the case where the subchannel call is never actually started,
+    // since the client load reporting filter will not be able to do it
+    // in that case.
+    class SubchannelCallTracker : public SubchannelCallTrackerInterface {
+     public:
+      SubchannelCallTracker(
+          RefCountedPtr<GrpcLbClientStats> client_stats,
+          std::unique_ptr<SubchannelCallTrackerInterface> original_call_tracker)
+          : client_stats_(std::move(client_stats)),
+            original_call_tracker_(std::move(original_call_tracker)) {}
+
+      void Start() override {
+        if (original_call_tracker_ != nullptr) {
+          original_call_tracker_->Start();
+        }
+        // If we're actually starting the subchannel call, then the
+        // client load reporting filter will take ownership of the ref
+        // passed down to it via metadata.
+        client_stats_.release();
+      }
+
+      void Finish(FinishArgs args) override {
+        if (original_call_tracker_ != nullptr) {
+          original_call_tracker_->Finish(args);
+        }
+      }
+
+     private:
+      RefCountedPtr<GrpcLbClientStats> client_stats_;
+      std::unique_ptr<SubchannelCallTrackerInterface> original_call_tracker_;
+    };
+
     // Serverlist to be used for determining drops.
     RefCountedPtr<Serverlist> serverlist_;
 
-    std::unique_ptr<SubchannelPicker> child_picker_;
+    RefCountedPtr<SubchannelPicker> child_picker_;
     RefCountedPtr<GrpcLbClientStats> client_stats_;
   };
 
@@ -349,11 +462,12 @@ class GrpcLb : public LoadBalancingPolicy {
         : parent_(std::move(parent)) {}
 
     RefCountedPtr<SubchannelInterface> CreateSubchannel(
-        ServerAddress address, const grpc_channel_args& args) override;
+        ServerAddress address, const ChannelArgs& args) override;
     void UpdateState(grpc_connectivity_state state, const absl::Status& status,
-                     std::unique_ptr<SubchannelPicker> picker) override;
+                     RefCountedPtr<SubchannelPicker> picker) override;
     void RequestReresolution() override;
     absl::string_view GetAuthority() override;
+    grpc_event_engine::experimental::EventEngine* GetEventEngine() override;
     void AddTraceEvent(TraceSeverity severity,
                        absl::string_view message) override;
 
@@ -381,7 +495,8 @@ class GrpcLb : public LoadBalancingPolicy {
                 "entering fallback mode",
                 parent_.get(), status.ToString().c_str());
         parent_->fallback_at_startup_checks_pending_ = false;
-        grpc_timer_cancel(&parent_->lb_fallback_timer_);
+        parent_->channel_control_helper()->GetEventEngine()->Cancel(
+            *parent_->lb_fallback_timer_handle_);
         parent_->fallback_mode_ = true;
         parent_->CreateOrUpdateChildPolicyLocked();
         // Cancel the watch, since we don't care about the channel state once we
@@ -393,39 +508,34 @@ class GrpcLb : public LoadBalancingPolicy {
     RefCountedPtr<GrpcLb> parent_;
   };
 
-  ~GrpcLb() override;
-
   void ShutdownLocked() override;
 
   // Helper functions used in UpdateLocked().
-  void UpdateBalancerChannelLocked(const grpc_channel_args& args);
+  absl::Status UpdateBalancerChannelLocked(const ChannelArgs& args);
 
   void CancelBalancerChannelConnectivityWatchLocked();
 
   // Methods for dealing with fallback state.
   void MaybeEnterFallbackModeAfterStartup();
-  static void OnFallbackTimer(void* arg, grpc_error_handle error);
-  void OnFallbackTimerLocked(grpc_error_handle error);
+  void OnFallbackTimerLocked();
 
   // Methods for dealing with the balancer call.
   void StartBalancerCallLocked();
   void StartBalancerCallRetryTimerLocked();
-  static void OnBalancerCallRetryTimer(void* arg, grpc_error_handle error);
-  void OnBalancerCallRetryTimerLocked(grpc_error_handle error);
+  void OnBalancerCallRetryTimerLocked();
 
   // Methods for dealing with the child policy.
-  grpc_channel_args* CreateChildPolicyArgsLocked(
+  ChannelArgs CreateChildPolicyArgsLocked(
       bool is_backend_from_grpclb_load_balancer);
   OrphanablePtr<LoadBalancingPolicy> CreateChildPolicyLocked(
-      const grpc_channel_args* args);
+      const ChannelArgs& args);
   void CreateOrUpdateChildPolicyLocked();
 
   // Subchannel caching.
   void CacheDeletedSubchannelLocked(
       RefCountedPtr<SubchannelInterface> subchannel);
   void StartSubchannelCacheTimerLocked();
-  static void OnSubchannelCacheTimer(void* arg, grpc_error_handle error);
-  void OnSubchannelCacheTimerLocked(grpc_error_handle error);
+  void OnSubchannelCacheTimerLocked();
 
   // Who the client is trying to communicate with.
   std::string server_name_;
@@ -433,7 +543,7 @@ class GrpcLb : public LoadBalancingPolicy {
   RefCountedPtr<GrpcLbConfig> config_;
 
   // Current channel args from the resolver.
-  grpc_channel_args* args_ = nullptr;
+  ChannelArgs args_;
 
   // Internal state.
   bool shutting_down_ = false;
@@ -456,9 +566,7 @@ class GrpcLb : public LoadBalancingPolicy {
   const Duration lb_call_timeout_;
   // Balancer call retry state.
   BackOff lb_call_backoff_;
-  bool retry_timer_callback_pending_ = false;
-  grpc_timer lb_call_retry_timer_;
-  grpc_closure lb_on_call_retry_;
+  absl::optional<EventEngine::TaskHandle> lb_call_retry_timer_handle_;
 
   // The deserialized response from the balancer. May be nullptr until one
   // such response has arrived.
@@ -476,8 +584,7 @@ class GrpcLb : public LoadBalancingPolicy {
   // we have not received a serverlist from the balancer.
   const Duration fallback_at_startup_timeout_;
   bool fallback_at_startup_checks_pending_ = false;
-  grpc_timer lb_fallback_timer_;
-  grpc_closure lb_on_fallback_;
+  absl::optional<EventEngine::TaskHandle> lb_fallback_timer_handle_;
 
   // The child policy to use for the backends.
   OrphanablePtr<LoadBalancingPolicy> child_policy_;
@@ -489,9 +596,7 @@ class GrpcLb : public LoadBalancingPolicy {
   std::map<Timestamp /*deletion time*/,
            std::vector<RefCountedPtr<SubchannelInterface>>>
       cached_subchannels_;
-  grpc_timer subchannel_cache_timer_;
-  grpc_closure on_subchannel_cache_timer_;
-  bool subchannel_cache_timer_pending_ = false;
+  absl::optional<EventEngine::TaskHandle> subchannel_cache_timer_handle_;
 };
 
 //
@@ -506,8 +611,8 @@ void ParseServer(const GrpcLbServer& server, grpc_resolved_address* addr) {
   memset(addr, 0, sizeof(*addr));
   if (server.drop) return;
   const uint16_t netorder_port = grpc_htons(static_cast<uint16_t>(server.port));
-  /* the addresses are given in binary format (a in(6)_addr struct) in
-   * server->ip_address.bytes. */
+  // the addresses are given in binary format (a in(6)_addr struct) in
+  // server->ip_address.bytes.
   if (server.ip_size == 4) {
     addr->len = static_cast<socklen_t>(sizeof(grpc_sockaddr_in));
     grpc_sockaddr_in* addr4 = reinterpret_cast<grpc_sockaddr_in*>(&addr->addr);
@@ -534,7 +639,8 @@ std::string GrpcLb::Serverlist::AsText() const {
     } else {
       grpc_resolved_address addr;
       ParseServer(server, &addr);
-      ipport = grpc_sockaddr_to_string(&addr, false);
+      auto addr_str = grpc_sockaddr_to_string(&addr, false);
+      ipport = addr_str.ok() ? *addr_str : addr_str.status().ToString();
     }
     entries.push_back(absl::StrFormat("  %" PRIuPTR ": %s token=%s\n", i,
                                       ipport, server.load_balance_token));
@@ -582,19 +688,21 @@ ServerAddressList GrpcLb::Serverlist::GetServerAddressList(
         server.load_balance_token, GPR_ARRAY_SIZE(server.load_balance_token));
     std::string lb_token(server.load_balance_token, lb_token_length);
     if (lb_token.empty()) {
+      auto addr_uri = grpc_sockaddr_to_uri(&addr);
       gpr_log(GPR_INFO,
               "Missing LB token for backend address '%s'. The empty token will "
               "be used instead",
-              grpc_sockaddr_to_uri(&addr).c_str());
+              addr_uri.ok() ? addr_uri->c_str()
+                            : addr_uri.status().ToString().c_str());
     }
     // Attach attribute to address containing LB token and stats object.
     std::map<const char*, std::unique_ptr<ServerAddress::AttributeInterface>>
         attributes;
     attributes[kGrpcLbAddressAttributeKey] =
-        absl::make_unique<TokenAndClientStatsAttribute>(std::move(lb_token),
-                                                        stats);
+        std::make_unique<TokenAndClientStatsAttribute>(std::move(lb_token),
+                                                       stats);
     // Add address.
-    addresses.emplace_back(addr, /*args=*/nullptr, std::move(attributes));
+    addresses.emplace_back(addr, ChannelArgs(), std::move(attributes));
   }
   return addresses;
 }
@@ -645,7 +753,10 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
     // client_load_reporting filter.
     GrpcLbClientStats* client_stats = subchannel_wrapper->client_stats();
     if (client_stats != nullptr) {
-      client_stats->Ref().release();  // Ref passed via metadata.
+      complete_pick->subchannel_call_tracker =
+          std::make_unique<SubchannelCallTracker>(
+              client_stats->Ref(),
+              std::move(complete_pick->subchannel_call_tracker));
       // The metadata value is a hack: we pretend the pointer points to
       // a string and rely on the client_load_reporting filter to know
       // how to interpret it.
@@ -676,16 +787,15 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
 //
 
 RefCountedPtr<SubchannelInterface> GrpcLb::Helper::CreateSubchannel(
-    ServerAddress address, const grpc_channel_args& args) {
+    ServerAddress address, const ChannelArgs& args) {
   if (parent_->shutting_down_) return nullptr;
   const TokenAndClientStatsAttribute* attribute =
       static_cast<const TokenAndClientStatsAttribute*>(
           address.GetAttribute(kGrpcLbAddressAttributeKey));
   if (attribute == nullptr) {
-    gpr_log(GPR_ERROR,
-            "[grpclb %p] no TokenAndClientStatsAttribute for address %p",
-            parent_.get(), address.ToString().c_str());
-    abort();
+    Crash(absl::StrFormat(
+        "[grpclb %p] no TokenAndClientStatsAttribute for address %p",
+        parent_.get(), address.ToString().c_str()));
   }
   std::string lb_token = attribute->lb_token();
   RefCountedPtr<GrpcLbClientStats> client_stats = attribute->client_stats();
@@ -698,7 +808,7 @@ RefCountedPtr<SubchannelInterface> GrpcLb::Helper::CreateSubchannel(
 
 void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
                                  const absl::Status& status,
-                                 std::unique_ptr<SubchannelPicker> picker) {
+                                 RefCountedPtr<SubchannelPicker> picker) {
   if (parent_->shutting_down_) return;
   // Record whether child policy reports READY.
   parent_->child_policy_ready_ = state == GRPC_CHANNEL_READY;
@@ -733,8 +843,8 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
   }
   parent_->channel_control_helper()->UpdateState(
       state, status,
-      absl::make_unique<Picker>(std::move(serverlist), std::move(picker),
-                                std::move(client_stats)));
+      MakeRefCounted<Picker>(std::move(serverlist), std::move(picker),
+                             std::move(client_stats)));
 }
 
 void GrpcLb::Helper::RequestReresolution() {
@@ -753,6 +863,10 @@ absl::string_view GrpcLb::Helper::GetAuthority() {
   return parent_->channel_control_helper()->GetAuthority();
 }
 
+grpc_event_engine::experimental::EventEngine* GrpcLb::Helper::GetEventEngine() {
+  return parent_->channel_control_helper()->GetEventEngine();
+}
+
 void GrpcLb::Helper::AddTraceEvent(TraceSeverity severity,
                                    absl::string_view message) {
   if (parent_->shutting_down_) return;
@@ -782,12 +896,12 @@ GrpcLb::BalancerCallState::BalancerCallState(
                     OnBalancerMessageReceived, this, grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&lb_on_balancer_status_received_, OnBalancerStatusReceived,
                     this, grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&client_load_report_closure_, MaybeSendClientLoadReport,
+  GRPC_CLOSURE_INIT(&client_load_report_done_closure_, ClientLoadReportDone,
                     this, grpc_schedule_on_exec_ctx);
   const Timestamp deadline =
       grpclb_policy()->lb_call_timeout_ == Duration::Zero()
           ? Timestamp::InfFuture()
-          : ExecCtx::Get()->Now() + grpclb_policy()->lb_call_timeout_;
+          : Timestamp::Now() + grpclb_policy()->lb_call_timeout_;
   lb_call_ = grpc_channel_create_pollset_set_call(
       grpclb_policy()->lb_channel_, nullptr, GRPC_PROPAGATE_DEFAULTS,
       grpclb_policy_->interested_parties(),
@@ -802,7 +916,7 @@ GrpcLb::BalancerCallState::BalancerCallState(
       arena.ptr());
   send_message_payload_ =
       grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_slice_unref_internal(request_payload_slice);
+  CSliceUnref(request_payload_slice);
   // Init other data associated with the LB call.
   grpc_metadata_array_init(&lb_initial_metadata_recv_);
   grpc_metadata_array_init(&lb_trailing_metadata_recv_);
@@ -815,7 +929,7 @@ GrpcLb::BalancerCallState::~BalancerCallState() {
   grpc_metadata_array_destroy(&lb_trailing_metadata_recv_);
   grpc_byte_buffer_destroy(send_message_payload_);
   grpc_byte_buffer_destroy(recv_message_payload_);
-  grpc_slice_unref_internal(lb_call_status_details_);
+  CSliceUnref(lb_call_status_details_);
 }
 
 void GrpcLb::BalancerCallState::Orphan() {
@@ -825,8 +939,10 @@ void GrpcLb::BalancerCallState::Orphan() {
   // up. Otherwise, we are here because grpclb_policy has to orphan a failed
   // call, then the following cancellation will be a no-op.
   grpc_call_cancel_internal(lb_call_);
-  if (client_load_report_timer_callback_pending_) {
-    grpc_timer_cancel(&client_load_report_timer_);
+  if (client_load_report_handle_.has_value() &&
+      grpclb_policy()->channel_control_helper()->GetEventEngine()->Cancel(
+          client_load_report_handle_.value())) {
+    Unref(DEBUG_LOCATION, "client_load_report cancelled");
   }
   // Note that the initial ref is hold by lb_on_balancer_status_received_
   // instead of the caller of this function. So the corresponding unref happens
@@ -910,34 +1026,20 @@ void GrpcLb::BalancerCallState::StartQuery() {
 }
 
 void GrpcLb::BalancerCallState::ScheduleNextClientLoadReportLocked() {
-  // InvalidateNow to avoid getting stuck re-initializing this timer
-  // in a loop while draining the currently-held WorkSerializer.
-  // Also see https://github.com/grpc/grpc/issues/26079.
-  ExecCtx::Get()->InvalidateNow();
-  const Timestamp next_client_load_report_time =
-      ExecCtx::Get()->Now() + client_stats_report_interval_;
-  GRPC_CLOSURE_INIT(&client_load_report_closure_, MaybeSendClientLoadReport,
-                    this, grpc_schedule_on_exec_ctx);
-  grpc_timer_init(&client_load_report_timer_, next_client_load_report_time,
-                  &client_load_report_closure_);
-  client_load_report_timer_callback_pending_ = true;
-}
-
-void GrpcLb::BalancerCallState::MaybeSendClientLoadReport(
-    void* arg, grpc_error_handle error) {
-  BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  lb_calld->grpclb_policy()->work_serializer()->Run(
-      [lb_calld, error]() { lb_calld->MaybeSendClientLoadReportLocked(error); },
-      DEBUG_LOCATION);
+  client_load_report_handle_ =
+      grpclb_policy()->channel_control_helper()->GetEventEngine()->RunAfter(
+          client_stats_report_interval_, [this] {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            grpclb_policy()->work_serializer()->Run(
+                [this] { MaybeSendClientLoadReportLocked(); }, DEBUG_LOCATION);
+          });
 }
 
-void GrpcLb::BalancerCallState::MaybeSendClientLoadReportLocked(
-    grpc_error_handle error) {
-  client_load_report_timer_callback_pending_ = false;
-  if (error != GRPC_ERROR_NONE || this != grpclb_policy()->lb_calld_.get()) {
+void GrpcLb::BalancerCallState::MaybeSendClientLoadReportLocked() {
+  client_load_report_handle_.reset();
+  if (this != grpclb_policy()->lb_calld_.get()) {
     Unref(DEBUG_LOCATION, "client_load_report");
-    GRPC_ERROR_UNREF(error);
     return;
   }
   // If we've already sent the initial request, then we can go ahead and send
@@ -984,16 +1086,14 @@ void GrpcLb::BalancerCallState::SendClientLoadReportLocked() {
       num_calls_finished_known_received, drop_token_counts.get(), arena.ptr());
   send_message_payload_ =
       grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_slice_unref_internal(request_payload_slice);
+  CSliceUnref(request_payload_slice);
   // Send the report.
   grpc_op op;
   memset(&op, 0, sizeof(op));
   op.op = GRPC_OP_SEND_MESSAGE;
   op.data.send_message.send_message = send_message_payload_;
-  GRPC_CLOSURE_INIT(&client_load_report_closure_, ClientLoadReportDone, this,
-                    grpc_schedule_on_exec_ctx);
   grpc_call_error call_error = grpc_call_start_batch_and_execute(
-      lb_call_, &op, 1, &client_load_report_closure_);
+      lb_call_, &op, 1, &client_load_report_done_closure_);
   if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) {
     gpr_log(GPR_ERROR,
             "[grpclb %p] lb_calld=%p call_error=%d sending client load report",
@@ -1005,7 +1105,6 @@ void GrpcLb::BalancerCallState::SendClientLoadReportLocked() {
 void GrpcLb::BalancerCallState::ClientLoadReportDone(void* arg,
                                                      grpc_error_handle error) {
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
   lb_calld->grpclb_policy()->work_serializer()->Run(
       [lb_calld, error]() { lb_calld->ClientLoadReportDoneLocked(error); },
       DEBUG_LOCATION);
@@ -1015,9 +1114,8 @@ void GrpcLb::BalancerCallState::ClientLoadReportDoneLocked(
     grpc_error_handle error) {
   grpc_byte_buffer_destroy(send_message_payload_);
   send_message_payload_ = nullptr;
-  if (error != GRPC_ERROR_NONE || this != grpclb_policy()->lb_calld_.get()) {
+  if (!error.ok() || this != grpclb_policy()->lb_calld_.get()) {
     Unref(DEBUG_LOCATION, "client_load_report");
-    GRPC_ERROR_UNREF(error);
     return;
   }
   ScheduleNextClientLoadReportLocked();
@@ -1157,7 +1255,8 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
           }
           if (grpclb_policy()->fallback_at_startup_checks_pending_) {
             grpclb_policy()->fallback_at_startup_checks_pending_ = false;
-            grpc_timer_cancel(&grpclb_policy()->lb_fallback_timer_);
+            grpclb_policy()->channel_control_helper()->GetEventEngine()->Cancel(
+                *grpclb_policy()->lb_fallback_timer_handle_);
             grpclb_policy()->CancelBalancerChannelConnectivityWatchLocked();
           }
           // Update the serverlist in the GrpcLb instance. This serverlist
@@ -1175,7 +1274,8 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
                   grpclb_policy());
           if (grpclb_policy()->fallback_at_startup_checks_pending_) {
             grpclb_policy()->fallback_at_startup_checks_pending_ = false;
-            grpc_timer_cancel(&grpclb_policy()->lb_fallback_timer_);
+            grpclb_policy()->channel_control_helper()->GetEventEngine()->Cancel(
+                *grpclb_policy()->lb_fallback_timer_handle_);
             grpclb_policy()->CancelBalancerChannelConnectivityWatchLocked();
           }
           grpclb_policy()->fallback_mode_ = true;
@@ -1189,7 +1289,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
       }
     }
   }
-  grpc_slice_unref_internal(response_slice);
+  CSliceUnref(response_slice);
   if (!grpclb_policy()->shutting_down_) {
     // Keep listening for serverlist updates.
     grpc_op op;
@@ -1210,7 +1310,6 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
 void GrpcLb::BalancerCallState::OnBalancerStatusReceived(
     void* arg, grpc_error_handle error) {
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  (void)GRPC_ERROR_REF(error);  // owned by lambda
   lb_calld->grpclb_policy()->work_serializer()->Run(
       [lb_calld, error]() { lb_calld->OnBalancerStatusReceivedLocked(error); },
       DEBUG_LOCATION);
@@ -1225,10 +1324,9 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
             "[grpclb %p] lb_calld=%p: Status from LB server received. "
             "Status = %d, details = '%s', (lb_call: %p), error '%s'",
             grpclb_policy(), this, lb_call_status_, status_details, lb_call_,
-            grpc_error_std_string(error).c_str());
+            StatusToString(error).c_str());
     gpr_free(status_details);
   }
-  GRPC_ERROR_UNREF(error);
   // If this lb_calld is still in use, this call ended because of a failure so
   // we want to retry connecting. Otherwise, we have deliberately ended this
   // call and no further action is required.
@@ -1236,6 +1334,7 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
     // If the fallback-at-startup checks are pending, go into fallback mode
     // immediately.  This short-circuits the timeout for the fallback-at-startup
     // case.
+    grpclb_policy()->lb_calld_.reset();
     if (grpclb_policy()->fallback_at_startup_checks_pending_) {
       GPR_ASSERT(!seen_serverlist_);
       gpr_log(GPR_INFO,
@@ -1243,7 +1342,8 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
               "serverlist; entering fallback mode",
               grpclb_policy());
       grpclb_policy()->fallback_at_startup_checks_pending_ = false;
-      grpc_timer_cancel(&grpclb_policy()->lb_fallback_timer_);
+      grpclb_policy()->channel_control_helper()->GetEventEngine()->Cancel(
+          *grpclb_policy()->lb_fallback_timer_handle_);
       grpclb_policy()->CancelBalancerChannelConnectivityWatchLocked();
       grpclb_policy()->fallback_mode_ = true;
       grpclb_policy()->CreateOrUpdateChildPolicyLocked();
@@ -1251,7 +1351,6 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
       // This handles the fallback-after-startup case.
       grpclb_policy()->MaybeEnterFallbackModeAfterStartup();
     }
-    grpclb_policy()->lb_calld_.reset();
     GPR_ASSERT(!grpclb_policy()->shutting_down_);
     grpclb_policy()->channel_control_helper()->RequestReresolution();
     if (seen_initial_response_) {
@@ -1272,102 +1371,104 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
 // helper code for creating balancer channel
 //
 
-ServerAddressList ExtractBalancerAddresses(const grpc_channel_args& args) {
+ServerAddressList ExtractBalancerAddresses(const ChannelArgs& args) {
   const ServerAddressList* addresses =
       FindGrpclbBalancerAddressesInChannelArgs(args);
   if (addresses != nullptr) return *addresses;
   return ServerAddressList();
 }
 
-/* Returns the channel args for the LB channel, used to create a bidirectional
- * stream for the reception of load balancing updates.
- *
- * Inputs:
- *   - \a response_generator: in order to propagate updates from the resolver
- *   above the grpclb policy.
- *   - \a args: other args inherited from the grpclb policy. */
-grpc_channel_args* BuildBalancerChannelArgs(
+// Returns the channel args for the LB channel, used to create a bidirectional
+// stream for the reception of load balancing updates.
+//
+// Inputs:
+//   - \a response_generator: in order to propagate updates from the resolver
+//   above the grpclb policy.
+//   - \a args: other args inherited from the grpclb policy.
+ChannelArgs BuildBalancerChannelArgs(
     FakeResolverResponseGenerator* response_generator,
-    const grpc_channel_args* args) {
-  // Channel args to remove.
-  static const char* args_to_remove[] = {
-      // LB policy name, since we want to use the default (pick_first) in
-      // the LB channel.
-      GRPC_ARG_LB_POLICY_NAME,
-      // Strip out the service config, since we don't want the LB policy
-      // config specified for the parent channel to affect the LB channel.
-      GRPC_ARG_SERVICE_CONFIG,
-      // The channel arg for the server URI, since that will be different for
-      // the LB channel than for the parent channel.  The client channel
-      // factory will re-add this arg with the right value.
-      GRPC_ARG_SERVER_URI,
-      // The fake resolver response generator, because we are replacing it
-      // with the one from the grpclb policy, used to propagate updates to
-      // the LB channel.
-      GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR,
-      // The LB channel should use the authority indicated by the target
-      // authority table (see \a ModifyGrpclbBalancerChannelArgs),
-      // as opposed to the authority from the parent channel.
-      GRPC_ARG_DEFAULT_AUTHORITY,
-      // Just as for \a GRPC_ARG_DEFAULT_AUTHORITY, the LB channel should be
-      // treated as a stand-alone channel and not inherit this argument from the
-      // args of the parent channel.
-      GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
-      // Don't want to pass down channelz node from parent; the balancer
-      // channel will get its own.
-      GRPC_ARG_CHANNELZ_CHANNEL_NODE,
-      // Remove the channel args for channel credentials and replace it
-      // with a version that does not contain call credentials. The loadbalancer
-      // is not necessarily trusted to handle bearer token credentials.
-      GRPC_ARG_CHANNEL_CREDENTIALS,
-  };
+    const ChannelArgs& args) {
+  ChannelArgs grpclb_channel_args;
+  const grpc_channel_args* lb_channel_specific_args =
+      args.GetPointer<grpc_channel_args>(
+          GRPC_ARG_EXPERIMENTAL_GRPCLB_CHANNEL_ARGS);
+  if (lb_channel_specific_args != nullptr) {
+    grpclb_channel_args = ChannelArgs::FromC(lb_channel_specific_args);
+  } else {
+    // Set grpclb_channel_args based on the parent channel's channel args.
+    grpclb_channel_args =
+        args
+            // LB policy name, since we want to use the default (pick_first) in
+            // the LB channel.
+            .Remove(GRPC_ARG_LB_POLICY_NAME)
+            // Strip out the service config, since we don't want the LB policy
+            // config specified for the parent channel to affect the LB channel.
+            .Remove(GRPC_ARG_SERVICE_CONFIG)
+            // The channel arg for the server URI, since that will be different
+            // for the LB channel than for the parent channel.  The client
+            // channel factory will re-add this arg with the right value.
+            .Remove(GRPC_ARG_SERVER_URI)
+            // The fake resolver response generator, because we are replacing it
+            // with the one from the grpclb policy, used to propagate updates to
+            // the LB channel.
+            .Remove(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR)
+            // The LB channel should use the authority indicated by the target
+            // authority table (see \a ModifyGrpclbBalancerChannelArgs),
+            // as opposed to the authority from the parent channel.
+            .Remove(GRPC_ARG_DEFAULT_AUTHORITY)
+            // Just as for \a GRPC_ARG_DEFAULT_AUTHORITY, the LB channel should
+            // be treated as a stand-alone channel and not inherit this argument
+            // from the args of the parent channel.
+            .Remove(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG)
+            // Don't want to pass down channelz node from parent; the balancer
+            // channel will get its own.
+            .Remove(GRPC_ARG_CHANNELZ_CHANNEL_NODE)
+            // Remove the channel args for channel credentials and replace it
+            // with a version that does not contain call credentials. The
+            // loadbalancer is not necessarily trusted to handle bearer token
+            // credentials.
+            .Remove(GRPC_ARG_CHANNEL_CREDENTIALS);
+  }
   // Create channel args for channel credentials that does not contain bearer
   // token credentials.
-  grpc_channel_credentials* channel_credentials =
-      grpc_channel_credentials_find_in_args(args);
+  auto* channel_credentials = args.GetObject<grpc_channel_credentials>();
   GPR_ASSERT(channel_credentials != nullptr);
   RefCountedPtr<grpc_channel_credentials> creds_sans_call_creds =
       channel_credentials->duplicate_without_call_credentials();
   GPR_ASSERT(creds_sans_call_creds != nullptr);
-  // Channel args to add.
-  absl::InlinedVector<grpc_arg, 4> args_to_add = {
-      // The fake resolver response generator, which we use to inject
-      // address updates into the LB channel.
-      FakeResolverResponseGenerator::MakeChannelArg(response_generator),
+  return grpclb_channel_args
       // A channel arg indicating the target is a grpclb load balancer.
-      grpc_channel_arg_integer_create(
-          const_cast<char*>(GRPC_ARG_ADDRESS_IS_GRPCLB_LOAD_BALANCER), 1),
+      .Set(GRPC_ARG_ADDRESS_IS_GRPCLB_LOAD_BALANCER, 1)
       // Tells channelz that this is an internal channel.
-      grpc_channel_arg_integer_create(
-          const_cast<char*>(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL), 1),
+      .Set(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL, 1)
       // A channel args for new channel credentials that does not contain bearer
       // tokens.
-      grpc_channel_credentials_to_arg(creds_sans_call_creds.get()),
-  };
-  return grpc_channel_args_copy_and_add_and_remove(
-      args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), args_to_add.data(),
-      args_to_add.size());
+      .SetObject(creds_sans_call_creds)
+      // The fake resolver response generator, which we use to inject
+      // address updates into the LB channel.
+      .SetObject(response_generator->Ref());
 }
 
 //
 // ctor and dtor
 //
 
-std::string GetServerNameFromChannelArgs(const grpc_channel_args* args) {
-  const char* server_uri =
-      grpc_channel_args_find_string(args, GRPC_ARG_SERVER_URI);
-  GPR_ASSERT(server_uri != nullptr);
-  absl::StatusOr<URI> uri = URI::Parse(server_uri);
+std::string GetServerNameFromChannelArgs(const ChannelArgs& args) {
+  absl::StatusOr<URI> uri =
+      URI::Parse(args.GetString(GRPC_ARG_SERVER_URI).value());
   GPR_ASSERT(uri.ok() && !uri->path().empty());
   return std::string(absl::StripPrefix(uri->path(), "/"));
 }
 
 GrpcLb::GrpcLb(Args args)
     : LoadBalancingPolicy(std::move(args)),
-      server_name_(GetServerNameFromChannelArgs(args.args)),
+      server_name_(GetServerNameFromChannelArgs(channel_args())),
       response_generator_(MakeRefCounted<FakeResolverResponseGenerator>()),
-      lb_call_timeout_(Duration::Milliseconds(grpc_channel_args_find_integer(
-          args.args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS, {0, 0, INT_MAX}))),
+      lb_call_timeout_(std::max(
+          Duration::Zero(),
+          channel_args()
+              .GetDurationFromIntMillis(GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS)
+              .value_or(Duration::Zero()))),
       lb_call_backoff_(
           BackOff::Options()
               .set_initial_backoff(Duration::Seconds(
@@ -1376,45 +1477,43 @@ GrpcLb::GrpcLb(Args args)
               .set_jitter(GRPC_GRPCLB_RECONNECT_JITTER)
               .set_max_backoff(Duration::Seconds(
                   GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS))),
-      fallback_at_startup_timeout_(
-          Duration::Milliseconds(grpc_channel_args_find_integer(
-              args.args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS,
-              {GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX}))),
-      subchannel_cache_interval_(
-          Duration::Milliseconds(grpc_channel_args_find_integer(
-              args.args, GRPC_ARG_GRPCLB_SUBCHANNEL_CACHE_INTERVAL_MS,
-              {GRPC_GRPCLB_DEFAULT_SUBCHANNEL_DELETION_DELAY_MS, 0,
-               INT_MAX}))) {
+      fallback_at_startup_timeout_(std::max(
+          Duration::Zero(),
+          channel_args()
+              .GetDurationFromIntMillis(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS)
+              .value_or(Duration::Milliseconds(
+                  GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS)))),
+      subchannel_cache_interval_(std::max(
+          Duration::Zero(),
+          channel_args()
+              .GetDurationFromIntMillis(
+                  GRPC_ARG_GRPCLB_SUBCHANNEL_CACHE_INTERVAL_MS)
+              .value_or(Duration::Milliseconds(
+                  GRPC_GRPCLB_DEFAULT_SUBCHANNEL_DELETION_DELAY_MS)))) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO,
             "[grpclb %p] Will use '%s' as the server name for LB request.",
             this, server_name_.c_str());
   }
-  // Closure Initialization
-  GRPC_CLOSURE_INIT(&lb_on_fallback_, &GrpcLb::OnFallbackTimer, this,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&lb_on_call_retry_, &GrpcLb::OnBalancerCallRetryTimer, this,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&on_subchannel_cache_timer_, &OnSubchannelCacheTimer, this,
-                    nullptr);
 }
 
-GrpcLb::~GrpcLb() { grpc_channel_args_destroy(args_); }
-
 void GrpcLb::ShutdownLocked() {
   shutting_down_ = true;
   lb_calld_.reset();
-  if (subchannel_cache_timer_pending_) {
-    subchannel_cache_timer_pending_ = false;
-    grpc_timer_cancel(&subchannel_cache_timer_);
+  if (subchannel_cache_timer_handle_.has_value()) {
+    channel_control_helper()->GetEventEngine()->Cancel(
+        *subchannel_cache_timer_handle_);
+    subchannel_cache_timer_handle_.reset();
   }
   cached_subchannels_.clear();
-  if (retry_timer_callback_pending_) {
-    grpc_timer_cancel(&lb_call_retry_timer_);
+  if (lb_call_retry_timer_handle_.has_value()) {
+    channel_control_helper()->GetEventEngine()->Cancel(
+        *lb_call_retry_timer_handle_);
   }
   if (fallback_at_startup_checks_pending_) {
     fallback_at_startup_checks_pending_ = false;
-    grpc_timer_cancel(&lb_fallback_timer_);
+    channel_control_helper()->GetEventEngine()->Cancel(
+        *lb_fallback_timer_handle_);
     CancelBalancerChannelConnectivityWatchLocked();
   }
   if (child_policy_ != nullptr) {
@@ -1451,7 +1550,7 @@ void GrpcLb::ResetBackoffLocked() {
   }
 }
 
-void GrpcLb::UpdateLocked(UpdateArgs args) {
+absl::Status GrpcLb::UpdateLocked(UpdateArgs args) {
   const bool is_initial_update = lb_channel_ == nullptr;
   config_ = args.config;
   GPR_ASSERT(config_ != nullptr);
@@ -1462,12 +1561,12 @@ void GrpcLb::UpdateLocked(UpdateArgs args) {
     for (ServerAddress& address : *fallback_backend_addresses_) {
       address = address.WithAttribute(
           kGrpcLbAddressAttributeKey,
-          absl::make_unique<TokenAndClientStatsAttribute>("", nullptr));
+          std::make_unique<TokenAndClientStatsAttribute>("", nullptr));
     }
   }
   resolution_note_ = std::move(args.resolution_note);
   // Update balancer channel.
-  UpdateBalancerChannelLocked(*args.args);
+  absl::Status status = UpdateBalancerChannelLocked(args.args);
   // Update the existing child policy, if any.
   if (child_policy_ != nullptr) CreateOrUpdateChildPolicyLocked();
   // If this is the initial update, start the fallback-at-startup checks
@@ -1475,13 +1574,23 @@ void GrpcLb::UpdateLocked(UpdateArgs args) {
   if (is_initial_update) {
     fallback_at_startup_checks_pending_ = true;
     // Start timer.
-    Timestamp deadline = ExecCtx::Get()->Now() + fallback_at_startup_timeout_;
-    Ref(DEBUG_LOCATION, "on_fallback_timer").release();  // Ref for callback
-    grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_);
+    lb_fallback_timer_handle_ =
+        channel_control_helper()->GetEventEngine()->RunAfter(
+            fallback_at_startup_timeout_,
+            [self = static_cast<RefCountedPtr<GrpcLb>>(
+                 Ref(DEBUG_LOCATION, "on_fallback_timer"))]() mutable {
+              ApplicationCallbackExecCtx callback_exec_ctx;
+              ExecCtx exec_ctx;
+              auto self_ptr = self.get();
+              self_ptr->work_serializer()->Run(
+                  [self = std::move(self)]() { self->OnFallbackTimerLocked(); },
+                  DEBUG_LOCATION);
+            });
     // Start watching the channel's connectivity state.  If the channel
     // goes into state TRANSIENT_FAILURE before the timer fires, we go into
     // fallback mode even if the fallback timeout has not elapsed.
-    ClientChannel* client_channel = ClientChannel::GetFromChannel(lb_channel_);
+    ClientChannel* client_channel =
+        ClientChannel::GetFromChannel(Channel::FromC(lb_channel_));
     GPR_ASSERT(client_channel != nullptr);
     // Ref held by callback.
     watcher_ = new StateWatcher(Ref(DEBUG_LOCATION, "StateWatcher"));
@@ -1491,43 +1600,40 @@ void GrpcLb::UpdateLocked(UpdateArgs args) {
     // Start balancer call.
     StartBalancerCallLocked();
   }
+  return status;
 }
 
 //
 // helpers for UpdateLocked()
 //
 
-void GrpcLb::UpdateBalancerChannelLocked(const grpc_channel_args& args) {
+absl::Status GrpcLb::UpdateBalancerChannelLocked(const ChannelArgs& args) {
   // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args,
   // since we use this to trigger the client_load_reporting filter.
-  static const char* args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME};
-  grpc_arg new_arg = grpc_channel_arg_string_create(
-      const_cast<char*>(GRPC_ARG_LB_POLICY_NAME), const_cast<char*>("grpclb"));
-  grpc_channel_args_destroy(args_);
-  args_ = grpc_channel_args_copy_and_add_and_remove(
-      &args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1);
-  // Construct args for balancer channel.
+  args_ = args.Set(GRPC_ARG_LB_POLICY_NAME, "grpclb");
+  // Get balancer addresses.
   ServerAddressList balancer_addresses = ExtractBalancerAddresses(args);
-  grpc_channel_args* lb_channel_args =
-      BuildBalancerChannelArgs(response_generator_.get(), &args);
+  absl::Status status;
+  if (balancer_addresses.empty()) {
+    status = absl::UnavailableError("balancer address list must be non-empty");
+  }
+  // Construct args for balancer channel.
+  ChannelArgs lb_channel_args =
+      BuildBalancerChannelArgs(response_generator_.get(), args);
   // Create balancer channel if needed.
   if (lb_channel_ == nullptr) {
     std::string uri_str = absl::StrCat("fake:///", server_name_);
-    grpc_channel_credentials* creds =
-        grpc_channel_credentials_find_in_args(lb_channel_args);
+    auto* creds = lb_channel_args.GetObject<grpc_channel_credentials>();
     GPR_ASSERT(creds != nullptr);
-    const char* arg_to_remove = GRPC_ARG_CHANNEL_CREDENTIALS;
-    grpc_channel_args* new_args =
-        grpc_channel_args_copy_and_remove(lb_channel_args, &arg_to_remove, 1);
-    lb_channel_ = grpc_channel_create(uri_str.c_str(), creds, new_args);
+    lb_channel_ = grpc_channel_create(
+        uri_str.c_str(), creds,
+        lb_channel_args.Remove(GRPC_ARG_CHANNEL_CREDENTIALS).ToC().get());
     GPR_ASSERT(lb_channel_ != nullptr);
-    grpc_channel_args_destroy(new_args);
     // Set up channelz linkage.
     channelz::ChannelNode* child_channelz_node =
         grpc_channel_get_channelz_node(lb_channel_);
     channelz::ChannelNode* parent_channelz_node =
-        grpc_channel_args_find_pointer<channelz::ChannelNode>(
-            &args, GRPC_ARG_CHANNELZ_CHANNEL_NODE);
+        args.GetObject<channelz::ChannelNode>();
     if (child_channelz_node != nullptr && parent_channelz_node != nullptr) {
       parent_channelz_node->AddChildChannel(child_channelz_node->uuid());
       parent_channelz_node_ = parent_channelz_node->Ref();
@@ -1539,10 +1645,13 @@ void GrpcLb::UpdateBalancerChannelLocked(const grpc_channel_args& args) {
   result.addresses = std::move(balancer_addresses);
   result.args = lb_channel_args;
   response_generator_->SetResponse(std::move(result));
+  // Return status.
+  return status;
 }
 
 void GrpcLb::CancelBalancerChannelConnectivityWatchLocked() {
-  ClientChannel* client_channel = ClientChannel::GetFromChannel(lb_channel_);
+  ClientChannel* client_channel =
+      ClientChannel::GetFromChannel(Channel::FromC(lb_channel_));
   GPR_ASSERT(client_channel != nullptr);
   client_channel->RemoveConnectivityWatcher(watcher_);
 }
@@ -1566,10 +1675,9 @@ void GrpcLb::StartBalancerCallLocked() {
 }
 
 void GrpcLb::StartBalancerCallRetryTimerLocked() {
-  Timestamp next_try = lb_call_backoff_.NextAttemptTime();
+  Duration timeout = lb_call_backoff_.NextAttemptTime() - Timestamp::Now();
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p] Connection to LB server lost...", this);
-    Duration timeout = next_try - ExecCtx::Get()->Now();
     if (timeout > Duration::Zero()) {
       gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active in %" PRId64 "ms.",
               this, timeout.millis());
@@ -1578,35 +1686,30 @@ void GrpcLb::StartBalancerCallRetryTimerLocked() {
               this);
     }
   }
-  // TODO(roth): We currently track this ref manually.  Once the
-  // ClosureRef API is ready, we should pass the RefCountedPtr<> along
-  // with the callback.
-  auto self = Ref(DEBUG_LOCATION, "on_balancer_call_retry_timer");
-  self.release();
-  retry_timer_callback_pending_ = true;
-  grpc_timer_init(&lb_call_retry_timer_, next_try, &lb_on_call_retry_);
-}
-
-void GrpcLb::OnBalancerCallRetryTimer(void* arg, grpc_error_handle error) {
-  GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  grpclb_policy->work_serializer()->Run(
-      [grpclb_policy, error]() {
-        grpclb_policy->OnBalancerCallRetryTimerLocked(error);
-      },
-      DEBUG_LOCATION);
+  lb_call_retry_timer_handle_ =
+      channel_control_helper()->GetEventEngine()->RunAfter(
+          timeout,
+          [self = static_cast<RefCountedPtr<GrpcLb>>(
+               Ref(DEBUG_LOCATION, "on_balancer_call_retry_timer"))]() mutable {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            auto self_ptr = self.get();
+            self_ptr->work_serializer()->Run(
+                [self = std::move(self)]() {
+                  self->OnBalancerCallRetryTimerLocked();
+                },
+                DEBUG_LOCATION);
+          });
 }
 
-void GrpcLb::OnBalancerCallRetryTimerLocked(grpc_error_handle error) {
-  retry_timer_callback_pending_ = false;
-  if (!shutting_down_ && error == GRPC_ERROR_NONE && lb_calld_ == nullptr) {
+void GrpcLb::OnBalancerCallRetryTimerLocked() {
+  lb_call_retry_timer_handle_.reset();
+  if (!shutting_down_ && lb_calld_ == nullptr) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO, "[grpclb %p] Restarting call to LB server", this);
     }
     StartBalancerCallLocked();
   }
-  Unref(DEBUG_LOCATION, "on_balancer_call_retry_timer");
-  GRPC_ERROR_UNREF(error);
 }
 
 //
@@ -1631,19 +1734,10 @@ void GrpcLb::MaybeEnterFallbackModeAfterStartup() {
   }
 }
 
-void GrpcLb::OnFallbackTimer(void* arg, grpc_error_handle error) {
-  GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
-  grpclb_policy->work_serializer()->Run(
-      [grpclb_policy, error]() { grpclb_policy->OnFallbackTimerLocked(error); },
-      DEBUG_LOCATION);
-}
-
-void GrpcLb::OnFallbackTimerLocked(grpc_error_handle error) {
+void GrpcLb::OnFallbackTimerLocked() {
   // If we receive a serverlist after the timer fires but before this callback
   // actually runs, don't fall back.
-  if (fallback_at_startup_checks_pending_ && !shutting_down_ &&
-      error == GRPC_ERROR_NONE) {
+  if (fallback_at_startup_checks_pending_ && !shutting_down_) {
     gpr_log(GPR_INFO,
             "[grpclb %p] No response from balancer after fallback timeout; "
             "entering fallback mode",
@@ -1653,34 +1747,30 @@ void GrpcLb::OnFallbackTimerLocked(grpc_error_handle error) {
     fallback_mode_ = true;
     CreateOrUpdateChildPolicyLocked();
   }
-  Unref(DEBUG_LOCATION, "on_fallback_timer");
-  GRPC_ERROR_UNREF(error);
 }
 
 //
 // code for interacting with the child policy
 //
 
-grpc_channel_args* GrpcLb::CreateChildPolicyArgsLocked(
+ChannelArgs GrpcLb::CreateChildPolicyArgsLocked(
     bool is_backend_from_grpclb_load_balancer) {
-  absl::InlinedVector<grpc_arg, 2> args_to_add;
-  args_to_add.emplace_back(grpc_channel_arg_integer_create(
-      const_cast<char*>(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
-      is_backend_from_grpclb_load_balancer));
+  ChannelArgs r =
+      args_.Set(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER,
+                is_backend_from_grpclb_load_balancer);
   if (is_backend_from_grpclb_load_balancer) {
-    args_to_add.emplace_back(grpc_channel_arg_integer_create(
-        const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1));
+    r = r.Set(GRPC_ARG_INHIBIT_HEALTH_CHECKING, 1);
   }
-  return grpc_channel_args_copy_and_add(args_, args_to_add.data(),
-                                        args_to_add.size());
+  return r;
 }
 
 OrphanablePtr<LoadBalancingPolicy> GrpcLb::CreateChildPolicyLocked(
-    const grpc_channel_args* args) {
+    const ChannelArgs& args) {
   LoadBalancingPolicy::Args lb_policy_args;
   lb_policy_args.work_serializer = work_serializer();
   lb_policy_args.args = args;
-  lb_policy_args.channel_control_helper = absl::make_unique<Helper>(Ref());
+  lb_policy_args.channel_control_helper =
+      std::make_unique<Helper>(Ref(DEBUG_LOCATION, "Helper"));
   OrphanablePtr<LoadBalancingPolicy> lb_policy =
       MakeOrphanable<ChildPolicyHandler>(std::move(lb_policy_args),
                                          &grpc_lb_glb_trace);
@@ -1703,9 +1793,10 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
   bool is_backend_from_grpclb_load_balancer = false;
   if (fallback_mode_) {
     // If CreateOrUpdateChildPolicyLocked() is invoked when we haven't
-    // received any serverlist from the balancer, we use the fallback backends
-    // returned by the resolver. Note that the fallback backend list may be
-    // empty, in which case the new child policy will fail the picks.
+    // received any serverlist from the balancer, we use the fallback
+    // backends returned by the resolver. Note that the fallback backend
+    // list may be empty, in which case the new child policy will fail the
+    // picks.
     update_args.addresses = fallback_backend_addresses_;
     if (fallback_backend_addresses_.ok() &&
         fallback_backend_addresses_->empty()) {
@@ -1720,7 +1811,7 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
   }
   update_args.args =
       CreateChildPolicyArgsLocked(is_backend_from_grpclb_load_balancer);
-  GPR_ASSERT(update_args.args != nullptr);
+  GPR_ASSERT(update_args.args != ChannelArgs());
   update_args.config = config_->child_policy();
   // Create child policy if needed.
   if (child_policy_ == nullptr) {
@@ -1731,7 +1822,9 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
     gpr_log(GPR_INFO, "[grpclb %p] Updating child policy handler %p", this,
             child_policy_.get());
   }
-  child_policy_->UpdateLocked(std::move(update_args));
+  // TODO(roth): If we're in fallback mode and the child policy rejects the
+  // update, we should propagate that failure back to the resolver somehow.
+  (void)child_policy_->UpdateLocked(std::move(update_args));
 }
 
 //
@@ -1740,31 +1833,34 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
 
 void GrpcLb::CacheDeletedSubchannelLocked(
     RefCountedPtr<SubchannelInterface> subchannel) {
-  Timestamp deletion_time = ExecCtx::Get()->Now() + subchannel_cache_interval_;
+  Timestamp deletion_time = Timestamp::Now() + subchannel_cache_interval_;
   cached_subchannels_[deletion_time].push_back(std::move(subchannel));
-  if (!subchannel_cache_timer_pending_) {
-    Ref(DEBUG_LOCATION, "OnSubchannelCacheTimer").release();
-    subchannel_cache_timer_pending_ = true;
+  if (!subchannel_cache_timer_handle_.has_value()) {
     StartSubchannelCacheTimerLocked();
   }
 }
 
 void GrpcLb::StartSubchannelCacheTimerLocked() {
   GPR_ASSERT(!cached_subchannels_.empty());
-  grpc_timer_init(&subchannel_cache_timer_, cached_subchannels_.begin()->first,
-                  &on_subchannel_cache_timer_);
-}
-
-void GrpcLb::OnSubchannelCacheTimer(void* arg, grpc_error_handle error) {
-  auto* self = static_cast<GrpcLb*>(arg);
-  (void)GRPC_ERROR_REF(error);
-  self->work_serializer()->Run(
-      [self, error]() { self->GrpcLb::OnSubchannelCacheTimerLocked(error); },
-      DEBUG_LOCATION);
+  subchannel_cache_timer_handle_ =
+      channel_control_helper()->GetEventEngine()->RunAfter(
+          cached_subchannels_.begin()->first - Timestamp::Now(),
+          [self = static_cast<RefCountedPtr<GrpcLb>>(
+               Ref(DEBUG_LOCATION, "OnSubchannelCacheTimer"))]() mutable {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            auto* self_ptr = self.get();
+            self_ptr->work_serializer()->Run(
+                [self = std::move(self)]() mutable {
+                  self->OnSubchannelCacheTimerLocked();
+                },
+                DEBUG_LOCATION);
+          });
 }
 
-void GrpcLb::OnSubchannelCacheTimerLocked(grpc_error_handle error) {
-  if (subchannel_cache_timer_pending_ && error == GRPC_ERROR_NONE) {
+void GrpcLb::OnSubchannelCacheTimerLocked() {
+  if (subchannel_cache_timer_handle_.has_value()) {
+    subchannel_cache_timer_handle_.reset();
     auto it = cached_subchannels_.begin();
     if (it != cached_subchannels_.end()) {
       if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
@@ -1778,10 +1874,7 @@ void GrpcLb::OnSubchannelCacheTimerLocked(grpc_error_handle error) {
       StartSubchannelCacheTimerLocked();
       return;
     }
-    subchannel_cache_timer_pending_ = false;
   }
-  Unref(DEBUG_LOCATION, "OnSubchannelCacheTimer");
-  GRPC_ERROR_UNREF(error);
 }
 
 //
@@ -1795,91 +1888,33 @@ class GrpcLbFactory : public LoadBalancingPolicyFactory {
     return MakeOrphanable<GrpcLb>(std::move(args));
   }
 
-  const char* name() const override { return kGrpclb; }
+  absl::string_view name() const override { return kGrpclb; }
 
-  RefCountedPtr<LoadBalancingPolicy::Config> ParseLoadBalancingConfig(
-      const Json& json, grpc_error_handle* error) const override {
-    GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE);
-    if (json.type() == Json::Type::JSON_NULL) {
-      return MakeRefCounted<GrpcLbConfig>(nullptr, "");
-    }
-    std::vector<grpc_error_handle> error_list;
-    Json child_policy_config_json_tmp;
-    const Json* child_policy_config_json;
-    std::string service_name;
-    auto it = json.object_value().find("serviceName");
-    if (it != json.object_value().end()) {
-      const Json& service_name_json = it->second;
-      if (service_name_json.type() != Json::Type::STRING) {
-        error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-            "field:serviceName error:type should be string"));
-      } else {
-        service_name = service_name_json.string_value();
-      }
-    }
-    it = json.object_value().find("childPolicy");
-    if (it == json.object_value().end()) {
-      child_policy_config_json_tmp = Json::Array{Json::Object{
-          {"round_robin", Json::Object()},
-      }};
-      child_policy_config_json = &child_policy_config_json_tmp;
-    } else {
-      child_policy_config_json = &it->second;
-    }
-    grpc_error_handle parse_error = GRPC_ERROR_NONE;
-    RefCountedPtr<LoadBalancingPolicy::Config> child_policy_config =
-        LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(
-            *child_policy_config_json, &parse_error);
-    if (parse_error != GRPC_ERROR_NONE) {
-      std::vector<grpc_error_handle> child_errors;
-      child_errors.push_back(parse_error);
-      error_list.push_back(
-          GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors));
-    }
-    if (error_list.empty()) {
-      return MakeRefCounted<GrpcLbConfig>(std::move(child_policy_config),
-                                          std::move(service_name));
-    } else {
-      *error = GRPC_ERROR_CREATE_FROM_VECTOR("GrpcLb Parser", &error_list);
-      return nullptr;
-    }
+  absl::StatusOr<RefCountedPtr<LoadBalancingPolicy::Config>>
+  ParseLoadBalancingConfig(const Json& json) const override {
+    return LoadRefCountedFromJson<GrpcLbConfig>(
+        json, JsonArgs(), "errors validating grpclb LB policy config");
   }
 };
 
 }  // namespace
 
-}  // namespace grpc_core
-
 //
 // Plugin registration
 //
 
-void grpc_lb_policy_grpclb_init() {
-  grpc_core::LoadBalancingPolicyRegistry::Builder::
-      RegisterLoadBalancingPolicyFactory(
-          absl::make_unique<grpc_core::GrpcLbFactory>());
-}
-
-void grpc_lb_policy_grpclb_shutdown() {}
-
-namespace grpc_core {
-void RegisterGrpcLbLoadReportingFilter(CoreConfiguration::Builder* builder) {
+void RegisterGrpcLbPolicy(CoreConfiguration::Builder* builder) {
+  builder->lb_policy_registry()->RegisterLoadBalancingPolicyFactory(
+      std::make_unique<GrpcLbFactory>());
   builder->channel_init()->RegisterStage(
       GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
       [](ChannelStackBuilder* builder) {
-        const grpc_channel_args* args = builder->channel_args();
-        const grpc_arg* channel_arg =
-            grpc_channel_args_find(args, GRPC_ARG_LB_POLICY_NAME);
-        if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_STRING &&
-            strcmp(channel_arg->value.string, "grpclb") == 0) {
-          // TODO(roth): When we get around to re-attempting
-          // https://github.com/grpc/grpc/pull/16214, we should try to keep
-          // this filter at the very top of the subchannel stack, since that
-          // will minimize the number of metadata elements that the filter
-          // needs to iterate through to find the ClientStats object.
-          builder->PrependFilter(&grpc_client_load_reporting_filter, nullptr);
+        if (builder->channel_args().GetString(GRPC_ARG_LB_POLICY_NAME) ==
+            "grpclb") {
+          builder->PrependFilter(&ClientLoadReportingFilter::kFilter);
         }
         return true;
       });
 }
+
 }  // namespace grpc_core