grpc 1.45.0 → 1.53.0

data/src/core/ext/xds/xds_client.cc

@@ -19,47 +19,32 @@
 #include "src/core/ext/xds/xds_client.h"
 
 #include <inttypes.h>
-#include <limits.h>
 #include <string.h>
 
-#include "absl/container/inlined_vector.h"
-#include "absl/strings/str_format.h"
+#include <algorithm>
+#include <type_traits>
+
+#include "absl/strings/match.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
+#include "absl/strings/str_split.h"
 #include "absl/strings/string_view.h"
+#include "absl/strings/strip.h"
+#include "absl/types/optional.h"
+#include "upb/arena.h"
 
-#include <grpc/byte_buffer_reader.h>
-#include <grpc/grpc.h>
-#include <grpc/support/alloc.h>
-#include <grpc/support/time.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/support/log.h>
 
-#include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/xds/xds_api.h"
 #include "src/core/ext/xds/xds_bootstrap.h"
-#include "src/core/ext/xds/xds_channel_args.h"
 #include "src/core/ext/xds/xds_client_stats.h"
-#include "src/core/ext/xds/xds_cluster.h"
-#include "src/core/ext/xds/xds_endpoint.h"
-#include "src/core/ext/xds/xds_http_filters.h"
-#include "src/core/ext/xds/xds_listener.h"
-#include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/backoff/backoff.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/config/core_configuration.h"
-#include "src/core/lib/gpr/env.h"
-#include "src/core/lib/gpr/string.h"
-#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
-#include "src/core/lib/iomgr/sockaddr.h"
-#include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/security/credentials/channel_creds_registry.h"
-#include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/surface/call.h"
-#include "src/core/lib/surface/channel.h"
-#include "src/core/lib/surface/lame_client.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/uri/uri_parser.h"
 
 #define GRPC_XDS_INITIAL_CONNECT_BACKOFF_SECONDS 1
@@ -70,56 +55,11 @@
 
 namespace grpc_core {
 
+using ::grpc_event_engine::experimental::EventEngine;
+
 TraceFlag grpc_xds_client_trace(false, "xds_client");
 TraceFlag grpc_xds_client_refcount_trace(false, "xds_client_refcount");
 
-namespace {
-
-Mutex* g_mu = nullptr;
-
-const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr;
-XdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr;
-char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr;
-
-}  // namespace
-
-class XdsClient::Notifier {
- public:
-  // Helper template function to invoke `OnError()` on a list of watchers \a
-  // watchers_list within \a work_serializer. Works with all 4 resource types.
-  template <class T>
-  static void ScheduleNotifyWatchersOnErrorInWorkSerializer(
-      XdsClient* xds_client, const T& watchers_list, grpc_error_handle error,
-      const DebugLocation& location) {
-    xds_client->work_serializer_.Schedule(
-        [watchers_list, error]()
-            ABSL_EXCLUSIVE_LOCKS_REQUIRED(&xds_client->work_serializer_) {
-              for (const auto& p : watchers_list) {
-                p.first->OnError(GRPC_ERROR_REF(error));
-              }
-              GRPC_ERROR_UNREF(error);
-            },
-        location);
-  }
-
-  // Helper template function to invoke `OnResourceDoesNotExist()` on a list of
-  // watchers \a watchers_list within \a work_serializer. Works with all 4
-  // resource types.
-  template <class T>
-  static void ScheduleNotifyWatchersOnResourceDoesNotExistInWorkSerializer(
-      XdsClient* xds_client, const T& watchers_list,
-      const DebugLocation& location) {
-    xds_client->work_serializer_.Schedule(
-        [watchers_list]()
-            ABSL_EXCLUSIVE_LOCKS_REQUIRED(&xds_client->work_serializer_) {
-              for (const auto& p : watchers_list) {
-                p.first->OnResourceDoesNotExist();
-              }
-            },
-        location);
-  }
-};
-
 //
 // Internal class declarations
 //
@@ -132,9 +72,12 @@ class XdsClient::ChannelState::RetryableCall
  public:
   explicit RetryableCall(WeakRefCountedPtr<ChannelState> chand);
 
-  void Orphan() override;
+  // Disable thread-safety analysis because this method is called via
+  // OrphanablePtr<>, but there's no way to pass the lock annotation
+  // through there.
+  void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS;
 
-  void OnCallFinishedLocked();
+  void OnCallFinishedLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
   T* calld() const { return calld_.get(); }
   ChannelState* chand() const { return chand_.get(); }
@@ -143,9 +86,9 @@ class XdsClient::ChannelState::RetryableCall
 
  private:
   void StartNewCallLocked();
-  void StartRetryTimerLocked();
-  static void OnRetryTimer(void* arg, grpc_error_handle error);
-  void OnRetryTimerLocked(grpc_error_handle error);
+  void StartRetryTimerLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+
+  void OnRetryTimer();
 
   // The wrapped xds call that talks to the xds server. It's instantiated
   // every time we start a new call. It's null during call retry backoff.
@@ -155,9 +98,8 @@ class XdsClient::ChannelState::RetryableCall
 
   // Retry state.
   BackOff backoff_;
-  grpc_timer retry_timer_;
-  grpc_closure on_retry_timer_;
-  bool retry_timer_callback_pending_ = false;
+  absl::optional<EventEngine::TaskHandle> timer_handle_
+      ABSL_GUARDED_BY(&XdsClient::mu_);
 
   bool shutting_down_ = false;
 };
@@ -168,7 +110,6 @@ class XdsClient::ChannelState::AdsCallState
  public:
   // The ctor and dtor should not be used directly.
   explicit AdsCallState(RefCountedPtr<RetryableCall<AdsCallState>> parent);
-  ~AdsCallState() override;
 
   void Orphan() override;
 
@@ -177,7 +118,8 @@ class XdsClient::ChannelState::AdsCallState
   XdsClient* xds_client() const { return chand()->xds_client(); }
   bool seen_response() const { return seen_response_; }
 
-  void SubscribeLocked(const XdsResourceType* type, const XdsResourceName& name)
+  void SubscribeLocked(const XdsResourceType* type, const XdsResourceName& name,
+                       bool delay_send)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
   void UnsubscribeLocked(const XdsResourceType* type,
                          const XdsResourceName& name, bool delay_unsubscription)
@@ -205,110 +147,159 @@ class XdsClient::ChannelState::AdsCallState
     absl::Status ProcessAdsResponseFields(AdsResponseFields fields) override
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
-    void ParseResource(const XdsEncodingContext& context, size_t idx,
-                       absl::string_view type_url,
+    void ParseResource(upb_Arena* arena, size_t idx, absl::string_view type_url,
+                       absl::string_view resource_name,
                        absl::string_view serialized_resource) override
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
+    void ResourceWrapperParsingFailed(size_t idx) override;
+
     Result TakeResult() { return std::move(result_); }
 
    private:
     XdsClient* xds_client() const { return ads_call_state_->xds_client(); }
 
     AdsCallState* ads_call_state_;
-    const Timestamp update_time_ = ExecCtx::Get()->Now();
+    const Timestamp update_time_ = Timestamp::Now();
     Result result_;
   };
 
   class ResourceTimer : public InternallyRefCounted<ResourceTimer> {
    public:
     ResourceTimer(const XdsResourceType* type, const XdsResourceName& name)
-        : type_(type), name_(name) {
-      GRPC_CLOSURE_INIT(&timer_callback_, OnTimer, this,
-                        grpc_schedule_on_exec_ctx);
-    }
+        : type_(type), name_(name) {}
 
-    void Orphan() override {
+    // Disable thread-safety analysis because this method is called via
+    // OrphanablePtr<>, but there's no way to pass the lock annotation
+    // through there.
+    void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS {
       MaybeCancelTimer();
       Unref(DEBUG_LOCATION, "Orphan");
     }
 
-    void MaybeStartTimer(RefCountedPtr<AdsCallState> ads_calld) {
-      if (timer_started_) return;
-      timer_started_ = true;
-      ads_calld_ = std::move(ads_calld);
-      Ref(DEBUG_LOCATION, "timer").release();
-      timer_pending_ = true;
-      grpc_timer_init(
-          &timer_,
-          ExecCtx::Get()->Now() + ads_calld_->xds_client()->request_timeout_,
-          &timer_callback_);
+    void MarkSubscriptionSendStarted()
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
+      subscription_sent_ = true;
     }
 
-    void MaybeCancelTimer() {
-      if (timer_pending_) {
-        grpc_timer_cancel(&timer_);
-        timer_pending_ = false;
-      }
+    void MaybeMarkSubscriptionSendComplete(
+        RefCountedPtr<AdsCallState> ads_calld)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
+      if (subscription_sent_) MaybeStartTimer(std::move(ads_calld));
     }
 
-   private:
-    static void OnTimer(void* arg, grpc_error_handle error) {
-      ResourceTimer* self = static_cast<ResourceTimer*>(arg);
-      {
-        MutexLock lock(&self->ads_calld_->xds_client()->mu_);
-        self->OnTimerLocked(GRPC_ERROR_REF(error));
+    void MarkSeen() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
+      resource_seen_ = true;
+      MaybeCancelTimer();
+    }
+
+    void MaybeCancelTimer() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
+      if (timer_handle_.has_value() &&
+          ads_calld_->xds_client()->engine()->Cancel(*timer_handle_)) {
+        timer_handle_.reset();
       }
-      self->ads_calld_->xds_client()->work_serializer_.DrainQueue();
-      self->ads_calld_.reset();
-      self->Unref(DEBUG_LOCATION, "timer");
     }
 
-    void OnTimerLocked(grpc_error_handle error)
+   private:
+    void MaybeStartTimer(RefCountedPtr<AdsCallState> ads_calld)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
-      if (error == GRPC_ERROR_NONE && timer_pending_) {
-        timer_pending_ = false;
-        grpc_error_handle watcher_error =
-            GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
-                "timeout obtaining resource {type=%s name=%s} from xds server",
-                type_->type_url(),
+      // Don't start timer if we've already either seen the resource or
+      // marked it as non-existing.
+      // Note: There are edge cases where we can have seen the resource
+      // before we have sent the initial subscription request, such as
+      // when we unsubscribe and then resubscribe to a given resource
+      // and then get a response containing that resource, all while a
+      // send_message op is in flight.
+      if (resource_seen_) return;
+      // Don't start timer if we haven't yet sent the initial subscription
+      // request for the resource.
+      if (!subscription_sent_) return;
+      // Don't start timer if it's already running.
+      if (timer_handle_.has_value()) return;
+      // Check if we already have a cached version of this resource
+      // (i.e., if this is the initial request for the resource after an
+      // ADS stream restart).  If so, we don't start the timer, because
+      // (a) we already have the resource and (b) the server may
+      // optimize by not resending the resource that we already have.
+      auto& authority_state =
+          ads_calld->xds_client()->authority_state_map_[name_.authority];
+      ResourceState& state = authority_state.resource_map[type_][name_.key];
+      if (state.resource != nullptr) return;
+      // Start timer.
+      ads_calld_ = std::move(ads_calld);
+      timer_handle_ = ads_calld_->xds_client()->engine()->RunAfter(
+          ads_calld_->xds_client()->request_timeout_,
+          [self = Ref(DEBUG_LOCATION, "timer")]() {
+            ApplicationCallbackExecCtx callback_exec_ctx;
+            ExecCtx exec_ctx;
+            self->OnTimer();
+          });
+    }
+
+    void OnTimer() {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_client %p] xds server %s: timeout obtaining resource "
+                "{type=%s name=%s} from xds server",
+                ads_calld_->xds_client(),
+                ads_calld_->chand()->server_.server_uri().c_str(),
+                std::string(type_->type_url()).c_str(),
                 XdsClient::ConstructFullXdsResourceName(
-                    name_.authority, type_->type_url(), name_.key)));
-        watcher_error = grpc_error_set_int(
-            watcher_error, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-          gpr_log(GPR_INFO, "[xds_client %p] xds server %s: %s",
-                  ads_calld_->xds_client(),
-                  ads_calld_->chand()->server_.server_uri.c_str(),
-                  grpc_error_std_string(watcher_error).c_str());
-        }
+                    name_.authority, type_->type_url(), name_.key)
+                    .c_str());
+      }
+      {
+        MutexLock lock(&ads_calld_->xds_client()->mu_);
+        timer_handle_.reset();
+        resource_seen_ = true;
         auto& authority_state =
             ads_calld_->xds_client()->authority_state_map_[name_.authority];
         ResourceState& state = authority_state.resource_map[type_][name_.key];
         state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
-        Notifier::ScheduleNotifyWatchersOnErrorInWorkSerializer(
-            ads_calld_->xds_client(), state.watchers, watcher_error,
-            DEBUG_LOCATION);
+        ads_calld_->xds_client()->NotifyWatchersOnResourceDoesNotExist(
+            state.watchers);
       }
-      GRPC_ERROR_UNREF(error);
+      ads_calld_->xds_client()->work_serializer_.DrainQueue();
+      ads_calld_.reset();
     }
 
     const XdsResourceType* type_;
     const XdsResourceName name_;
 
     RefCountedPtr<AdsCallState> ads_calld_;
-    bool timer_started_ = false;
-    bool timer_pending_ = false;
-    grpc_timer timer_;
-    grpc_closure timer_callback_;
+    // True if we have sent the initial subscription request for this
+    // resource on this ADS stream.
+    bool subscription_sent_ ABSL_GUARDED_BY(&XdsClient::mu_) = false;
+    // True if we have either (a) seen the resource in a response on this
+    // stream or (b) declared the resource to not exist due to the timer
+    // firing.
+    bool resource_seen_ ABSL_GUARDED_BY(&XdsClient::mu_) = false;
+    absl::optional<EventEngine::TaskHandle> timer_handle_
+        ABSL_GUARDED_BY(&XdsClient::mu_);
   };
 
-  struct ResourceTypeState {
-    ~ResourceTypeState() { GRPC_ERROR_UNREF(error); }
+  class StreamEventHandler
+      : public XdsTransportFactory::XdsTransport::StreamingCall::EventHandler {
+   public:
+    explicit StreamEventHandler(RefCountedPtr<AdsCallState> ads_calld)
+        : ads_calld_(std::move(ads_calld)) {}
 
-    // Nonce and error for this resource type.
+    void OnRequestSent(bool ok) override { ads_calld_->OnRequestSent(ok); }
+    void OnRecvMessage(absl::string_view payload) override {
+      ads_calld_->OnRecvMessage(payload);
+    }
+    void OnStatusReceived(absl::Status status) override {
+      ads_calld_->OnStatusReceived(std::move(status));
+    }
+
+   private:
+    RefCountedPtr<AdsCallState> ads_calld_;
+  };
+
+  struct ResourceTypeState {
+    // Nonce and status for this resource type.
     std::string nonce;
-    grpc_error_handle error = GRPC_ERROR_NONE;
+    absl::Status status;
 
     // Subscribed resources of this type.
     std::map<std::string /*authority*/,
@@ -319,47 +310,27 @@ class XdsClient::ChannelState::AdsCallState
   void SendMessageLocked(const XdsResourceType* type)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
-  static void OnRequestSent(void* arg, grpc_error_handle error);
-  void OnRequestSentLocked(grpc_error_handle error)
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-  static void OnResponseReceived(void* arg, grpc_error_handle error);
-  bool OnResponseReceivedLocked()
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-  static void OnStatusReceived(void* arg, grpc_error_handle error);
-  void OnStatusReceivedLocked(grpc_error_handle error)
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+  void OnRequestSent(bool ok);
+  void OnRecvMessage(absl::string_view payload);
+  void OnStatusReceived(absl::Status status);
 
   bool IsCurrentCallOnChannel() const;
 
   // Constructs a list of resource names of a given type for an ADS
   // request.  Also starts the timer for each resource if needed.
-  std::vector<std::string> ResourceNamesForRequest(const XdsResourceType* type);
+  std::vector<std::string> ResourceNamesForRequest(const XdsResourceType* type)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
   // The owning RetryableCall<>.
   RefCountedPtr<RetryableCall<AdsCallState>> parent_;
 
+  OrphanablePtr<XdsTransportFactory::XdsTransport::StreamingCall> call_;
+
   bool sent_initial_message_ = false;
   bool seen_response_ = false;
 
-  // Always non-NULL.
-  grpc_call* call_;
-
-  // recv_initial_metadata
-  grpc_metadata_array initial_metadata_recv_;
-
-  // send_message
-  grpc_byte_buffer* send_message_payload_ = nullptr;
-  grpc_closure on_request_sent_;
-
-  // recv_message
-  grpc_byte_buffer* recv_message_payload_ = nullptr;
-  grpc_closure on_response_received_;
-
-  // recv_trailing_metadata
-  grpc_metadata_array trailing_metadata_recv_;
-  grpc_status_code status_code_;
-  grpc_slice status_details_;
-  grpc_closure on_status_received_;
+  const XdsResourceType* send_message_pending_
+      ABSL_GUARDED_BY(&XdsClient::mu_) = nullptr;
 
   // Resource types for which requests need to be sent.
   std::set<const XdsResourceType*> buffered_requests_;
@@ -374,11 +345,11 @@ class XdsClient::ChannelState::LrsCallState
  public:
   // The ctor and dtor should not be used directly.
   explicit LrsCallState(RefCountedPtr<RetryableCall<LrsCallState>> parent);
-  ~LrsCallState() override;
 
   void Orphan() override;
 
-  void MaybeStartReportingLocked();
+  void MaybeStartReportingLocked()
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
   RetryableCall<LrsCallState>* parent() { return parent_.get(); }
   ChannelState* chand() const { return parent_->chand(); }
@@ -386,30 +357,44 @@ class XdsClient::ChannelState::LrsCallState
   bool seen_response() const { return seen_response_; }
 
  private:
+  class StreamEventHandler
+      : public XdsTransportFactory::XdsTransport::StreamingCall::EventHandler {
+   public:
+    explicit StreamEventHandler(RefCountedPtr<LrsCallState> lrs_calld)
+        : lrs_calld_(std::move(lrs_calld)) {}
+
+    void OnRequestSent(bool ok) override { lrs_calld_->OnRequestSent(ok); }
+    void OnRecvMessage(absl::string_view payload) override {
+      lrs_calld_->OnRecvMessage(payload);
+    }
+    void OnStatusReceived(absl::Status status) override {
+      lrs_calld_->OnStatusReceived(std::move(status));
+    }
+
+   private:
+    RefCountedPtr<LrsCallState> lrs_calld_;
+  };
+
   // Reports client-side load stats according to a fixed interval.
   class Reporter : public InternallyRefCounted<Reporter> {
    public:
     Reporter(RefCountedPtr<LrsCallState> parent, Duration report_interval)
         : parent_(std::move(parent)), report_interval_(report_interval) {
-      GRPC_CLOSURE_INIT(&on_next_report_timer_, OnNextReportTimer, this,
-                        grpc_schedule_on_exec_ctx);
-      GRPC_CLOSURE_INIT(&on_report_done_, OnReportDone, this,
-                        grpc_schedule_on_exec_ctx);
       ScheduleNextReportLocked();
     }
 
-    void Orphan() override;
+    // Disable thread-safety analysis because this method is called via
+    // OrphanablePtr<>, but there's no way to pass the lock annotation
+    // through there.
+    void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS;
+
+    void OnReportDoneLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
    private:
     void ScheduleNextReportLocked()
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-    static void OnNextReportTimer(void* arg, grpc_error_handle error);
-    bool OnNextReportTimerLocked(grpc_error_handle error)
-        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+    bool OnNextReportTimer();
     bool SendReportLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-    static void OnReportDone(void* arg, grpc_error_handle error);
-    bool OnReportDoneLocked(grpc_error_handle error)
-        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
     bool IsCurrentReporterOnCall() const {
       return this == parent_->reporter_.get();
@@ -422,47 +407,23 @@ class XdsClient::ChannelState::LrsCallState
     // The load reporting state.
     const Duration report_interval_;
     bool last_report_counters_were_zero_ = false;
-    bool next_report_timer_callback_pending_ = false;
-    grpc_timer next_report_timer_;
-    grpc_closure on_next_report_timer_;
-    grpc_closure on_report_done_;
+    absl::optional<EventEngine::TaskHandle> timer_handle_
+        ABSL_GUARDED_BY(&XdsClient::mu_);
   };
 
-  static void OnInitialRequestSent(void* arg, grpc_error_handle error);
-  void OnInitialRequestSentLocked()
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-  static void OnResponseReceived(void* arg, grpc_error_handle error);
-  bool OnResponseReceivedLocked()
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-  static void OnStatusReceived(void* arg, grpc_error_handle error);
-  void OnStatusReceivedLocked(grpc_error_handle error)
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
+  void OnRequestSent(bool ok);
+  void OnRecvMessage(absl::string_view payload);
+  void OnStatusReceived(absl::Status status);
 
   bool IsCurrentCallOnChannel() const;
 
   // The owning RetryableCall<>.
   RefCountedPtr<RetryableCall<LrsCallState>> parent_;
-  bool seen_response_ = false;
-
-  // Always non-NULL.
-  grpc_call* call_;
 
-  // recv_initial_metadata
-  grpc_metadata_array initial_metadata_recv_;
+  OrphanablePtr<XdsTransportFactory::XdsTransport::StreamingCall> call_;
 
-  // send_message
-  grpc_byte_buffer* send_message_payload_ = nullptr;
-  grpc_closure on_initial_request_sent_;
-
-  // recv_message
-  grpc_byte_buffer* recv_message_payload_ = nullptr;
-  grpc_closure on_response_received_;
-
-  // recv_trailing_metadata
-  grpc_metadata_array trailing_metadata_recv_;
-  grpc_status_code status_code_;
-  grpc_slice status_details_;
-  grpc_closure on_status_received_;
+  bool seen_response_ = false;
+  bool send_message_pending_ ABSL_GUARDED_BY(&XdsClient::mu_) = false;
 
   // Load reporting state.
   bool send_all_clusters_ = false;
@@ -471,58 +432,10 @@ class XdsClient::ChannelState::LrsCallState
   OrphanablePtr<Reporter> reporter_;
 };
 
-//
-// XdsClient::ChannelState::StateWatcher
-//
-
-class XdsClient::ChannelState::StateWatcher
-    : public AsyncConnectivityStateWatcherInterface {
- public:
-  explicit StateWatcher(WeakRefCountedPtr<ChannelState> parent)
-      : parent_(std::move(parent)) {}
-
- private:
-  void OnConnectivityStateChange(grpc_connectivity_state new_state,
-                                 const absl::Status& status) override {
-    {
-      MutexLock lock(&parent_->xds_client_->mu_);
-      if (!parent_->shutting_down_ &&
-          new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-        // In TRANSIENT_FAILURE.  Notify all watchers of error.
-        gpr_log(GPR_INFO,
-                "[xds_client %p] xds channel for server %s in "
-                "state TRANSIENT_FAILURE: %s",
-                parent_->xds_client(), parent_->server_.server_uri.c_str(),
-                status.ToString().c_str());
-        parent_->xds_client_->NotifyOnErrorLocked(
-            GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
-                "xds channel in TRANSIENT_FAILURE, connectivity error: ",
-                status.ToString())));
-      }
-    }
-    parent_->xds_client()->work_serializer_.DrainQueue();
-  }
-
-  WeakRefCountedPtr<ChannelState> parent_;
-};
-
 //
 // XdsClient::ChannelState
 //
 
-namespace {
-
-grpc_channel* CreateXdsChannel(grpc_channel_args* args,
-                               const XdsBootstrap::XdsServer& server) {
-  RefCountedPtr<grpc_channel_credentials> channel_creds =
-      CoreConfiguration::Get().channel_creds_registry().CreateChannelCreds(
-          server.channel_creds_type, server.channel_creds_config);
-  return grpc_channel_create(server.server_uri.c_str(), channel_creds.get(),
-                             args);
-}
-
-}  // namespace
-
 XdsClient::ChannelState::ChannelState(WeakRefCountedPtr<XdsClient> xds_client,
                                       const XdsBootstrap::XdsServer& server)
     : DualRefCounted<ChannelState>(
@@ -533,37 +446,45 @@ XdsClient::ChannelState::ChannelState(WeakRefCountedPtr<XdsClient> xds_client,
       server_(server) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO, "[xds_client %p] creating channel to %s",
-            xds_client_.get(), server.server_uri.c_str());
-  }
-  channel_ = CreateXdsChannel(xds_client_->args_, server);
-  GPR_ASSERT(channel_ != nullptr);
-  StartConnectivityWatchLocked();
+            xds_client_.get(), server.server_uri().c_str());
+  }
+  absl::Status status;
+  transport_ = xds_client_->transport_factory_->Create(
+      server,
+      [self = WeakRef(DEBUG_LOCATION, "OnConnectivityFailure")](
+          absl::Status status) {
+        self->OnConnectivityFailure(std::move(status));
+      },
+      &status);
+  GPR_ASSERT(transport_ != nullptr);
+  if (!status.ok()) SetChannelStatusLocked(std::move(status));
 }
 
 XdsClient::ChannelState::~ChannelState() {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO, "[xds_client %p] destroying xds channel %p for server %s",
-            xds_client(), this, server_.server_uri.c_str());
+            xds_client(), this, server_.server_uri().c_str());
   }
-  grpc_channel_destroy(channel_);
   xds_client_.reset(DEBUG_LOCATION, "ChannelState");
 }
 
 // This method should only ever be called when holding the lock, but we can't
 // use a ABSL_EXCLUSIVE_LOCKS_REQUIRED annotation, because Orphan() will be
-// called from DualRefCounted::Unref, which cannot have a lock annotation for a
-// lock in this subclass.
+// called from DualRefCounted::Unref, which cannot have a lock annotation for
+// a lock in this subclass.
 void XdsClient::ChannelState::Orphan() ABSL_NO_THREAD_SAFETY_ANALYSIS {
   shutting_down_ = true;
-  CancelConnectivityWatchLocked();
+  transport_.reset();
   // At this time, all strong refs are removed, remove from channel map to
-  // prevent subsequent subscription from trying to use this ChannelState as it
-  // is shutting down.
-  xds_client_->xds_server_channel_map_.erase(server_);
+  // prevent subsequent subscription from trying to use this ChannelState as
+  // it is shutting down.
+  xds_client_->xds_server_channel_map_.erase(&server_);
   ads_calld_.reset();
   lrs_calld_.reset();
 }
 
+void XdsClient::ChannelState::ResetBackoff() { transport_->ResetBackoff(); }
+
 XdsClient::ChannelState::AdsCallState* XdsClient::ChannelState::ads_calld()
     const {
   return ads_calld_->calld();
@@ -574,10 +495,6 @@ XdsClient::ChannelState::LrsCallState* XdsClient::ChannelState::lrs_calld()
   return lrs_calld_->calld();
 }
 
-bool XdsClient::ChannelState::HasActiveAdsCall() const {
-  return ads_calld_ != nullptr && ads_calld_->calld() != nullptr;
-}
-
 void XdsClient::ChannelState::MaybeStartLrsCall() {
   if (lrs_calld_ != nullptr) return;
   lrs_calld_.reset(new RetryableCall<LrsCallState>(
@@ -585,43 +502,10 @@ void XdsClient::ChannelState::MaybeStartLrsCall() {
 }
 
 void XdsClient::ChannelState::StopLrsCallLocked() {
-  xds_client_->xds_load_report_server_map_.erase(server_);
+  xds_client_->xds_load_report_server_map_.erase(&server_);
   lrs_calld_.reset();
 }
 
-namespace {
-
-bool IsLameChannel(grpc_channel* channel) {
-  grpc_channel_element* elem =
-      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
-  return elem->filter == &grpc_lame_filter;
-}
-
-}  // namespace
-
-void XdsClient::ChannelState::StartConnectivityWatchLocked() {
-  if (IsLameChannel(channel_)) {
-    xds_client()->NotifyOnErrorLocked(
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("xds client has a lame channel"));
-    return;
-  }
-  ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_);
-  GPR_ASSERT(client_channel != nullptr);
-  watcher_ = new StateWatcher(WeakRef(DEBUG_LOCATION, "ChannelState+watch"));
-  client_channel->AddConnectivityWatcher(
-      GRPC_CHANNEL_IDLE,
-      OrphanablePtr<AsyncConnectivityStateWatcherInterface>(watcher_));
-}
-
-void XdsClient::ChannelState::CancelConnectivityWatchLocked() {
-  if (IsLameChannel(channel_)) {
-    return;
-  }
-  ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_);
-  GPR_ASSERT(client_channel != nullptr);
-  client_channel->RemoveConnectivityWatcher(watcher_);
-}
-
 void XdsClient::ChannelState::SubscribeLocked(const XdsResourceType* type,
                                               const XdsResourceName& name) {
   if (ads_calld_ == nullptr) {
@@ -637,7 +521,7 @@ void XdsClient::ChannelState::SubscribeLocked(const XdsResourceType* type,
   // because when the call is restarted it will resend all necessary requests.
   if (ads_calld() == nullptr) return;
   // Subscribe to this resource if the ADS call is active.
-  ads_calld()->SubscribeLocked(type, name);
+  ads_calld()->SubscribeLocked(type, name, /*delay_send=*/false);
 }
 
 void XdsClient::ChannelState::UnsubscribeLocked(const XdsResourceType* type,
@@ -654,6 +538,56 @@ void XdsClient::ChannelState::UnsubscribeLocked(const XdsResourceType* type,
   }
 }
 
+void XdsClient::ChannelState::OnConnectivityFailure(absl::Status status) {
+  {
+    MutexLock lock(&xds_client_->mu_);
+    SetChannelStatusLocked(std::move(status));
+  }
+  xds_client_->work_serializer_.DrainQueue();
+}
+
+void XdsClient::ChannelState::SetChannelStatusLocked(absl::Status status) {
+  if (shutting_down_) return;
+  status = absl::Status(status.code(), absl::StrCat("xDS channel for server ",
+                                                    server_.server_uri(), ": ",
+                                                    status.message()));
+  gpr_log(GPR_INFO, "[xds_client %p] %s", xds_client(),
+          status.ToString().c_str());
+  // If the node ID is set, append that to the status message that we send to
+  // the watchers, so that it will appear in log messages visible to users.
+  const auto* node = xds_client_->bootstrap_->node();
+  if (node != nullptr) {
+    status = absl::Status(
+        status.code(),
+        absl::StrCat(status.message(),
+                     " (node ID:", xds_client_->bootstrap_->node()->id(), ")"));
+  }
+  // Save status in channel, so that we can immediately generate an
+  // error for any new watchers that may be started.
+  status_ = status;
+  // Find all watchers for this channel.
+  std::set<RefCountedPtr<ResourceWatcherInterface>> watchers;
+  for (const auto& a : xds_client_->authority_state_map_) {  // authority
+    if (a.second.channel_state != this) continue;
+    for (const auto& t : a.second.resource_map) {  // type
+      for (const auto& r : t.second) {             // resource id
+        for (const auto& w : r.second.watchers) {  // watchers
+          watchers.insert(w.second);
+        }
+      }
+    }
+  }
+  // Enqueue notification for the watchers.
+  xds_client_->work_serializer_.Schedule(
+      [watchers = std::move(watchers), status = std::move(status)]()
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(xds_client_->work_serializer_) {
+            for (const auto& watcher : watchers) {
+              watcher->OnError(status);
+            }
+          },
+      DEBUG_LOCATION);
+}
+
 //
 // XdsClient::ChannelState::RetryableCall<>
 //
@@ -669,9 +603,6 @@ XdsClient::ChannelState::RetryableCall<T>::RetryableCall(
                    .set_jitter(GRPC_XDS_RECONNECT_JITTER)
                    .set_max_backoff(Duration::Seconds(
                        GRPC_XDS_RECONNECT_MAX_BACKOFF_SECONDS))) {
-  // Closure Initialization
-  GRPC_CLOSURE_INIT(&on_retry_timer_, OnRetryTimer, this,
-                    grpc_schedule_on_exec_ctx);
   StartNewCallLocked();
 }
 
@@ -679,35 +610,32 @@ template <typename T>
 void XdsClient::ChannelState::RetryableCall<T>::Orphan() {
   shutting_down_ = true;
   calld_.reset();
-  if (retry_timer_callback_pending_) grpc_timer_cancel(&retry_timer_);
+  if (timer_handle_.has_value()) {
+    chand()->xds_client()->engine()->Cancel(*timer_handle_);
+    timer_handle_.reset();
+  }
   this->Unref(DEBUG_LOCATION, "RetryableCall+orphaned");
 }
 
 template <typename T>
 void XdsClient::ChannelState::RetryableCall<T>::OnCallFinishedLocked() {
-  const bool seen_response = calld_->seen_response();
+  // If we saw a response on the current stream, reset backoff.
+  if (calld_->seen_response()) backoff_.Reset();
   calld_.reset();
-  if (seen_response) {
-    // If we lost connection to the xds server, reset backoff and restart the
-    // call immediately.
-    backoff_.Reset();
-    StartNewCallLocked();
-  } else {
-    // If we failed to connect to the xds server, retry later.
-    StartRetryTimerLocked();
-  }
+  // Start retry timer.
+  StartRetryTimerLocked();
 }
 
 template <typename T>
 void XdsClient::ChannelState::RetryableCall<T>::StartNewCallLocked() {
   if (shutting_down_) return;
-  GPR_ASSERT(chand_->channel_ != nullptr);
+  GPR_ASSERT(chand_->transport_ != nullptr);
   GPR_ASSERT(calld_ == nullptr);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-    gpr_log(
-        GPR_INFO,
-        "[xds_client %p] xds server %s: start new call from retryable call %p",
-        chand()->xds_client(), chand()->server_.server_uri.c_str(), this);
+    gpr_log(GPR_INFO,
+            "[xds_client %p] xds server %s: start new call from retryable "
+            "call %p",
+            chand()->xds_client(), chand()->server_.server_uri().c_str(), this);
   }
   calld_ = MakeOrphanable<T>(
       this->Ref(DEBUG_LOCATION, "RetryableCall+start_new_call"));
@@ -717,45 +645,39 @@ template <typename T>
 void XdsClient::ChannelState::RetryableCall<T>::StartRetryTimerLocked() {
   if (shutting_down_) return;
   const Timestamp next_attempt_time = backoff_.NextAttemptTime();
+  const Duration timeout =
+      std::max(next_attempt_time - Timestamp::Now(), Duration::Zero());
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-    Duration timeout =
-        std::max(next_attempt_time - ExecCtx::Get()->Now(), Duration::Zero());
     gpr_log(GPR_INFO,
             "[xds_client %p] xds server %s: call attempt failed; "
             "retry timer will fire in %" PRId64 "ms.",
-            chand()->xds_client(), chand()->server_.server_uri.c_str(),
+            chand()->xds_client(), chand()->server_.server_uri().c_str(),
             timeout.millis());
   }
-  this->Ref(DEBUG_LOCATION, "RetryableCall+retry_timer_start").release();
-  grpc_timer_init(&retry_timer_, next_attempt_time, &on_retry_timer_);
-  retry_timer_callback_pending_ = true;
+  timer_handle_ = chand()->xds_client()->engine()->RunAfter(
+      timeout,
+      [self = this->Ref(DEBUG_LOCATION, "RetryableCall+retry_timer_start")]() {
+        ApplicationCallbackExecCtx callback_exec_ctx;
+        ExecCtx exec_ctx;
+        self->OnRetryTimer();
+      });
 }
 
 template <typename T>
-void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer(
-    void* arg, grpc_error_handle error) {
-  RetryableCall* calld = static_cast<RetryableCall*>(arg);
-  {
-    MutexLock lock(&calld->chand_->xds_client()->mu_);
-    calld->OnRetryTimerLocked(GRPC_ERROR_REF(error));
-  }
-  calld->Unref(DEBUG_LOCATION, "RetryableCall+retry_timer_done");
-}
-
-template <typename T>
-void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimerLocked(
-    grpc_error_handle error) {
-  retry_timer_callback_pending_ = false;
-  if (!shutting_down_ && error == GRPC_ERROR_NONE) {
+void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer() {
+  MutexLock lock(&chand_->xds_client()->mu_);
+  if (timer_handle_.has_value()) {
+    timer_handle_.reset();
+    if (shutting_down_) return;
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
       gpr_log(GPR_INFO,
               "[xds_client %p] xds server %s: retry timer fired (retryable "
               "call: %p)",
-              chand()->xds_client(), chand()->server_.server_uri.c_str(), this);
+              chand()->xds_client(), chand()->server_.server_uri().c_str(),
+              this);
     }
     StartNewCallLocked();
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 //
@@ -770,7 +692,7 @@ absl::Status XdsClient::ChannelState::AdsCallState::AdsResponseParser::
         "[xds_client %p] xds server %s: received ADS response: type_url=%s, "
         "version=%s, nonce=%s, num_resources=%" PRIuPTR,
         ads_call_state_->xds_client(),
-        ads_call_state_->chand()->server_.server_uri.c_str(),
+        ads_call_state_->chand()->server_.server_uri().c_str(),
         fields.type_url.c_str(), fields.version.c_str(), fields.nonce.c_str(),
         fields.num_resources);
   }
@@ -813,48 +735,68 @@ void UpdateResourceMetadataNacked(const std::string& version,
 }  // namespace
 
 void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource(
-    const XdsEncodingContext& context, size_t idx, absl::string_view type_url,
-    absl::string_view serialized_resource) {
+    upb_Arena* arena, size_t idx, absl::string_view type_url,
+    absl::string_view resource_name, absl::string_view serialized_resource) {
+  std::string error_prefix = absl::StrCat(
+      "resource index ", idx, ": ",
+      resource_name.empty() ? "" : absl::StrCat(resource_name, ": "));
   // Check the type_url of the resource.
-  bool is_v2 = false;
-  if (!result_.type->IsType(type_url, &is_v2)) {
+  if (result_.type_url != type_url) {
     result_.errors.emplace_back(
-        absl::StrCat("resource index ", idx, ": incorrect resource type ",
-                     type_url, " (should be ", result_.type_url, ")"));
+        absl::StrCat(error_prefix, "incorrect resource type \"", type_url,
+                     "\" (should be \"", result_.type_url, "\")"));
     return;
   }
   // Parse the resource.
-  absl::StatusOr<XdsResourceType::DecodeResult> result =
-      result_.type->Decode(context, serialized_resource, is_v2);
-  if (!result.ok()) {
+  XdsResourceType::DecodeContext context = {
+      xds_client(), ads_call_state_->chand()->server_, &grpc_xds_client_trace,
+      xds_client()->symtab_.ptr(), arena};
+  XdsResourceType::DecodeResult decode_result =
+      result_.type->Decode(context, serialized_resource);
+  // If we didn't already have the resource name from the Resource
+  // wrapper, try to get it from the decoding result.
+  if (resource_name.empty()) {
+    if (decode_result.name.has_value()) {
+      resource_name = *decode_result.name;
+      error_prefix =
+          absl::StrCat("resource index ", idx, ": ", resource_name, ": ");
+    } else {
+      // We don't have any way of determining the resource name, so
+      // there's nothing more we can do here.
+      result_.errors.emplace_back(absl::StrCat(
+          error_prefix, decode_result.resource.status().ToString()));
+      return;
+    }
+  }
+  // If decoding failed, make sure we include the error in the NACK.
+  const absl::Status& decode_status = decode_result.resource.status();
+  if (!decode_status.ok()) {
     result_.errors.emplace_back(
-        absl::StrCat("resource index ", idx, ": ", result.status().ToString()));
-    return;
+        absl::StrCat(error_prefix, decode_status.ToString()));
   }
   // Check the resource name.
-  auto resource_name =
-      XdsClient::ParseXdsResourceName(result->name, result_.type);
-  if (!resource_name.ok()) {
-    result_.errors.emplace_back(absl::StrCat(
-        "resource index ", idx, ": Cannot parse xDS resource name \"",
-        result->name, "\""));
+  auto parsed_resource_name =
+      xds_client()->ParseXdsResourceName(resource_name, result_.type);
+  if (!parsed_resource_name.ok()) {
+    result_.errors.emplace_back(
+        absl::StrCat(error_prefix, "Cannot parse xDS resource name"));
     return;
   }
   // Cancel resource-does-not-exist timer, if needed.
   auto timer_it = ads_call_state_->state_map_.find(result_.type);
   if (timer_it != ads_call_state_->state_map_.end()) {
-    auto it =
-        timer_it->second.subscribed_resources.find(resource_name->authority);
+    auto it = timer_it->second.subscribed_resources.find(
+        parsed_resource_name->authority);
     if (it != timer_it->second.subscribed_resources.end()) {
-      auto res_it = it->second.find(resource_name->key);
+      auto res_it = it->second.find(parsed_resource_name->key);
       if (res_it != it->second.end()) {
-        res_it->second->MaybeCancelTimer();
+        res_it->second->MarkSeen();
       }
     }
   }
   // Lookup the authority in the cache.
   auto authority_it =
-      xds_client()->authority_state_map_.find(resource_name->authority);
+      xds_client()->authority_state_map_.find(parsed_resource_name->authority);
   if (authority_it == xds_client()->authority_state_map_.end()) {
     return;  // Skip resource -- we don't have a subscription for it.
   }
@@ -866,29 +808,35 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource(
   }
   auto& type_map = type_it->second;
   // Found type, so look up resource key.
-  auto it = type_map.find(resource_name->key);
+  auto it = type_map.find(parsed_resource_name->key);
   if (it == type_map.end()) {
     return;  // Skip resource -- we don't have a subscription for it.
   }
   ResourceState& resource_state = it->second;
   // If needed, record that we've seen this resource.
   if (result_.type->AllResourcesRequiredInSotW()) {
-    result_.resources_seen[resource_name->authority].insert(resource_name->key);
+    result_.resources_seen[parsed_resource_name->authority].insert(
+        parsed_resource_name->key);
+  }
+  // If we previously ignored the resource's deletion, log that we're
+  // now re-adding it.
+  if (resource_state.ignored_deletion) {
+    gpr_log(GPR_INFO,
+            "[xds_client %p] xds server %s: server returned new version of "
+            "resource for which we previously ignored a deletion: type %s "
+            "name %s",
+            xds_client(),
+            ads_call_state_->chand()->server_.server_uri().c_str(),
+            std::string(type_url).c_str(), std::string(resource_name).c_str());
+    resource_state.ignored_deletion = false;
   }
   // Update resource state based on whether the resource is valid.
-  if (!result->resource.ok()) {
-    result_.errors.emplace_back(absl::StrCat(
-        "resource index ", idx, ": ", result->name,
-        ": validation error: ", result->resource.status().ToString()));
-    Notifier::ScheduleNotifyWatchersOnErrorInWorkSerializer(
-        xds_client(), resource_state.watchers,
-        grpc_error_set_int(
-            GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
-                "invalid resource: ", result->resource.status().ToString())),
-            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
-        DEBUG_LOCATION);
-    UpdateResourceMetadataNacked(result_.version,
-                                 result->resource.status().ToString(),
+  if (!decode_status.ok()) {
+    xds_client()->NotifyWatchersOnErrorLocked(
+        resource_state.watchers,
+        absl::UnavailableError(
+            absl::StrCat("invalid resource: ", decode_status.ToString())));
+    UpdateResourceMetadataNacked(result_.version, decode_status.ToString(),
                                  update_time_, &resource_state.meta);
     return;
   }
@@ -897,16 +845,17 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource(
   // If it didn't change, ignore it.
   if (resource_state.resource != nullptr &&
       result_.type->ResourcesEqual(resource_state.resource.get(),
-                                   result->resource->get())) {
+                                   decode_result.resource->get())) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
       gpr_log(GPR_INFO,
               "[xds_client %p] %s resource %s identical to current, ignoring.",
-              xds_client(), result_.type_url.c_str(), result->name.c_str());
+              xds_client(), result_.type_url.c_str(),
+              std::string(resource_name).c_str());
     }
     return;
   }
   // Update the resource state.
-  resource_state.resource = std::move(*result->resource);
+  resource_state.resource = std::move(*decode_result.resource);
   resource_state.meta = CreateResourceMetadataAcked(
       std::string(serialized_resource), result_.version, update_time_);
   // Notify watchers.
@@ -924,6 +873,12 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource(
       DEBUG_LOCATION);
 }
 
+void XdsClient::ChannelState::AdsCallState::AdsResponseParser::
+    ResourceWrapperParsingFailed(size_t idx) {
+  result_.errors.emplace_back(absl::StrCat(
+      "resource index ", idx, ": Can't decode Resource proto wrapper"));
+}
+
 //
 // XdsClient::ChannelState::AdsCallState
 //
@@ -935,51 +890,27 @@ XdsClient::ChannelState::AdsCallState::AdsCallState(
               ? "AdsCallState"
               : nullptr),
       parent_(std::move(parent)) {
-  // Init the ADS call. Note that the call will progress every time there's
-  // activity in xds_client()->interested_parties_, which is comprised of
-  // the polling entities from client_channel.
   GPR_ASSERT(xds_client() != nullptr);
-  // Create a call with the specified method name.
+  // Init the ADS call.
   const char* method =
-      chand()->server_.ShouldUseV3()
-          ? "/envoy.service.discovery.v3.AggregatedDiscoveryService/"
-            "StreamAggregatedResources"
-          : "/envoy.service.discovery.v2.AggregatedDiscoveryService/"
-            "StreamAggregatedResources";
-  call_ = grpc_channel_create_pollset_set_call(
-      chand()->channel_, nullptr, GRPC_PROPAGATE_DEFAULTS,
-      xds_client()->interested_parties_,
-      StaticSlice::FromStaticString(method).c_slice(), nullptr,
-      Timestamp::InfFuture(), nullptr);
+      "/envoy.service.discovery.v3.AggregatedDiscoveryService/"
+      "StreamAggregatedResources";
+  call_ = chand()->transport_->CreateStreamingCall(
+      method, std::make_unique<StreamEventHandler>(
+                  // Passing the initial ref here.  This ref will go away when
+                  // the StreamEventHandler is destroyed.
+                  RefCountedPtr<AdsCallState>(this)));
   GPR_ASSERT(call_ != nullptr);
-  // Init data associated with the call.
-  grpc_metadata_array_init(&initial_metadata_recv_);
-  grpc_metadata_array_init(&trailing_metadata_recv_);
   // Start the call.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
             "[xds_client %p] xds server %s: starting ADS call "
             "(calld: %p, call: %p)",
-            xds_client(), chand()->server_.server_uri.c_str(), this, call_);
-  }
-  // Create the ops.
-  grpc_call_error call_error;
-  grpc_op ops[3];
-  memset(ops, 0, sizeof(ops));
-  // Op: send initial metadata.
-  grpc_op* op = ops;
-  op->op = GRPC_OP_SEND_INITIAL_METADATA;
-  op->data.send_initial_metadata.count = 0;
-  op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY |
-              GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET;
-  op->reserved = nullptr;
-  op++;
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), nullptr);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  // Op: send request message.
-  GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this,
-                    grpc_schedule_on_exec_ctx);
+            xds_client(), chand()->server_.server_uri().c_str(), this,
+            call_.get());
+  }
+  // If this is a reconnect, add any necessary subscriptions from what's
+  // already in the cache.
   for (const auto& a : xds_client()->authority_state_map_) {
     const std::string& authority = a.first;
     // Skip authorities that are not using this xDS channel.
@@ -988,129 +919,57 @@ XdsClient::ChannelState::AdsCallState::AdsCallState(
       const XdsResourceType* type = t.first;
       for (const auto& r : t.second) {
         const XdsResourceKey& resource_key = r.first;
-        SubscribeLocked(type, {authority, resource_key});
+        SubscribeLocked(type, {authority, resource_key}, /*delay_send=*/true);
       }
     }
   }
-  // Op: recv initial metadata.
-  op = ops;
-  op->op = GRPC_OP_RECV_INITIAL_METADATA;
-  op->data.recv_initial_metadata.recv_initial_metadata =
-      &initial_metadata_recv_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  // Op: recv response.
-  op->op = GRPC_OP_RECV_MESSAGE;
-  op->data.recv_message.recv_message = &recv_message_payload_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  Ref(DEBUG_LOCATION, "ADS+OnResponseReceivedLocked").release();
-  GRPC_CLOSURE_INIT(&on_response_received_, OnResponseReceived, this,
-                    grpc_schedule_on_exec_ctx);
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), &on_response_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  // Op: recv server status.
-  op = ops;
-  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
-  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv_;
-  op->data.recv_status_on_client.status = &status_code_;
-  op->data.recv_status_on_client.status_details = &status_details_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  // This callback signals the end of the call, so it relies on the initial
-  // ref instead of a new ref. When it's invoked, it's the initial ref that is
-  // unreffed.
-  GRPC_CLOSURE_INIT(&on_status_received_, OnStatusReceived, this,
-                    grpc_schedule_on_exec_ctx);
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), &on_status_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-}
-
-XdsClient::ChannelState::AdsCallState::~AdsCallState() {
-  grpc_metadata_array_destroy(&initial_metadata_recv_);
-  grpc_metadata_array_destroy(&trailing_metadata_recv_);
-  grpc_byte_buffer_destroy(send_message_payload_);
-  grpc_byte_buffer_destroy(recv_message_payload_);
-  grpc_slice_unref_internal(status_details_);
-  GPR_ASSERT(call_ != nullptr);
-  grpc_call_unref(call_);
+  // Send initial message if we added any subscriptions above.
+  for (const auto& p : state_map_) {
+    SendMessageLocked(p.first);
+  }
 }
 
 void XdsClient::ChannelState::AdsCallState::Orphan() {
-  GPR_ASSERT(call_ != nullptr);
-  // If we are here because xds_client wants to cancel the call,
-  // on_status_received_ will complete the cancellation and clean up. Otherwise,
-  // we are here because xds_client has to orphan a failed call, then the
-  // following cancellation will be a no-op.
-  grpc_call_cancel_internal(call_);
   state_map_.clear();
-  // Note that the initial ref is hold by on_status_received_. So the
-  // corresponding unref happens in on_status_received_ instead of here.
+  // Note that the initial ref is held by the StreamEventHandler, which
+  // will be destroyed when call_ is destroyed, which may not happen
+  // here, since there may be other refs held to call_ by internal callbacks.
+  call_.reset();
 }
 
 void XdsClient::ChannelState::AdsCallState::SendMessageLocked(
     const XdsResourceType* type)
     ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) {
   // Buffer message sending if an existing message is in flight.
-  if (send_message_payload_ != nullptr) {
+  if (send_message_pending_ != nullptr) {
     buffered_requests_.insert(type);
     return;
   }
   auto& state = state_map_[type];
-  grpc_slice request_payload_slice;
-  request_payload_slice = xds_client()->api_.CreateAdsRequest(
-      chand()->server_,
-      chand()->server_.ShouldUseV3() ? type->type_url() : type->v2_type_url(),
-      chand()->resource_type_version_map_[type], state.nonce,
-      ResourceNamesForRequest(type), GRPC_ERROR_REF(state.error),
-      !sent_initial_message_);
+  std::string serialized_message = xds_client()->api_.CreateAdsRequest(
+      type->type_url(), chand()->resource_type_version_map_[type], state.nonce,
+      ResourceNamesForRequest(type), state.status, !sent_initial_message_);
   sent_initial_message_ = true;
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
             "[xds_client %p] xds server %s: sending ADS request: type=%s "
             "version=%s nonce=%s error=%s",
-            xds_client(), chand()->server_.server_uri.c_str(),
+            xds_client(), chand()->server_.server_uri().c_str(),
             std::string(type->type_url()).c_str(),
             chand()->resource_type_version_map_[type].c_str(),
-            state.nonce.c_str(), grpc_error_std_string(state.error).c_str());
-  }
-  GRPC_ERROR_UNREF(state.error);
-  state.error = GRPC_ERROR_NONE;
-  // Create message payload.
-  send_message_payload_ =
-      grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_slice_unref_internal(request_payload_slice);
-  // Send the message.
-  grpc_op op;
-  memset(&op, 0, sizeof(op));
-  op.op = GRPC_OP_SEND_MESSAGE;
-  op.data.send_message.send_message = send_message_payload_;
-  Ref(DEBUG_LOCATION, "ADS+OnRequestSentLocked").release();
-  GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this,
-                    grpc_schedule_on_exec_ctx);
-  grpc_call_error call_error =
-      grpc_call_start_batch_and_execute(call_, &op, 1, &on_request_sent_);
-  if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) {
-    gpr_log(GPR_ERROR,
-            "[xds_client %p] xds server %s: error starting ADS send_message "
-            "batch on calld=%p: call_error=%d",
-            xds_client(), chand()->server_.server_uri.c_str(), this,
-            call_error);
-    GPR_ASSERT(GRPC_CALL_OK == call_error);
+            state.nonce.c_str(), state.status.ToString().c_str());
   }
+  state.status = absl::OkStatus();
+  call_->SendMessage(std::move(serialized_message));
+  send_message_pending_ = type;
 }
 
 void XdsClient::ChannelState::AdsCallState::SubscribeLocked(
-    const XdsResourceType* type, const XdsResourceName& name) {
+    const XdsResourceType* type, const XdsResourceName& name, bool delay_send) {
   auto& state = state_map_[type].subscribed_resources[name.authority][name.key];
   if (state == nullptr) {
     state = MakeOrphanable<ResourceTimer>(type, name);
-    SendMessageLocked(type);
+    if (!delay_send) SendMessageLocked(type);
   }
 }
 
@@ -1123,7 +982,12 @@ void XdsClient::ChannelState::AdsCallState::UnsubscribeLocked(
   if (authority_map.empty()) {
     type_state_map.subscribed_resources.erase(name.authority);
   }
-  if (!delay_unsubscription) SendMessageLocked(type);
+  // Don't need to send unsubscription message if this was the last
+  // resource we were subscribed to, since we'll be closing the stream
+  // immediately in that case.
+  if (!delay_unsubscription && HasSubscribedResources()) {
+    SendMessageLocked(type);
+  }
 }
 
 bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const {
@@ -1133,22 +997,21 @@ bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const {
   return false;
 }
 
-void XdsClient::ChannelState::AdsCallState::OnRequestSent(
-    void* arg, grpc_error_handle error) {
-  AdsCallState* ads_calld = static_cast<AdsCallState*>(arg);
-  {
-    MutexLock lock(&ads_calld->xds_client()->mu_);
-    ads_calld->OnRequestSentLocked(GRPC_ERROR_REF(error));
+void XdsClient::ChannelState::AdsCallState::OnRequestSent(bool ok) {
+  MutexLock lock(&xds_client()->mu_);
+  // For each resource that was in the message we just sent, start the
+  // resource timer if needed.
+  if (ok) {
+    auto& resource_type_state = state_map_[send_message_pending_];
+    for (const auto& p : resource_type_state.subscribed_resources) {
+      for (auto& q : p.second) {
+        q.second->MaybeMarkSubscriptionSendComplete(
+            Ref(DEBUG_LOCATION, "ResourceTimer"));
+      }
+    }
   }
-  ads_calld->Unref(DEBUG_LOCATION, "ADS+OnRequestSentLocked");
-}
-
-void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked(
-    grpc_error_handle error) {
-  if (IsCurrentCallOnChannel() && error == GRPC_ERROR_NONE) {
-    // Clean up the sent message.
-    grpc_byte_buffer_destroy(send_message_payload_);
-    send_message_payload_ = nullptr;
+  send_message_pending_ = nullptr;
+  if (ok && IsCurrentCallOnChannel()) {
     // Continue to send another pending message if any.
     // TODO(roth): The current code to handle buffered messages has the
     // advantage of sending only the most recent list of resource names for
@@ -1164,173 +1027,149 @@ void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked(
       buffered_requests_.erase(it);
     }
   }
-  GRPC_ERROR_UNREF(error);
 }
 
-void XdsClient::ChannelState::AdsCallState::OnResponseReceived(
-    void* arg, grpc_error_handle /* error */) {
-  AdsCallState* ads_calld = static_cast<AdsCallState*>(arg);
-  bool done;
+void XdsClient::ChannelState::AdsCallState::OnRecvMessage(
+    absl::string_view payload) {
   {
-    MutexLock lock(&ads_calld->xds_client()->mu_);
-    done = ads_calld->OnResponseReceivedLocked();
-  }
-  ads_calld->xds_client()->work_serializer_.DrainQueue();
-  if (done) ads_calld->Unref(DEBUG_LOCATION, "ADS+OnResponseReceivedLocked");
-}
-
-bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() {
-  // Empty payload means the call was cancelled.
-  if (!IsCurrentCallOnChannel() || recv_message_payload_ == nullptr) {
-    return true;
-  }
-  // Read the response.
-  grpc_byte_buffer_reader bbr;
-  grpc_byte_buffer_reader_init(&bbr, recv_message_payload_);
-  grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr);
-  grpc_byte_buffer_reader_destroy(&bbr);
-  grpc_byte_buffer_destroy(recv_message_payload_);
-  recv_message_payload_ = nullptr;
-  // Parse and validate the response.
-  AdsResponseParser parser(this);
-  absl::Status status = xds_client()->api_.ParseAdsResponse(
-      chand()->server_, response_slice, &parser);
-  grpc_slice_unref_internal(response_slice);
-  if (!status.ok()) {
-    // Ignore unparsable response.
-    gpr_log(GPR_ERROR,
-            "[xds_client %p] xds server %s: error parsing ADS response (%s) "
-            "-- ignoring",
-            xds_client(), chand()->server_.server_uri.c_str(),
-            status.ToString().c_str());
-  } else {
-    AdsResponseParser::Result result = parser.TakeResult();
-    // Update nonce.
-    auto& state = state_map_[result.type];
-    state.nonce = result.nonce;
-    // If we got an error, set state.error so that we'll NACK the update.
-    if (!result.errors.empty()) {
-      std::string error = absl::StrJoin(result.errors, "; ");
-      gpr_log(
-          GPR_ERROR,
-          "[xds_client %p] xds server %s: ADS response invalid for resource "
-          "type %s version %s, will NACK: nonce=%s error=%s",
-          xds_client(), chand()->server_.server_uri.c_str(),
-          result.type_url.c_str(), result.version.c_str(), state.nonce.c_str(),
-          error.c_str());
-      GRPC_ERROR_UNREF(state.error);
-      state.error = grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(error),
-                                       GRPC_ERROR_INT_GRPC_STATUS,
-                                       GRPC_STATUS_UNAVAILABLE);
-    }
-    // Delete resources not seen in update if needed.
-    if (result.type->AllResourcesRequiredInSotW()) {
-      for (auto& a : xds_client()->authority_state_map_) {
-        const std::string& authority = a.first;
-        AuthorityState& authority_state = a.second;
-        // Skip authorities that are not using this xDS channel.
-        if (authority_state.channel_state != chand()) continue;
-        auto seen_authority_it = result.resources_seen.find(authority);
-        // Find this resource type.
-        auto type_it = authority_state.resource_map.find(result.type);
-        if (type_it == authority_state.resource_map.end()) continue;
-        // Iterate over resource ids.
-        for (auto& r : type_it->second) {
-          const XdsResourceKey& resource_key = r.first;
-          ResourceState& resource_state = r.second;
-          if (seen_authority_it == result.resources_seen.end() ||
-              seen_authority_it->second.find(resource_key) ==
-                  seen_authority_it->second.end()) {
-            // If the resource was newly requested but has not yet been
-            // received, we don't want to generate an error for the watchers,
-            // because this ADS response may be in reaction to an earlier
-            // request that did not yet request the new resource, so its absence
-            // from the response does not necessarily indicate that the resource
-            // does not exist.  For that case, we rely on the request timeout
-            // instead.
-            if (resource_state.resource == nullptr) continue;
-            resource_state.resource.reset();
-            Notifier::
-                ScheduleNotifyWatchersOnResourceDoesNotExistInWorkSerializer(
-                    xds_client(), resource_state.watchers, DEBUG_LOCATION);
+    MutexLock lock(&xds_client()->mu_);
+    if (!IsCurrentCallOnChannel()) return;
+    // Parse and validate the response.
+    AdsResponseParser parser(this);
+    absl::Status status = xds_client()->api_.ParseAdsResponse(payload, &parser);
+    if (!status.ok()) {
+      // Ignore unparsable response.
+      gpr_log(GPR_ERROR,
+              "[xds_client %p] xds server %s: error parsing ADS response (%s) "
+              "-- ignoring",
+              xds_client(), chand()->server_.server_uri().c_str(),
+              status.ToString().c_str());
+    } else {
+      seen_response_ = true;
+      chand()->status_ = absl::OkStatus();
+      AdsResponseParser::Result result = parser.TakeResult();
+      // Update nonce.
+      auto& state = state_map_[result.type];
+      state.nonce = result.nonce;
+      // If we got an error, set state.status so that we'll NACK the update.
+      if (!result.errors.empty()) {
+        state.status = absl::UnavailableError(
+            absl::StrCat("xDS response validation errors: [",
+                         absl::StrJoin(result.errors, "; "), "]"));
+        gpr_log(GPR_ERROR,
+                "[xds_client %p] xds server %s: ADS response invalid for "
+                "resource "
+                "type %s version %s, will NACK: nonce=%s status=%s",
+                xds_client(), chand()->server_.server_uri().c_str(),
+                result.type_url.c_str(), result.version.c_str(),
+                state.nonce.c_str(), state.status.ToString().c_str());
+      }
+      // Delete resources not seen in update if needed.
+      if (result.type->AllResourcesRequiredInSotW()) {
+        for (auto& a : xds_client()->authority_state_map_) {
+          const std::string& authority = a.first;
+          AuthorityState& authority_state = a.second;
+          // Skip authorities that are not using this xDS channel.
+          if (authority_state.channel_state != chand()) continue;
+          auto seen_authority_it = result.resources_seen.find(authority);
+          // Find this resource type.
+          auto type_it = authority_state.resource_map.find(result.type);
+          if (type_it == authority_state.resource_map.end()) continue;
+          // Iterate over resource ids.
+          for (auto& r : type_it->second) {
+            const XdsResourceKey& resource_key = r.first;
+            ResourceState& resource_state = r.second;
+            if (seen_authority_it == result.resources_seen.end() ||
+                seen_authority_it->second.find(resource_key) ==
+                    seen_authority_it->second.end()) {
+              // If the resource was newly requested but has not yet been
+              // received, we don't want to generate an error for the
+              // watchers, because this ADS response may be in reaction to an
+              // earlier request that did not yet request the new resource, so
+              // its absence from the response does not necessarily indicate
+              // that the resource does not exist.  For that case, we rely on
+              // the request timeout instead.
+              if (resource_state.resource == nullptr) continue;
+              if (chand()->server_.IgnoreResourceDeletion()) {
+                if (!resource_state.ignored_deletion) {
+                  gpr_log(GPR_ERROR,
+                          "[xds_client %p] xds server %s: ignoring deletion "
+                          "for resource type %s name %s",
+                          xds_client(), chand()->server_.server_uri().c_str(),
+                          result.type_url.c_str(),
+                          XdsClient::ConstructFullXdsResourceName(
+                              authority, result.type_url.c_str(), resource_key)
+                              .c_str());
+                  resource_state.ignored_deletion = true;
+                }
+              } else {
+                resource_state.resource.reset();
+                resource_state.meta.client_status =
+                    XdsApi::ResourceMetadata::DOES_NOT_EXIST;
+                xds_client()->NotifyWatchersOnResourceDoesNotExist(
+                    resource_state.watchers);
+              }
+            }
           }
         }
       }
-    }
-    // If we had valid resources, update the version.
-    if (result.have_valid_resources) {
-      seen_response_ = true;
-      chand()->resource_type_version_map_[result.type] =
-          std::move(result.version);
-      // Start load reporting if needed.
-      auto& lrs_call = chand()->lrs_calld_;
-      if (lrs_call != nullptr) {
-        LrsCallState* lrs_calld = lrs_call->calld();
-        if (lrs_calld != nullptr) lrs_calld->MaybeStartReportingLocked();
+      // If we had valid resources or the update was empty, update the version.
+      if (result.have_valid_resources || result.errors.empty()) {
+        chand()->resource_type_version_map_[result.type] =
+            std::move(result.version);
+        // Start load reporting if needed.
+        auto& lrs_call = chand()->lrs_calld_;
+        if (lrs_call != nullptr) {
+          LrsCallState* lrs_calld = lrs_call->calld();
+          if (lrs_calld != nullptr) lrs_calld->MaybeStartReportingLocked();
+        }
       }
+      // Send ACK or NACK.
+      SendMessageLocked(result.type);
     }
-    // Send ACK or NACK.
-    SendMessageLocked(result.type);
-  }
-  if (xds_client()->shutting_down_) return true;
-  // Keep listening for updates.
-  grpc_op op;
-  memset(&op, 0, sizeof(op));
-  op.op = GRPC_OP_RECV_MESSAGE;
-  op.data.recv_message.recv_message = &recv_message_payload_;
-  op.flags = 0;
-  op.reserved = nullptr;
-  GPR_ASSERT(call_ != nullptr);
-  // Reuse the "ADS+OnResponseReceivedLocked" ref taken in ctor.
-  const grpc_call_error call_error =
-      grpc_call_start_batch_and_execute(call_, &op, 1, &on_response_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  return false;
+  }
+  xds_client()->work_serializer_.DrainQueue();
 }
 
 void XdsClient::ChannelState::AdsCallState::OnStatusReceived(
-    void* arg, grpc_error_handle error) {
-  AdsCallState* ads_calld = static_cast<AdsCallState*>(arg);
+    absl::Status status) {
   {
-    MutexLock lock(&ads_calld->xds_client()->mu_);
-    ads_calld->OnStatusReceivedLocked(GRPC_ERROR_REF(error));
-  }
-  ads_calld->xds_client()->work_serializer_.DrainQueue();
-  ads_calld->Unref(DEBUG_LOCATION, "ADS+OnStatusReceivedLocked");
-}
-
-void XdsClient::ChannelState::AdsCallState::OnStatusReceivedLocked(
-    grpc_error_handle error) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-    char* status_details = grpc_slice_to_c_string(status_details_);
-    gpr_log(GPR_INFO,
-            "[xds_client %p] xds server %s: ADS call status received "
-            "(chand=%p, ads_calld=%p, call=%p): "
-            "status=%d, details='%s', error='%s'",
-            xds_client(), chand()->server_.server_uri.c_str(), chand(), this,
-            call_, status_code_, status_details,
-            grpc_error_std_string(error).c_str());
-    gpr_free(status_details);
-  }
-  // Ignore status from a stale call.
-  if (IsCurrentCallOnChannel()) {
-    // Try to restart the call.
-    parent_->OnCallFinishedLocked();
-    // Send error to all watchers.
-    xds_client()->NotifyOnErrorLocked(
-        GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
-            "xDS call failed: xDS server: %s, ADS call status code=%d, "
-            "details='%s', error='%s'",
-            chand()->server_.server_uri, status_code_,
-            StringViewFromSlice(status_details_),
-            grpc_error_std_string(error))));
+    MutexLock lock(&xds_client()->mu_);
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+      gpr_log(GPR_INFO,
+              "[xds_client %p] xds server %s: ADS call status received "
+              "(chand=%p, ads_calld=%p, call=%p): %s",
+              xds_client(), chand()->server_.server_uri().c_str(), chand(),
+              this, call_.get(), status.ToString().c_str());
+    }
+    // Cancel any does-not-exist timers that may be pending.
+    for (const auto& p : state_map_) {
+      for (const auto& q : p.second.subscribed_resources) {
+        for (auto& r : q.second) {
+          r.second->MaybeCancelTimer();
+        }
+      }
+    }
+    // Ignore status from a stale call.
+    if (IsCurrentCallOnChannel()) {
+      // Try to restart the call.
+      parent_->OnCallFinishedLocked();
+      // If we didn't receive a response on the stream, report the
+      // stream failure as a connectivity failure, which will report the
+      // error to all watchers of resources on this channel.
+      if (!seen_response_) {
+        chand()->SetChannelStatusLocked(absl::UnavailableError(
+            absl::StrCat("xDS call failed with no responses received; status: ",
+                         status.ToString())));
+      }
+    }
   }
-  GRPC_ERROR_UNREF(error);
+  xds_client()->work_serializer_.DrainQueue();
 }
 
 bool XdsClient::ChannelState::AdsCallState::IsCurrentCallOnChannel() const {
-  // If the retryable ADS call is null (which only happens when the xds channel
-  // is shutting down), all the ADS calls are stale.
+  // If the retryable ADS call is null (which only happens when the xds
+  // channel is shutting down), all the ADS calls are stale.
   if (chand()->ads_calld_ == nullptr) return false;
   return this == chand()->ads_calld_->calld();
 }
@@ -1348,7 +1187,7 @@ XdsClient::ChannelState::AdsCallState::ResourceNamesForRequest(
         resource_names.emplace_back(XdsClient::ConstructFullXdsResourceName(
             authority, type->type_url(), resource_key));
         OrphanablePtr<ResourceTimer>& resource_timer = p.second;
-        resource_timer->MaybeStartTimer(Ref(DEBUG_LOCATION, "ResourceTimer"));
+        resource_timer->MarkSubscriptionSendStarted();
       }
     }
   }
@@ -1360,38 +1199,30 @@ XdsClient::ChannelState::AdsCallState::ResourceNamesForRequest(
 //
 
 void XdsClient::ChannelState::LrsCallState::Reporter::Orphan() {
-  if (next_report_timer_callback_pending_) {
-    grpc_timer_cancel(&next_report_timer_);
+  if (timer_handle_.has_value() &&
+      xds_client()->engine()->Cancel(*timer_handle_)) {
+    timer_handle_.reset();
+    Unref(DEBUG_LOCATION, "Orphan");
   }
 }
 
 void XdsClient::ChannelState::LrsCallState::Reporter::
     ScheduleNextReportLocked() {
-  const Timestamp next_report_time = ExecCtx::Get()->Now() + report_interval_;
-  grpc_timer_init(&next_report_timer_, next_report_time,
-                  &on_next_report_timer_);
-  next_report_timer_callback_pending_ = true;
-}
-
-void XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer(
-    void* arg, grpc_error_handle error) {
-  Reporter* self = static_cast<Reporter*>(arg);
-  bool done;
-  {
-    MutexLock lock(&self->xds_client()->mu_);
-    done = self->OnNextReportTimerLocked(GRPC_ERROR_REF(error));
-  }
-  if (done) self->Unref(DEBUG_LOCATION, "Reporter+timer");
+  timer_handle_ = xds_client()->engine()->RunAfter(report_interval_, [this]() {
+    ApplicationCallbackExecCtx callback_exec_ctx;
+    ExecCtx exec_ctx;
+    if (OnNextReportTimer()) {
+      Unref(DEBUG_LOCATION, "OnNextReportTimer()");
+    }
+  });
 }
 
-bool XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimerLocked(
-    grpc_error_handle error) {
-  next_report_timer_callback_pending_ = false;
-  if (error != GRPC_ERROR_NONE || !IsCurrentReporterOnCall()) {
-    GRPC_ERROR_UNREF(error);
-    return true;
-  }
-  return SendReportLocked();
+bool XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer() {
+  MutexLock lock(&xds_client()->mu_);
+  timer_handle_.reset();
+  if (!IsCurrentReporterOnCall()) return true;
+  SendReportLocked();
+  return false;
 }
 
 namespace {
@@ -1422,7 +1253,7 @@ bool XdsClient::ChannelState::LrsCallState::Reporter::SendReportLocked() {
   last_report_counters_were_zero_ = LoadReportCountersAreZero(snapshot);
   if (old_val && last_report_counters_were_zero_) {
     auto it = xds_client()->xds_load_report_server_map_.find(
-        parent_->chand()->server_);
+        &parent_->chand()->server_);
     if (it == xds_client()->xds_load_report_server_map_.end() ||
         it->second.load_report_map.empty()) {
       it->second.channel_state->StopLrsCallLocked();
@@ -1431,65 +1262,34 @@ bool XdsClient::ChannelState::LrsCallState::Reporter::SendReportLocked() {
     ScheduleNextReportLocked();
     return false;
   }
-  // Create a request that contains the snapshot.
-  grpc_slice request_payload_slice =
+  // Send a request that contains the snapshot.
+  std::string serialized_payload =
       xds_client()->api_.CreateLrsRequest(std::move(snapshot));
-  parent_->send_message_payload_ =
-      grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_slice_unref_internal(request_payload_slice);
-  // Send the report.
-  grpc_op op;
-  memset(&op, 0, sizeof(op));
-  op.op = GRPC_OP_SEND_MESSAGE;
-  op.data.send_message.send_message = parent_->send_message_payload_;
-  grpc_call_error call_error = grpc_call_start_batch_and_execute(
-      parent_->call_, &op, 1, &on_report_done_);
-  if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) {
-    gpr_log(GPR_ERROR,
-            "[xds_client %p] xds server %s: error starting LRS send_message "
-            "batch on calld=%p: call_error=%d",
-            xds_client(), parent_->chand()->server_.server_uri.c_str(), this,
-            call_error);
-    GPR_ASSERT(GRPC_CALL_OK == call_error);
-  }
+  parent_->call_->SendMessage(std::move(serialized_payload));
+  parent_->send_message_pending_ = true;
   return false;
 }
 
-void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDone(
-    void* arg, grpc_error_handle error) {
-  Reporter* self = static_cast<Reporter*>(arg);
-  bool done;
-  {
-    MutexLock lock(&self->xds_client()->mu_);
-    done = self->OnReportDoneLocked(GRPC_ERROR_REF(error));
-  }
-  if (done) self->Unref(DEBUG_LOCATION, "Reporter+report_done");
-}
-
-bool XdsClient::ChannelState::LrsCallState::Reporter::OnReportDoneLocked(
-    grpc_error_handle error) {
-  grpc_byte_buffer_destroy(parent_->send_message_payload_);
-  parent_->send_message_payload_ = nullptr;
+void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDoneLocked() {
+  // If a reporter starts a send_message op, then the reporting interval
+  // changes and we destroy that reporter and create a new one, and then
+  // the send_message op started by the old reporter finishes, this
+  // method will be called even though it was for a completion started
+  // by the old reporter.  In that case, the timer will be pending, so
+  // we just ignore the completion and wait for the timer to fire.
+  if (timer_handle_.has_value()) return;
   // If there are no more registered stats to report, cancel the call.
-  auto it =
-      xds_client()->xds_load_report_server_map_.find(parent_->chand()->server_);
-  if (it == xds_client()->xds_load_report_server_map_.end() ||
-      it->second.load_report_map.empty()) {
-    it->second.channel_state->StopLrsCallLocked();
-    GRPC_ERROR_UNREF(error);
-    return true;
-  }
-  if (error != GRPC_ERROR_NONE || !IsCurrentReporterOnCall()) {
-    GRPC_ERROR_UNREF(error);
-    // If this reporter is no longer the current one on the call, the reason
-    // might be that it was orphaned for a new one due to config update.
-    if (!IsCurrentReporterOnCall()) {
-      parent_->MaybeStartReportingLocked();
+  auto it = xds_client()->xds_load_report_server_map_.find(
+      &parent_->chand()->server_);
+  if (it == xds_client()->xds_load_report_server_map_.end()) return;
+  if (it->second.load_report_map.empty()) {
+    if (it->second.channel_state != nullptr) {
+      it->second.channel_state->StopLrsCallLocked();
     }
-    return true;
+    return;
   }
+  // Otherwise, schedule the next load report.
   ScheduleNextReportLocked();
-  return false;
 }
 
 //
@@ -1508,123 +1308,41 @@ XdsClient::ChannelState::LrsCallState::LrsCallState(
   // the polling entities from client_channel.
   GPR_ASSERT(xds_client() != nullptr);
   const char* method =
-      chand()->server_.ShouldUseV3()
-          ? "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats"
-          : "/envoy.service.load_stats.v2.LoadReportingService/StreamLoadStats";
-  call_ = grpc_channel_create_pollset_set_call(
-      chand()->channel_, nullptr, GRPC_PROPAGATE_DEFAULTS,
-      xds_client()->interested_parties_,
-      Slice::FromStaticString(method).c_slice(), nullptr,
-      Timestamp::InfFuture(), nullptr);
+      "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats";
+  call_ = chand()->transport_->CreateStreamingCall(
+      method, std::make_unique<StreamEventHandler>(
+                  // Passing the initial ref here.  This ref will go away when
+                  // the StreamEventHandler is destroyed.
+                  RefCountedPtr<LrsCallState>(this)));
   GPR_ASSERT(call_ != nullptr);
-  // Init the request payload.
-  grpc_slice request_payload_slice =
-      xds_client()->api_.CreateLrsInitialRequest(chand()->server_);
-  send_message_payload_ =
-      grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_slice_unref_internal(request_payload_slice);
-  // Init other data associated with the LRS call.
-  grpc_metadata_array_init(&initial_metadata_recv_);
-  grpc_metadata_array_init(&trailing_metadata_recv_);
   // Start the call.
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-    gpr_log(
-        GPR_INFO,
-        "[xds_client %p] xds server %s: starting LRS call (calld=%p, call=%p)",
-        xds_client(), chand()->server_.server_uri.c_str(), this, call_);
-  }
-  // Create the ops.
-  grpc_call_error call_error;
-  grpc_op ops[3];
-  memset(ops, 0, sizeof(ops));
-  // Op: send initial metadata.
-  grpc_op* op = ops;
-  op->op = GRPC_OP_SEND_INITIAL_METADATA;
-  op->data.send_initial_metadata.count = 0;
-  op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY |
-              GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET;
-  op->reserved = nullptr;
-  op++;
-  // Op: send request message.
-  GPR_ASSERT(send_message_payload_ != nullptr);
-  op->op = GRPC_OP_SEND_MESSAGE;
-  op->data.send_message.send_message = send_message_payload_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  Ref(DEBUG_LOCATION, "LRS+OnInitialRequestSentLocked").release();
-  GRPC_CLOSURE_INIT(&on_initial_request_sent_, OnInitialRequestSent, this,
-                    grpc_schedule_on_exec_ctx);
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), &on_initial_request_sent_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  // Op: recv initial metadata.
-  op = ops;
-  op->op = GRPC_OP_RECV_INITIAL_METADATA;
-  op->data.recv_initial_metadata.recv_initial_metadata =
-      &initial_metadata_recv_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  // Op: recv response.
-  op->op = GRPC_OP_RECV_MESSAGE;
-  op->data.recv_message.recv_message = &recv_message_payload_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  Ref(DEBUG_LOCATION, "LRS+OnResponseReceivedLocked").release();
-  GRPC_CLOSURE_INIT(&on_response_received_, OnResponseReceived, this,
-                    grpc_schedule_on_exec_ctx);
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), &on_response_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  // Op: recv server status.
-  op = ops;
-  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
-  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv_;
-  op->data.recv_status_on_client.status = &status_code_;
-  op->data.recv_status_on_client.status_details = &status_details_;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  // This callback signals the end of the call, so it relies on the initial
-  // ref instead of a new ref. When it's invoked, it's the initial ref that is
-  // unreffed.
-  GRPC_CLOSURE_INIT(&on_status_received_, OnStatusReceived, this,
-                    grpc_schedule_on_exec_ctx);
-  call_error = grpc_call_start_batch_and_execute(
-      call_, ops, static_cast<size_t>(op - ops), &on_status_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-}
-
-XdsClient::ChannelState::LrsCallState::~LrsCallState() {
-  grpc_metadata_array_destroy(&initial_metadata_recv_);
-  grpc_metadata_array_destroy(&trailing_metadata_recv_);
-  grpc_byte_buffer_destroy(send_message_payload_);
-  grpc_byte_buffer_destroy(recv_message_payload_);
-  grpc_slice_unref_internal(status_details_);
-  GPR_ASSERT(call_ != nullptr);
-  grpc_call_unref(call_);
+    gpr_log(GPR_INFO,
+            "[xds_client %p] xds server %s: starting LRS call (calld=%p, "
+            "call=%p)",
+            xds_client(), chand()->server_.server_uri().c_str(), this,
+            call_.get());
+  }
+  // Send the initial request.
+  std::string serialized_payload = xds_client()->api_.CreateLrsInitialRequest();
+  call_->SendMessage(std::move(serialized_payload));
+  send_message_pending_ = true;
 }
 
 void XdsClient::ChannelState::LrsCallState::Orphan() {
   reporter_.reset();
-  GPR_ASSERT(call_ != nullptr);
-  // If we are here because xds_client wants to cancel the call,
-  // on_status_received_ will complete the cancellation and clean up. Otherwise,
-  // we are here because xds_client has to orphan a failed call, then the
-  // following cancellation will be a no-op.
-  grpc_call_cancel_internal(call_);
-  // Note that the initial ref is hold by on_status_received_. So the
-  // corresponding unref happens in on_status_received_ instead of here.
+  // Note that the initial ref is held by the StreamEventHandler, which
+  // will be destroyed when call_ is destroyed, which may not happen
+  // here, since there may be other refs held to call_ by internal callbacks.
+  call_.reset();
 }
 
 void XdsClient::ChannelState::LrsCallState::MaybeStartReportingLocked() {
   // Don't start again if already started.
   if (reporter_ != nullptr) return;
-  // Don't start if the previous send_message op (of the initial request or the
-  // last report of the previous reporter) hasn't completed.
-  if (send_message_payload_ != nullptr) return;
+  // Don't start if the previous send_message op (of the initial request or
+  // the last report of the previous reporter) hasn't completed.
+  if (call_ != nullptr && send_message_pending_) return;
   // Don't start if no LRS response has arrived.
   if (!seen_response()) return;
   // Don't start if the ADS call hasn't received any valid response. Note that
@@ -1640,166 +1358,105 @@ void XdsClient::ChannelState::LrsCallState::MaybeStartReportingLocked() {
       Ref(DEBUG_LOCATION, "LRS+load_report+start"), load_reporting_interval_);
 }
 
-void XdsClient::ChannelState::LrsCallState::OnInitialRequestSent(
-    void* arg, grpc_error_handle /*error*/) {
-  LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg);
-  {
-    MutexLock lock(&lrs_calld->xds_client()->mu_);
-    lrs_calld->OnInitialRequestSentLocked();
+void XdsClient::ChannelState::LrsCallState::OnRequestSent(bool /*ok*/) {
+  MutexLock lock(&xds_client()->mu_);
+  send_message_pending_ = false;
+  if (reporter_ != nullptr) {
+    reporter_->OnReportDoneLocked();
+  } else {
+    MaybeStartReportingLocked();
   }
-  lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnInitialRequestSentLocked");
 }
 
-void XdsClient::ChannelState::LrsCallState::OnInitialRequestSentLocked() {
-  // Clear the send_message_payload_.
-  grpc_byte_buffer_destroy(send_message_payload_);
-  send_message_payload_ = nullptr;
-  MaybeStartReportingLocked();
-}
-
-void XdsClient::ChannelState::LrsCallState::OnResponseReceived(
-    void* arg, grpc_error_handle /*error*/) {
-  LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg);
-  bool done;
-  {
-    MutexLock lock(&lrs_calld->xds_client()->mu_);
-    done = lrs_calld->OnResponseReceivedLocked();
-  }
-  if (done) lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnResponseReceivedLocked");
-}
-
-bool XdsClient::ChannelState::LrsCallState::OnResponseReceivedLocked() {
-  // Empty payload means the call was cancelled.
-  if (!IsCurrentCallOnChannel() || recv_message_payload_ == nullptr) {
-    return true;
-  }
-  // Read the response.
-  grpc_byte_buffer_reader bbr;
-  grpc_byte_buffer_reader_init(&bbr, recv_message_payload_);
-  grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr);
-  grpc_byte_buffer_reader_destroy(&bbr);
-  grpc_byte_buffer_destroy(recv_message_payload_);
-  recv_message_payload_ = nullptr;
-  // This anonymous lambda is a hack to avoid the usage of goto.
-  [&]() {
-    // Parse the response.
-    bool send_all_clusters = false;
-    std::set<std::string> new_cluster_names;
-    Duration new_load_reporting_interval;
-    grpc_error_handle parse_error = xds_client()->api_.ParseLrsResponse(
-        response_slice, &send_all_clusters, &new_cluster_names,
-        &new_load_reporting_interval);
-    if (parse_error != GRPC_ERROR_NONE) {
-      gpr_log(GPR_ERROR,
-              "[xds_client %p] xds server %s: LRS response parsing failed: %s",
-              xds_client(), chand()->server_.server_uri.c_str(),
-              grpc_error_std_string(parse_error).c_str());
-      GRPC_ERROR_UNREF(parse_error);
-      return;
+void XdsClient::ChannelState::LrsCallState::OnRecvMessage(
+    absl::string_view payload) {
+  MutexLock lock(&xds_client()->mu_);
+  // If we're no longer the current call, ignore the result.
+  if (!IsCurrentCallOnChannel()) return;
+  // Parse the response.
+  bool send_all_clusters = false;
+  std::set<std::string> new_cluster_names;
+  Duration new_load_reporting_interval;
+  absl::Status status = xds_client()->api_.ParseLrsResponse(
+      payload, &send_all_clusters, &new_cluster_names,
+      &new_load_reporting_interval);
+  if (!status.ok()) {
+    gpr_log(GPR_ERROR,
+            "[xds_client %p] xds server %s: LRS response parsing failed: %s",
+            xds_client(), chand()->server_.server_uri().c_str(),
+            status.ToString().c_str());
+    return;
+  }
+  seen_response_ = true;
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+    gpr_log(
+        GPR_INFO,
+        "[xds_client %p] xds server %s: LRS response received, %" PRIuPTR
+        " cluster names, send_all_clusters=%d, load_report_interval=%" PRId64
+        "ms",
+        xds_client(), chand()->server_.server_uri().c_str(),
+        new_cluster_names.size(), send_all_clusters,
+        new_load_reporting_interval.millis());
+    size_t i = 0;
+    for (const auto& name : new_cluster_names) {
+      gpr_log(GPR_INFO, "[xds_client %p] cluster_name %" PRIuPTR ": %s",
+              xds_client(), i++, name.c_str());
     }
-    seen_response_ = true;
+  }
+  if (new_load_reporting_interval <
+      Duration::Milliseconds(GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS)) {
+    new_load_reporting_interval =
+        Duration::Milliseconds(GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS);
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-      gpr_log(
-          GPR_INFO,
-          "[xds_client %p] xds server %s: LRS response received, %" PRIuPTR
-          " cluster names, send_all_clusters=%d, load_report_interval=%" PRId64
-          "ms",
-          xds_client(), chand()->server_.server_uri.c_str(),
-          new_cluster_names.size(), send_all_clusters,
-          new_load_reporting_interval.millis());
-      size_t i = 0;
-      for (const auto& name : new_cluster_names) {
-        gpr_log(GPR_INFO, "[xds_client %p] cluster_name %" PRIuPTR ": %s",
-                xds_client(), i++, name.c_str());
-      }
-    }
-    if (new_load_reporting_interval <
-        Duration::Milliseconds(
-            GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS)) {
-      new_load_reporting_interval = Duration::Milliseconds(
-          GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS);
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-        gpr_log(GPR_INFO,
-                "[xds_client %p] xds server %s: increased load_report_interval "
-                "to minimum value %dms",
-                xds_client(), chand()->server_.server_uri.c_str(),
-                GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS);
-      }
+      gpr_log(GPR_INFO,
+              "[xds_client %p] xds server %s: increased load_report_interval "
+              "to minimum value %dms",
+              xds_client(), chand()->server_.server_uri().c_str(),
+              GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS);
     }
-    // Ignore identical update.
-    if (send_all_clusters == send_all_clusters_ &&
-        cluster_names_ == new_cluster_names &&
-        load_reporting_interval_ == new_load_reporting_interval) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-        gpr_log(
-            GPR_INFO,
-            "[xds_client %p] xds server %s: incoming LRS response identical "
-            "to current, ignoring.",
-            xds_client(), chand()->server_.server_uri.c_str());
-      }
-      return;
+  }
+  // Ignore identical update.
+  if (send_all_clusters == send_all_clusters_ &&
+      cluster_names_ == new_cluster_names &&
+      load_reporting_interval_ == new_load_reporting_interval) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+      gpr_log(GPR_INFO,
+              "[xds_client %p] xds server %s: incoming LRS response identical "
+              "to current, ignoring.",
+              xds_client(), chand()->server_.server_uri().c_str());
     }
-    // Stop current load reporting (if any) to adopt the new config.
-    reporter_.reset();
-    // Record the new config.
-    send_all_clusters_ = send_all_clusters;
-    cluster_names_ = std::move(new_cluster_names);
-    load_reporting_interval_ = new_load_reporting_interval;
-    // Try starting sending load report.
-    MaybeStartReportingLocked();
-  }();
-  grpc_slice_unref_internal(response_slice);
-  if (xds_client()->shutting_down_) return true;
-  // Keep listening for LRS config updates.
-  grpc_op op;
-  memset(&op, 0, sizeof(op));
-  op.op = GRPC_OP_RECV_MESSAGE;
-  op.data.recv_message.recv_message = &recv_message_payload_;
-  op.flags = 0;
-  op.reserved = nullptr;
-  GPR_ASSERT(call_ != nullptr);
-  // Reuse the "OnResponseReceivedLocked" ref taken in ctor.
-  const grpc_call_error call_error =
-      grpc_call_start_batch_and_execute(call_, &op, 1, &on_response_received_);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-  return false;
-}
-
-void XdsClient::ChannelState::LrsCallState::OnStatusReceived(
-    void* arg, grpc_error_handle error) {
-  LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg);
-  {
-    MutexLock lock(&lrs_calld->xds_client()->mu_);
-    lrs_calld->OnStatusReceivedLocked(GRPC_ERROR_REF(error));
+    return;
   }
-  lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnStatusReceivedLocked");
+  // Stop current load reporting (if any) to adopt the new config.
+  reporter_.reset();
+  // Record the new config.
+  send_all_clusters_ = send_all_clusters;
+  cluster_names_ = std::move(new_cluster_names);
+  load_reporting_interval_ = new_load_reporting_interval;
+  // Try starting sending load report.
+  MaybeStartReportingLocked();
 }
 
-void XdsClient::ChannelState::LrsCallState::OnStatusReceivedLocked(
-    grpc_error_handle error) {
-  GPR_ASSERT(call_ != nullptr);
+void XdsClient::ChannelState::LrsCallState::OnStatusReceived(
+    absl::Status status) {
+  MutexLock lock(&xds_client()->mu_);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-    char* status_details = grpc_slice_to_c_string(status_details_);
     gpr_log(GPR_INFO,
             "[xds_client %p] xds server %s: LRS call status received "
-            "(chand=%p, calld=%p, call=%p): "
-            "status=%d, details='%s', error='%s'",
-            xds_client(), chand()->server_.server_uri.c_str(), chand(), this,
-            call_, status_code_, status_details,
-            grpc_error_std_string(error).c_str());
-    gpr_free(status_details);
+            "(chand=%p, calld=%p, call=%p): %s",
+            xds_client(), chand()->server_.server_uri().c_str(), chand(), this,
+            call_.get(), status.ToString().c_str());
   }
   // Ignore status from a stale call.
   if (IsCurrentCallOnChannel()) {
     // Try to restart the call.
     parent_->OnCallFinishedLocked();
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 bool XdsClient::ChannelState::LrsCallState::IsCurrentCallOnChannel() const {
-  // If the retryable LRS call is null (which only happens when the xds channel
-  // is shutting down), all the LRS calls are stale.
+  // If the retryable LRS call is null (which only happens when the xds
+  // channel is shutting down), all the LRS calls are stale.
   if (chand()->lrs_calld_ == nullptr) return false;
   return this == chand()->lrs_calld_->calld();
 }
@@ -1808,85 +1465,66 @@ bool XdsClient::ChannelState::LrsCallState::IsCurrentCallOnChannel() const {
 // XdsClient
 //
 
-namespace {
-
-Duration GetRequestTimeout(const grpc_channel_args* args) {
-  return Duration::Milliseconds(grpc_channel_args_find_integer(
-      args, GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS,
-      {15000, 0, INT_MAX}));
-}
-
-grpc_channel_args* ModifyChannelArgs(const grpc_channel_args* args) {
-  absl::InlinedVector<grpc_arg, 1> args_to_add = {
-      grpc_channel_arg_integer_create(
-          const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS),
-          5 * 60 * GPR_MS_PER_SEC),
-  };
-  return grpc_channel_args_copy_and_add(args, args_to_add.data(),
-                                        args_to_add.size());
-}
-
-}  // namespace
-
-XdsClient::XdsClient(std::unique_ptr<XdsBootstrap> bootstrap,
-                     const grpc_channel_args* args)
+XdsClient::XdsClient(
+    std::unique_ptr<XdsBootstrap> bootstrap,
+    OrphanablePtr<XdsTransportFactory> transport_factory,
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine,
+    std::string user_agent_name, std::string user_agent_version,
+    Duration resource_request_timeout)
     : DualRefCounted<XdsClient>(
           GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_refcount_trace) ? "XdsClient"
                                                                   : nullptr),
       bootstrap_(std::move(bootstrap)),
-      args_(ModifyChannelArgs(args)),
-      request_timeout_(GetRequestTimeout(args)),
-      interested_parties_(grpc_pollset_set_create()),
-      certificate_provider_store_(MakeOrphanable<CertificateProviderStore>(
-          bootstrap_->certificate_providers())),
-      api_(this, &grpc_xds_client_trace, bootstrap_->node(),
-           &bootstrap_->certificate_providers(), &symtab_) {
+      transport_factory_(std::move(transport_factory)),
+      request_timeout_(resource_request_timeout),
+      xds_federation_enabled_(XdsFederationEnabled()),
+      api_(this, &grpc_xds_client_trace, bootstrap_->node(), &symtab_,
+           std::move(user_agent_name), std::move(user_agent_version)),
+      engine_(std::move(engine)) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO, "[xds_client %p] creating xds client", this);
   }
-  // Calling grpc_init to ensure gRPC does not shut down until the XdsClient is
-  // destroyed.
-  grpc_init();
+  GPR_ASSERT(bootstrap_ != nullptr);
+  if (bootstrap_->node() != nullptr) {
+    gpr_log(GPR_INFO, "[xds_client %p] xDS node ID: %s", this,
+            bootstrap_->node()->id().c_str());
+  }
 }
 
 XdsClient::~XdsClient() {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO, "[xds_client %p] destroying xds client", this);
   }
-  grpc_channel_args_destroy(args_);
-  grpc_pollset_set_destroy(interested_parties_);
-  // Calling grpc_shutdown to ensure gRPC does not shut down until the XdsClient
-  // is destroyed.
-  grpc_shutdown();
 }
 
 void XdsClient::Orphan() {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO, "[xds_client %p] shutting down xds client", this);
   }
-  {
-    MutexLock lock(g_mu);
-    if (g_xds_client == this) g_xds_client = nullptr;
-  }
-  {
-    MutexLock lock(&mu_);
-    shutting_down_ = true;
-    // Clear cache and any remaining watchers that may not have been cancelled.
-    authority_state_map_.clear();
-    invalid_watchers_.clear();
+  MutexLock lock(&mu_);
+  shutting_down_ = true;
+  // Clear cache and any remaining watchers that may not have been cancelled.
+  authority_state_map_.clear();
+  invalid_watchers_.clear();
+  // We may still be sending lingering queued load report data, so don't
+  // just clear the load reporting map, but we do want to clear the refs
+  // we're holding to the ChannelState objects, to make sure that
+  // everything shuts down properly.
+  for (auto& p : xds_load_report_server_map_) {
+    p.second.channel_state.reset(DEBUG_LOCATION, "XdsClient::Orphan()");
   }
 }
 
 RefCountedPtr<XdsClient::ChannelState> XdsClient::GetOrCreateChannelStateLocked(
-    const XdsBootstrap::XdsServer& server) {
-  auto it = xds_server_channel_map_.find(server);
+    const XdsBootstrap::XdsServer& server, const char* reason) {
+  auto it = xds_server_channel_map_.find(&server);
   if (it != xds_server_channel_map_.end()) {
-    return it->second->Ref(DEBUG_LOCATION, "Authority");
+    return it->second->Ref(DEBUG_LOCATION, reason);
   }
   // Channel not found, so create a new one.
   auto channel_state = MakeRefCounted<ChannelState>(
       WeakRef(DEBUG_LOCATION, "ChannelState"), server);
-  xds_server_channel_map_[server] = channel_state.get();
+  xds_server_channel_map_[&server] = channel_state.get();
   return channel_state;
 }
 
@@ -1895,24 +1533,23 @@ void XdsClient::WatchResource(const XdsResourceType* type,
                               RefCountedPtr<ResourceWatcherInterface> watcher) {
   ResourceWatcherInterface* w = watcher.get();
   // Lambda for handling failure cases.
-  auto fail = [&](grpc_error_handle error) mutable {
+  auto fail = [&](absl::Status status) mutable {
     {
       MutexLock lock(&mu_);
       MaybeRegisterResourceTypeLocked(type);
       invalid_watchers_[w] = watcher;
     }
     work_serializer_.Run(
-        // TODO(yashykt): When we move to C++14, capture watcher using
-        // std::move()
-        [watcher, error]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
-          watcher->OnError(error);
-        },
+        [watcher = std::move(watcher), status = std::move(status)]()
+            ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
+              watcher->OnError(status);
+            },
         DEBUG_LOCATION);
   };
   auto resource_name = ParseXdsResourceName(name, type);
   if (!resource_name.ok()) {
-    fail(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
-        "Unable to parse resource name for listener %s", name)));
+    fail(absl::UnavailableError(
+        absl::StrCat("Unable to parse resource name ", name)));
     return;
   }
   // Find server to use.
@@ -1921,14 +1558,12 @@ void XdsClient::WatchResource(const XdsResourceType* type,
   if (absl::ConsumePrefix(&authority_name, "xdstp:")) {
     auto* authority = bootstrap_->LookupAuthority(std::string(authority_name));
     if (authority == nullptr) {
-      fail(GRPC_ERROR_CREATE_FROM_CPP_STRING(
+      fail(absl::UnavailableError(
           absl::StrCat("authority \"", authority_name,
                        "\" not present in bootstrap config")));
       return;
     }
-    if (!authority->xds_servers.empty()) {
-      xds_server = &authority->xds_servers[0];
-    }
+    xds_server = authority->server();
   }
   if (xds_server == nullptr) xds_server = &bootstrap_->server();
   {
@@ -1954,12 +1589,60 @@ void XdsClient::WatchResource(const XdsResourceType* type,
             delete value;
           },
           DEBUG_LOCATION);
+    } else if (resource_state.meta.client_status ==
+               XdsApi::ResourceMetadata::DOES_NOT_EXIST) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_client %p] reporting cached does-not-exist for %s", this,
+                std::string(name).c_str());
+      }
+      work_serializer_.Schedule(
+          [watcher]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
+            watcher->OnResourceDoesNotExist();
+          },
+          DEBUG_LOCATION);
+    } else if (resource_state.meta.client_status ==
+               XdsApi::ResourceMetadata::NACKED) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+        gpr_log(
+            GPR_INFO,
+            "[xds_client %p] reporting cached validation failure for %s: %s",
+            this, std::string(name).c_str(),
+            resource_state.meta.failed_details.c_str());
+      }
+      std::string details = resource_state.meta.failed_details;
+      const auto* node = bootstrap_->node();
+      if (node != nullptr) {
+        absl::StrAppend(&details, " (node ID:", bootstrap_->node()->id(), ")");
+      }
+      work_serializer_.Schedule(
+          [watcher, details = std::move(details)]()
+              ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
+                watcher->OnError(absl::UnavailableError(
+                    absl::StrCat("invalid resource: ", details)));
+              },
+          DEBUG_LOCATION);
     }
     // If the authority doesn't yet have a channel, set it, creating it if
     // needed.
     if (authority_state.channel_state == nullptr) {
       authority_state.channel_state =
-          GetOrCreateChannelStateLocked(*xds_server);
+          GetOrCreateChannelStateLocked(*xds_server, "start watch");
+    }
+    absl::Status channel_status = authority_state.channel_state->status();
+    if (!channel_status.ok()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_client %p] returning cached channel error for %s: %s",
+                this, std::string(name).c_str(),
+                channel_status.ToString().c_str());
+      }
+      work_serializer_.Schedule(
+          [watcher = std::move(watcher), status = std::move(channel_status)]()
+              ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) mutable {
+                watcher->OnError(std::move(status));
+              },
+          DEBUG_LOCATION);
     }
     authority_state.channel_state->SubscribeLocked(type, *resource_name);
   }
@@ -1972,12 +1655,11 @@ void XdsClient::CancelResourceWatch(const XdsResourceType* type,
                                     bool delay_unsubscription) {
   auto resource_name = ParseXdsResourceName(name, type);
   MutexLock lock(&mu_);
-  if (!resource_name.ok()) {
-    invalid_watchers_.erase(watcher);
-    return;
-  }
-  if (shutting_down_) return;
+  // We cannot be sure whether the watcher is in invalid_watchers_ or in
+  // authority_state_map_, so we check both, just to be safe.
+  invalid_watchers_.erase(watcher);
   // Find authority.
+  if (!resource_name.ok()) return;
   auto authority_it = authority_state_map_.find(resource_name->authority);
   if (authority_it == authority_state_map_.end()) return;
   AuthorityState& authority_state = authority_it->second;
@@ -1993,6 +1675,13 @@ void XdsClient::CancelResourceWatch(const XdsResourceType* type,
   resource_state.watchers.erase(watcher);
   // Clean up empty map entries, if any.
   if (resource_state.watchers.empty()) {
+    if (resource_state.ignored_deletion) {
+      gpr_log(GPR_INFO,
+              "[xds_client %p] unsubscribing from a resource for which we "
+              "previously ignored a deletion: type %s name %s",
+              this, std::string(type->type_url()).c_str(),
+              std::string(name).c_str());
+    }
     authority_state.channel_state->UnsubscribeLocked(type, *resource_name,
                                                      delay_unsubscription);
     type_map.erase(resource_it);
@@ -2013,24 +1702,22 @@ void XdsClient::MaybeRegisterResourceTypeLocked(
     return;
   }
   resource_types_.emplace(resource_type->type_url(), resource_type);
-  v2_resource_types_.emplace(resource_type->v2_type_url(), resource_type);
-  resource_type->InitUpbSymtab(symtab_.ptr());
+  resource_type->InitUpbSymtab(this, symtab_.ptr());
 }
 
 const XdsResourceType* XdsClient::GetResourceTypeLocked(
     absl::string_view resource_type) {
   auto it = resource_types_.find(resource_type);
   if (it != resource_types_.end()) return it->second;
-  auto it2 = v2_resource_types_.find(resource_type);
-  if (it2 != v2_resource_types_.end()) return it2->second;
   return nullptr;
 }
 
 absl::StatusOr<XdsClient::XdsResourceName> XdsClient::ParseXdsResourceName(
     absl::string_view name, const XdsResourceType* type) {
   // Old-style names use the empty string for authority.
-  // authority is prefixed with "old:" to indicate that it's an old-style name.
-  if (!absl::StartsWith(name, "xdstp:")) {
+  // authority is prefixed with "old:" to indicate that it's an old-style
+  // name.
+  if (!xds_federation_enabled_ || !absl::StartsWith(name, "xdstp:")) {
     return XdsResourceName{"old:", {std::string(name), {}}};
   }
   // New style name.  Parse URI.
@@ -2039,7 +1726,7 @@ absl::StatusOr<XdsClient::XdsResourceName> XdsClient::ParseXdsResourceName(
   // Split the resource type off of the path to get the id.
   std::pair<absl::string_view, absl::string_view> path_parts = absl::StrSplit(
       absl::StripPrefix(uri->path(), "/"), absl::MaxSplits('/', 1));
-  if (!type->IsType(path_parts.first, nullptr)) {
+  if (type->type_url() != path_parts.first) {
     return absl::InvalidArgumentError(
         "xdstp URI path must indicate valid xDS resource type");
   }
@@ -2071,40 +1758,45 @@ std::string XdsClient::ConstructFullXdsResourceName(
 RefCountedPtr<XdsClusterDropStats> XdsClient::AddClusterDropStats(
     const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
     absl::string_view eds_service_name) {
-  if (!bootstrap_->XdsServerExists(xds_server)) return nullptr;
+  const auto* server = bootstrap_->FindXdsServer(xds_server);
+  if (server == nullptr) return nullptr;
   auto key =
       std::make_pair(std::string(cluster_name), std::string(eds_service_name));
-  MutexLock lock(&mu_);
-  // We jump through some hoops here to make sure that the const
-  // XdsBootstrap::XdsServer& and absl::string_views
-  // stored in the XdsClusterDropStats object point to the
-  // XdsBootstrap::XdsServer and strings
-  // in the load_report_map_ key, so that they have the same lifetime.
-  auto server_it =
-      xds_load_report_server_map_.emplace(xds_server, LoadReportServer()).first;
-  if (server_it->second.channel_state == nullptr) {
-    server_it->second.channel_state = GetOrCreateChannelStateLocked(xds_server);
-  }
-  auto load_report_it = server_it->second.load_report_map
-                            .emplace(std::move(key), LoadReportState())
-                            .first;
-  LoadReportState& load_report_state = load_report_it->second;
   RefCountedPtr<XdsClusterDropStats> cluster_drop_stats;
-  if (load_report_state.drop_stats != nullptr) {
-    cluster_drop_stats = load_report_state.drop_stats->RefIfNonZero();
-  }
-  if (cluster_drop_stats == nullptr) {
+  {
+    MutexLock lock(&mu_);
+    // We jump through some hoops here to make sure that the const
+    // XdsBootstrap::XdsServer& and absl::string_views
+    // stored in the XdsClusterDropStats object point to the
+    // XdsBootstrap::XdsServer and strings
+    // in the load_report_map_ key, so that they have the same lifetime.
+    auto server_it =
+        xds_load_report_server_map_.emplace(server, LoadReportServer()).first;
+    if (server_it->second.channel_state == nullptr) {
+      server_it->second.channel_state = GetOrCreateChannelStateLocked(
+          *server, "load report map (drop stats)");
+    }
+    auto load_report_it = server_it->second.load_report_map
+                              .emplace(std::move(key), LoadReportState())
+                              .first;
+    LoadReportState& load_report_state = load_report_it->second;
     if (load_report_state.drop_stats != nullptr) {
-      load_report_state.deleted_drop_stats +=
-          load_report_state.drop_stats->GetSnapshotAndReset();
+      cluster_drop_stats = load_report_state.drop_stats->RefIfNonZero();
+    }
+    if (cluster_drop_stats == nullptr) {
+      if (load_report_state.drop_stats != nullptr) {
+        load_report_state.deleted_drop_stats +=
+            load_report_state.drop_stats->GetSnapshotAndReset();
+      }
+      cluster_drop_stats = MakeRefCounted<XdsClusterDropStats>(
+          Ref(DEBUG_LOCATION, "DropStats"), *server,
+          load_report_it->first.first /*cluster_name*/,
+          load_report_it->first.second /*eds_service_name*/);
+      load_report_state.drop_stats = cluster_drop_stats.get();
     }
-    cluster_drop_stats = MakeRefCounted<XdsClusterDropStats>(
-        Ref(DEBUG_LOCATION, "DropStats"), server_it->first,
-        load_report_it->first.first /*cluster_name*/,
-        load_report_it->first.second /*eds_service_name*/);
-    load_report_state.drop_stats = cluster_drop_stats.get();
+    server_it->second.channel_state->MaybeStartLrsCall();
   }
-  server_it->second.channel_state->MaybeStartLrsCall();
+  work_serializer_.DrainQueue();
   return cluster_drop_stats;
 }
 
@@ -2112,8 +1804,10 @@ void XdsClient::RemoveClusterDropStats(
     const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
     absl::string_view eds_service_name,
     XdsClusterDropStats* cluster_drop_stats) {
+  const auto* server = bootstrap_->FindXdsServer(xds_server);
+  if (server == nullptr) return;
   MutexLock lock(&mu_);
-  auto server_it = xds_load_report_server_map_.find(xds_server);
+  auto server_it = xds_load_report_server_map_.find(server);
   if (server_it == xds_load_report_server_map_.end()) return;
   auto load_report_it = server_it->second.load_report_map.find(
       std::make_pair(std::string(cluster_name), std::string(eds_service_name)));
@@ -2132,42 +1826,48 @@ RefCountedPtr<XdsClusterLocalityStats> XdsClient::AddClusterLocalityStats(
     const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
     absl::string_view eds_service_name,
     RefCountedPtr<XdsLocalityName> locality) {
-  if (!bootstrap_->XdsServerExists(xds_server)) return nullptr;
+  const auto* server = bootstrap_->FindXdsServer(xds_server);
+  if (server == nullptr) return nullptr;
   auto key =
       std::make_pair(std::string(cluster_name), std::string(eds_service_name));
-  MutexLock lock(&mu_);
-  // We jump through some hoops here to make sure that the const
-  // XdsBootstrap::XdsServer& and absl::string_views
-  // stored in the XdsClusterDropStats object point to the
-  // XdsBootstrap::XdsServer and strings
-  // in the load_report_map_ key, so that they have the same lifetime.
-  auto server_it =
-      xds_load_report_server_map_.emplace(xds_server, LoadReportServer()).first;
-  if (server_it->second.channel_state == nullptr) {
-    server_it->second.channel_state = GetOrCreateChannelStateLocked(xds_server);
-  }
-  auto load_report_it = server_it->second.load_report_map
-                            .emplace(std::move(key), LoadReportState())
-                            .first;
-  LoadReportState& load_report_state = load_report_it->second;
-  LoadReportState::LocalityState& locality_state =
-      load_report_state.locality_stats[locality];
   RefCountedPtr<XdsClusterLocalityStats> cluster_locality_stats;
-  if (locality_state.locality_stats != nullptr) {
-    cluster_locality_stats = locality_state.locality_stats->RefIfNonZero();
-  }
-  if (cluster_locality_stats == nullptr) {
+  {
+    MutexLock lock(&mu_);
+    // We jump through some hoops here to make sure that the const
+    // XdsBootstrap::XdsServer& and absl::string_views
+    // stored in the XdsClusterDropStats object point to the
+    // XdsBootstrap::XdsServer and strings
+    // in the load_report_map_ key, so that they have the same lifetime.
+    auto server_it =
+        xds_load_report_server_map_.emplace(server, LoadReportServer()).first;
+    if (server_it->second.channel_state == nullptr) {
+      server_it->second.channel_state = GetOrCreateChannelStateLocked(
+          *server, "load report map (locality stats)");
+    }
+    auto load_report_it = server_it->second.load_report_map
+                              .emplace(std::move(key), LoadReportState())
+                              .first;
+    LoadReportState& load_report_state = load_report_it->second;
+    LoadReportState::LocalityState& locality_state =
+        load_report_state.locality_stats[locality];
     if (locality_state.locality_stats != nullptr) {
-      locality_state.deleted_locality_stats +=
-          locality_state.locality_stats->GetSnapshotAndReset();
+      cluster_locality_stats = locality_state.locality_stats->RefIfNonZero();
+    }
+    if (cluster_locality_stats == nullptr) {
+      if (locality_state.locality_stats != nullptr) {
+        locality_state.deleted_locality_stats +=
+            locality_state.locality_stats->GetSnapshotAndReset();
+      }
+      cluster_locality_stats = MakeRefCounted<XdsClusterLocalityStats>(
+          Ref(DEBUG_LOCATION, "LocalityStats"), *server,
+          load_report_it->first.first /*cluster_name*/,
+          load_report_it->first.second /*eds_service_name*/,
+          std::move(locality));
+      locality_state.locality_stats = cluster_locality_stats.get();
     }
-    cluster_locality_stats = MakeRefCounted<XdsClusterLocalityStats>(
-        Ref(DEBUG_LOCATION, "LocalityStats"), server_it->first,
-        load_report_it->first.first /*cluster_name*/,
-        load_report_it->first.second /*eds_service_name*/, std::move(locality));
-    locality_state.locality_stats = cluster_locality_stats.get();
+    server_it->second.channel_state->MaybeStartLrsCall();
   }
-  server_it->second.channel_state->MaybeStartLrsCall();
+  work_serializer_.DrainQueue();
   return cluster_locality_stats;
 }
 
@@ -2176,8 +1876,10 @@ void XdsClient::RemoveClusterLocalityStats(
     absl::string_view eds_service_name,
     const RefCountedPtr<XdsLocalityName>& locality,
     XdsClusterLocalityStats* cluster_locality_stats) {
+  const auto* server = bootstrap_->FindXdsServer(xds_server);
+  if (server == nullptr) return;
   MutexLock lock(&mu_);
-  auto server_it = xds_load_report_server_map_.find(xds_server);
+  auto server_it = xds_load_report_server_map_.find(server);
   if (server_it == xds_load_report_server_map_.end()) return;
   auto load_report_it = server_it->second.load_report_map.find(
       std::make_pair(std::string(cluster_name), std::string(eds_service_name)));
@@ -2198,29 +1900,38 @@ void XdsClient::RemoveClusterLocalityStats(
 void XdsClient::ResetBackoff() {
   MutexLock lock(&mu_);
   for (auto& p : xds_server_channel_map_) {
-    grpc_channel_reset_connect_backoff(p.second->channel());
+    p.second->ResetBackoff();
   }
 }
 
-void XdsClient::NotifyOnErrorLocked(grpc_error_handle error) {
-  std::set<RefCountedPtr<ResourceWatcherInterface>> watchers;
-  for (const auto& a : authority_state_map_) {     // authority
-    for (const auto& t : a.second.resource_map) {  // type
-      for (const auto& r : t.second) {             // resource id
-        for (const auto& w : r.second.watchers) {  // watchers
-          watchers.insert(w.second);
-        }
-      }
-    }
+void XdsClient::NotifyWatchersOnErrorLocked(
+    const std::map<ResourceWatcherInterface*,
+                   RefCountedPtr<ResourceWatcherInterface>>& watchers,
+    absl::Status status) {
+  const auto* node = bootstrap_->node();
+  if (node != nullptr) {
+    status = absl::Status(
+        status.code(),
+        absl::StrCat(status.message(), " (node ID:", node->id(), ")"));
   }
   work_serializer_.Schedule(
-      // TODO(yashykt): When we move to C++14, capture watchers using
-      // std::move()
-      [watchers, error]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(work_serializer_) {
-        for (const auto& watcher : watchers) {
-          watcher->OnError(GRPC_ERROR_REF(error));
+      [watchers, status = std::move(status)]()
+          ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
+            for (const auto& p : watchers) {
+              p.first->OnError(status);
+            }
+          },
+      DEBUG_LOCATION);
+}
+
+void XdsClient::NotifyWatchersOnResourceDoesNotExist(
+    const std::map<ResourceWatcherInterface*,
+                   RefCountedPtr<ResourceWatcherInterface>>& watchers) {
+  work_serializer_.Schedule(
+      [watchers]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) {
+        for (const auto& p : watchers) {
+          p.first->OnResourceDoesNotExist();
         }
-        GRPC_ERROR_UNREF(error);
       },
       DEBUG_LOCATION);
 }
@@ -2232,7 +1943,7 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked(
     gpr_log(GPR_INFO, "[xds_client %p] start building load report", this);
   }
   XdsApi::ClusterLoadReportMap snapshot_map;
-  auto server_it = xds_load_report_server_map_.find(xds_server);
+  auto server_it = xds_load_report_server_map_.find(&xds_server);
   if (server_it == xds_load_report_server_map_.end()) return snapshot_map;
   auto& load_report_map = server_it->second.load_report_map;
   for (auto load_report_it = load_report_map.begin();
@@ -2291,7 +2002,7 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked(
       }
     }
     // Compute load report interval.
-    const Timestamp now = ExecCtx::Get()->Now();
+    const Timestamp now = Timestamp::Now();
     snapshot.load_report_interval = now - load_report.last_report_time;
     load_report.last_report_time = now;
     // Record snapshot.
@@ -2331,190 +2042,4 @@ std::string XdsClient::DumpClientConfigBinary() {
   return api_.AssembleClientConfig(resource_type_metadata_map);
 }
 
-//
-// accessors for global state
-//
-
-void XdsClientGlobalInit() {
-  g_mu = new Mutex;
-  XdsHttpFilterRegistry::Init();
-}
-
-// TODO(roth): Find a better way to clear the fallback config that does
-// not require using ABSL_NO_THREAD_SAFETY_ANALYSIS.
-void XdsClientGlobalShutdown() ABSL_NO_THREAD_SAFETY_ANALYSIS {
-  gpr_free(g_fallback_bootstrap_config);
-  g_fallback_bootstrap_config = nullptr;
-  delete g_mu;
-  g_mu = nullptr;
-  XdsHttpFilterRegistry::Shutdown();
-}
-
-namespace {
-
-std::string GetBootstrapContents(const char* fallback_config,
-                                 grpc_error_handle* error) {
-  // First, try GRPC_XDS_BOOTSTRAP env var.
-  UniquePtr<char> path(gpr_getenv("GRPC_XDS_BOOTSTRAP"));
-  if (path != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-      gpr_log(GPR_INFO,
-              "Got bootstrap file location from GRPC_XDS_BOOTSTRAP "
-              "environment variable: %s",
-              path.get());
-    }
-    grpc_slice contents;
-    *error =
-        grpc_load_file(path.get(), /*add_null_terminator=*/true, &contents);
-    if (*error != GRPC_ERROR_NONE) return "";
-    std::string contents_str(StringViewFromSlice(contents));
-    grpc_slice_unref_internal(contents);
-    return contents_str;
-  }
-  // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var.
-  UniquePtr<char> env_config(gpr_getenv("GRPC_XDS_BOOTSTRAP_CONFIG"));
-  if (env_config != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-      gpr_log(GPR_INFO,
-              "Got bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG "
-              "environment variable");
-    }
-    return env_config.get();
-  }
-  // Finally, try fallback config.
-  if (fallback_config != nullptr) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-      gpr_log(GPR_INFO, "Got bootstrap contents from fallback config");
-    }
-    return fallback_config;
-  }
-  // No bootstrap config found.
-  *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-      "Environment variables GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG "
-      "not defined");
-  return "";
-}
-
-}  // namespace
-
-RefCountedPtr<XdsClient> XdsClient::GetOrCreate(const grpc_channel_args* args,
-                                                grpc_error_handle* error) {
-  RefCountedPtr<XdsClient> xds_client;
-  // If getting bootstrap from channel args, create a local XdsClient
-  // instance for the channel or server instead of using the global instance.
-  const char* bootstrap_config = grpc_channel_args_find_string(
-      args, GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG);
-  if (bootstrap_config != nullptr) {
-    std::unique_ptr<XdsBootstrap> bootstrap =
-        XdsBootstrap::Create(bootstrap_config, error);
-    if (*error == GRPC_ERROR_NONE) {
-      grpc_channel_args* xds_channel_args =
-          grpc_channel_args_find_pointer<grpc_channel_args>(
-              args,
-              GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS);
-      return MakeRefCounted<XdsClient>(std::move(bootstrap), xds_channel_args);
-    }
-    return nullptr;
-  }
-  // Otherwise, use the global instance.
-  {
-    MutexLock lock(g_mu);
-    if (g_xds_client != nullptr) {
-      auto xds_client = g_xds_client->RefIfNonZero();
-      if (xds_client != nullptr) return xds_client;
-    }
-    // Find bootstrap contents.
-    std::string bootstrap_contents =
-        GetBootstrapContents(g_fallback_bootstrap_config, error);
-    if (*error != GRPC_ERROR_NONE) return nullptr;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
-      gpr_log(GPR_INFO, "xDS bootstrap contents: %s",
-              bootstrap_contents.c_str());
-    }
-    // Parse bootstrap.
-    std::unique_ptr<XdsBootstrap> bootstrap =
-        XdsBootstrap::Create(bootstrap_contents, error);
-    if (*error != GRPC_ERROR_NONE) return nullptr;
-    // Instantiate XdsClient.
-    xds_client =
-        MakeRefCounted<XdsClient>(std::move(bootstrap), g_channel_args);
-    g_xds_client = xds_client.get();
-  }
-  return xds_client;
-}
-
-namespace internal {
-
-void SetXdsChannelArgsForTest(grpc_channel_args* args) {
-  MutexLock lock(g_mu);
-  g_channel_args = args;
-}
-
-void UnsetGlobalXdsClientForTest() {
-  MutexLock lock(g_mu);
-  g_xds_client = nullptr;
-}
-
-void SetXdsFallbackBootstrapConfig(const char* config) {
-  MutexLock lock(g_mu);
-  gpr_free(g_fallback_bootstrap_config);
-  g_fallback_bootstrap_config = gpr_strdup(config);
-}
-
-}  // namespace internal
-
-//
-// embedding XdsClient in channel args
-//
-
-#define GRPC_ARG_XDS_CLIENT "grpc.internal.xds_client"
-
-namespace {
-
-void* XdsClientArgCopy(void* p) {
-  XdsClient* xds_client = static_cast<XdsClient*>(p);
-  xds_client->Ref(DEBUG_LOCATION, "channel arg").release();
-  return p;
-}
-
-void XdsClientArgDestroy(void* p) {
-  XdsClient* xds_client = static_cast<XdsClient*>(p);
-  xds_client->Unref(DEBUG_LOCATION, "channel arg");
-}
-
-int XdsClientArgCmp(void* p, void* q) { return QsortCompare(p, q); }
-
-const grpc_arg_pointer_vtable kXdsClientArgVtable = {
-    XdsClientArgCopy, XdsClientArgDestroy, XdsClientArgCmp};
-
-}  // namespace
-
-grpc_arg XdsClient::MakeChannelArg() const {
-  return grpc_channel_arg_pointer_create(const_cast<char*>(GRPC_ARG_XDS_CLIENT),
-                                         const_cast<XdsClient*>(this),
-                                         &kXdsClientArgVtable);
-}
-
-RefCountedPtr<XdsClient> XdsClient::GetFromChannelArgs(
-    const grpc_channel_args& args) {
-  XdsClient* xds_client =
-      grpc_channel_args_find_pointer<XdsClient>(&args, GRPC_ARG_XDS_CLIENT);
-  if (xds_client == nullptr) return nullptr;
-  return xds_client->Ref(DEBUG_LOCATION, "GetFromChannelArgs");
-}
-
 }  // namespace grpc_core
-
-// The returned bytes may contain NULL(0), so we can't use c-string.
-grpc_slice grpc_dump_xds_configs() {
-  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
-  grpc_core::ExecCtx exec_ctx;
-  grpc_error_handle error = GRPC_ERROR_NONE;
-  auto xds_client = grpc_core::XdsClient::GetOrCreate(nullptr, &error);
-  if (error != GRPC_ERROR_NONE) {
-    // If we isn't using xDS, just return an empty string.
-    GRPC_ERROR_UNREF(error);
-    return grpc_empty_slice();
-  }
-  return grpc_slice_from_cpp_string(xds_client->DumpClientConfigBinary());
-}