grpc 1.31.1 → 1.35.0.pre1

data/src/core/ext/filters/client_channel/lb_policy_registry.cc

@@ -23,7 +23,9 @@
 #include <string.h>
 
 #include "absl/container/inlined_vector.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
 
 #include "src/core/lib/gpr/string.h"
 
@@ -126,6 +128,7 @@ grpc_error* ParseLoadBalancingConfigHelper(
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING("type should be array");
   }
   // Find the first LB policy that this client supports.
+  std::vector<absl::string_view> policies_tried;
   for (const Json& lb_config : lb_config_array.array_value()) {
     if (lb_config.type() != Json::Type::OBJECT) {
       return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
@@ -149,8 +152,12 @@ grpc_error* ParseLoadBalancingConfigHelper(
       *result = it;
       return GRPC_ERROR_NONE;
     }
+    policies_tried.push_back(it->first);
   }
-  return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No known policy");
+  return GRPC_ERROR_CREATE_FROM_COPIED_STRING(
+      absl::StrCat("No known policies in list: ",
+                   absl::StrJoin(policies_tried, " "))
+          .c_str());
 }
 
 }  // namespace

data/src/core/ext/filters/client_channel/resolver.cc

@@ -31,7 +31,9 @@ namespace grpc_core {
 
 Resolver::Resolver(std::shared_ptr<WorkSerializer> work_serializer,
                    std::unique_ptr<ResultHandler> result_handler)
-    : InternallyRefCounted(&grpc_trace_resolver_refcount),
+    : InternallyRefCounted(GRPC_TRACE_FLAG_ENABLED(grpc_trace_resolver_refcount)
+                               ? "Resolver"
+                               : nullptr),
       work_serializer_(std::move(work_serializer)),
       result_handler_(std::move(result_handler)) {}
 

data/src/core/ext/filters/client_channel/resolver.h

@@ -32,6 +32,9 @@
 
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_resolver_refcount;
 
+// Name associated with individual address, if available.
+#define GRPC_ARG_ADDRESS_NAME "grpc.address_name"
+
 namespace grpc_core {
 
 /// Interface for name resolution.
@@ -87,7 +90,7 @@ class Resolver : public InternallyRefCounted<Resolver> {
   // Not copyable nor movable.
   Resolver(const Resolver&) = delete;
   Resolver& operator=(const Resolver&) = delete;
-  virtual ~Resolver() = default;
+  ~Resolver() override = default;
 
   /// Starts resolving.
   virtual void StartLocked() = 0;

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -75,7 +75,7 @@ class AresDnsResolver : public Resolver {
   void ShutdownLocked() override;
 
  private:
-  virtual ~AresDnsResolver();
+  ~AresDnsResolver() override;
 
   void MaybeStartResolvingLocked();
   void StartResolvingLocked();
@@ -86,9 +86,9 @@ class AresDnsResolver : public Resolver {
   void OnResolvedLocked(grpc_error* error);
 
   /// DNS server to use (if not system default)
-  char* dns_server_;
+  std::string dns_server_;
   /// name to resolve (usually the same as target_name)
-  char* name_to_resolve_;
+  std::string name_to_resolve_;
   /// channel args
   grpc_channel_args* channel_args_;
   /// whether to request the service config
@@ -139,14 +139,9 @@ AresDnsResolver::AresDnsResolver(ResolverArgs args)
                     grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&on_resolved_, OnResolved, this, grpc_schedule_on_exec_ctx);
   // Get name to resolve from URI path.
-  const char* path = args.uri->path;
-  if (path[0] == '/') ++path;
-  name_to_resolve_ = gpr_strdup(path);
+  name_to_resolve_ = std::string(absl::StripPrefix(args.uri.path(), "/"));
   // Get DNS server from URI authority.
-  dns_server_ = nullptr;
-  if (0 != strcmp(args.uri->authority, "")) {
-    dns_server_ = gpr_strdup(args.uri->authority);
-  }
+  dns_server_ = args.uri.authority();
   channel_args_ = grpc_channel_args_copy(args.args);
   // Disable service config option
   const grpc_arg* arg = grpc_channel_args_find(
@@ -175,8 +170,6 @@ AresDnsResolver::AresDnsResolver(ResolverArgs args)
 AresDnsResolver::~AresDnsResolver() {
   GRPC_CARES_TRACE_LOG("resolver:%p destroying AresDnsResolver", this);
   grpc_pollset_set_destroy(interested_parties_);
-  gpr_free(dns_server_);
-  gpr_free(name_to_resolve_);
   grpc_channel_args_destroy(channel_args_);
 }
 
@@ -352,7 +345,7 @@ void AresDnsResolver::OnResolvedLocked(grpc_error* error) {
         GRPC_CARES_TRACE_LOG("resolver:%p selected service config choice: %s",
                              this, service_config_string.c_str());
         result.service_config = ServiceConfig::Create(
-            service_config_string, &result.service_config_error);
+            channel_args_, service_config_string, &result.service_config_error);
       }
     }
     absl::InlinedVector<grpc_arg, 1> new_args;
@@ -439,8 +432,8 @@ void AresDnsResolver::StartResolvingLocked() {
   resolving_ = true;
   service_config_json_ = nullptr;
   pending_request_ = grpc_dns_lookup_ares_locked(
-      dns_server_, name_to_resolve_, kDefaultPort, interested_parties_,
-      &on_resolved_, &addresses_,
+      dns_server_.c_str(), name_to_resolve_.c_str(), kDefaultPort,
+      interested_parties_, &on_resolved_, &addresses_,
       enable_srv_queries_ ? &balancer_addresses_ : nullptr,
       request_service_config_ ? &service_config_json_ : nullptr,
       query_timeout_ms_, work_serializer());
@@ -455,7 +448,7 @@ void AresDnsResolver::StartResolvingLocked() {
 
 class AresDnsResolverFactory : public ResolverFactory {
  public:
-  bool IsValidUri(const grpc_uri* /*uri*/) const override { return true; }
+  bool IsValidUri(const URI& /*uri*/) const override { return true; }
 
   OrphanablePtr<Resolver> CreateResolver(ResolverArgs args) const override {
     return MakeOrphanable<AresDnsResolver>(std::move(args));

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -22,39 +22,8 @@
 #include <grpc/support/port_platform.h>
 
 #include <ares.h>
-#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-
-typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
-
-/* Start \a ev_driver. It will keep working until all IO on its ares_channel is
-   done, or grpc_ares_ev_driver_destroy() is called. It may notify the callbacks
-   bound to its ares_channel when necessary. */
-void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver);
-
-/* Returns the ares_channel owned by \a ev_driver. To bind a c-ares query to
-   \a ev_driver, use the ares_channel owned by \a ev_driver as the arg of the
-   query. */
-ares_channel* grpc_ares_ev_driver_get_channel_locked(
-    grpc_ares_ev_driver* ev_driver);
-
-/* Creates a new grpc_ares_ev_driver. Returns GRPC_ERROR_NONE if \a ev_driver is
-   created successfully. */
-grpc_error* grpc_ares_ev_driver_create_locked(
-    grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set,
-    int query_timeout_ms,
-    std::shared_ptr<grpc_core::WorkSerializer> work_serializer,
-    grpc_ares_request* request);
-
-/* Called back when all DNS lookups have completed. */
-void grpc_ares_ev_driver_on_queries_complete_locked(
-    grpc_ares_ev_driver* ev_driver);
-
-/* Shutdown all the grpc_fds used by \a ev_driver */
-void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver);
-
-/* Exposed in this header for C-core tests only */
-extern void (*grpc_ares_test_only_inject_config)(ares_channel channel);
+#include "src/core/lib/iomgr/work_serializer.h"
 
 namespace grpc_core {
 

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc

@@ -43,13 +43,13 @@ namespace grpc_core {
 class GrpcPolledFdPosix : public GrpcPolledFd {
  public:
   GrpcPolledFdPosix(ares_socket_t as, grpc_pollset_set* driver_pollset_set)
-      : name_(absl::StrCat("c-ares fd: ", (int)as)), as_(as) {
-    fd_ = grpc_fd_create((int)as, name_.c_str(), false);
+      : name_(absl::StrCat("c-ares fd: ", static_cast<int>(as))), as_(as) {
+    fd_ = grpc_fd_create(static_cast<int>(as), name_.c_str(), false);
     driver_pollset_set_ = driver_pollset_set;
     grpc_pollset_set_add_fd(driver_pollset_set_, fd_);
   }
 
-  ~GrpcPolledFdPosix() {
+  ~GrpcPolledFdPosix() override {
     grpc_pollset_set_del_fd(driver_pollset_set_, fd_);
     /* c-ares library will close the fd inside grpc_fd. This fd may be picked up
        immediately by another thread, and should not be closed by the following

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc

@@ -37,8 +37,6 @@
 #include <grpc/support/time.h>
 
 #include <address_sorting/address_sorting.h>
-#include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h"
-#include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/host_port.h"
@@ -46,7 +44,10 @@
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/nameser.h"
+#include "src/core/lib/iomgr/parse_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/transport/authority_override.h"
 
 using grpc_core::ServerAddress;
 using grpc_core::ServerAddressList;
@@ -56,6 +57,8 @@ grpc_core::TraceFlag grpc_trace_cares_address_sorting(false,
 
 grpc_core::TraceFlag grpc_trace_cares_resolver(false, "cares_resolver");
 
+typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
+
 struct grpc_ares_request {
   /** indicates the DNS server to use, if specified */
   struct ares_addr_port_node dns_server_addr;
@@ -77,6 +80,60 @@ struct grpc_ares_request {
   grpc_error* error;
 };
 
+typedef struct fd_node {
+  /** the owner of this fd node */
+  grpc_ares_ev_driver* ev_driver;
+  /** a closure wrapping on_readable_locked, which should be
+     invoked when the grpc_fd in this node becomes readable. */
+  grpc_closure read_closure;
+  /** a closure wrapping on_writable_locked, which should be
+     invoked when the grpc_fd in this node becomes writable. */
+  grpc_closure write_closure;
+  /** next fd node in the list */
+  struct fd_node* next;
+
+  /** wrapped fd that's polled by grpc's poller for the current platform */
+  grpc_core::GrpcPolledFd* grpc_polled_fd;
+  /** if the readable closure has been registered */
+  bool readable_registered;
+  /** if the writable closure has been registered */
+  bool writable_registered;
+  /** if the fd has been shutdown yet from grpc iomgr perspective */
+  bool already_shutdown;
+} fd_node;
+
+struct grpc_ares_ev_driver {
+  /** the ares_channel owned by this event driver */
+  ares_channel channel;
+  /** pollset set for driving the IO events of the channel */
+  grpc_pollset_set* pollset_set;
+  /** refcount of the event driver */
+  gpr_refcount refs;
+
+  /** work_serializer to synchronize c-ares and I/O callbacks on */
+  std::shared_ptr<grpc_core::WorkSerializer> work_serializer;
+  /** a list of grpc_fd that this event driver is currently using. */
+  fd_node* fds;
+  /** is this event driver currently working? */
+  bool working;
+  /** is this event driver being shut down */
+  bool shutting_down;
+  /** request object that's using this ev driver */
+  grpc_ares_request* request;
+  /** Owned by the ev_driver. Creates new GrpcPolledFd's */
+  std::unique_ptr<grpc_core::GrpcPolledFdFactory> polled_fd_factory;
+  /** query timeout in milliseconds */
+  int query_timeout_ms;
+  /** alarm to cancel active queries */
+  grpc_timer query_timeout;
+  /** cancels queries on a timeout */
+  grpc_closure on_timeout_locked;
+  /** alarm to poll ares_process on in case fd events don't happen */
+  grpc_timer ares_backup_poll_alarm;
+  /** polls ares_process on a periodic timer */
+  grpc_closure on_ares_backup_poll_alarm_locked;
+};
+
 // TODO(apolcyn): make grpc_ares_hostbyname_request a sub-class
 // of GrpcAresQuery.
 typedef struct grpc_ares_hostbyname_request {
@@ -121,6 +178,390 @@ class GrpcAresQuery {
   const std::string name_;
 };
 
+static grpc_ares_ev_driver* grpc_ares_ev_driver_ref(
+    grpc_ares_ev_driver* ev_driver) {
+  GRPC_CARES_TRACE_LOG("request:%p Ref ev_driver %p", ev_driver->request,
+                       ev_driver);
+  gpr_ref(&ev_driver->refs);
+  return ev_driver;
+}
+
+static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) {
+  GRPC_CARES_TRACE_LOG("request:%p Unref ev_driver %p", ev_driver->request,
+                       ev_driver);
+  if (gpr_unref(&ev_driver->refs)) {
+    GRPC_CARES_TRACE_LOG("request:%p destroy ev_driver %p", ev_driver->request,
+                         ev_driver);
+    GPR_ASSERT(ev_driver->fds == nullptr);
+    ares_destroy(ev_driver->channel);
+    grpc_ares_complete_request_locked(ev_driver->request);
+    delete ev_driver;
+  }
+}
+
+static void fd_node_destroy_locked(fd_node* fdn) {
+  GRPC_CARES_TRACE_LOG("request:%p delete fd: %s", fdn->ev_driver->request,
+                       fdn->grpc_polled_fd->GetName());
+  GPR_ASSERT(!fdn->readable_registered);
+  GPR_ASSERT(!fdn->writable_registered);
+  GPR_ASSERT(fdn->already_shutdown);
+  delete fdn->grpc_polled_fd;
+  gpr_free(fdn);
+}
+
+static void fd_node_shutdown_locked(fd_node* fdn, const char* reason) {
+  if (!fdn->already_shutdown) {
+    fdn->already_shutdown = true;
+    fdn->grpc_polled_fd->ShutdownLocked(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING(reason));
+  }
+}
+
+void grpc_ares_ev_driver_on_queries_complete_locked(
+    grpc_ares_ev_driver* ev_driver) {
+  // We mark the event driver as being shut down. If the event driver
+  // is working, grpc_ares_notify_on_event_locked will shut down the
+  // fds; if it's not working, there are no fds to shut down.
+  ev_driver->shutting_down = true;
+  grpc_timer_cancel(&ev_driver->query_timeout);
+  grpc_timer_cancel(&ev_driver->ares_backup_poll_alarm);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver) {
+  ev_driver->shutting_down = true;
+  fd_node* fn = ev_driver->fds;
+  while (fn != nullptr) {
+    fd_node_shutdown_locked(fn, "grpc_ares_ev_driver_shutdown");
+    fn = fn->next;
+  }
+}
+
+// Search fd in the fd_node list head. This is an O(n) search, the max possible
+// value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests.
+static fd_node* pop_fd_node_locked(fd_node** head, ares_socket_t as) {
+  fd_node dummy_head;
+  dummy_head.next = *head;
+  fd_node* node = &dummy_head;
+  while (node->next != nullptr) {
+    if (node->next->grpc_polled_fd->GetWrappedAresSocketLocked() == as) {
+      fd_node* ret = node->next;
+      node->next = node->next->next;
+      *head = dummy_head.next;
+      return ret;
+    }
+    node = node->next;
+  }
+  return nullptr;
+}
+
+static grpc_millis calculate_next_ares_backup_poll_alarm_ms(
+    grpc_ares_ev_driver* driver) {
+  // An alternative here could be to use ares_timeout to try to be more
+  // accurate, but that would require using "struct timeval"'s, which just makes
+  // things a bit more complicated. So just poll every second, as suggested
+  // by the c-ares code comments.
+  grpc_millis ms_until_next_ares_backup_poll_alarm = 1000;
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p. next ares process poll time in "
+      "%" PRId64 " ms",
+      driver->request, driver, ms_until_next_ares_backup_poll_alarm);
+  return ms_until_next_ares_backup_poll_alarm +
+         grpc_core::ExecCtx::Get()->Now();
+}
+
+static void on_timeout_locked(grpc_ares_ev_driver* driver, grpc_error* error) {
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p on_timeout_locked. driver->shutting_down=%d. "
+      "err=%s",
+      driver->request, driver, driver->shutting_down, grpc_error_string(error));
+  if (!driver->shutting_down && error == GRPC_ERROR_NONE) {
+    grpc_ares_ev_driver_shutdown_locked(driver);
+  }
+  grpc_ares_ev_driver_unref(driver);
+  GRPC_ERROR_UNREF(error);
+}
+
+static void on_timeout(void* arg, grpc_error* error) {
+  grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
+  GRPC_ERROR_REF(error);  // ref owned by lambda
+  driver->work_serializer->Run(
+      [driver, error]() { on_timeout_locked(driver, error); }, DEBUG_LOCATION);
+}
+
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
+
+static void on_ares_backup_poll_alarm_locked(grpc_ares_ev_driver* driver,
+                                             grpc_error* error);
+
+static void on_ares_backup_poll_alarm(void* arg, grpc_error* error) {
+  grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
+  GRPC_ERROR_REF(error);
+  driver->work_serializer->Run(
+      [driver, error]() { on_ares_backup_poll_alarm_locked(driver, error); },
+      DEBUG_LOCATION);
+}
+
+/* In case of non-responsive DNS servers, dropped packets, etc., c-ares has
+ * intelligent timeout and retry logic, which we can take advantage of by
+ * polling ares_process_fd on time intervals. Overall, the c-ares library is
+ * meant to be called into and given a chance to proceed name resolution:
+ *   a) when fd events happen
+ *   b) when some time has passed without fd events having happened
+ * For the latter, we use this backup poller. Also see
+ * https://github.com/grpc/grpc/pull/17688 description for more details. */
+static void on_ares_backup_poll_alarm_locked(grpc_ares_ev_driver* driver,
+                                             grpc_error* error) {
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked. "
+      "driver->shutting_down=%d. "
+      "err=%s",
+      driver->request, driver, driver->shutting_down, grpc_error_string(error));
+  if (!driver->shutting_down && error == GRPC_ERROR_NONE) {
+    fd_node* fdn = driver->fds;
+    while (fdn != nullptr) {
+      if (!fdn->already_shutdown) {
+        GRPC_CARES_TRACE_LOG(
+            "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked; "
+            "ares_process_fd. fd=%s",
+            driver->request, driver, fdn->grpc_polled_fd->GetName());
+        ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+        ares_process_fd(driver->channel, as, as);
+      }
+      fdn = fdn->next;
+    }
+    if (!driver->shutting_down) {
+      grpc_millis next_ares_backup_poll_alarm =
+          calculate_next_ares_backup_poll_alarm_ms(driver);
+      grpc_ares_ev_driver_ref(driver);
+      GRPC_CLOSURE_INIT(&driver->on_ares_backup_poll_alarm_locked,
+                        on_ares_backup_poll_alarm, driver,
+                        grpc_schedule_on_exec_ctx);
+      grpc_timer_init(&driver->ares_backup_poll_alarm,
+                      next_ares_backup_poll_alarm,
+                      &driver->on_ares_backup_poll_alarm_locked);
+    }
+    grpc_ares_notify_on_event_locked(driver);
+  }
+  grpc_ares_ev_driver_unref(driver);
+  GRPC_ERROR_UNREF(error);
+}
+
+static void on_readable_locked(fd_node* fdn, grpc_error* error) {
+  GPR_ASSERT(fdn->readable_registered);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  const ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+  fdn->readable_registered = false;
+  GRPC_CARES_TRACE_LOG("request:%p readable on %s", fdn->ev_driver->request,
+                       fdn->grpc_polled_fd->GetName());
+  if (error == GRPC_ERROR_NONE) {
+    do {
+      ares_process_fd(ev_driver->channel, as, ARES_SOCKET_BAD);
+    } while (fdn->grpc_polled_fd->IsFdStillReadableLocked());
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  grpc_ares_notify_on_event_locked(ev_driver);
+  grpc_ares_ev_driver_unref(ev_driver);
+  GRPC_ERROR_UNREF(error);
+}
+
+static void on_readable(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  GRPC_ERROR_REF(error); /* ref owned by lambda */
+  fdn->ev_driver->work_serializer->Run(
+      [fdn, error]() { on_readable_locked(fdn, error); }, DEBUG_LOCATION);
+}
+
+static void on_writable_locked(fd_node* fdn, grpc_error* error) {
+  GPR_ASSERT(fdn->writable_registered);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  const ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+  fdn->writable_registered = false;
+  GRPC_CARES_TRACE_LOG("request:%p writable on %s", ev_driver->request,
+                       fdn->grpc_polled_fd->GetName());
+  if (error == GRPC_ERROR_NONE) {
+    ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, as);
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  grpc_ares_notify_on_event_locked(ev_driver);
+  grpc_ares_ev_driver_unref(ev_driver);
+  GRPC_ERROR_UNREF(error);
+}
+
+static void on_writable(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  GRPC_ERROR_REF(error); /* ref owned by lambda */
+  fdn->ev_driver->work_serializer->Run(
+      [fdn, error]() { on_writable_locked(fdn, error); }, DEBUG_LOCATION);
+}
+
+// Get the file descriptors used by the ev_driver's ares channel, register
+// driver_closure with these filedescriptors.
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) {
+  fd_node* new_list = nullptr;
+  if (!ev_driver->shutting_down) {
+    ares_socket_t socks[ARES_GETSOCK_MAXNUM];
+    int socks_bitmask =
+        ares_getsock(ev_driver->channel, socks, ARES_GETSOCK_MAXNUM);
+    for (size_t i = 0; i < ARES_GETSOCK_MAXNUM; i++) {
+      if (ARES_GETSOCK_READABLE(socks_bitmask, i) ||
+          ARES_GETSOCK_WRITABLE(socks_bitmask, i)) {
+        fd_node* fdn = pop_fd_node_locked(&ev_driver->fds, socks[i]);
+        // Create a new fd_node if sock[i] is not in the fd_node list.
+        if (fdn == nullptr) {
+          fdn = static_cast<fd_node*>(gpr_malloc(sizeof(fd_node)));
+          fdn->grpc_polled_fd =
+              ev_driver->polled_fd_factory->NewGrpcPolledFdLocked(
+                  socks[i], ev_driver->pollset_set, ev_driver->work_serializer);
+          GRPC_CARES_TRACE_LOG("request:%p new fd: %s", ev_driver->request,
+                               fdn->grpc_polled_fd->GetName());
+          fdn->ev_driver = ev_driver;
+          fdn->readable_registered = false;
+          fdn->writable_registered = false;
+          fdn->already_shutdown = false;
+        }
+        fdn->next = new_list;
+        new_list = fdn;
+        // Register read_closure if the socket is readable and read_closure has
+        // not been registered with this socket.
+        if (ARES_GETSOCK_READABLE(socks_bitmask, i) &&
+            !fdn->readable_registered) {
+          grpc_ares_ev_driver_ref(ev_driver);
+          GRPC_CARES_TRACE_LOG("request:%p notify read on: %s",
+                               ev_driver->request,
+                               fdn->grpc_polled_fd->GetName());
+          GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable, fdn,
+                            grpc_schedule_on_exec_ctx);
+          fdn->grpc_polled_fd->RegisterForOnReadableLocked(&fdn->read_closure);
+          fdn->readable_registered = true;
+        }
+        // Register write_closure if the socket is writable and write_closure
+        // has not been registered with this socket.
+        if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) &&
+            !fdn->writable_registered) {
+          GRPC_CARES_TRACE_LOG("request:%p notify write on: %s",
+                               ev_driver->request,
+                               fdn->grpc_polled_fd->GetName());
+          grpc_ares_ev_driver_ref(ev_driver);
+          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable, fdn,
+                            grpc_schedule_on_exec_ctx);
+          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable, fdn,
+                            grpc_schedule_on_exec_ctx);
+          fdn->grpc_polled_fd->RegisterForOnWriteableLocked(
+              &fdn->write_closure);
+          fdn->writable_registered = true;
+        }
+      }
+    }
+  }
+  // Any remaining fds in ev_driver->fds were not returned by ares_getsock() and
+  // are therefore no longer in use, so they can be shut down and removed from
+  // the list.
+  while (ev_driver->fds != nullptr) {
+    fd_node* cur = ev_driver->fds;
+    ev_driver->fds = ev_driver->fds->next;
+    fd_node_shutdown_locked(cur, "c-ares fd shutdown");
+    if (!cur->readable_registered && !cur->writable_registered) {
+      fd_node_destroy_locked(cur);
+    } else {
+      cur->next = new_list;
+      new_list = cur;
+    }
+  }
+  ev_driver->fds = new_list;
+  // If the ev driver has no working fd, all the tasks are done.
+  if (new_list == nullptr) {
+    ev_driver->working = false;
+    GRPC_CARES_TRACE_LOG("request:%p ev driver stop working",
+                         ev_driver->request);
+  }
+}
+
+void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
+  if (!ev_driver->working) {
+    ev_driver->working = true;
+    grpc_ares_notify_on_event_locked(ev_driver);
+    // Initialize overall DNS resolution timeout alarm
+    grpc_millis timeout =
+        ev_driver->query_timeout_ms == 0
+            ? GRPC_MILLIS_INF_FUTURE
+            : ev_driver->query_timeout_ms + grpc_core::ExecCtx::Get()->Now();
+    GRPC_CARES_TRACE_LOG(
+        "request:%p ev_driver=%p grpc_ares_ev_driver_start_locked. timeout in "
+        "%" PRId64 " ms",
+        ev_driver->request, ev_driver, timeout);
+    grpc_ares_ev_driver_ref(ev_driver);
+    GRPC_CLOSURE_INIT(&ev_driver->on_timeout_locked, on_timeout, ev_driver,
+                      grpc_schedule_on_exec_ctx);
+    grpc_timer_init(&ev_driver->query_timeout, timeout,
+                    &ev_driver->on_timeout_locked);
+    // Initialize the backup poll alarm
+    grpc_millis next_ares_backup_poll_alarm =
+        calculate_next_ares_backup_poll_alarm_ms(ev_driver);
+    grpc_ares_ev_driver_ref(ev_driver);
+    GRPC_CLOSURE_INIT(&ev_driver->on_ares_backup_poll_alarm_locked,
+                      on_ares_backup_poll_alarm, ev_driver,
+                      grpc_schedule_on_exec_ctx);
+    grpc_timer_init(&ev_driver->ares_backup_poll_alarm,
+                    next_ares_backup_poll_alarm,
+                    &ev_driver->on_ares_backup_poll_alarm_locked);
+  }
+}
+
+static void noop_inject_channel_config(ares_channel /*channel*/) {}
+
+void (*grpc_ares_test_only_inject_config)(ares_channel channel) =
+    noop_inject_channel_config;
+
+grpc_error* grpc_ares_ev_driver_create_locked(
+    grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set,
+    int query_timeout_ms,
+    std::shared_ptr<grpc_core::WorkSerializer> work_serializer,
+    grpc_ares_request* request) {
+  *ev_driver = new grpc_ares_ev_driver();
+  ares_options opts;
+  memset(&opts, 0, sizeof(opts));
+  opts.flags |= ARES_FLAG_STAYOPEN;
+  int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS);
+  grpc_ares_test_only_inject_config((*ev_driver)->channel);
+  GRPC_CARES_TRACE_LOG("request:%p grpc_ares_ev_driver_create_locked", request);
+  if (status != ARES_SUCCESS) {
+    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(
+        absl::StrCat("Failed to init ares channel. C-ares error: ",
+                     ares_strerror(status))
+            .c_str());
+    gpr_free(*ev_driver);
+    return err;
+  }
+  (*ev_driver)->work_serializer = std::move(work_serializer);
+  gpr_ref_init(&(*ev_driver)->refs, 1);
+  (*ev_driver)->pollset_set = pollset_set;
+  (*ev_driver)->fds = nullptr;
+  (*ev_driver)->working = false;
+  (*ev_driver)->shutting_down = false;
+  (*ev_driver)->request = request;
+  (*ev_driver)->polled_fd_factory =
+      grpc_core::NewGrpcPolledFdFactory((*ev_driver)->work_serializer);
+  (*ev_driver)
+      ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel);
+  (*ev_driver)->query_timeout_ms = query_timeout_ms;
+  return GRPC_ERROR_NONE;
+}
+
 static void log_address_sorting_list(const grpc_ares_request* r,
                                      const ServerAddressList& addresses,
                                      const char* input_output_str) {
@@ -139,8 +580,8 @@ void grpc_cares_wrapper_address_sorting_sort(const grpc_ares_request* r,
   if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_cares_address_sorting)) {
     log_address_sorting_list(r, *addresses, "input");
   }
-  address_sorting_sortable* sortables = (address_sorting_sortable*)gpr_zalloc(
-      sizeof(address_sorting_sortable) * addresses->size());
+  address_sorting_sortable* sortables = static_cast<address_sorting_sortable*>(
+      gpr_zalloc(sizeof(address_sorting_sortable) * addresses->size()));
   for (size_t i = 0; i < addresses->size(); ++i) {
     sortables[i].user_data = &(*addresses)[i];
     memcpy(&sortables[i].dest_addr.addr, &(*addresses)[i].address().addr,
@@ -237,7 +678,7 @@ static void on_hostbyname_done_locked(void* arg, int status, int /*timeouts*/,
       absl::InlinedVector<grpc_arg, 1> args_to_add;
       if (hr->is_balancer) {
         args_to_add.emplace_back(
-            grpc_core::CreateGrpclbBalancerNameArg(hr->host));
+            grpc_core::CreateAuthorityOverrideChannelArg(hr->host));
       }
       grpc_channel_args* args = grpc_channel_args_copy_and_add(
           nullptr, args_to_add.data(), args_to_add.size());
@@ -303,20 +744,18 @@ static void on_srv_query_done_locked(void* arg, int status, int /*timeouts*/,
     GRPC_CARES_TRACE_LOG("request:%p ares_parse_srv_reply: %d", r,
                          parse_status);
     if (parse_status == ARES_SUCCESS) {
-      ares_channel* channel =
-          grpc_ares_ev_driver_get_channel_locked(r->ev_driver);
       for (struct ares_srv_reply* srv_it = reply; srv_it != nullptr;
            srv_it = srv_it->next) {
         if (grpc_ares_query_ipv6()) {
           grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked(
               r, srv_it->host, htons(srv_it->port), true /* is_balancer */,
               "AAAA");
-          ares_gethostbyname(*channel, hr->host, AF_INET6,
+          ares_gethostbyname(r->ev_driver->channel, hr->host, AF_INET6,
                              on_hostbyname_done_locked, hr);
         }
         grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked(
             r, srv_it->host, htons(srv_it->port), true /* is_balancer */, "A");
-        ares_gethostbyname(*channel, hr->host, AF_INET,
+        ares_gethostbyname(r->ev_driver->channel, hr->host, AF_INET,
                            on_hostbyname_done_locked, hr);
         grpc_ares_ev_driver_start_locked(r->ev_driver);
       }
@@ -400,7 +839,6 @@ void grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked(
     std::shared_ptr<grpc_core::WorkSerializer> work_serializer) {
   grpc_error* error = GRPC_ERROR_NONE;
   grpc_ares_hostbyname_request* hr = nullptr;
-  ares_channel* channel = nullptr;
   /* parse name, splitting it into host and port parts */
   std::string host;
   std::string port;
@@ -423,9 +861,8 @@ void grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked(
                                             query_timeout_ms,
                                             std::move(work_serializer), r);
   if (error != GRPC_ERROR_NONE) goto error_cleanup;
-  channel = grpc_ares_ev_driver_get_channel_locked(r->ev_driver);
   // If dns_server is specified, use it.
-  if (dns_server != nullptr) {
+  if (dns_server != nullptr && dns_server[0] != '\0') {
     GRPC_CARES_TRACE_LOG("request:%p Using DNS server %s", r, dns_server);
     grpc_resolved_address addr;
     if (grpc_parse_ipv4_hostport(dns_server, &addr, false /* log_errors */)) {
@@ -450,7 +887,8 @@ void grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked(
           GRPC_ERROR_STR_TARGET_ADDRESS, grpc_slice_from_copied_string(name));
       goto error_cleanup;
     }
-    int status = ares_set_servers_ports(*channel, &r->dns_server_addr);
+    int status =
+        ares_set_servers_ports(r->ev_driver->channel, &r->dns_server_addr);
     if (status != ARES_SUCCESS) {
       error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(
           absl::StrCat("C-ares status is not ARES_SUCCESS: ",
@@ -464,25 +902,25 @@ void grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked(
     hr = create_hostbyname_request_locked(r, host.c_str(),
                                           grpc_strhtons(port.c_str()),
                                           /*is_balancer=*/false, "AAAA");
-    ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_locked,
-                       hr);
+    ares_gethostbyname(r->ev_driver->channel, hr->host, AF_INET6,
+                       on_hostbyname_done_locked, hr);
   }
   hr = create_hostbyname_request_locked(r, host.c_str(),
                                         grpc_strhtons(port.c_str()),
                                         /*is_balancer=*/false, "A");
-  ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_locked,
-                     hr);
+  ares_gethostbyname(r->ev_driver->channel, hr->host, AF_INET,
+                     on_hostbyname_done_locked, hr);
   if (r->balancer_addresses_out != nullptr) {
     /* Query the SRV record */
     std::string service_name = absl::StrCat("_grpclb._tcp.", host);
     GrpcAresQuery* srv_query = new GrpcAresQuery(r, service_name);
-    ares_query(*channel, service_name.c_str(), ns_c_in, ns_t_srv,
+    ares_query(r->ev_driver->channel, service_name.c_str(), ns_c_in, ns_t_srv,
                on_srv_query_done_locked, srv_query);
   }
   if (r->service_config_json_out != nullptr) {
     std::string config_name = absl::StrCat("_grpc_config.", host);
     GrpcAresQuery* txt_query = new GrpcAresQuery(r, config_name);
-    ares_search(*channel, config_name.c_str(), ns_c_in, ns_t_txt,
+    ares_search(r->ev_driver->channel, config_name.c_str(), ns_c_in, ns_t_txt,
                 on_txt_done_locked, txt_query);
   }
   grpc_ares_ev_driver_start_locked(r->ev_driver);