grpc-z 1.11.1

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -0,0 +1,53 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H
+#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H
+
+#include <grpc/support/port_platform.h>
+
+#include <ares.h>
+#include "src/core/lib/iomgr/pollset_set.h"
+
+typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
+
+/* Start \a ev_driver. It will keep working until all IO on its ares_channel is
+   done, or grpc_ares_ev_driver_destroy() is called. It may notify the callbacks
+   bound to its ares_channel when necessary. */
+void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver);
+
+/* Returns the ares_channel owned by \a ev_driver. To bind a c-ares query to
+   \a ev_driver, use the ares_channel owned by \a ev_driver as the arg of the
+   query. */
+ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver);
+
+/* Creates a new grpc_ares_ev_driver. Returns GRPC_ERROR_NONE if \a ev_driver is
+   created successfully. */
+grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
+                                       grpc_pollset_set* pollset_set);
+
+/* Destroys \a ev_driver asynchronously. Pending lookups made on \a ev_driver
+   will be cancelled and their on_done callbacks will be invoked with a status
+   of ARES_ECANCELLED. */
+void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver);
+
+/* Shutdown all the grpc_fds used by \a ev_driver */
+void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver);
+
+#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H \
+        */

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc

@@ -0,0 +1,351 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/port.h"
+#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET)
+
+#include <ares.h>
+#include <sys/ioctl.h>
+
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+
+typedef struct fd_node {
+  /** the owner of this fd node */
+  grpc_ares_ev_driver* ev_driver;
+  /** a closure wrapping on_readable_cb, which should be invoked when the
+      grpc_fd in this node becomes readable. */
+  grpc_closure read_closure;
+  /** a closure wrapping on_writable_cb, which should be invoked when the
+      grpc_fd in this node becomes writable. */
+  grpc_closure write_closure;
+  /** next fd node in the list */
+  struct fd_node* next;
+
+  /** mutex guarding the rest of the state */
+  gpr_mu mu;
+  /** the grpc_fd owned by this fd node */
+  grpc_fd* fd;
+  /** if the readable closure has been registered */
+  bool readable_registered;
+  /** if the writable closure has been registered */
+  bool writable_registered;
+  /** if the fd is being shut down */
+  bool shutting_down;
+} fd_node;
+
+struct grpc_ares_ev_driver {
+  /** the ares_channel owned by this event driver */
+  ares_channel channel;
+  /** pollset set for driving the IO events of the channel */
+  grpc_pollset_set* pollset_set;
+  /** refcount of the event driver */
+  gpr_refcount refs;
+
+  /** mutex guarding the rest of the state */
+  gpr_mu mu;
+  /** a list of grpc_fd that this event driver is currently using. */
+  fd_node* fds;
+  /** is this event driver currently working? */
+  bool working;
+  /** is this event driver being shut down */
+  bool shutting_down;
+};
+
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
+
+static grpc_ares_ev_driver* grpc_ares_ev_driver_ref(
+    grpc_ares_ev_driver* ev_driver) {
+  gpr_log(GPR_DEBUG, "Ref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+  gpr_ref(&ev_driver->refs);
+  return ev_driver;
+}
+
+static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) {
+  gpr_log(GPR_DEBUG, "Unref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+  if (gpr_unref(&ev_driver->refs)) {
+    gpr_log(GPR_DEBUG, "destroy ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
+    GPR_ASSERT(ev_driver->fds == nullptr);
+    gpr_mu_destroy(&ev_driver->mu);
+    ares_destroy(ev_driver->channel);
+    gpr_free(ev_driver);
+  }
+}
+
+static void fd_node_destroy(fd_node* fdn) {
+  gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->fd));
+  GPR_ASSERT(!fdn->readable_registered);
+  GPR_ASSERT(!fdn->writable_registered);
+  gpr_mu_destroy(&fdn->mu);
+  /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up
+     immediately by another thread, and should not be closed by the following
+     grpc_fd_orphan. */
+  grpc_fd_orphan(fdn->fd, nullptr, nullptr, true /* already_closed */,
+                 "c-ares query finished");
+  gpr_free(fdn);
+}
+
+static void fd_node_shutdown(fd_node* fdn) {
+  gpr_mu_lock(&fdn->mu);
+  fdn->shutting_down = true;
+  if (!fdn->readable_registered && !fdn->writable_registered) {
+    gpr_mu_unlock(&fdn->mu);
+    fd_node_destroy(fdn);
+  } else {
+    grpc_fd_shutdown(
+        fdn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown"));
+    gpr_mu_unlock(&fdn->mu);
+  }
+}
+
+grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
+                                       grpc_pollset_set* pollset_set) {
+  *ev_driver = static_cast<grpc_ares_ev_driver*>(
+      gpr_malloc(sizeof(grpc_ares_ev_driver)));
+  int status = ares_init(&(*ev_driver)->channel);
+  gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create");
+  if (status != ARES_SUCCESS) {
+    char* err_msg;
+    gpr_asprintf(&err_msg, "Failed to init ares channel. C-ares error: %s",
+                 ares_strerror(status));
+    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(err_msg);
+    gpr_free(err_msg);
+    gpr_free(*ev_driver);
+    return err;
+  }
+  gpr_mu_init(&(*ev_driver)->mu);
+  gpr_ref_init(&(*ev_driver)->refs, 1);
+  (*ev_driver)->pollset_set = pollset_set;
+  (*ev_driver)->fds = nullptr;
+  (*ev_driver)->working = false;
+  (*ev_driver)->shutting_down = false;
+  return GRPC_ERROR_NONE;
+}
+
+void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver) {
+  // It's not safe to shut down remaining fds here directly, becauses
+  // ares_host_callback does not provide an exec_ctx. We mark the event driver
+  // as being shut down. If the event driver is working,
+  // grpc_ares_notify_on_event_locked will shut down the fds; if it's not
+  // working, there are no fds to shut down.
+  gpr_mu_lock(&ev_driver->mu);
+  ev_driver->shutting_down = true;
+  gpr_mu_unlock(&ev_driver->mu);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver) {
+  gpr_mu_lock(&ev_driver->mu);
+  ev_driver->shutting_down = true;
+  fd_node* fn = ev_driver->fds;
+  while (fn != nullptr) {
+    grpc_fd_shutdown(fn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                                 "grpc_ares_ev_driver_shutdown"));
+    fn = fn->next;
+  }
+  gpr_mu_unlock(&ev_driver->mu);
+}
+
+// Search fd in the fd_node list head. This is an O(n) search, the max possible
+// value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests.
+static fd_node* pop_fd_node(fd_node** head, int fd) {
+  fd_node dummy_head;
+  dummy_head.next = *head;
+  fd_node* node = &dummy_head;
+  while (node->next != nullptr) {
+    if (grpc_fd_wrapped_fd(node->next->fd) == fd) {
+      fd_node* ret = node->next;
+      node->next = node->next->next;
+      *head = dummy_head.next;
+      return ret;
+    }
+    node = node->next;
+  }
+  return nullptr;
+}
+
+/* Check if \a fd is still readable */
+static bool grpc_ares_is_fd_still_readable(grpc_ares_ev_driver* ev_driver,
+                                           int fd) {
+  size_t bytes_available = 0;
+  return ioctl(fd, FIONREAD, &bytes_available) == 0 && bytes_available > 0;
+}
+
+static void on_readable_cb(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  gpr_mu_lock(&fdn->mu);
+  const int fd = grpc_fd_wrapped_fd(fdn->fd);
+  fdn->readable_registered = false;
+  if (fdn->shutting_down && !fdn->writable_registered) {
+    gpr_mu_unlock(&fdn->mu);
+    fd_node_destroy(fdn);
+    grpc_ares_ev_driver_unref(ev_driver);
+    return;
+  }
+  gpr_mu_unlock(&fdn->mu);
+
+  gpr_log(GPR_DEBUG, "readable on %d", fd);
+  if (error == GRPC_ERROR_NONE) {
+    do {
+      ares_process_fd(ev_driver->channel, fd, ARES_SOCKET_BAD);
+    } while (grpc_ares_is_fd_still_readable(ev_driver, fd));
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  gpr_mu_lock(&ev_driver->mu);
+  grpc_ares_notify_on_event_locked(ev_driver);
+  gpr_mu_unlock(&ev_driver->mu);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+static void on_writable_cb(void* arg, grpc_error* error) {
+  fd_node* fdn = static_cast<fd_node*>(arg);
+  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
+  gpr_mu_lock(&fdn->mu);
+  const int fd = grpc_fd_wrapped_fd(fdn->fd);
+  fdn->writable_registered = false;
+  if (fdn->shutting_down && !fdn->readable_registered) {
+    gpr_mu_unlock(&fdn->mu);
+    fd_node_destroy(fdn);
+    grpc_ares_ev_driver_unref(ev_driver);
+    return;
+  }
+  gpr_mu_unlock(&fdn->mu);
+
+  gpr_log(GPR_DEBUG, "writable on %d", fd);
+  if (error == GRPC_ERROR_NONE) {
+    ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, fd);
+  } else {
+    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
+    // timed out. The pending lookups made on this ev_driver will be cancelled
+    // by the following ares_cancel() and the on_done callbacks will be invoked
+    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
+    // ev_driver will be cleaned up in the follwing
+    // grpc_ares_notify_on_event_locked().
+    ares_cancel(ev_driver->channel);
+  }
+  gpr_mu_lock(&ev_driver->mu);
+  grpc_ares_notify_on_event_locked(ev_driver);
+  gpr_mu_unlock(&ev_driver->mu);
+  grpc_ares_ev_driver_unref(ev_driver);
+}
+
+ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver) {
+  return &ev_driver->channel;
+}
+
+// Get the file descriptors used by the ev_driver's ares channel, register
+// driver_closure with these filedescriptors.
+static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) {
+  fd_node* new_list = nullptr;
+  if (!ev_driver->shutting_down) {
+    ares_socket_t socks[ARES_GETSOCK_MAXNUM];
+    int socks_bitmask =
+        ares_getsock(ev_driver->channel, socks, ARES_GETSOCK_MAXNUM);
+    for (size_t i = 0; i < ARES_GETSOCK_MAXNUM; i++) {
+      if (ARES_GETSOCK_READABLE(socks_bitmask, i) ||
+          ARES_GETSOCK_WRITABLE(socks_bitmask, i)) {
+        fd_node* fdn = pop_fd_node(&ev_driver->fds, socks[i]);
+        // Create a new fd_node if sock[i] is not in the fd_node list.
+        if (fdn == nullptr) {
+          char* fd_name;
+          gpr_asprintf(&fd_name, "ares_ev_driver-%" PRIuPTR, i);
+          fdn = static_cast<fd_node*>(gpr_malloc(sizeof(fd_node)));
+          gpr_log(GPR_DEBUG, "new fd: %d", socks[i]);
+          fdn->fd = grpc_fd_create(socks[i], fd_name);
+          fdn->ev_driver = ev_driver;
+          fdn->readable_registered = false;
+          fdn->writable_registered = false;
+          fdn->shutting_down = false;
+          gpr_mu_init(&fdn->mu);
+          GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_cb, fdn,
+                            grpc_schedule_on_exec_ctx);
+          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_cb, fdn,
+                            grpc_schedule_on_exec_ctx);
+          grpc_pollset_set_add_fd(ev_driver->pollset_set, fdn->fd);
+          gpr_free(fd_name);
+        }
+        fdn->next = new_list;
+        new_list = fdn;
+        gpr_mu_lock(&fdn->mu);
+        // Register read_closure if the socket is readable and read_closure has
+        // not been registered with this socket.
+        if (ARES_GETSOCK_READABLE(socks_bitmask, i) &&
+            !fdn->readable_registered) {
+          grpc_ares_ev_driver_ref(ev_driver);
+          gpr_log(GPR_DEBUG, "notify read on: %d", grpc_fd_wrapped_fd(fdn->fd));
+          grpc_fd_notify_on_read(fdn->fd, &fdn->read_closure);
+          fdn->readable_registered = true;
+        }
+        // Register write_closure if the socket is writable and write_closure
+        // has not been registered with this socket.
+        if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) &&
+            !fdn->writable_registered) {
+          gpr_log(GPR_DEBUG, "notify write on: %d",
+                  grpc_fd_wrapped_fd(fdn->fd));
+          grpc_ares_ev_driver_ref(ev_driver);
+          grpc_fd_notify_on_write(fdn->fd, &fdn->write_closure);
+          fdn->writable_registered = true;
+        }
+        gpr_mu_unlock(&fdn->mu);
+      }
+    }
+  }
+  // Any remaining fds in ev_driver->fds were not returned by ares_getsock() and
+  // are therefore no longer in use, so they can be shut down and removed from
+  // the list.
+  while (ev_driver->fds != nullptr) {
+    fd_node* cur = ev_driver->fds;
+    ev_driver->fds = ev_driver->fds->next;
+    fd_node_shutdown(cur);
+  }
+  ev_driver->fds = new_list;
+  // If the ev driver has no working fd, all the tasks are done.
+  if (new_list == nullptr) {
+    ev_driver->working = false;
+    gpr_log(GPR_DEBUG, "ev driver stop working");
+  }
+}
+
+void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver) {
+  gpr_mu_lock(&ev_driver->mu);
+  if (!ev_driver->working) {
+    ev_driver->working = true;
+    grpc_ares_notify_on_event_locked(ev_driver);
+  }
+  gpr_mu_unlock(&ev_driver->mu);
+}
+
+#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) */

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc

@@ -0,0 +1,593 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#if GRPC_ARES == 1 && !defined(GRPC_UV)
+
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/lib/iomgr/sockaddr.h"
+#include "src/core/lib/iomgr/socket_utils_posix.h"
+
+#include <string.h>
+#include <sys/types.h>
+
+#include <ares.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
+
+#include <address_sorting/address_sorting.h>
+#include "src/core/ext/filters/client_channel/parse_address.h"
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
+#include "src/core/lib/gpr/host_port.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/nameser.h"
+#include "src/core/lib/iomgr/sockaddr_utils.h"
+
+static gpr_once g_basic_init = GPR_ONCE_INIT;
+static gpr_mu g_init_mu;
+
+grpc_core::TraceFlag grpc_trace_cares_address_sorting(false,
+                                                      "cares_address_sorting");
+
+struct grpc_ares_request {
+  /** indicates the DNS server to use, if specified */
+  struct ares_addr_port_node dns_server_addr;
+  /** following members are set in grpc_resolve_address_ares_impl */
+  /** closure to call when the request completes */
+  grpc_closure* on_done;
+  /** the pointer to receive the resolved addresses */
+  grpc_lb_addresses** lb_addrs_out;
+  /** the pointer to receive the service config in JSON */
+  char** service_config_json_out;
+  /** the evernt driver used by this request */
+  grpc_ares_ev_driver* ev_driver;
+  /** number of ongoing queries */
+  gpr_refcount pending_queries;
+
+  /** mutex guarding the rest of the state */
+  gpr_mu mu;
+  /** is there at least one successful query, set in on_done_cb */
+  bool success;
+  /** the errors explaining the request failure, set in on_done_cb */
+  grpc_error* error;
+};
+
+typedef struct grpc_ares_hostbyname_request {
+  /** following members are set in create_hostbyname_request */
+  /** the top-level request instance */
+  grpc_ares_request* parent_request;
+  /** host to resolve, parsed from the name to resolve */
+  char* host;
+  /** port to fill in sockaddr_in, parsed from the name to resolve */
+  uint16_t port;
+  /** is it a grpclb address */
+  bool is_balancer;
+} grpc_ares_hostbyname_request;
+
+static void do_basic_init(void) { gpr_mu_init(&g_init_mu); }
+
+static uint16_t strhtons(const char* port) {
+  if (strcmp(port, "http") == 0) {
+    return htons(80);
+  } else if (strcmp(port, "https") == 0) {
+    return htons(443);
+  }
+  return htons(static_cast<unsigned short>(atoi(port)));
+}
+
+static void grpc_ares_request_ref(grpc_ares_request* r) {
+  gpr_ref(&r->pending_queries);
+}
+
+static void log_address_sorting_list(grpc_lb_addresses* lb_addrs,
+                                     const char* input_output_str) {
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    char* addr_str;
+    if (grpc_sockaddr_to_string(&addr_str, &lb_addrs->addresses[i].address,
+                                true)) {
+      gpr_log(GPR_DEBUG, "c-ares address sorting: %s[%" PRIuPTR "]=%s",
+              input_output_str, i, addr_str);
+      gpr_free(addr_str);
+    } else {
+      gpr_log(GPR_DEBUG,
+              "c-ares address sorting: %s[%" PRIuPTR "]=<unprintable>",
+              input_output_str, i);
+    }
+  }
+}
+
+void grpc_cares_wrapper_address_sorting_sort(grpc_lb_addresses* lb_addrs) {
+  if (grpc_trace_cares_address_sorting.enabled()) {
+    log_address_sorting_list(lb_addrs, "input");
+  }
+  address_sorting_sortable* sortables = (address_sorting_sortable*)gpr_zalloc(
+      sizeof(address_sorting_sortable) * lb_addrs->num_addresses);
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    sortables[i].user_data = &lb_addrs->addresses[i];
+    memcpy(&sortables[i].dest_addr.addr, &lb_addrs->addresses[i].address.addr,
+           lb_addrs->addresses[i].address.len);
+    sortables[i].dest_addr.len = lb_addrs->addresses[i].address.len;
+  }
+  address_sorting_rfc_6724_sort(sortables, lb_addrs->num_addresses);
+  grpc_lb_address* sorted_lb_addrs = (grpc_lb_address*)gpr_zalloc(
+      sizeof(grpc_lb_address) * lb_addrs->num_addresses);
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    sorted_lb_addrs[i] = *(grpc_lb_address*)sortables[i].user_data;
+  }
+  gpr_free(sortables);
+  gpr_free(lb_addrs->addresses);
+  lb_addrs->addresses = sorted_lb_addrs;
+  if (grpc_trace_cares_address_sorting.enabled()) {
+    log_address_sorting_list(lb_addrs, "output");
+  }
+}
+
+/* Allow tests to access grpc_ares_wrapper_address_sorting_sort */
+void grpc_cares_wrapper_test_only_address_sorting_sort(
+    grpc_lb_addresses* lb_addrs) {
+  grpc_cares_wrapper_address_sorting_sort(lb_addrs);
+}
+
+static void grpc_ares_request_unref(grpc_ares_request* r) {
+  /* If there are no pending queries, invoke on_done callback and destroy the
+     request */
+  if (gpr_unref(&r->pending_queries)) {
+    grpc_cares_wrapper_address_sorting_sort(*(r->lb_addrs_out));
+    GRPC_CLOSURE_SCHED(r->on_done, r->error);
+    gpr_mu_destroy(&r->mu);
+    grpc_ares_ev_driver_destroy(r->ev_driver);
+    gpr_free(r);
+  }
+}
+
+static grpc_ares_hostbyname_request* create_hostbyname_request(
+    grpc_ares_request* parent_request, char* host, uint16_t port,
+    bool is_balancer) {
+  grpc_ares_hostbyname_request* hr = static_cast<grpc_ares_hostbyname_request*>(
+      gpr_zalloc(sizeof(grpc_ares_hostbyname_request)));
+  hr->parent_request = parent_request;
+  hr->host = gpr_strdup(host);
+  hr->port = port;
+  hr->is_balancer = is_balancer;
+  grpc_ares_request_ref(parent_request);
+  return hr;
+}
+
+static void destroy_hostbyname_request(grpc_ares_hostbyname_request* hr) {
+  grpc_ares_request_unref(hr->parent_request);
+  gpr_free(hr->host);
+  gpr_free(hr);
+}
+
+static void on_hostbyname_done_cb(void* arg, int status, int timeouts,
+                                  struct hostent* hostent) {
+  grpc_ares_hostbyname_request* hr =
+      static_cast<grpc_ares_hostbyname_request*>(arg);
+  grpc_ares_request* r = hr->parent_request;
+  gpr_mu_lock(&r->mu);
+  if (status == ARES_SUCCESS) {
+    GRPC_ERROR_UNREF(r->error);
+    r->error = GRPC_ERROR_NONE;
+    r->success = true;
+    grpc_lb_addresses** lb_addresses = r->lb_addrs_out;
+    if (*lb_addresses == nullptr) {
+      *lb_addresses = grpc_lb_addresses_create(0, nullptr);
+    }
+    size_t prev_naddr = (*lb_addresses)->num_addresses;
+    size_t i;
+    for (i = 0; hostent->h_addr_list[i] != nullptr; i++) {
+    }
+    (*lb_addresses)->num_addresses += i;
+    (*lb_addresses)->addresses = static_cast<grpc_lb_address*>(
+        gpr_realloc((*lb_addresses)->addresses,
+                    sizeof(grpc_lb_address) * (*lb_addresses)->num_addresses));
+    for (i = prev_naddr; i < (*lb_addresses)->num_addresses; i++) {
+      switch (hostent->h_addrtype) {
+        case AF_INET6: {
+          size_t addr_len = sizeof(struct sockaddr_in6);
+          struct sockaddr_in6 addr;
+          memset(&addr, 0, addr_len);
+          memcpy(&addr.sin6_addr, hostent->h_addr_list[i - prev_naddr],
+                 sizeof(struct in6_addr));
+          addr.sin6_family = static_cast<sa_family_t>(hostent->h_addrtype);
+          addr.sin6_port = hr->port;
+          grpc_lb_addresses_set_address(
+              *lb_addresses, i, &addr, addr_len,
+              hr->is_balancer /* is_balancer */,
+              hr->is_balancer ? hr->host : nullptr /* balancer_name */,
+              nullptr /* user_data */);
+          char output[INET6_ADDRSTRLEN];
+          ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN);
+          gpr_log(GPR_DEBUG,
+                  "c-ares resolver gets a AF_INET6 result: \n"
+                  "  addr: %s\n  port: %d\n  sin6_scope_id: %d\n",
+                  output, ntohs(hr->port), addr.sin6_scope_id);
+          break;
+        }
+        case AF_INET: {
+          size_t addr_len = sizeof(struct sockaddr_in);
+          struct sockaddr_in addr;
+          memset(&addr, 0, addr_len);
+          memcpy(&addr.sin_addr, hostent->h_addr_list[i - prev_naddr],
+                 sizeof(struct in_addr));
+          addr.sin_family = static_cast<sa_family_t>(hostent->h_addrtype);
+          addr.sin_port = hr->port;
+          grpc_lb_addresses_set_address(
+              *lb_addresses, i, &addr, addr_len,
+              hr->is_balancer /* is_balancer */,
+              hr->is_balancer ? hr->host : nullptr /* balancer_name */,
+              nullptr /* user_data */);
+          char output[INET_ADDRSTRLEN];
+          ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN);
+          gpr_log(GPR_DEBUG,
+                  "c-ares resolver gets a AF_INET result: \n"
+                  "  addr: %s\n  port: %d\n",
+                  output, ntohs(hr->port));
+          break;
+        }
+      }
+    }
+  } else if (!r->success) {
+    char* error_msg;
+    gpr_asprintf(&error_msg, "C-ares status is not ARES_SUCCESS: %s",
+                 ares_strerror(status));
+    grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+    gpr_free(error_msg);
+    if (r->error == GRPC_ERROR_NONE) {
+      r->error = error;
+    } else {
+      r->error = grpc_error_add_child(error, r->error);
+    }
+  }
+  gpr_mu_unlock(&r->mu);
+  destroy_hostbyname_request(hr);
+}
+
+static void on_srv_query_done_cb(void* arg, int status, int timeouts,
+                                 unsigned char* abuf, int alen) {
+  grpc_ares_request* r = static_cast<grpc_ares_request*>(arg);
+  grpc_core::ExecCtx exec_ctx;
+  gpr_log(GPR_DEBUG, "on_query_srv_done_cb");
+  if (status == ARES_SUCCESS) {
+    gpr_log(GPR_DEBUG, "on_query_srv_done_cb ARES_SUCCESS");
+    struct ares_srv_reply* reply;
+    const int parse_status = ares_parse_srv_reply(abuf, alen, &reply);
+    if (parse_status == ARES_SUCCESS) {
+      ares_channel* channel = grpc_ares_ev_driver_get_channel(r->ev_driver);
+      for (struct ares_srv_reply* srv_it = reply; srv_it != nullptr;
+           srv_it = srv_it->next) {
+        if (grpc_ipv6_loopback_available()) {
+          grpc_ares_hostbyname_request* hr = create_hostbyname_request(
+              r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
+          ares_gethostbyname(*channel, hr->host, AF_INET6,
+                             on_hostbyname_done_cb, hr);
+        }
+        grpc_ares_hostbyname_request* hr = create_hostbyname_request(
+            r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
+        ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb,
+                           hr);
+        grpc_ares_ev_driver_start(r->ev_driver);
+      }
+    }
+    if (reply != nullptr) {
+      ares_free_data(reply);
+    }
+  } else if (!r->success) {
+    char* error_msg;
+    gpr_asprintf(&error_msg, "C-ares status is not ARES_SUCCESS: %s",
+                 ares_strerror(status));
+    grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+    gpr_free(error_msg);
+    if (r->error == GRPC_ERROR_NONE) {
+      r->error = error;
+    } else {
+      r->error = grpc_error_add_child(error, r->error);
+    }
+  }
+  grpc_ares_request_unref(r);
+}
+
+static const char g_service_config_attribute_prefix[] = "grpc_config=";
+
+static void on_txt_done_cb(void* arg, int status, int timeouts,
+                           unsigned char* buf, int len) {
+  gpr_log(GPR_DEBUG, "on_txt_done_cb");
+  char* error_msg;
+  grpc_ares_request* r = static_cast<grpc_ares_request*>(arg);
+  const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1;
+  struct ares_txt_ext* result = nullptr;
+  struct ares_txt_ext* reply = nullptr;
+  grpc_error* error = GRPC_ERROR_NONE;
+  gpr_mu_lock(&r->mu);
+  if (status != ARES_SUCCESS) goto fail;
+  status = ares_parse_txt_reply_ext(buf, len, &reply);
+  if (status != ARES_SUCCESS) goto fail;
+  // Find service config in TXT record.
+  for (result = reply; result != nullptr; result = result->next) {
+    if (result->record_start &&
+        memcmp(result->txt, g_service_config_attribute_prefix, prefix_len) ==
+            0) {
+      break;
+    }
+  }
+  // Found a service config record.
+  if (result != nullptr) {
+    size_t service_config_len = result->length - prefix_len;
+    *r->service_config_json_out =
+        static_cast<char*>(gpr_malloc(service_config_len + 1));
+    memcpy(*r->service_config_json_out, result->txt + prefix_len,
+           service_config_len);
+    for (result = result->next; result != nullptr && !result->record_start;
+         result = result->next) {
+      *r->service_config_json_out = static_cast<char*>(
+          gpr_realloc(*r->service_config_json_out,
+                      service_config_len + result->length + 1));
+      memcpy(*r->service_config_json_out + service_config_len, result->txt,
+             result->length);
+      service_config_len += result->length;
+    }
+    (*r->service_config_json_out)[service_config_len] = '\0';
+    gpr_log(GPR_INFO, "found service config: %s", *r->service_config_json_out);
+  }
+  // Clean up.
+  ares_free_data(reply);
+  goto done;
+fail:
+  gpr_asprintf(&error_msg, "C-ares TXT lookup status is not ARES_SUCCESS: %s",
+               ares_strerror(status));
+  error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+  gpr_free(error_msg);
+  if (r->error == GRPC_ERROR_NONE) {
+    r->error = error;
+  } else {
+    r->error = grpc_error_add_child(error, r->error);
+  }
+done:
+  gpr_mu_unlock(&r->mu);
+  grpc_ares_request_unref(r);
+}
+
+static grpc_ares_request* grpc_dns_lookup_ares_impl(
+    const char* dns_server, const char* name, const char* default_port,
+    grpc_pollset_set* interested_parties, grpc_closure* on_done,
+    grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json) {
+  grpc_error* error = GRPC_ERROR_NONE;
+  grpc_ares_hostbyname_request* hr = nullptr;
+  grpc_ares_request* r = nullptr;
+  ares_channel* channel = nullptr;
+  /* TODO(zyc): Enable tracing after #9603 is checked in */
+  /* if (grpc_dns_trace) {
+      gpr_log(GPR_DEBUG, "resolve_address (blocking): name=%s, default_port=%s",
+              name, default_port);
+     } */
+
+  /* parse name, splitting it into host and port parts */
+  char* host;
+  char* port;
+  gpr_split_host_port(name, &host, &port);
+  if (host == nullptr) {
+    error = grpc_error_set_str(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING("unparseable host:port"),
+        GRPC_ERROR_STR_TARGET_ADDRESS, grpc_slice_from_copied_string(name));
+    goto error_cleanup;
+  } else if (port == nullptr) {
+    if (default_port == nullptr) {
+      error = grpc_error_set_str(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("no port in name"),
+          GRPC_ERROR_STR_TARGET_ADDRESS, grpc_slice_from_copied_string(name));
+      goto error_cleanup;
+    }
+    port = gpr_strdup(default_port);
+  }
+
+  grpc_ares_ev_driver* ev_driver;
+  error = grpc_ares_ev_driver_create(&ev_driver, interested_parties);
+  if (error != GRPC_ERROR_NONE) goto error_cleanup;
+
+  r = static_cast<grpc_ares_request*>(gpr_zalloc(sizeof(grpc_ares_request)));
+  gpr_mu_init(&r->mu);
+  r->ev_driver = ev_driver;
+  r->on_done = on_done;
+  r->lb_addrs_out = addrs;
+  r->service_config_json_out = service_config_json;
+  r->success = false;
+  r->error = GRPC_ERROR_NONE;
+  channel = grpc_ares_ev_driver_get_channel(r->ev_driver);
+
+  // If dns_server is specified, use it.
+  if (dns_server != nullptr) {
+    gpr_log(GPR_INFO, "Using DNS server %s", dns_server);
+    grpc_resolved_address addr;
+    if (grpc_parse_ipv4_hostport(dns_server, &addr, false /* log_errors */)) {
+      r->dns_server_addr.family = AF_INET;
+      struct sockaddr_in* in = reinterpret_cast<struct sockaddr_in*>(addr.addr);
+      memcpy(&r->dns_server_addr.addr.addr4, &in->sin_addr,
+             sizeof(struct in_addr));
+      r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr);
+      r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr);
+    } else if (grpc_parse_ipv6_hostport(dns_server, &addr,
+                                        false /* log_errors */)) {
+      r->dns_server_addr.family = AF_INET6;
+      struct sockaddr_in6* in6 =
+          reinterpret_cast<struct sockaddr_in6*>(addr.addr);
+      memcpy(&r->dns_server_addr.addr.addr6, &in6->sin6_addr,
+             sizeof(struct in6_addr));
+      r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr);
+      r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr);
+    } else {
+      error = grpc_error_set_str(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("cannot parse authority"),
+          GRPC_ERROR_STR_TARGET_ADDRESS, grpc_slice_from_copied_string(name));
+      gpr_free(r);
+      goto error_cleanup;
+    }
+    int status = ares_set_servers_ports(*channel, &r->dns_server_addr);
+    if (status != ARES_SUCCESS) {
+      char* error_msg;
+      gpr_asprintf(&error_msg, "C-ares status is not ARES_SUCCESS: %s",
+                   ares_strerror(status));
+      error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+      gpr_free(error_msg);
+      gpr_free(r);
+      goto error_cleanup;
+    }
+  }
+  gpr_ref_init(&r->pending_queries, 1);
+  if (grpc_ipv6_loopback_available()) {
+    hr = create_hostbyname_request(r, host, strhtons(port),
+                                   false /* is_balancer */);
+    ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_cb, hr);
+  }
+  hr = create_hostbyname_request(r, host, strhtons(port),
+                                 false /* is_balancer */);
+  ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, hr);
+  if (check_grpclb) {
+    /* Query the SRV record */
+    grpc_ares_request_ref(r);
+    char* service_name;
+    gpr_asprintf(&service_name, "_grpclb._tcp.%s", host);
+    ares_query(*channel, service_name, ns_c_in, ns_t_srv, on_srv_query_done_cb,
+               r);
+    gpr_free(service_name);
+  }
+  if (service_config_json != nullptr) {
+    grpc_ares_request_ref(r);
+    char* config_name;
+    gpr_asprintf(&config_name, "_grpc_config.%s", host);
+    ares_search(*channel, config_name, ns_c_in, ns_t_txt, on_txt_done_cb, r);
+    gpr_free(config_name);
+  }
+  /* TODO(zyc): Handle CNAME records here. */
+  grpc_ares_ev_driver_start(r->ev_driver);
+  grpc_ares_request_unref(r);
+  gpr_free(host);
+  gpr_free(port);
+  return r;
+
+error_cleanup:
+  GRPC_CLOSURE_SCHED(on_done, error);
+  gpr_free(host);
+  gpr_free(port);
+  return nullptr;
+}
+
+grpc_ares_request* (*grpc_dns_lookup_ares)(
+    const char* dns_server, const char* name, const char* default_port,
+    grpc_pollset_set* interested_parties, grpc_closure* on_done,
+    grpc_lb_addresses** addrs, bool check_grpclb,
+    char** service_config_json) = grpc_dns_lookup_ares_impl;
+
+void grpc_cancel_ares_request(grpc_ares_request* r) {
+  if (grpc_dns_lookup_ares == grpc_dns_lookup_ares_impl) {
+    grpc_ares_ev_driver_shutdown(r->ev_driver);
+  }
+}
+
+grpc_error* grpc_ares_init(void) {
+  gpr_once_init(&g_basic_init, do_basic_init);
+  gpr_mu_lock(&g_init_mu);
+  int status = ares_library_init(ARES_LIB_INIT_ALL);
+  gpr_mu_unlock(&g_init_mu);
+
+  if (status != ARES_SUCCESS) {
+    char* error_msg;
+    gpr_asprintf(&error_msg, "ares_library_init failed: %s",
+                 ares_strerror(status));
+    grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg);
+    gpr_free(error_msg);
+    return error;
+  }
+  return GRPC_ERROR_NONE;
+}
+
+void grpc_ares_cleanup(void) {
+  gpr_mu_lock(&g_init_mu);
+  ares_library_cleanup();
+  gpr_mu_unlock(&g_init_mu);
+}
+
+/*
+ * grpc_resolve_address_ares related structs and functions
+ */
+
+typedef struct grpc_resolve_address_ares_request {
+  /** the pointer to receive the resolved addresses */
+  grpc_resolved_addresses** addrs_out;
+  /** currently resolving lb addresses */
+  grpc_lb_addresses* lb_addrs;
+  /** closure to call when the resolve_address_ares request completes */
+  grpc_closure* on_resolve_address_done;
+  /** a closure wrapping on_dns_lookup_done_cb, which should be invoked when the
+      grpc_dns_lookup_ares operation is done. */
+  grpc_closure on_dns_lookup_done;
+} grpc_resolve_address_ares_request;
+
+static void on_dns_lookup_done_cb(void* arg, grpc_error* error) {
+  grpc_resolve_address_ares_request* r =
+      static_cast<grpc_resolve_address_ares_request*>(arg);
+  grpc_resolved_addresses** resolved_addresses = r->addrs_out;
+  if (r->lb_addrs == nullptr || r->lb_addrs->num_addresses == 0) {
+    *resolved_addresses = nullptr;
+  } else {
+    *resolved_addresses = static_cast<grpc_resolved_addresses*>(
+        gpr_zalloc(sizeof(grpc_resolved_addresses)));
+    (*resolved_addresses)->naddrs = r->lb_addrs->num_addresses;
+    (*resolved_addresses)->addrs =
+        static_cast<grpc_resolved_address*>(gpr_zalloc(
+            sizeof(grpc_resolved_address) * (*resolved_addresses)->naddrs));
+    for (size_t i = 0; i < (*resolved_addresses)->naddrs; i++) {
+      GPR_ASSERT(!r->lb_addrs->addresses[i].is_balancer);
+      memcpy(&(*resolved_addresses)->addrs[i],
+             &r->lb_addrs->addresses[i].address, sizeof(grpc_resolved_address));
+    }
+  }
+  GRPC_CLOSURE_SCHED(r->on_resolve_address_done, GRPC_ERROR_REF(error));
+  if (r->lb_addrs != nullptr) grpc_lb_addresses_destroy(r->lb_addrs);
+  gpr_free(r);
+}
+
+static void grpc_resolve_address_ares_impl(const char* name,
+                                           const char* default_port,
+                                           grpc_pollset_set* interested_parties,
+                                           grpc_closure* on_done,
+                                           grpc_resolved_addresses** addrs) {
+  grpc_resolve_address_ares_request* r =
+      static_cast<grpc_resolve_address_ares_request*>(
+          gpr_zalloc(sizeof(grpc_resolve_address_ares_request)));
+  r->addrs_out = addrs;
+  r->on_resolve_address_done = on_done;
+  GRPC_CLOSURE_INIT(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r,
+                    grpc_schedule_on_exec_ctx);
+  grpc_dns_lookup_ares(nullptr /* dns_server */, name, default_port,
+                       interested_parties, &r->on_dns_lookup_done, &r->lb_addrs,
+                       false /* check_grpclb */,
+                       nullptr /* service_config_json */);
+}
+
+void (*grpc_resolve_address_ares)(
+    const char* name, const char* default_port,
+    grpc_pollset_set* interested_parties, grpc_closure* on_done,
+    grpc_resolved_addresses** addrs) = grpc_resolve_address_ares_impl;
+
+#endif /* GRPC_ARES == 1 && !defined(GRPC_UV) */