grpc-flamingo 1.11.0 → 1.15.0

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -22,6 +22,8 @@
 #include <grpc/support/port_platform.h>
 
 #include <ares.h>
+#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/lib/gprpp/abstract.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 
 typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
@@ -29,25 +31,76 @@ typedef struct grpc_ares_ev_driver grpc_ares_ev_driver;
 /* Start \a ev_driver. It will keep working until all IO on its ares_channel is
    done, or grpc_ares_ev_driver_destroy() is called. It may notify the callbacks
    bound to its ares_channel when necessary. */
-void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver);
+void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver);
 
 /* Returns the ares_channel owned by \a ev_driver. To bind a c-ares query to
    \a ev_driver, use the ares_channel owned by \a ev_driver as the arg of the
    query. */
-ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver);
+ares_channel* grpc_ares_ev_driver_get_channel_locked(
+    grpc_ares_ev_driver* ev_driver);
 
 /* Creates a new grpc_ares_ev_driver. Returns GRPC_ERROR_NONE if \a ev_driver is
    created successfully. */
-grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
-                                       grpc_pollset_set* pollset_set);
+grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
+                                              grpc_pollset_set* pollset_set,
+                                              grpc_combiner* combiner,
+                                              grpc_ares_request* request);
 
-/* Destroys \a ev_driver asynchronously. Pending lookups made on \a ev_driver
-   will be cancelled and their on_done callbacks will be invoked with a status
-   of ARES_ECANCELLED. */
-void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver);
+/* Called back when all DNS lookups have completed. */
+void grpc_ares_ev_driver_on_queries_complete_locked(
+    grpc_ares_ev_driver* ev_driver);
 
 /* Shutdown all the grpc_fds used by \a ev_driver */
-void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver);
+void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver);
+
+namespace grpc_core {
+
+/* A wrapped fd that integrates with the grpc iomgr of the current platform.
+ * A GrpcPolledFd knows how to create grpc platform-specific iomgr endpoints
+ * from "ares_socket_t" sockets, and then sign up for readability/writeability
+ * with that poller, and do shutdown and destruction. */
+class GrpcPolledFd {
+ public:
+  virtual ~GrpcPolledFd() {}
+  /* Called when c-ares library is interested and there's no pending callback */
+  virtual void RegisterForOnReadableLocked(grpc_closure* read_closure)
+      GRPC_ABSTRACT;
+  /* Called when c-ares library is interested and there's no pending callback */
+  virtual void RegisterForOnWriteableLocked(grpc_closure* write_closure)
+      GRPC_ABSTRACT;
+  /* Indicates if there is data left even after just being read from */
+  virtual bool IsFdStillReadableLocked() GRPC_ABSTRACT;
+  /* Called once and only once. Must cause cancellation of any pending
+   * read/write callbacks. */
+  virtual void ShutdownLocked(grpc_error* error) GRPC_ABSTRACT;
+  /* Get the underlying ares_socket_t that this was created from */
+  virtual ares_socket_t GetWrappedAresSocketLocked() GRPC_ABSTRACT;
+  /* A unique name, for logging */
+  virtual const char* GetName() GRPC_ABSTRACT;
+
+  GRPC_ABSTRACT_BASE_CLASS
+};
+
+/* A GrpcPolledFdFactory is 1-to-1 with and owned by the
+ * ares event driver. It knows how to create GrpcPolledFd's
+ * for the current platform, and the ares driver uses it for all of
+ * its fd's. */
+class GrpcPolledFdFactory {
+ public:
+  virtual ~GrpcPolledFdFactory() {}
+  /* Creates a new wrapped fd for the current platform */
+  virtual GrpcPolledFd* NewGrpcPolledFdLocked(
+      ares_socket_t as, grpc_pollset_set* driver_pollset_set,
+      grpc_combiner* combiner) GRPC_ABSTRACT;
+  /* Optionally configures the ares channel after creation */
+  virtual void ConfigureAresChannelLocked(ares_channel channel) GRPC_ABSTRACT;
+
+  GRPC_ABSTRACT_BASE_CLASS
+};
+
+UniquePtr<GrpcPolledFdFactory> NewGrpcPolledFdFactory(grpc_combiner* combiner);
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H \
         */

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc

@@ -18,9 +18,10 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/iomgr/port.h"
-#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET)
+#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER)
 
 #include <ares.h>
+#include <string.h>
 #include <sys/ioctl.h>
 
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
@@ -35,317 +36,71 @@
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 
-typedef struct fd_node {
-  /** the owner of this fd node */
-  grpc_ares_ev_driver* ev_driver;
-  /** a closure wrapping on_readable_cb, which should be invoked when the
-      grpc_fd in this node becomes readable. */
-  grpc_closure read_closure;
-  /** a closure wrapping on_writable_cb, which should be invoked when the
-      grpc_fd in this node becomes writable. */
-  grpc_closure write_closure;
-  /** next fd node in the list */
-  struct fd_node* next;
+namespace grpc_core {
 
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** the grpc_fd owned by this fd node */
-  grpc_fd* fd;
-  /** if the readable closure has been registered */
-  bool readable_registered;
-  /** if the writable closure has been registered */
-  bool writable_registered;
-  /** if the fd is being shut down */
-  bool shutting_down;
-} fd_node;
-
-struct grpc_ares_ev_driver {
-  /** the ares_channel owned by this event driver */
-  ares_channel channel;
-  /** pollset set for driving the IO events of the channel */
-  grpc_pollset_set* pollset_set;
-  /** refcount of the event driver */
-  gpr_refcount refs;
-
-  /** mutex guarding the rest of the state */
-  gpr_mu mu;
-  /** a list of grpc_fd that this event driver is currently using. */
-  fd_node* fds;
-  /** is this event driver currently working? */
-  bool working;
-  /** is this event driver being shut down */
-  bool shutting_down;
-};
-
-static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
-
-static grpc_ares_ev_driver* grpc_ares_ev_driver_ref(
-    grpc_ares_ev_driver* ev_driver) {
-  gpr_log(GPR_DEBUG, "Ref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-  gpr_ref(&ev_driver->refs);
-  return ev_driver;
-}
-
-static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) {
-  gpr_log(GPR_DEBUG, "Unref ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-  if (gpr_unref(&ev_driver->refs)) {
-    gpr_log(GPR_DEBUG, "destroy ev_driver %" PRIuPTR, (uintptr_t)ev_driver);
-    GPR_ASSERT(ev_driver->fds == nullptr);
-    gpr_mu_destroy(&ev_driver->mu);
-    ares_destroy(ev_driver->channel);
-    gpr_free(ev_driver);
+class GrpcPolledFdPosix : public GrpcPolledFd {
+ public:
+  GrpcPolledFdPosix(ares_socket_t as, grpc_pollset_set* driver_pollset_set)
+      : as_(as) {
+    gpr_asprintf(&name_, "c-ares fd: %d", (int)as);
+    fd_ = grpc_fd_create((int)as, name_, false);
+    driver_pollset_set_ = driver_pollset_set;
+    grpc_pollset_set_add_fd(driver_pollset_set_, fd_);
   }
-}
 
-static void fd_node_destroy(fd_node* fdn) {
-  gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->fd));
-  GPR_ASSERT(!fdn->readable_registered);
-  GPR_ASSERT(!fdn->writable_registered);
-  gpr_mu_destroy(&fdn->mu);
-  /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up
-     immediately by another thread, and should not be closed by the following
-     grpc_fd_orphan. */
-  grpc_fd_orphan(fdn->fd, nullptr, nullptr, true /* already_closed */,
-                 "c-ares query finished");
-  gpr_free(fdn);
-}
-
-static void fd_node_shutdown(fd_node* fdn) {
-  gpr_mu_lock(&fdn->mu);
-  fdn->shutting_down = true;
-  if (!fdn->readable_registered && !fdn->writable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-  } else {
-    grpc_fd_shutdown(
-        fdn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown"));
-    gpr_mu_unlock(&fdn->mu);
+  ~GrpcPolledFdPosix() {
+    gpr_free(name_);
+    grpc_pollset_set_del_fd(driver_pollset_set_, fd_);
+    /* c-ares library will close the fd inside grpc_fd. This fd may be picked up
+       immediately by another thread, and should not be closed by the following
+       grpc_fd_orphan. */
+    int dummy_release_fd;
+    grpc_fd_orphan(fd_, nullptr, &dummy_release_fd, "c-ares query finished");
   }
-}
 
-grpc_error* grpc_ares_ev_driver_create(grpc_ares_ev_driver** ev_driver,
-                                       grpc_pollset_set* pollset_set) {
-  *ev_driver = static_cast<grpc_ares_ev_driver*>(
-      gpr_malloc(sizeof(grpc_ares_ev_driver)));
-  int status = ares_init(&(*ev_driver)->channel);
-  gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create");
-  if (status != ARES_SUCCESS) {
-    char* err_msg;
-    gpr_asprintf(&err_msg, "Failed to init ares channel. C-ares error: %s",
-                 ares_strerror(status));
-    grpc_error* err = GRPC_ERROR_CREATE_FROM_COPIED_STRING(err_msg);
-    gpr_free(err_msg);
-    gpr_free(*ev_driver);
-    return err;
+  void RegisterForOnReadableLocked(grpc_closure* read_closure) override {
+    grpc_fd_notify_on_read(fd_, read_closure);
   }
-  gpr_mu_init(&(*ev_driver)->mu);
-  gpr_ref_init(&(*ev_driver)->refs, 1);
-  (*ev_driver)->pollset_set = pollset_set;
-  (*ev_driver)->fds = nullptr;
-  (*ev_driver)->working = false;
-  (*ev_driver)->shutting_down = false;
-  return GRPC_ERROR_NONE;
-}
 
-void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver* ev_driver) {
-  // It's not safe to shut down remaining fds here directly, becauses
-  // ares_host_callback does not provide an exec_ctx. We mark the event driver
-  // as being shut down. If the event driver is working,
-  // grpc_ares_notify_on_event_locked will shut down the fds; if it's not
-  // working, there are no fds to shut down.
-  gpr_mu_lock(&ev_driver->mu);
-  ev_driver->shutting_down = true;
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
-
-void grpc_ares_ev_driver_shutdown(grpc_ares_ev_driver* ev_driver) {
-  gpr_mu_lock(&ev_driver->mu);
-  ev_driver->shutting_down = true;
-  fd_node* fn = ev_driver->fds;
-  while (fn != nullptr) {
-    grpc_fd_shutdown(fn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                                 "grpc_ares_ev_driver_shutdown"));
-    fn = fn->next;
+  void RegisterForOnWriteableLocked(grpc_closure* write_closure) override {
+    grpc_fd_notify_on_write(fd_, write_closure);
   }
-  gpr_mu_unlock(&ev_driver->mu);
-}
 
-// Search fd in the fd_node list head. This is an O(n) search, the max possible
-// value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests.
-static fd_node* pop_fd_node(fd_node** head, int fd) {
-  fd_node dummy_head;
-  dummy_head.next = *head;
-  fd_node* node = &dummy_head;
-  while (node->next != nullptr) {
-    if (grpc_fd_wrapped_fd(node->next->fd) == fd) {
-      fd_node* ret = node->next;
-      node->next = node->next->next;
-      *head = dummy_head.next;
-      return ret;
-    }
-    node = node->next;
+  bool IsFdStillReadableLocked() override {
+    size_t bytes_available = 0;
+    return ioctl(grpc_fd_wrapped_fd(fd_), FIONREAD, &bytes_available) == 0 &&
+           bytes_available > 0;
   }
-  return nullptr;
-}
 
-/* Check if \a fd is still readable */
-static bool grpc_ares_is_fd_still_readable(grpc_ares_ev_driver* ev_driver,
-                                           int fd) {
-  size_t bytes_available = 0;
-  return ioctl(fd, FIONREAD, &bytes_available) == 0 && bytes_available > 0;
-}
-
-static void on_readable_cb(void* arg, grpc_error* error) {
-  fd_node* fdn = static_cast<fd_node*>(arg);
-  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
-  gpr_mu_lock(&fdn->mu);
-  const int fd = grpc_fd_wrapped_fd(fdn->fd);
-  fdn->readable_registered = false;
-  if (fdn->shutting_down && !fdn->writable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-    grpc_ares_ev_driver_unref(ev_driver);
-    return;
+  void ShutdownLocked(grpc_error* error) override {
+    grpc_fd_shutdown(fd_, error);
   }
-  gpr_mu_unlock(&fdn->mu);
 
-  gpr_log(GPR_DEBUG, "readable on %d", fd);
-  if (error == GRPC_ERROR_NONE) {
-    do {
-      ares_process_fd(ev_driver->channel, fd, ARES_SOCKET_BAD);
-    } while (grpc_ares_is_fd_still_readable(ev_driver, fd));
-  } else {
-    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
-    // timed out. The pending lookups made on this ev_driver will be cancelled
-    // by the following ares_cancel() and the on_done callbacks will be invoked
-    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
-    // ev_driver will be cleaned up in the follwing
-    // grpc_ares_notify_on_event_locked().
-    ares_cancel(ev_driver->channel);
-  }
-  gpr_mu_lock(&ev_driver->mu);
-  grpc_ares_notify_on_event_locked(ev_driver);
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
+  ares_socket_t GetWrappedAresSocketLocked() override { return as_; }
 
-static void on_writable_cb(void* arg, grpc_error* error) {
-  fd_node* fdn = static_cast<fd_node*>(arg);
-  grpc_ares_ev_driver* ev_driver = fdn->ev_driver;
-  gpr_mu_lock(&fdn->mu);
-  const int fd = grpc_fd_wrapped_fd(fdn->fd);
-  fdn->writable_registered = false;
-  if (fdn->shutting_down && !fdn->readable_registered) {
-    gpr_mu_unlock(&fdn->mu);
-    fd_node_destroy(fdn);
-    grpc_ares_ev_driver_unref(ev_driver);
-    return;
-  }
-  gpr_mu_unlock(&fdn->mu);
+  const char* GetName() override { return name_; }
+
+  char* name_;
+  ares_socket_t as_;
+  grpc_fd* fd_;
+  grpc_pollset_set* driver_pollset_set_;
+};
 
-  gpr_log(GPR_DEBUG, "writable on %d", fd);
-  if (error == GRPC_ERROR_NONE) {
-    ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, fd);
-  } else {
-    // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or
-    // timed out. The pending lookups made on this ev_driver will be cancelled
-    // by the following ares_cancel() and the on_done callbacks will be invoked
-    // with a status of ARES_ECANCELLED. The remaining file descriptors in this
-    // ev_driver will be cleaned up in the follwing
-    // grpc_ares_notify_on_event_locked().
-    ares_cancel(ev_driver->channel);
+class GrpcPolledFdFactoryPosix : public GrpcPolledFdFactory {
+ public:
+  GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as,
+                                      grpc_pollset_set* driver_pollset_set,
+                                      grpc_combiner* combiner) override {
+    return New<GrpcPolledFdPosix>(as, driver_pollset_set);
   }
-  gpr_mu_lock(&ev_driver->mu);
-  grpc_ares_notify_on_event_locked(ev_driver);
-  gpr_mu_unlock(&ev_driver->mu);
-  grpc_ares_ev_driver_unref(ev_driver);
-}
 
-ares_channel* grpc_ares_ev_driver_get_channel(grpc_ares_ev_driver* ev_driver) {
-  return &ev_driver->channel;
-}
+  void ConfigureAresChannelLocked(ares_channel channel) override {}
+};
 
-// Get the file descriptors used by the ev_driver's ares channel, register
-// driver_closure with these filedescriptors.
-static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) {
-  fd_node* new_list = nullptr;
-  if (!ev_driver->shutting_down) {
-    ares_socket_t socks[ARES_GETSOCK_MAXNUM];
-    int socks_bitmask =
-        ares_getsock(ev_driver->channel, socks, ARES_GETSOCK_MAXNUM);
-    for (size_t i = 0; i < ARES_GETSOCK_MAXNUM; i++) {
-      if (ARES_GETSOCK_READABLE(socks_bitmask, i) ||
-          ARES_GETSOCK_WRITABLE(socks_bitmask, i)) {
-        fd_node* fdn = pop_fd_node(&ev_driver->fds, socks[i]);
-        // Create a new fd_node if sock[i] is not in the fd_node list.
-        if (fdn == nullptr) {
-          char* fd_name;
-          gpr_asprintf(&fd_name, "ares_ev_driver-%" PRIuPTR, i);
-          fdn = static_cast<fd_node*>(gpr_malloc(sizeof(fd_node)));
-          gpr_log(GPR_DEBUG, "new fd: %d", socks[i]);
-          fdn->fd = grpc_fd_create(socks[i], fd_name);
-          fdn->ev_driver = ev_driver;
-          fdn->readable_registered = false;
-          fdn->writable_registered = false;
-          fdn->shutting_down = false;
-          gpr_mu_init(&fdn->mu);
-          GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_cb, fdn,
-                            grpc_schedule_on_exec_ctx);
-          GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_cb, fdn,
-                            grpc_schedule_on_exec_ctx);
-          grpc_pollset_set_add_fd(ev_driver->pollset_set, fdn->fd);
-          gpr_free(fd_name);
-        }
-        fdn->next = new_list;
-        new_list = fdn;
-        gpr_mu_lock(&fdn->mu);
-        // Register read_closure if the socket is readable and read_closure has
-        // not been registered with this socket.
-        if (ARES_GETSOCK_READABLE(socks_bitmask, i) &&
-            !fdn->readable_registered) {
-          grpc_ares_ev_driver_ref(ev_driver);
-          gpr_log(GPR_DEBUG, "notify read on: %d", grpc_fd_wrapped_fd(fdn->fd));
-          grpc_fd_notify_on_read(fdn->fd, &fdn->read_closure);
-          fdn->readable_registered = true;
-        }
-        // Register write_closure if the socket is writable and write_closure
-        // has not been registered with this socket.
-        if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) &&
-            !fdn->writable_registered) {
-          gpr_log(GPR_DEBUG, "notify write on: %d",
-                  grpc_fd_wrapped_fd(fdn->fd));
-          grpc_ares_ev_driver_ref(ev_driver);
-          grpc_fd_notify_on_write(fdn->fd, &fdn->write_closure);
-          fdn->writable_registered = true;
-        }
-        gpr_mu_unlock(&fdn->mu);
-      }
-    }
-  }
-  // Any remaining fds in ev_driver->fds were not returned by ares_getsock() and
-  // are therefore no longer in use, so they can be shut down and removed from
-  // the list.
-  while (ev_driver->fds != nullptr) {
-    fd_node* cur = ev_driver->fds;
-    ev_driver->fds = ev_driver->fds->next;
-    fd_node_shutdown(cur);
-  }
-  ev_driver->fds = new_list;
-  // If the ev driver has no working fd, all the tasks are done.
-  if (new_list == nullptr) {
-    ev_driver->working = false;
-    gpr_log(GPR_DEBUG, "ev driver stop working");
-  }
+UniquePtr<GrpcPolledFdFactory> NewGrpcPolledFdFactory(grpc_combiner* combiner) {
+  return UniquePtr<GrpcPolledFdFactory>(New<GrpcPolledFdFactoryPosix>());
 }
 
-void grpc_ares_ev_driver_start(grpc_ares_ev_driver* ev_driver) {
-  gpr_mu_lock(&ev_driver->mu);
-  if (!ev_driver->working) {
-    ev_driver->working = true;
-    grpc_ares_notify_on_event_locked(ev_driver);
-  }
-  gpr_mu_unlock(&ev_driver->mu);
-}
+}  // namespace grpc_core
 
-#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) */
+#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */