grpc 1.10.0 → 1.11.0.pre2

data/src/core/lib/iomgr/ev_epoll1_linux.h

@@ -19,6 +19,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H
 #define GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/port.h"
 

data/src/core/lib/iomgr/ev_epollex_linux.cc

@@ -16,6 +16,8 @@
  *
  */
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/port.h"
 
 #include <grpc/support/log.h>
@@ -57,7 +59,7 @@
 //#define GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP 1
 
 #define MAX_EPOLL_EVENTS 100
-#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 16
+#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 1
 
 grpc_core::DebugOnlyTraceFlag grpc_trace_pollable_refcount(false,
                                                            "pollable_refcount");
@@ -196,7 +198,6 @@ struct grpc_pollset_worker {
 
 struct grpc_pollset {
   gpr_mu mu;
-  gpr_atm worker_count;
   pollable* active_pollable;
   bool kicked_without_poller;
   grpc_closure* shutdown_closure;
@@ -684,7 +685,6 @@ static grpc_error* pollset_kick_all(grpc_pollset* pollset) {
 
 static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
   gpr_mu_init(&pollset->mu);
-  gpr_atm_no_barrier_store(&pollset->worker_count, 0);
   pollset->active_pollable = POLLABLE_REF(g_empty_pollable, "pollset");
   pollset->kicked_without_poller = false;
   pollset->shutdown_closure = nullptr;
@@ -758,20 +758,8 @@ static grpc_error* pollable_process_events(grpc_pollset* pollset,
                                            pollable* pollable_obj, bool drain) {
   GPR_TIMER_SCOPE("pollable_process_events", 0);
   static const char* err_desc = "pollset_process_events";
-  // Use a simple heuristic to determine how many fd events to process
-  // per loop iteration.  (events/workers)
-  int handle_count = 1;
-  int worker_count = gpr_atm_no_barrier_load(&pollset->worker_count);
-  GPR_ASSERT(worker_count > 0);
-  handle_count =
-      (pollable_obj->event_count - pollable_obj->event_cursor) / worker_count;
-  if (handle_count == 0) {
-    handle_count = 1;
-  } else if (handle_count > MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) {
-    handle_count = MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL;
-  }
   grpc_error* error = GRPC_ERROR_NONE;
-  for (int i = 0; (drain || i < handle_count) &&
+  for (int i = 0; (drain || i < MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) &&
                   pollable_obj->event_cursor != pollable_obj->event_count;
        i++) {
     int n = pollable_obj->event_cursor++;
@@ -896,7 +884,6 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker,
   GPR_TIMER_SCOPE("begin_worker", 0);
   bool do_poll =
       (pollset->shutdown_closure == nullptr && !pollset->already_shutdown);
-  gpr_atm_no_barrier_fetch_add(&pollset->worker_count, 1);
   if (worker_hdl != nullptr) *worker_hdl = worker;
   worker->initialized_cv = false;
   worker->kicked = false;
@@ -977,7 +964,6 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker,
   if (worker->initialized_cv) {
     gpr_cv_destroy(&worker->cv);
   }
-  gpr_atm_no_barrier_fetch_add(&pollset->worker_count, -1);
 }
 
 #ifndef NDEBUG

data/src/core/lib/iomgr/ev_epollex_linux.h

@@ -19,6 +19,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H
 #define GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/port.h"
 

data/src/core/lib/iomgr/ev_epollsig_linux.cc

@@ -16,6 +16,8 @@
  *
  */
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/port.h"
 
 #include <grpc/grpc_posix.h>

data/src/core/lib/iomgr/ev_epollsig_linux.h

@@ -19,6 +19,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H
 #define GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/port.h"
 

data/src/core/lib/iomgr/ev_poll_posix.cc

@@ -16,6 +16,8 @@
  *
  */
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/port.h"
 
 #ifdef GRPC_POSIX_SOCKET
@@ -36,9 +38,9 @@
 
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/murmur_hash.h"
-#include "src/core/lib/gpr/thd.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/thd.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/wakeup_fd_cv.h"
@@ -175,7 +177,6 @@ struct grpc_pollset {
   int called_shutdown;
   int kicked_without_pollers;
   grpc_closure* shutdown_done;
-  grpc_closure_list idle_jobs;
   int pollset_set_count;
   /* all polled fds */
   size_t fd_count;
@@ -253,8 +254,13 @@ typedef struct poll_result {
 } poll_result;
 
 typedef struct poll_args {
+  grpc_core::Thread poller_thd;
   gpr_cv trigger;
   int trigger_set;
+  bool harvestable;
+  gpr_cv harvest;
+  bool joinable;
+  gpr_cv join;
   struct pollfd* fds;
   nfds_t nfds;
   poll_result* result;
@@ -264,15 +270,17 @@ typedef struct poll_args {
 
 // This is a 2-tiered cache, we mantain a hash table
 // of active poll calls, so we can wait on the result
-// of that call.  We also maintain a freelist of inactive
-// poll threads.
+// of that call.  We also maintain freelists of inactive
+// poll args and of dead poller threads.
 typedef struct poll_hash_table {
   poll_args* free_pollers;
   poll_args** active_pollers;
+  poll_args* dead_pollers;
   unsigned int size;
   unsigned int count;
 } poll_hash_table;
 
+// TODO(kpayson64): Eliminate use of global non-POD variables
 poll_hash_table poll_cache;
 grpc_cv_fd_table g_cvfds;
 
@@ -803,7 +811,6 @@ static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
-  pollset->idle_jobs.head = pollset->idle_jobs.tail = nullptr;
   pollset->local_wakeup_cache = nullptr;
   pollset->kicked_without_pollers = 0;
   pollset->fd_count = 0;
@@ -814,7 +821,6 @@ static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
 
 static void pollset_destroy(grpc_pollset* pollset) {
   GPR_ASSERT(!pollset_has_workers(pollset));
-  GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   while (pollset->local_wakeup_cache) {
     grpc_cached_wakeup_fd* next = pollset->local_wakeup_cache->next;
     grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd);
@@ -846,7 +852,6 @@ exit:
 }
 
 static void finish_shutdown(grpc_pollset* pollset) {
-  GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs));
   size_t i;
   for (i = 0; i < pollset->fd_count; i++) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
@@ -867,7 +872,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
                                 grpc_pollset_worker** worker_hdl,
                                 grpc_millis deadline) {
   GPR_TIMER_SCOPE("pollset_work", 0);
-
   grpc_pollset_worker worker;
   if (worker_hdl) *worker_hdl = &worker;
   grpc_error* error = GRPC_ERROR_NONE;
@@ -898,14 +902,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
     }
   }
   worker.kicked_specifically = 0;
-  /* If there's work waiting for the pollset to be idle, and the
-     pollset is idle, then do that work */
-  if (!pollset_has_workers(pollset) &&
-      !grpc_closure_list_empty(pollset->idle_jobs)) {
-    GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
-    GRPC_CLOSURE_LIST_SCHED(&pollset->idle_jobs);
-    goto done;
-  }
   /* If we're shutting down then we don't execute any extended work */
   if (pollset->shutting_down) {
     GPR_TIMER_MARK("pollset_work.shutting_down", 0);
@@ -918,7 +914,8 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
   gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
   while (keep_polling) {
     keep_polling = 0;
-    if (!pollset->kicked_without_pollers) {
+    if (!pollset->kicked_without_pollers ||
+        deadline <= grpc_core::ExecCtx::Get()->Now()) {
       if (!added_worker) {
         push_front_worker(pollset, &worker);
         added_worker = 1;
@@ -1092,11 +1089,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
        * pollset_work.
        * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
       gpr_mu_lock(&pollset->mu);
-    } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
-      GRPC_CLOSURE_LIST_SCHED(&pollset->idle_jobs);
-      gpr_mu_unlock(&pollset->mu);
-      grpc_core::ExecCtx::Get()->Flush();
-      gpr_mu_lock(&pollset->mu);
     }
   }
   if (worker_hdl) *worker_hdl = nullptr;
@@ -1109,9 +1101,6 @@ static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
   pollset->shutting_down = 1;
   pollset->shutdown_done = closure;
   pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
-  if (!pollset_has_workers(pollset)) {
-    GRPC_CLOSURE_LIST_SCHED(&pollset->idle_jobs);
-  }
   if (!pollset->called_shutdown && !pollset_has_observers(pollset)) {
     pollset->called_shutdown = 1;
     finish_shutdown(pollset);
@@ -1299,6 +1288,7 @@ static void pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) {
 
 static void run_poll(void* args);
 static void cache_poller_locked(poll_args* args);
+static void cache_harvest_locked();
 
 static void cache_insert_locked(poll_args* args) {
   uint32_t key = gpr_murmur_hash3(args->fds, args->nfds * sizeof(struct pollfd),
@@ -1361,6 +1351,10 @@ static poll_args* get_poller_locked(struct pollfd* fds, nfds_t count) {
   poll_args* pargs =
       static_cast<poll_args*>(gpr_malloc(sizeof(struct poll_args)));
   gpr_cv_init(&pargs->trigger);
+  gpr_cv_init(&pargs->harvest);
+  gpr_cv_init(&pargs->join);
+  pargs->harvestable = false;
+  pargs->joinable = false;
   pargs->fds = fds;
   pargs->nfds = count;
   pargs->next = nullptr;
@@ -1368,11 +1362,9 @@ static poll_args* get_poller_locked(struct pollfd* fds, nfds_t count) {
   pargs->trigger_set = 0;
   init_result(pargs);
   cache_poller_locked(pargs);
-  gpr_thd_id t_id;
-  gpr_thd_options opt = gpr_thd_options_default();
   gpr_ref(&g_cvfds.pollcount);
-  gpr_thd_options_set_detached(&opt);
-  GPR_ASSERT(gpr_thd_new(&t_id, "grpc_poller", &run_poll, pargs, &opt));
+  pargs->poller_thd = grpc_core::Thread("grpc_poller", &run_poll, pargs);
+  pargs->poller_thd.Start();
   return pargs;
 }
 
@@ -1437,7 +1429,33 @@ static void cache_destroy_locked(poll_args* args) {
     poll_cache.free_pollers = args->next;
   }
 
-  gpr_free(args);
+  // Now move this args to the dead poller list for later join
+  if (poll_cache.dead_pollers != nullptr) {
+    poll_cache.dead_pollers->prev = args;
+  }
+  args->prev = nullptr;
+  args->next = poll_cache.dead_pollers;
+  poll_cache.dead_pollers = args;
+}
+
+static void cache_harvest_locked() {
+  while (poll_cache.dead_pollers) {
+    poll_args* args = poll_cache.dead_pollers;
+    poll_cache.dead_pollers = poll_cache.dead_pollers->next;
+    // Keep the list consistent in case new dead pollers get added when we
+    // release the lock below to wait on joining
+    if (poll_cache.dead_pollers) {
+      poll_cache.dead_pollers->prev = nullptr;
+    }
+    args->harvestable = true;
+    gpr_cv_signal(&args->harvest);
+    while (!args->joinable) {
+      gpr_cv_wait(&args->join, &g_cvfds.mu,
+                  gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    }
+    args->poller_thd.Join();
+    gpr_free(args);
+  }
 }
 
 static void decref_poll_result(poll_result* res) {
@@ -1469,6 +1487,7 @@ static void run_poll(void* args) {
     poll_result* result = pargs->result;
     int retval = g_cvfds.poll(result->fds, result->nfds, CV_POLL_PERIOD_MS);
     gpr_mu_lock(&g_cvfds.mu);
+    cache_harvest_locked();
     if (retval != 0) {
       result->completed = 1;
       result->retval = retval;
@@ -1488,6 +1507,7 @@ static void run_poll(void* args) {
       deadline = gpr_time_add(deadline, thread_grace);
       pargs->trigger_set = 0;
       gpr_cv_wait(&pargs->trigger, &g_cvfds.mu, deadline);
+      cache_harvest_locked();
       if (!pargs->trigger_set) {
         cache_destroy_locked(pargs);
         break;
@@ -1496,10 +1516,15 @@ static void run_poll(void* args) {
     gpr_mu_unlock(&g_cvfds.mu);
   }
 
-  // We still have the lock here
   if (gpr_unref(&g_cvfds.pollcount)) {
     gpr_cv_signal(&g_cvfds.shutdown_cv);
   }
+  while (!pargs->harvestable) {
+    gpr_cv_wait(&pargs->harvest, &g_cvfds.mu,
+                gpr_inf_future(GPR_CLOCK_MONOTONIC));
+  }
+  pargs->joinable = true;
+  gpr_cv_signal(&pargs->join);
   gpr_mu_unlock(&g_cvfds.mu);
 }
 
@@ -1512,6 +1537,7 @@ static int cvfd_poll(struct pollfd* fds, nfds_t nfds, int timeout) {
   nfds_t nsockfds = 0;
   poll_result* result = nullptr;
   gpr_mu_lock(&g_cvfds.mu);
+  cache_harvest_locked();
   pollcv = static_cast<grpc_cv_node*>(gpr_malloc(sizeof(grpc_cv_node)));
   pollcv->next = nullptr;
   gpr_cv pollcv_cv;
@@ -1575,12 +1601,14 @@ static int cvfd_poll(struct pollfd* fds, nfds_t nfds, int timeout) {
     pargs->trigger_set = 1;
     gpr_cv_signal(&pargs->trigger);
     gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
+    cache_harvest_locked();
     res = result->retval;
     errno = result->err;
     result->watchcount--;
     remove_cvn(&result->watchers, pollcv);
   } else if (!skip_poll) {
     gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline);
+    cache_harvest_locked();
   }
 
   idx = 0;
@@ -1637,6 +1665,7 @@ static void global_cv_fd_table_init() {
   for (unsigned int i = 0; i < poll_cache.size; i++) {
     poll_cache.active_pollers[i] = nullptr;
   }
+  poll_cache.dead_pollers = nullptr;
 
   gpr_mu_unlock(&g_cvfds.mu);
 }
@@ -1655,6 +1684,7 @@ static void global_cv_fd_table_shutdown() {
   grpc_poll_function = g_cvfds.poll;
   gpr_free(g_cvfds.cvfds);
 
+  cache_harvest_locked();
   gpr_free(poll_cache.active_pollers);
 
   gpr_mu_unlock(&g_cvfds.mu);

data/src/core/lib/iomgr/ev_poll_posix.h

@@ -19,6 +19,8 @@
 #ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H
 #define GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/ev_posix.h"
 
 const grpc_event_engine_vtable* grpc_init_poll_posix(bool explicit_request);

data/src/core/lib/iomgr/ev_posix.cc

@@ -16,6 +16,8 @@
  *
  */
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/iomgr/port.h"
 
 #ifdef GRPC_POSIX_SOCKET
@@ -222,26 +224,26 @@ void grpc_fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) {
   g_event_engine->fd_notify_on_write(fd, closure);
 }
 
-size_t grpc_pollset_size(void) { return g_event_engine->pollset_size; }
+static size_t pollset_size(void) { return g_event_engine->pollset_size; }
 
-void grpc_pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
+static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
   GRPC_POLLING_API_TRACE("pollset_init(%p)", pollset);
   g_event_engine->pollset_init(pollset, mu);
 }
 
-void grpc_pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
+static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
   GRPC_POLLING_API_TRACE("pollset_shutdown(%p)", pollset);
   g_event_engine->pollset_shutdown(pollset, closure);
 }
 
-void grpc_pollset_destroy(grpc_pollset* pollset) {
+static void pollset_destroy(grpc_pollset* pollset) {
   GRPC_POLLING_API_TRACE("pollset_destroy(%p)", pollset);
   g_event_engine->pollset_destroy(pollset);
 }
 
-grpc_error* grpc_pollset_work(grpc_pollset* pollset,
-                              grpc_pollset_worker** worker,
-                              grpc_millis deadline) {
+static grpc_error* pollset_work(grpc_pollset* pollset,
+                                grpc_pollset_worker** worker,
+                                grpc_millis deadline) {
   GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRIdPTR ") begin", pollset,
                          deadline);
   grpc_error* err = g_event_engine->pollset_work(pollset, worker, deadline);
@@ -250,8 +252,8 @@ grpc_error* grpc_pollset_work(grpc_pollset* pollset,
   return err;
 }
 
-grpc_error* grpc_pollset_kick(grpc_pollset* pollset,
-                              grpc_pollset_worker* specific_worker) {
+static grpc_error* pollset_kick(grpc_pollset* pollset,
+                                grpc_pollset_worker* specific_worker) {
   GRPC_POLLING_API_TRACE("pollset_kick(%p, %p)", pollset, specific_worker);
   return g_event_engine->pollset_kick(pollset, specific_worker);
 }
@@ -262,43 +264,57 @@ void grpc_pollset_add_fd(grpc_pollset* pollset, struct grpc_fd* fd) {
   g_event_engine->pollset_add_fd(pollset, fd);
 }
 
-grpc_pollset_set* grpc_pollset_set_create(void) {
+void pollset_global_init() {}
+void pollset_global_shutdown() {}
+
+grpc_pollset_vtable grpc_posix_pollset_vtable = {
+    pollset_global_init, pollset_global_shutdown,
+    pollset_init,        pollset_shutdown,
+    pollset_destroy,     pollset_work,
+    pollset_kick,        pollset_size};
+
+static grpc_pollset_set* pollset_set_create(void) {
   grpc_pollset_set* pss = g_event_engine->pollset_set_create();
   GRPC_POLLING_API_TRACE("pollset_set_create(%p)", pss);
   return pss;
 }
 
-void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {
+static void pollset_set_destroy(grpc_pollset_set* pollset_set) {
   GRPC_POLLING_API_TRACE("pollset_set_destroy(%p)", pollset_set);
   g_event_engine->pollset_set_destroy(pollset_set);
 }
 
-void grpc_pollset_set_add_pollset(grpc_pollset_set* pollset_set,
-                                  grpc_pollset* pollset) {
+static void pollset_set_add_pollset(grpc_pollset_set* pollset_set,
+                                    grpc_pollset* pollset) {
   GRPC_POLLING_API_TRACE("pollset_set_add_pollset(%p, %p)", pollset_set,
                          pollset);
   g_event_engine->pollset_set_add_pollset(pollset_set, pollset);
 }
 
-void grpc_pollset_set_del_pollset(grpc_pollset_set* pollset_set,
-                                  grpc_pollset* pollset) {
+static void pollset_set_del_pollset(grpc_pollset_set* pollset_set,
+                                    grpc_pollset* pollset) {
   GRPC_POLLING_API_TRACE("pollset_set_del_pollset(%p, %p)", pollset_set,
                          pollset);
   g_event_engine->pollset_set_del_pollset(pollset_set, pollset);
 }
 
-void grpc_pollset_set_add_pollset_set(grpc_pollset_set* bag,
-                                      grpc_pollset_set* item) {
+static void pollset_set_add_pollset_set(grpc_pollset_set* bag,
+                                        grpc_pollset_set* item) {
   GRPC_POLLING_API_TRACE("pollset_set_add_pollset_set(%p, %p)", bag, item);
   g_event_engine->pollset_set_add_pollset_set(bag, item);
 }
 
-void grpc_pollset_set_del_pollset_set(grpc_pollset_set* bag,
-                                      grpc_pollset_set* item) {
+static void pollset_set_del_pollset_set(grpc_pollset_set* bag,
+                                        grpc_pollset_set* item) {
   GRPC_POLLING_API_TRACE("pollset_set_del_pollset_set(%p, %p)", bag, item);
   g_event_engine->pollset_set_del_pollset_set(bag, item);
 }
 
+grpc_pollset_set_vtable grpc_posix_pollset_set_vtable = {
+    pollset_set_create,          pollset_set_destroy,
+    pollset_set_add_pollset,     pollset_set_del_pollset,
+    pollset_set_add_pollset_set, pollset_set_del_pollset_set};
+
 void grpc_pollset_set_add_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) {
   GRPC_POLLING_API_TRACE("pollset_set_add_fd(%p, %d)", pollset_set,
                          grpc_fd_wrapped_fd(fd));