grpc 1.13.0 → 1.14.0

data/src/core/lib/iomgr/ev_epollsig_linux.cc

@@ -132,6 +132,7 @@ struct grpc_fd {
 
   grpc_core::ManualConstructor<grpc_core::LockfreeEvent> read_closure;
   grpc_core::ManualConstructor<grpc_core::LockfreeEvent> write_closure;
+  grpc_core::ManualConstructor<grpc_core::LockfreeEvent> error_closure;
 
   struct grpc_fd* freelist_next;
   grpc_closure* on_done_closure;
@@ -141,6 +142,9 @@ struct grpc_fd {
   gpr_atm read_notifier_pollset;
 
   grpc_iomgr_object iomgr_object;
+
+  /* Do we need to track EPOLLERR events separately? */
+  bool track_err;
 };
 
 /* Reference counting for fds */
@@ -352,7 +356,10 @@ static void polling_island_add_fds_locked(polling_island* pi, grpc_fd** fds,
 
   for (i = 0; i < fd_count; i++) {
     ev.events = static_cast<uint32_t>(EPOLLIN | EPOLLOUT | EPOLLET);
-    ev.data.ptr = fds[i];
+    /* Use the least significant bit of ev.data.ptr to store track_err to avoid
+     * synchronization issues when accessing it after receiving an event */
+    ev.data.ptr = reinterpret_cast<void*>(reinterpret_cast<intptr_t>(fds[i]) |
+                                          (fds[i]->track_err ? 1 : 0));
     err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev);
 
     if (err < 0) {
@@ -435,7 +442,6 @@ static void polling_island_remove_all_fds_locked(polling_island* pi,
 
 /* The caller is expected to hold pi->mu lock before calling this function */
 static void polling_island_remove_fd_locked(polling_island* pi, grpc_fd* fd,
-                                            bool is_fd_closed,
                                             grpc_error** error) {
   int err;
   size_t i;
@@ -444,16 +450,14 @@ static void polling_island_remove_fd_locked(polling_island* pi, grpc_fd* fd,
 
   /* If fd is already closed, then it would have been automatically been removed
      from the epoll set */
-  if (!is_fd_closed) {
-    err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, nullptr);
-    if (err < 0 && errno != ENOENT) {
-      gpr_asprintf(
-          &err_msg,
-          "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)",
-          pi->epoll_fd, fd->fd, errno, strerror(errno));
-      append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
-      gpr_free(err_msg);
-    }
+  err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, nullptr);
+  if (err < 0 && errno != ENOENT) {
+    gpr_asprintf(
+        &err_msg,
+        "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)",
+        pi->epoll_fd, fd->fd, errno, strerror(errno));
+    append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+    gpr_free(err_msg);
   }
 
   for (i = 0; i < pi->fd_cnt; i++) {
@@ -769,6 +773,7 @@ static void unref_by(grpc_fd* fd, int n) {
 
     fd->read_closure->DestroyEvent();
     fd->write_closure->DestroyEvent();
+    fd->error_closure->DestroyEvent();
 
     gpr_mu_unlock(&fd_freelist_mu);
   } else {
@@ -806,7 +811,7 @@ static void fd_global_shutdown(void) {
   gpr_mu_destroy(&fd_freelist_mu);
 }
 
-static grpc_fd* fd_create(int fd, const char* name) {
+static grpc_fd* fd_create(int fd, const char* name, bool track_err) {
   grpc_fd* new_fd = nullptr;
 
   gpr_mu_lock(&fd_freelist_mu);
@@ -821,6 +826,7 @@ static grpc_fd* fd_create(int fd, const char* name) {
     gpr_mu_init(&new_fd->po.mu);
     new_fd->read_closure.Init();
     new_fd->write_closure.Init();
+    new_fd->error_closure.Init();
   }
 
   /* Note: It is not really needed to get the new_fd->po.mu lock here. If this
@@ -837,6 +843,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   new_fd->orphaned = false;
   new_fd->read_closure->InitEvent();
   new_fd->write_closure->InitEvent();
+  new_fd->error_closure->InitEvent();
+  new_fd->track_err = track_err;
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;
@@ -863,7 +871,7 @@ static int fd_wrapped_fd(grpc_fd* fd) {
 }
 
 static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
-                      bool already_closed, const char* reason) {
+                      const char* reason) {
   grpc_error* error = GRPC_ERROR_NONE;
   polling_island* unref_pi = nullptr;
 
@@ -884,7 +892,7 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
        before doing this.) */
   if (fd->po.pi != nullptr) {
     polling_island* pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
+    polling_island_remove_fd_locked(pi_latest, fd, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
@@ -933,6 +941,7 @@ static void fd_shutdown(grpc_fd* fd, grpc_error* why) {
   if (fd->read_closure->SetShutdown(GRPC_ERROR_REF(why))) {
     shutdown(fd->fd, SHUT_RDWR);
     fd->write_closure->SetShutdown(GRPC_ERROR_REF(why));
+    fd->error_closure->SetShutdown(GRPC_ERROR_REF(why));
   }
   GRPC_ERROR_UNREF(why);
 }
@@ -945,6 +954,10 @@ static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) {
   fd->write_closure->NotifyOn(closure);
 }
 
+static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) {
+  fd->error_closure->NotifyOn(closure);
+}
+
 /*******************************************************************************
  * Pollset Definitions
  */
@@ -1116,6 +1129,8 @@ static void fd_become_readable(grpc_fd* fd, grpc_pollset* notifier) {
 
 static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); }
 
+static void fd_has_errors(grpc_fd* fd) { fd->error_closure->SetReady(); }
+
 static void pollset_release_polling_island(grpc_pollset* ps,
                                            const char* reason) {
   if (ps->po.pi != nullptr) {
@@ -1254,14 +1269,23 @@ static void pollset_work_and_unlock(grpc_pollset* pollset,
          to the function pollset_work_and_unlock() will pick up the correct
          epoll_fd */
     } else {
-      grpc_fd* fd = static_cast<grpc_fd*>(data_ptr);
-      int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
-      int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
-      int write_ev = ep_ev[i].events & EPOLLOUT;
-      if (read_ev || cancel) {
+      grpc_fd* fd = reinterpret_cast<grpc_fd*>(
+          reinterpret_cast<intptr_t>(data_ptr) & ~static_cast<intptr_t>(1));
+      bool track_err =
+          reinterpret_cast<intptr_t>(data_ptr) & ~static_cast<intptr_t>(1);
+      bool cancel = (ep_ev[i].events & EPOLLHUP) != 0;
+      bool error = (ep_ev[i].events & EPOLLERR) != 0;
+      bool read_ev = (ep_ev[i].events & (EPOLLIN | EPOLLPRI)) != 0;
+      bool write_ev = (ep_ev[i].events & EPOLLOUT) != 0;
+      bool err_fallback = error && !track_err;
+
+      if (error && !err_fallback) {
+        fd_has_errors(fd);
+      }
+      if (read_ev || cancel || err_fallback) {
         fd_become_readable(fd, pollset);
       }
-      if (write_ev || cancel) {
+      if (write_ev || cancel || err_fallback) {
         fd_become_writable(fd);
       }
     }
@@ -1634,6 +1658,7 @@ static void shutdown_engine(void) {
 
 static const grpc_event_engine_vtable vtable = {
     sizeof(grpc_pollset),
+    true,
 
     fd_create,
     fd_wrapped_fd,
@@ -1641,6 +1666,7 @@ static const grpc_event_engine_vtable vtable = {
     fd_shutdown,
     fd_notify_on_read,
     fd_notify_on_write,
+    fd_notify_on_error,
     fd_is_shutdown,
     fd_get_read_notifier_pollset,
 

data/src/core/lib/iomgr/ev_poll_posix.cc

@@ -330,7 +330,8 @@ static void unref_by(grpc_fd* fd, int n) {
   }
 }
 
-static grpc_fd* fd_create(int fd, const char* name) {
+static grpc_fd* fd_create(int fd, const char* name, bool track_err) {
+  GPR_DEBUG_ASSERT(track_err == false);
   grpc_fd* r = static_cast<grpc_fd*>(gpr_malloc(sizeof(*r)));
   gpr_mu_init(&r->mu);
   gpr_atm_rel_store(&r->refst, 1);
@@ -424,14 +425,12 @@ static int fd_wrapped_fd(grpc_fd* fd) {
 }
 
 static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
-                      bool already_closed, const char* reason) {
+                      const char* reason) {
   fd->on_done_closure = on_done;
   fd->released = release_fd != nullptr;
   if (release_fd != nullptr) {
     *release_fd = fd->fd;
     fd->released = true;
-  } else if (already_closed) {
-    fd->released = true;
   }
   gpr_mu_lock(&fd->mu);
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
@@ -553,6 +552,11 @@ static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) {
   gpr_mu_unlock(&fd->mu);
 }
 
+static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) {
+  gpr_log(GPR_ERROR, "Polling engine does not support tracking errors.");
+  abort();
+}
+
 static uint32_t fd_begin_poll(grpc_fd* fd, grpc_pollset* pollset,
                               grpc_pollset_worker* worker, uint32_t read_mask,
                               uint32_t write_mask, grpc_fd_watcher* watcher) {
@@ -1710,6 +1714,7 @@ static void shutdown_engine(void) {
 
 static const grpc_event_engine_vtable vtable = {
     sizeof(grpc_pollset),
+    false,
 
     fd_create,
     fd_wrapped_fd,
@@ -1717,6 +1722,7 @@ static const grpc_event_engine_vtable vtable = {
     fd_shutdown,
     fd_notify_on_read,
     fd_notify_on_write,
+    fd_notify_on_error,
     fd_is_shutdown,
     fd_get_read_notifier_pollset,
 

data/src/core/lib/iomgr/ev_posix.cc

@@ -193,10 +193,15 @@ void grpc_event_engine_shutdown(void) {
   g_event_engine = nullptr;
 }
 
-grpc_fd* grpc_fd_create(int fd, const char* name) {
-  GRPC_POLLING_API_TRACE("fd_create(%d, %s)", fd, name);
-  GRPC_FD_TRACE("fd_create(%d, %s)", fd, name);
-  return g_event_engine->fd_create(fd, name);
+bool grpc_event_engine_can_track_errors(void) {
+  return g_event_engine->can_track_err;
+}
+
+grpc_fd* grpc_fd_create(int fd, const char* name, bool track_err) {
+  GRPC_POLLING_API_TRACE("fd_create(%d, %s, %d)", fd, name, track_err);
+  GRPC_FD_TRACE("fd_create(%d, %s, %d)", fd, name, track_err);
+  GPR_DEBUG_ASSERT(!track_err || g_event_engine->can_track_err);
+  return g_event_engine->fd_create(fd, name, track_err);
 }
 
 int grpc_fd_wrapped_fd(grpc_fd* fd) {
@@ -204,13 +209,12 @@ int grpc_fd_wrapped_fd(grpc_fd* fd) {
 }
 
 void grpc_fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
-                    bool already_closed, const char* reason) {
-  GRPC_POLLING_API_TRACE("fd_orphan(%d, %p, %p, %d, %s)",
-                         grpc_fd_wrapped_fd(fd), on_done, release_fd,
-                         already_closed, reason);
+                    const char* reason) {
+  GRPC_POLLING_API_TRACE("fd_orphan(%d, %p, %p, %s)", grpc_fd_wrapped_fd(fd),
+                         on_done, release_fd, reason);
   GRPC_FD_TRACE("grpc_fd_orphan, fd:%d closed", grpc_fd_wrapped_fd(fd));
 
-  g_event_engine->fd_orphan(fd, on_done, release_fd, already_closed, reason);
+  g_event_engine->fd_orphan(fd, on_done, release_fd, reason);
 }
 
 void grpc_fd_shutdown(grpc_fd* fd, grpc_error* why) {
@@ -231,6 +235,10 @@ void grpc_fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) {
   g_event_engine->fd_notify_on_write(fd, closure);
 }
 
+void grpc_fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) {
+  g_event_engine->fd_notify_on_error(fd, closure);
+}
+
 static size_t pollset_size(void) { return g_event_engine->pollset_size; }
 
 static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {

data/src/core/lib/iomgr/ev_posix.h

@@ -41,14 +41,16 @@ typedef struct grpc_fd grpc_fd;
 
 typedef struct grpc_event_engine_vtable {
   size_t pollset_size;
+  bool can_track_err;
 
-  grpc_fd* (*fd_create)(int fd, const char* name);
+  grpc_fd* (*fd_create)(int fd, const char* name, bool track_err);
   int (*fd_wrapped_fd)(grpc_fd* fd);
   void (*fd_orphan)(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
-                    bool already_closed, const char* reason);
+                    const char* reason);
   void (*fd_shutdown)(grpc_fd* fd, grpc_error* why);
   void (*fd_notify_on_read)(grpc_fd* fd, grpc_closure* closure);
   void (*fd_notify_on_write)(grpc_fd* fd, grpc_closure* closure);
+  void (*fd_notify_on_error)(grpc_fd* fd, grpc_closure* closure);
   bool (*fd_is_shutdown)(grpc_fd* fd);
   grpc_pollset* (*fd_get_read_notifier_pollset)(grpc_fd* fd);
 
@@ -84,10 +86,20 @@ void grpc_event_engine_shutdown(void);
 /* Return the name of the poll strategy */
 const char* grpc_get_poll_strategy_name();
 
+/* Returns true if polling engine can track errors separately, false otherwise.
+ * If this is true, fd can be created with track_err set. After this, error
+ * events will be reported using fd_notify_on_error. If it is not set, errors
+ * will continue to be reported through fd_notify_on_read and
+ * fd_notify_on_write.
+ */
+bool grpc_event_engine_can_track_errors();
+
 /* Create a wrapped file descriptor.
    Requires fd is a non-blocking file descriptor.
+   \a track_err if true means that error events would be tracked separately
+   using grpc_fd_notify_on_error. Currently, valid only for linux systems.
    This takes ownership of closing fd. */
-grpc_fd* grpc_fd_create(int fd, const char* name);
+grpc_fd* grpc_fd_create(int fd, const char* name, bool track_err);
 
 /* Return the wrapped fd, or -1 if it has been released or closed. */
 int grpc_fd_wrapped_fd(grpc_fd* fd);
@@ -100,7 +112,7 @@ int grpc_fd_wrapped_fd(grpc_fd* fd);
    notify_on_write.
    MUST NOT be called with a pollset lock taken */
 void grpc_fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
-                    bool already_closed, const char* reason);
+                    const char* reason);
 
 /* Has grpc_fd_shutdown been called on an fd? */
 bool grpc_fd_is_shutdown(grpc_fd* fd);
@@ -126,6 +138,10 @@ void grpc_fd_notify_on_read(grpc_fd* fd, grpc_closure* closure);
 /* Exactly the same semantics as above, except based on writable events.  */
 void grpc_fd_notify_on_write(grpc_fd* fd, grpc_closure* closure);
 
+/* Exactly the same semantics as above, except based on error events. track_err
+ * needs to have been set on grpc_fd_create */
+void grpc_fd_notify_on_error(grpc_fd* fd, grpc_closure* closure);
+
 /* Return the read notifier pollset from the fd */
 grpc_pollset* grpc_fd_get_read_notifier_pollset(grpc_fd* fd);
 

data/src/core/lib/iomgr/executor.cc

@@ -28,52 +28,43 @@
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/gprpp/thd.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 #define MAX_DEPTH 2
 
-typedef struct {
-  gpr_mu mu;
-  gpr_cv cv;
-  grpc_closure_list elems;
-  size_t depth;
-  bool shutdown;
-  bool queued_long_job;
-  grpc_core::Thread thd;
-} thread_state;
-
-static thread_state* g_thread_state;
-static size_t g_max_threads;
-static gpr_atm g_cur_threads;
-static gpr_spinlock g_adding_thread_lock = GPR_SPINLOCK_STATIC_INITIALIZER;
+#define EXECUTOR_TRACE(format, ...)                     \
+  if (executor_trace.enabled()) {                       \
+    gpr_log(GPR_INFO, "EXECUTOR " format, __VA_ARGS__); \
+  }
+
+grpc_core::TraceFlag executor_trace(false, "executor");
 
 GPR_TLS_DECL(g_this_thread_state);
 
-grpc_core::TraceFlag executor_trace(false, "executor");
+GrpcExecutor::GrpcExecutor(const char* executor_name) : name_(executor_name) {
+  adding_thread_lock_ = GPR_SPINLOCK_STATIC_INITIALIZER;
+  gpr_atm_no_barrier_store(&num_threads_, 0);
+  max_threads_ = GPR_MAX(1, 2 * gpr_cpu_num_cores());
+}
 
-static void executor_thread(void* arg);
+void GrpcExecutor::Init() { SetThreading(true); }
 
-static size_t run_closures(grpc_closure_list list) {
+size_t GrpcExecutor::RunClosures(grpc_closure_list list) {
   size_t n = 0;
 
   grpc_closure* c = list.head;
   while (c != nullptr) {
     grpc_closure* next = c->next_data.next;
     grpc_error* error = c->error_data.error;
-    if (executor_trace.enabled()) {
-#ifndef NDEBUG
-      gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
-              c->file_created, c->line_created);
-#else
-      gpr_log(GPR_INFO, "EXECUTOR: run %p", c);
-#endif
-    }
 #ifndef NDEBUG
+    EXECUTOR_TRACE("run %p [created by %s:%d]", c, c->file_created,
+                   c->line_created);
     c->scheduled = false;
+#else
+    EXECUTOR_TRACE("run %p", c);
 #endif
     c->cb(c->cb_arg, error);
     GRPC_ERROR_UNREF(error);
@@ -85,217 +76,282 @@ static size_t run_closures(grpc_closure_list list) {
   return n;
 }
 
-bool grpc_executor_is_threaded() {
-  return gpr_atm_no_barrier_load(&g_cur_threads) > 0;
+bool GrpcExecutor::IsThreaded() const {
+  return gpr_atm_no_barrier_load(&num_threads_) > 0;
 }
 
-void grpc_executor_set_threading(bool threading) {
-  gpr_atm cur_threads = gpr_atm_no_barrier_load(&g_cur_threads);
+void GrpcExecutor::SetThreading(bool threading) {
+  gpr_atm curr_num_threads = gpr_atm_no_barrier_load(&num_threads_);
+
   if (threading) {
-    if (cur_threads > 0) return;
-    g_max_threads = GPR_MAX(1, 2 * gpr_cpu_num_cores());
-    gpr_atm_no_barrier_store(&g_cur_threads, 1);
+    if (curr_num_threads > 0) return;
+
+    GPR_ASSERT(num_threads_ == 0);
+    gpr_atm_no_barrier_store(&num_threads_, 1);
     gpr_tls_init(&g_this_thread_state);
-    g_thread_state = static_cast<thread_state*>(
-        gpr_zalloc(sizeof(thread_state) * g_max_threads));
-    for (size_t i = 0; i < g_max_threads; i++) {
-      gpr_mu_init(&g_thread_state[i].mu);
-      gpr_cv_init(&g_thread_state[i].cv);
-      g_thread_state[i].thd = grpc_core::Thread();
-      g_thread_state[i].elems = GRPC_CLOSURE_LIST_INIT;
+    thd_state_ = static_cast<ThreadState*>(
+        gpr_zalloc(sizeof(ThreadState) * max_threads_));
+
+    for (size_t i = 0; i < max_threads_; i++) {
+      gpr_mu_init(&thd_state_[i].mu);
+      gpr_cv_init(&thd_state_[i].cv);
+      thd_state_[i].id = i;
+      thd_state_[i].thd = grpc_core::Thread();
+      thd_state_[i].elems = GRPC_CLOSURE_LIST_INIT;
     }
 
-    g_thread_state[0].thd =
-        grpc_core::Thread("grpc_executor", executor_thread, &g_thread_state[0]);
-    g_thread_state[0].thd.Start();
-  } else {
-    if (cur_threads == 0) return;
-    for (size_t i = 0; i < g_max_threads; i++) {
-      gpr_mu_lock(&g_thread_state[i].mu);
-      g_thread_state[i].shutdown = true;
-      gpr_cv_signal(&g_thread_state[i].cv);
-      gpr_mu_unlock(&g_thread_state[i].mu);
+    thd_state_[0].thd =
+        grpc_core::Thread(name_, &GrpcExecutor::ThreadMain, &thd_state_[0]);
+    thd_state_[0].thd.Start();
+  } else {  // !threading
+    if (curr_num_threads == 0) return;
+
+    for (size_t i = 0; i < max_threads_; i++) {
+      gpr_mu_lock(&thd_state_[i].mu);
+      thd_state_[i].shutdown = true;
+      gpr_cv_signal(&thd_state_[i].cv);
+      gpr_mu_unlock(&thd_state_[i].mu);
     }
-    /* ensure no thread is adding a new thread... once this is past, then
-       no thread will try to add a new one either (since shutdown is true) */
-    gpr_spinlock_lock(&g_adding_thread_lock);
-    gpr_spinlock_unlock(&g_adding_thread_lock);
-    for (gpr_atm i = 0; i < g_cur_threads; i++) {
-      g_thread_state[i].thd.Join();
+
+    /* Ensure no thread is adding a new thread. Once this is past, then no
+     * thread will try to add a new one either (since shutdown is true) */
+    gpr_spinlock_lock(&adding_thread_lock_);
+    gpr_spinlock_unlock(&adding_thread_lock_);
+
+    curr_num_threads = gpr_atm_no_barrier_load(&num_threads_);
+    for (gpr_atm i = 0; i < curr_num_threads; i++) {
+      thd_state_[i].thd.Join();
+      EXECUTOR_TRACE(" Thread %" PRIdPTR " of %" PRIdPTR " joined", i,
+                     curr_num_threads);
     }
-    gpr_atm_no_barrier_store(&g_cur_threads, 0);
-    for (size_t i = 0; i < g_max_threads; i++) {
-      gpr_mu_destroy(&g_thread_state[i].mu);
-      gpr_cv_destroy(&g_thread_state[i].cv);
-      run_closures(g_thread_state[i].elems);
+
+    gpr_atm_no_barrier_store(&num_threads_, 0);
+    for (size_t i = 0; i < max_threads_; i++) {
+      gpr_mu_destroy(&thd_state_[i].mu);
+      gpr_cv_destroy(&thd_state_[i].cv);
+      RunClosures(thd_state_[i].elems);
     }
-    gpr_free(g_thread_state);
+
+    gpr_free(thd_state_);
     gpr_tls_destroy(&g_this_thread_state);
   }
 }
 
-void grpc_executor_init() {
-  gpr_atm_no_barrier_store(&g_cur_threads, 0);
-  grpc_executor_set_threading(true);
-}
+void GrpcExecutor::Shutdown() { SetThreading(false); }
 
-void grpc_executor_shutdown() { grpc_executor_set_threading(false); }
-
-static void executor_thread(void* arg) {
-  thread_state* ts = static_cast<thread_state*>(arg);
-  gpr_tls_set(&g_this_thread_state, (intptr_t)ts);
+void GrpcExecutor::ThreadMain(void* arg) {
+  ThreadState* ts = static_cast<ThreadState*>(arg);
+  gpr_tls_set(&g_this_thread_state, reinterpret_cast<intptr_t>(ts));
 
   grpc_core::ExecCtx exec_ctx(GRPC_EXEC_CTX_FLAG_IS_INTERNAL_THREAD);
 
   size_t subtract_depth = 0;
   for (;;) {
-    if (executor_trace.enabled()) {
-      gpr_log(GPR_INFO, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")",
-              static_cast<int>(ts - g_thread_state), subtract_depth);
-    }
+    EXECUTOR_TRACE("[%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", ts->id,
+                   subtract_depth);
+
     gpr_mu_lock(&ts->mu);
     ts->depth -= subtract_depth;
+    // Wait for closures to be enqueued or for the executor to be shutdown
     while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) {
       ts->queued_long_job = false;
       gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_MONOTONIC));
     }
+
     if (ts->shutdown) {
-      if (executor_trace.enabled()) {
-        gpr_log(GPR_INFO, "EXECUTOR[%d]: shutdown",
-                static_cast<int>(ts - g_thread_state));
-      }
+      EXECUTOR_TRACE("[%" PRIdPTR "]: shutdown", ts->id);
       gpr_mu_unlock(&ts->mu);
       break;
     }
+
     GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED();
-    grpc_closure_list exec = ts->elems;
+    grpc_closure_list closures = ts->elems;
     ts->elems = GRPC_CLOSURE_LIST_INIT;
     gpr_mu_unlock(&ts->mu);
-    if (executor_trace.enabled()) {
-      gpr_log(GPR_INFO, "EXECUTOR[%d]: execute",
-              static_cast<int>(ts - g_thread_state));
-    }
+
+    EXECUTOR_TRACE("[%" PRIdPTR "]: execute", ts->id);
 
     grpc_core::ExecCtx::Get()->InvalidateNow();
-    subtract_depth = run_closures(exec);
+    subtract_depth = RunClosures(closures);
   }
 }
 
-static void executor_push(grpc_closure* closure, grpc_error* error,
-                          bool is_short) {
+void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error,
+                           bool is_short) {
   bool retry_push;
   if (is_short) {
     GRPC_STATS_INC_EXECUTOR_SCHEDULED_SHORT_ITEMS();
   } else {
     GRPC_STATS_INC_EXECUTOR_SCHEDULED_LONG_ITEMS();
   }
+
   do {
     retry_push = false;
     size_t cur_thread_count =
-        static_cast<size_t>(gpr_atm_no_barrier_load(&g_cur_threads));
+        static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_));
+
+    // If the number of threads is zero(i.e either the executor is not threaded
+    // or already shutdown), then queue the closure on the exec context itself
     if (cur_thread_count == 0) {
-      if (executor_trace.enabled()) {
 #ifndef NDEBUG
-        gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p (created %s:%d) inline",
-                closure, closure->file_created, closure->line_created);
+      EXECUTOR_TRACE("schedule %p (created %s:%d) inline", closure,
+                     closure->file_created, closure->line_created);
 #else
-        gpr_log(GPR_INFO, "EXECUTOR: schedule %p inline", closure);
+      EXECUTOR_TRACE("schedule %p inline", closure);
 #endif
-      }
       grpc_closure_list_append(grpc_core::ExecCtx::Get()->closure_list(),
                                closure, error);
       return;
     }
-    thread_state* ts = (thread_state*)gpr_tls_get(&g_this_thread_state);
+
+    ThreadState* ts = (ThreadState*)gpr_tls_get(&g_this_thread_state);
     if (ts == nullptr) {
-      ts = &g_thread_state[GPR_HASH_POINTER(grpc_core::ExecCtx::Get(),
-                                            cur_thread_count)];
+      ts = &thd_state_[GPR_HASH_POINTER(grpc_core::ExecCtx::Get(),
+                                        cur_thread_count)];
     } else {
       GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF();
     }
-    thread_state* orig_ts = ts;
 
-    bool try_new_thread;
+    ThreadState* orig_ts = ts;
+
+    bool try_new_thread = false;
     for (;;) {
-      if (executor_trace.enabled()) {
 #ifndef NDEBUG
-        gpr_log(
-            GPR_DEBUG,
-            "EXECUTOR: try to schedule %p (%s) (created %s:%d) to thread %d",
-            closure, is_short ? "short" : "long", closure->file_created,
-            closure->line_created, static_cast<int>(ts - g_thread_state));
+      EXECUTOR_TRACE(
+          "try to schedule %p (%s) (created %s:%d) to thread "
+          "%" PRIdPTR,
+          closure, is_short ? "short" : "long", closure->file_created,
+          closure->line_created, ts->id);
 #else
-        gpr_log(GPR_INFO, "EXECUTOR: try to schedule %p (%s) to thread %d",
-                closure, is_short ? "short" : "long",
-                (int)(ts - g_thread_state));
+      EXECUTOR_TRACE("try to schedule %p (%s) to thread %" PRIdPTR, closure,
+                     is_short ? "short" : "long", ts->id);
 #endif
-      }
+
       gpr_mu_lock(&ts->mu);
       if (ts->queued_long_job) {
         // if there's a long job queued, we never queue anything else to this
         // queue (since long jobs can take 'infinite' time and we need to
-        // guarantee no starvation)
-        // ... spin through queues and try again
+        // guarantee no starvation). Spin through queues and try again
         gpr_mu_unlock(&ts->mu);
-        size_t idx = static_cast<size_t>(ts - g_thread_state);
-        ts = &g_thread_state[(idx + 1) % cur_thread_count];
+        size_t idx = ts->id;
+        ts = &thd_state_[(idx + 1) % cur_thread_count];
         if (ts == orig_ts) {
+          // We cycled through all the threads. Retry enqueue again (by creating
+          // a new thread)
           retry_push = true;
+          // TODO (sreek): What if the executor is shutdown OR if
+          // cur_thread_count is already equal to max_threads ? (currently - as
+          // of July 2018, we do not run in to this issue because there is only
+          // one instance of long job in gRPC. This has to be fixed soon)
           try_new_thread = true;
           break;
         }
+
         continue;
       }
+
+      // == Found the thread state (i.e thread) to enqueue this closure! ==
+
+      // Also, if this thread has been waiting for closures, wake it up.
+      // - If grpc_closure_list_empty() is true and the Executor is not
+      //   shutdown, it means that the thread must be waiting in ThreadMain()
+      // - Note that gpr_cv_signal() won't immediately wakeup the thread. That
+      //   happens after we release the mutex &ts->mu a few lines below
       if (grpc_closure_list_empty(ts->elems) && !ts->shutdown) {
         GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED();
         gpr_cv_signal(&ts->cv);
       }
+
       grpc_closure_list_append(&ts->elems, closure, error);
+
+      // If we already queued more than MAX_DEPTH number of closures on this
+      // thread, use this as a hint to create more threads
       ts->depth++;
       try_new_thread = ts->depth > MAX_DEPTH &&
-                       cur_thread_count < g_max_threads && !ts->shutdown;
-      if (!is_short) ts->queued_long_job = true;
+                       cur_thread_count < max_threads_ && !ts->shutdown;
+
+      ts->queued_long_job = !is_short;
+
       gpr_mu_unlock(&ts->mu);
       break;
     }
-    if (try_new_thread && gpr_spinlock_trylock(&g_adding_thread_lock)) {
+
+    if (try_new_thread && gpr_spinlock_trylock(&adding_thread_lock_)) {
       cur_thread_count =
-          static_cast<size_t>(gpr_atm_no_barrier_load(&g_cur_threads));
-      if (cur_thread_count < g_max_threads) {
-        gpr_atm_no_barrier_store(&g_cur_threads, cur_thread_count + 1);
-
-        g_thread_state[cur_thread_count].thd =
-            grpc_core::Thread("grpc_executor", executor_thread,
-                              &g_thread_state[cur_thread_count]);
-        g_thread_state[cur_thread_count].thd.Start();
+          static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_));
+      if (cur_thread_count < max_threads_) {
+        // Increment num_threads (Safe to do a no_barrier_store instead of a
+        // cas because we always increment num_threads under the
+        // 'adding_thread_lock')
+        gpr_atm_no_barrier_store(&num_threads_, cur_thread_count + 1);
+
+        thd_state_[cur_thread_count].thd = grpc_core::Thread(
+            name_, &GrpcExecutor::ThreadMain, &thd_state_[cur_thread_count]);
+        thd_state_[cur_thread_count].thd.Start();
       }
-      gpr_spinlock_unlock(&g_adding_thread_lock);
+      gpr_spinlock_unlock(&adding_thread_lock_);
     }
+
     if (retry_push) {
       GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES();
     }
   } while (retry_push);
 }
 
-static void executor_push_short(grpc_closure* closure, grpc_error* error) {
-  executor_push(closure, error, true);
+static GrpcExecutor* global_executor;
+
+void enqueue_long(grpc_closure* closure, grpc_error* error) {
+  global_executor->Enqueue(closure, error, false /* is_short */);
+}
+
+void enqueue_short(grpc_closure* closure, grpc_error* error) {
+  global_executor->Enqueue(closure, error, true /* is_short */);
+}
+
+// Short-Job executor scheduler
+static const grpc_closure_scheduler_vtable global_executor_vtable_short = {
+    enqueue_short, enqueue_short, "executor-short"};
+static grpc_closure_scheduler global_scheduler_short = {
+    &global_executor_vtable_short};
+
+// Long-job executor scheduler
+static const grpc_closure_scheduler_vtable global_executor_vtable_long = {
+    enqueue_long, enqueue_long, "executor-long"};
+static grpc_closure_scheduler global_scheduler_long = {
+    &global_executor_vtable_long};
+
+// grpc_executor_init() and grpc_executor_shutdown() functions are called in the
+// the grpc_init() and grpc_shutdown() code paths which are protected by a
+// global mutex. So it is okay to assume that these functions are thread-safe
+void grpc_executor_init() {
+  if (global_executor != nullptr) {
+    // grpc_executor_init() already called once (and grpc_executor_shutdown()
+    // wasn't called)
+    return;
+  }
+
+  global_executor = grpc_core::New<GrpcExecutor>("global-executor");
+  global_executor->Init();
 }
 
-static void executor_push_long(grpc_closure* closure, grpc_error* error) {
-  executor_push(closure, error, false);
+void grpc_executor_shutdown() {
+  // Shutdown already called
+  if (global_executor == nullptr) {
+    return;
+  }
+
+  global_executor->Shutdown();
+  grpc_core::Delete<GrpcExecutor>(global_executor);
+  global_executor = nullptr;
 }
 
-static const grpc_closure_scheduler_vtable executor_vtable_short = {
-    executor_push_short, executor_push_short, "executor"};
-static grpc_closure_scheduler executor_scheduler_short = {
-    &executor_vtable_short};
+bool grpc_executor_is_threaded() { return global_executor->IsThreaded(); }
 
-static const grpc_closure_scheduler_vtable executor_vtable_long = {
-    executor_push_long, executor_push_long, "executor"};
-static grpc_closure_scheduler executor_scheduler_long = {&executor_vtable_long};
+void grpc_executor_set_threading(bool enable) {
+  global_executor->SetThreading(enable);
+}
 
-grpc_closure_scheduler* grpc_executor_scheduler(
-    grpc_executor_job_length length) {
-  return length == GRPC_EXECUTOR_SHORT ? &executor_scheduler_short
-                                       : &executor_scheduler_long;
+grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) {
+  return job_type == GRPC_EXECUTOR_SHORT ? &global_scheduler_short
+                                         : &global_scheduler_long;
 }