grpc 1.6.7 → 1.7.0.pre1

data/src/core/lib/iomgr/ev_posix.c

@@ -31,8 +31,6 @@
 
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
-#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h"
-#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h"
 #include "src/core/lib/iomgr/ev_epollex_linux.h"
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
@@ -64,10 +62,8 @@ typedef struct {
 } event_engine_factory;
 
 static const event_engine_factory g_factories[] = {
-    {"epollsig", grpc_init_epollsig_linux},
     {"epoll1", grpc_init_epoll1_linux},
-    {"epoll-threadpool", grpc_init_epoll_thread_pool_linux},
-    {"epoll-limited", grpc_init_epoll_limited_pollers_linux},
+    {"epollsig", grpc_init_epollsig_linux},
     {"poll", grpc_init_poll_posix},
     {"poll-cv", grpc_init_poll_cv_posix},
     {"epollex", grpc_init_epollex_linux},
@@ -80,10 +76,10 @@ static void add(const char *beg, const char *end, char ***ss, size_t *ns) {
   size_t len;
   GPR_ASSERT(end >= beg);
   len = (size_t)(end - beg);
-  s = gpr_malloc(len + 1);
+  s = (char *)gpr_malloc(len + 1);
   memcpy(s, beg, len);
   s[len] = 0;
-  *ss = gpr_realloc(*ss, sizeof(char **) * np);
+  *ss = (char **)gpr_realloc(*ss, sizeof(char **) * np);
   (*ss)[n] = s;
   *ns = np;
 }
@@ -214,9 +210,9 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline);
 }
 
-grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
+grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker *specific_worker) {
-  return g_event_engine->pollset_kick(pollset, specific_worker);
+  return g_event_engine->pollset_kick(exec_ctx, pollset, specific_worker);
 }
 
 void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,

data/src/core/lib/iomgr/ev_posix.h

@@ -54,7 +54,7 @@ typedef struct grpc_event_engine_vtable {
   grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker **worker, gpr_timespec now,
                               gpr_timespec deadline);
-  grpc_error *(*pollset_kick)(grpc_pollset *pollset,
+  grpc_error *(*pollset_kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker *specific_worker);
   void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                          struct grpc_fd *fd);

data/src/core/lib/iomgr/exec_ctx.h

@@ -19,6 +19,7 @@
 #ifndef GRPC_CORE_LIB_IOMGR_EXEC_CTX_H
 #define GRPC_CORE_LIB_IOMGR_EXEC_CTX_H
 
+#include <grpc/support/cpu.h>
 #include "src/core/lib/iomgr/closure.h"
 
 /* #define GRPC_EXECUTION_CONTEXT_SANITIZER 1 */
@@ -62,6 +63,7 @@ struct grpc_exec_ctx {
   /** last active combiner in the active combiner list */
   grpc_combiner *last_combiner;
   uintptr_t flags;
+  unsigned starting_cpu;
   void *check_ready_to_finish_arg;
   bool (*check_ready_to_finish)(grpc_exec_ctx *exec_ctx, void *arg);
 };
@@ -69,7 +71,10 @@ struct grpc_exec_ctx {
 /* initializer for grpc_exec_ctx:
    prefer to use GRPC_EXEC_CTX_INIT whenever possible */
 #define GRPC_EXEC_CTX_INITIALIZER(flags, finish_check, finish_check_arg) \
-  { GRPC_CLOSURE_LIST_INIT, NULL, NULL, flags, finish_check_arg, finish_check }
+  {                                                                      \
+    GRPC_CLOSURE_LIST_INIT, NULL, NULL, flags, gpr_cpu_current_cpu(),    \
+        finish_check_arg, finish_check                                   \
+  }
 
 /* initialize an execution context at the top level of an API call into grpc
    (this is safe to use elsewhere, though possibly not as efficient) */

data/src/core/lib/iomgr/executor.c

@@ -28,6 +28,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/spinlock.h"
 
@@ -39,6 +40,7 @@ typedef struct {
   grpc_closure_list elems;
   size_t depth;
   bool shutdown;
+  bool queued_long_job;
   gpr_thd_id id;
 } thread_state;
 
@@ -49,6 +51,9 @@ static gpr_spinlock g_adding_thread_lock = GPR_SPINLOCK_STATIC_INITIALIZER;
 
 GPR_TLS_DECL(g_this_thread_state);
 
+static grpc_tracer_flag executor_trace =
+    GRPC_TRACER_INITIALIZER(false, "executor");
+
 static void executor_thread(void *arg);
 
 static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
@@ -58,6 +63,14 @@ static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
   while (c != NULL) {
     grpc_closure *next = c->next_data.next;
     grpc_error *error = c->error_data.error;
+    if (GRPC_TRACER_ON(executor_trace)) {
+#ifndef NDEBUG
+      gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
+              c->file_created, c->line_created);
+#else
+      gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c);
+#endif
+    }
 #ifndef NDEBUG
     c->scheduled = false;
 #endif
@@ -65,6 +78,7 @@ static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
     GRPC_ERROR_UNREF(error);
     c = next;
     n++;
+    grpc_exec_ctx_flush(exec_ctx);
   }
 
   return n;
@@ -81,7 +95,8 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) {
     g_max_threads = GPR_MAX(1, 2 * gpr_cpu_num_cores());
     gpr_atm_no_barrier_store(&g_cur_threads, 1);
     gpr_tls_init(&g_this_thread_state);
-    g_thread_state = gpr_zalloc(sizeof(thread_state) * g_max_threads);
+    g_thread_state =
+        (thread_state *)gpr_zalloc(sizeof(thread_state) * g_max_threads);
     for (size_t i = 0; i < g_max_threads; i++) {
       gpr_mu_init(&g_thread_state[i].mu);
       gpr_cv_init(&g_thread_state[i].cv);
@@ -119,6 +134,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) {
 }
 
 void grpc_executor_init(grpc_exec_ctx *exec_ctx) {
+  grpc_register_tracer(&executor_trace);
   gpr_atm_no_barrier_store(&g_cur_threads, 0);
   grpc_executor_set_threading(exec_ctx, true);
 }
@@ -128,7 +144,7 @@ void grpc_executor_shutdown(grpc_exec_ctx *exec_ctx) {
 }
 
 static void executor_thread(void *arg) {
-  thread_state *ts = arg;
+  thread_state *ts = (thread_state *)arg;
   gpr_tls_set(&g_this_thread_state, (intptr_t)ts);
 
   grpc_exec_ctx exec_ctx =
@@ -136,60 +152,150 @@ static void executor_thread(void *arg) {
 
   size_t subtract_depth = 0;
   for (;;) {
+    if (GRPC_TRACER_ON(executor_trace)) {
+      gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")",
+              (int)(ts - g_thread_state), subtract_depth);
+    }
     gpr_mu_lock(&ts->mu);
     ts->depth -= subtract_depth;
     while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) {
+      ts->queued_long_job = false;
       gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
     }
     if (ts->shutdown) {
+      if (GRPC_TRACER_ON(executor_trace)) {
+        gpr_log(GPR_DEBUG, "EXECUTOR[%d]: shutdown",
+                (int)(ts - g_thread_state));
+      }
       gpr_mu_unlock(&ts->mu);
       break;
     }
+    GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx);
     grpc_closure_list exec = ts->elems;
     ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
     gpr_mu_unlock(&ts->mu);
+    if (GRPC_TRACER_ON(executor_trace)) {
+      gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state));
+    }
 
     subtract_depth = run_closures(&exec_ctx, exec);
-    grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                          grpc_error *error) {
-  size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
-  if (cur_thread_count == 0) {
-    grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
-    return;
-  }
-  thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
-  if (ts == NULL) {
-    ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
-  }
-  gpr_mu_lock(&ts->mu);
-  if (grpc_closure_list_empty(ts->elems)) {
-    gpr_cv_signal(&ts->cv);
+                          grpc_error *error, bool is_short) {
+  bool retry_push;
+  if (is_short) {
+    GRPC_STATS_INC_EXECUTOR_SCHEDULED_SHORT_ITEMS(exec_ctx);
+  } else {
+    GRPC_STATS_INC_EXECUTOR_SCHEDULED_LONG_ITEMS(exec_ctx);
   }
-  grpc_closure_list_append(&ts->elems, closure, error);
-  ts->depth++;
-  bool try_new_thread = ts->depth > MAX_DEPTH &&
-                        cur_thread_count < g_max_threads && !ts->shutdown;
-  gpr_mu_unlock(&ts->mu);
-  if (try_new_thread && gpr_spinlock_trylock(&g_adding_thread_lock)) {
-    cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
-    if (cur_thread_count < g_max_threads) {
-      gpr_atm_no_barrier_store(&g_cur_threads, cur_thread_count + 1);
-
-      gpr_thd_options opt = gpr_thd_options_default();
-      gpr_thd_options_set_joinable(&opt);
-      gpr_thd_new(&g_thread_state[cur_thread_count].id, executor_thread,
-                  &g_thread_state[cur_thread_count], &opt);
+  do {
+    retry_push = false;
+    size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
+    if (cur_thread_count == 0) {
+      if (GRPC_TRACER_ON(executor_trace)) {
+#ifndef NDEBUG
+        gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p (created %s:%d) inline",
+                closure, closure->file_created, closure->line_created);
+#else
+        gpr_log(GPR_DEBUG, "EXECUTOR: schedule %p inline", closure);
+#endif
+      }
+      grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
+      return;
     }
-    gpr_spinlock_unlock(&g_adding_thread_lock);
-  }
+    thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
+    if (ts == NULL) {
+      ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
+    } else {
+      GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx);
+    }
+    thread_state *orig_ts = ts;
+
+    bool try_new_thread;
+    for (;;) {
+      if (GRPC_TRACER_ON(executor_trace)) {
+#ifndef NDEBUG
+        gpr_log(
+            GPR_DEBUG,
+            "EXECUTOR: try to schedule %p (%s) (created %s:%d) to thread %d",
+            closure, is_short ? "short" : "long", closure->file_created,
+            closure->line_created, (int)(ts - g_thread_state));
+#else
+        gpr_log(GPR_DEBUG, "EXECUTOR: try to schedule %p (%s) to thread %d",
+                closure, is_short ? "short" : "long",
+                (int)(ts - g_thread_state));
+#endif
+      }
+      gpr_mu_lock(&ts->mu);
+      if (ts->queued_long_job) {
+        // if there's a long job queued, we never queue anything else to this
+        // queue (since long jobs can take 'infinite' time and we need to
+        // guarantee no starvation)
+        // ... spin through queues and try again
+        gpr_mu_unlock(&ts->mu);
+        size_t idx = (size_t)(ts - g_thread_state);
+        ts = &g_thread_state[(idx + 1) % cur_thread_count];
+        if (ts == orig_ts) {
+          retry_push = true;
+          try_new_thread = true;
+          break;
+        }
+        continue;
+      }
+      if (grpc_closure_list_empty(ts->elems)) {
+        GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(exec_ctx);
+        gpr_cv_signal(&ts->cv);
+      }
+      grpc_closure_list_append(&ts->elems, closure, error);
+      ts->depth++;
+      try_new_thread = ts->depth > MAX_DEPTH &&
+                       cur_thread_count < g_max_threads && !ts->shutdown;
+      if (!is_short) ts->queued_long_job = true;
+      gpr_mu_unlock(&ts->mu);
+      break;
+    }
+    if (try_new_thread && gpr_spinlock_trylock(&g_adding_thread_lock)) {
+      cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
+      if (cur_thread_count < g_max_threads) {
+        gpr_atm_no_barrier_store(&g_cur_threads, cur_thread_count + 1);
+
+        gpr_thd_options opt = gpr_thd_options_default();
+        gpr_thd_options_set_joinable(&opt);
+        gpr_thd_new(&g_thread_state[cur_thread_count].id, executor_thread,
+                    &g_thread_state[cur_thread_count], &opt);
+      }
+      gpr_spinlock_unlock(&g_adding_thread_lock);
+    }
+    if (retry_push) {
+      GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(exec_ctx);
+    }
+  } while (retry_push);
 }
 
-static const grpc_closure_scheduler_vtable executor_vtable = {
-    executor_push, executor_push, "executor"};
-static grpc_closure_scheduler executor_scheduler = {&executor_vtable};
-grpc_closure_scheduler *grpc_executor_scheduler = &executor_scheduler;
+static void executor_push_short(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                                grpc_error *error) {
+  executor_push(exec_ctx, closure, error, true);
+}
+
+static void executor_push_long(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                               grpc_error *error) {
+  executor_push(exec_ctx, closure, error, false);
+}
+
+static const grpc_closure_scheduler_vtable executor_vtable_short = {
+    executor_push_short, executor_push_short, "executor"};
+static grpc_closure_scheduler executor_scheduler_short = {
+    &executor_vtable_short};
+
+static const grpc_closure_scheduler_vtable executor_vtable_long = {
+    executor_push_long, executor_push_long, "executor"};
+static grpc_closure_scheduler executor_scheduler_long = {&executor_vtable_long};
+
+grpc_closure_scheduler *grpc_executor_scheduler(
+    grpc_executor_job_length length) {
+  return length == GRPC_EXECUTOR_SHORT ? &executor_scheduler_short
+                                       : &executor_scheduler_long;
+}

data/src/core/lib/iomgr/executor.h

@@ -21,6 +21,11 @@
 
 #include "src/core/lib/iomgr/closure.h"
 
+typedef enum {
+  GRPC_EXECUTOR_SHORT,
+  GRPC_EXECUTOR_LONG
+} grpc_executor_job_length;
+
 /** Initialize the global executor.
  *
  * This mechanism is meant to outsource work (grpc_closure instances) to a
@@ -28,7 +33,7 @@
  * non-blocking solution available. */
 void grpc_executor_init(grpc_exec_ctx *exec_ctx);
 
-extern grpc_closure_scheduler *grpc_executor_scheduler;
+grpc_closure_scheduler *grpc_executor_scheduler(grpc_executor_job_length);
 
 /** Shutdown the executor, running all pending work as part of the call */
 void grpc_executor_shutdown(grpc_exec_ctx *exec_ctx);

data/src/core/lib/iomgr/fork_posix.c

@@ -0,0 +1,88 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_FORK
+
+#include <string.h>
+
+#include <grpc/fork.h>
+#include <grpc/support/log.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/timer_manager.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/fork.h"
+#include "src/core/lib/support/thd_internal.h"
+#include "src/core/lib/surface/init.h"
+
+/*
+ * NOTE: FORKING IS NOT GENERALLY SUPPORTED, THIS IS ONLY INTENDED TO WORK
+ *       AROUND VERY SPECIFIC USE CASES.
+ */
+
+void grpc_prefork() {
+  if (!grpc_fork_support_enabled()) {
+    gpr_log(GPR_ERROR,
+            "Fork support not enabled; try running with the "
+            "environment variable GRPC_ENABLE_FORK_SUPPORT=1");
+    return;
+  }
+  if (grpc_is_initialized()) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_timer_manager_set_threading(false);
+    grpc_executor_set_threading(&exec_ctx, false);
+    grpc_exec_ctx_finish(&exec_ctx);
+    if (!gpr_await_threads(
+            gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
+                         gpr_time_from_seconds(3, GPR_TIMESPAN)))) {
+      gpr_log(GPR_ERROR, "gRPC thread still active! Cannot fork!");
+    }
+  }
+}
+
+void grpc_postfork_parent() {
+  if (grpc_is_initialized()) {
+    grpc_timer_manager_set_threading(true);
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_executor_set_threading(&exec_ctx, true);
+    grpc_exec_ctx_finish(&exec_ctx);
+  }
+}
+
+void grpc_postfork_child() {
+  if (grpc_is_initialized()) {
+    grpc_timer_manager_set_threading(true);
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_executor_set_threading(&exec_ctx, true);
+    grpc_exec_ctx_finish(&exec_ctx);
+  }
+}
+
+void grpc_fork_handlers_auto_register() {
+  if (grpc_fork_support_enabled()) {
+    pthread_atfork(grpc_prefork, grpc_postfork_parent, grpc_postfork_child);
+  }
+}
+
+#endif  // GRPC_POSIX_FORK

data/src/core/lib/iomgr/fork_windows.c

@@ -0,0 +1,39 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifndef GRPC_POSIX_FORK
+
+#include <grpc/fork.h>
+#include <grpc/support/log.h>
+
+/*
+ * NOTE: FORKING IS NOT GENERALLY SUPPORTED, THIS IS ONLY INTENDED TO WORK
+ *       AROUND VERY SPECIFIC USE CASES.
+ */
+
+void grpc_prefork() { gpr_log(GPR_ERROR, "Forking not supported on Windows"); }
+
+void grpc_postfork_parent() {}
+
+void grpc_postfork_child() {}
+
+void grpc_fork_handlers_auto_register() {}
+
+#endif  // GRPC_POSIX_FORK