grpc 1.3.4 → 1.4.0

data/src/core/lib/iomgr/timer_manager.c

@@ -0,0 +1,276 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/timer_manager.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/thd.h>
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/timer.h"
+
+typedef struct completed_thread {
+  gpr_thd_id t;
+  struct completed_thread *next;
+} completed_thread;
+
+extern grpc_tracer_flag grpc_timer_check_trace;
+
+// global mutex
+static gpr_mu g_mu;
+// are we multi-threaded
+static bool g_threaded;
+// cv to wait until a thread is needed
+static gpr_cv g_cv_wait;
+// cv for notification when threading ends
+static gpr_cv g_cv_shutdown;
+// number of threads in the system
+static int g_thread_count;
+// number of threads sitting around waiting
+static int g_waiter_count;
+// linked list of threads that have completed (and need joining)
+static completed_thread *g_completed_threads;
+// was the manager kicked by the timer system
+static bool g_kicked;
+// is there a thread waiting until the next timer should fire?
+static bool g_has_timed_waiter;
+// generation counter to track which thread is waiting for the next timer
+static uint64_t g_timed_waiter_generation;
+
+static void timer_thread(void *unused);
+
+static void gc_completed_threads(void) {
+  if (g_completed_threads != NULL) {
+    completed_thread *to_gc = g_completed_threads;
+    g_completed_threads = NULL;
+    gpr_mu_unlock(&g_mu);
+    while (to_gc != NULL) {
+      gpr_thd_join(to_gc->t);
+      completed_thread *next = to_gc->next;
+      gpr_free(to_gc);
+      to_gc = next;
+    }
+    gpr_mu_lock(&g_mu);
+  }
+}
+
+static void start_timer_thread_and_unlock(void) {
+  GPR_ASSERT(g_threaded);
+  ++g_waiter_count;
+  ++g_thread_count;
+  gpr_mu_unlock(&g_mu);
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+    gpr_log(GPR_DEBUG, "Spawn timer thread");
+  }
+  gpr_thd_id thd;
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_thd_options_set_joinable(&opt);
+  gpr_thd_new(&thd, timer_thread, NULL, &opt);
+}
+
+void grpc_timer_manager_tick() {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+  grpc_timer_check(&exec_ctx, now, &next);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void timer_thread(void *unused) {
+  // this threads exec_ctx: we try to run things through to completion here
+  // since it's easy to spin up new threads
+  grpc_exec_ctx exec_ctx =
+      GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
+  const gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  for (;;) {
+    gpr_timespec next = inf_future;
+    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+    // check timer state, updates next to the next time to run a check
+    if (grpc_timer_check(&exec_ctx, now, &next)) {
+      // if there's something to execute...
+      gpr_mu_lock(&g_mu);
+      // remove a waiter from the pool, and start another thread if necessary
+      --g_waiter_count;
+      if (g_waiter_count == 0 && g_threaded) {
+        start_timer_thread_and_unlock();
+      } else {
+        // if there's no thread waiting with a timeout, kick an existing waiter
+        // so that the next deadline is not missed
+        if (!g_has_timed_waiter) {
+          if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+            gpr_log(GPR_DEBUG, "kick untimed waiter");
+          }
+          gpr_cv_signal(&g_cv_wait);
+        }
+        gpr_mu_unlock(&g_mu);
+      }
+      // without our lock, flush the exec_ctx
+      grpc_exec_ctx_flush(&exec_ctx);
+      gpr_mu_lock(&g_mu);
+      // garbage collect any threads hanging out that are dead
+      gc_completed_threads();
+      // get ready to wait again
+      ++g_waiter_count;
+      gpr_mu_unlock(&g_mu);
+    } else {
+      gpr_mu_lock(&g_mu);
+      // if we're not threaded anymore, leave
+      if (!g_threaded) break;
+      // if there's no timed waiter, we should become one: that waiter waits
+      // only until the next timer should expire
+      // all other timers wait forever
+      uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1;
+      if (!g_has_timed_waiter) {
+        g_has_timed_waiter = true;
+        // we use a generation counter to track the timed waiter so we can
+        // cancel an existing one quickly (and when it actually times out it'll
+        // figure stuff out instead of incurring a wakeup)
+        my_timed_waiter_generation = ++g_timed_waiter_generation;
+        if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+          gpr_log(GPR_DEBUG, "sleep for a while");
+        }
+      } else {
+        next = inf_future;
+        if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+          gpr_log(GPR_DEBUG, "sleep until kicked");
+        }
+      }
+      gpr_cv_wait(&g_cv_wait, &g_mu, next);
+      if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+        gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d",
+                my_timed_waiter_generation == g_timed_waiter_generation,
+                g_kicked);
+      }
+      // if this was the timed waiter, then we need to check timers, and flag
+      // that there's now no timed waiter... we'll look for a replacement if
+      // there's work to do after checking timers (code above)
+      if (my_timed_waiter_generation == g_timed_waiter_generation) {
+        g_has_timed_waiter = false;
+      }
+      // if this was a kick from the timer system, consume it (and don't stop
+      // this thread yet)
+      if (g_kicked) {
+        grpc_timer_consume_kick();
+        g_kicked = false;
+      }
+      gpr_mu_unlock(&g_mu);
+    }
+  }
+  // terminate the thread: drop the waiter count, thread count, and let whomever
+  // stopped the threading stuff know that we're done
+  --g_waiter_count;
+  --g_thread_count;
+  if (0 == g_thread_count) {
+    gpr_cv_signal(&g_cv_shutdown);
+  }
+  completed_thread *ct = gpr_malloc(sizeof(*ct));
+  ct->t = gpr_thd_currentid();
+  ct->next = g_completed_threads;
+  g_completed_threads = ct;
+  gpr_mu_unlock(&g_mu);
+  grpc_exec_ctx_finish(&exec_ctx);
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+    gpr_log(GPR_DEBUG, "End timer thread");
+  }
+}
+
+static void start_threads(void) {
+  gpr_mu_lock(&g_mu);
+  if (!g_threaded) {
+    g_threaded = true;
+    start_timer_thread_and_unlock();
+  } else {
+    g_threaded = false;
+    gpr_mu_unlock(&g_mu);
+  }
+}
+
+void grpc_timer_manager_init(void) {
+  gpr_mu_init(&g_mu);
+  gpr_cv_init(&g_cv_wait);
+  gpr_cv_init(&g_cv_shutdown);
+  g_threaded = false;
+  g_thread_count = 0;
+  g_waiter_count = 0;
+  g_completed_threads = NULL;
+
+  start_threads();
+}
+
+static void stop_threads(void) {
+  gpr_mu_lock(&g_mu);
+  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+    gpr_log(GPR_DEBUG, "stop timer threads: threaded=%d", g_threaded);
+  }
+  if (g_threaded) {
+    g_threaded = false;
+    gpr_cv_broadcast(&g_cv_wait);
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+      gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count);
+    }
+    while (g_thread_count > 0) {
+      gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME));
+      if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+        gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count);
+      }
+      gc_completed_threads();
+    }
+  }
+  gpr_mu_unlock(&g_mu);
+}
+
+void grpc_timer_manager_shutdown(void) {
+  stop_threads();
+
+  gpr_mu_destroy(&g_mu);
+  gpr_cv_destroy(&g_cv_wait);
+  gpr_cv_destroy(&g_cv_shutdown);
+}
+
+void grpc_timer_manager_set_threading(bool threaded) {
+  if (threaded) {
+    start_threads();
+  } else {
+    stop_threads();
+  }
+}
+
+void grpc_kick_poller(void) {
+  gpr_mu_lock(&g_mu);
+  g_kicked = true;
+  g_has_timed_waiter = false;
+  ++g_timed_waiter_generation;
+  gpr_cv_signal(&g_cv_wait);
+  gpr_mu_unlock(&g_mu);
+}

data/src/core/lib/iomgr/timer_manager.h

@@ -0,0 +1,52 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H
+#define GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H
+
+#include <stdbool.h>
+
+/* Timer Manager tries to keep one thread waiting for the next timeout at all
+   times */
+
+void grpc_timer_manager_init(void);
+void grpc_timer_manager_shutdown(void);
+
+/* enable/disable threading - must be called after grpc_timer_manager_init and
+ * before grpc_timer_manager_shutdown */
+void grpc_timer_manager_set_threading(bool enabled);
+/* explicitly perform one tick of the timer system - for when threading is
+ * disabled */
+void grpc_timer_manager_tick(void);
+
+#endif /* GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H */

data/src/core/lib/iomgr/timer_uv.c

@@ -38,10 +38,14 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/timer.h"
 
 #include <uv.h>
 
+grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false);
+
 static void timer_close_callback(uv_handle_t *handle) { gpr_free(handle); }
 
 static void stop_uv_timer(uv_timer_t *handle) {
@@ -100,4 +104,6 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now,
 void grpc_timer_list_init(gpr_timespec now) {}
 void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {}
 
+void grpc_timer_consume_kick(void) {}
+
 #endif /* GRPC_UV */

data/src/core/lib/iomgr/udp_server.c

@@ -79,14 +79,24 @@ struct grpc_udp_listener {
   grpc_resolved_address addr;
   grpc_closure read_closure;
   grpc_closure write_closure;
+  // To be called when corresponding QuicGrpcServer closes all active
+  // connections.
+  grpc_closure orphan_fd_closure;
   grpc_closure destroyed_closure;
   grpc_udp_server_read_cb read_cb;
   grpc_udp_server_write_cb write_cb;
   grpc_udp_server_orphan_cb orphan_cb;
+  // True if orphan_cb is trigered.
+  bool orphan_notified;
 
   struct grpc_udp_listener *next;
 };
 
+struct shutdown_fd_args {
+  grpc_fd *fd;
+  gpr_mu *server_mu;
+};
+
 /* the overall server */
 struct grpc_udp_server {
   gpr_mu mu;
@@ -146,6 +156,19 @@ grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args) {
   return s;
 }
 
+static void shutdown_fd(grpc_exec_ctx *exec_ctx, void *args,
+                        grpc_error *error) {
+  struct shutdown_fd_args *shutdown_args = (struct shutdown_fd_args *)args;
+  gpr_mu_lock(shutdown_args->server_mu);
+  grpc_fd_shutdown(exec_ctx, shutdown_args->fd, GRPC_ERROR_REF(error));
+  gpr_mu_unlock(shutdown_args->server_mu);
+  gpr_free(shutdown_args);
+}
+
+static void dummy_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  // No-op.
+}
+
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
   if (s->shutdown_complete != NULL) {
     grpc_closure_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE);
@@ -195,12 +218,16 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
 
       grpc_closure_init(&sp->destroyed_closure, destroyed_port, s,
                         grpc_schedule_on_exec_ctx);
-
-      /* Call the orphan_cb to signal that the FD is about to be closed and
-       * should no longer be used. */
-      GPR_ASSERT(sp->orphan_cb);
-      sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data);
-
+      if (!sp->orphan_notified) {
+        /* Call the orphan_cb to signal that the FD is about to be closed and
+         * should no longer be used. Because at this point, all listening ports
+         * have been shutdown already, no need to shutdown again.*/
+        grpc_closure_init(&sp->orphan_fd_closure, dummy_cb, sp->emfd,
+                          grpc_schedule_on_exec_ctx);
+        GPR_ASSERT(sp->orphan_cb);
+        sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure,
+                      sp->server->user_data);
+      }
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
                      "udp_listener_shutdown");
     }
@@ -225,9 +252,14 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s,
   if (s->active_ports) {
     for (sp = s->head; sp; sp = sp->next) {
       GPR_ASSERT(sp->orphan_cb);
-      sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data);
-      grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                                               "Server destroyed"));
+      struct shutdown_fd_args *args = gpr_malloc(sizeof(*args));
+      args->fd = sp->emfd;
+      args->server_mu = &s->mu;
+      grpc_closure_init(&sp->orphan_fd_closure, shutdown_fd, args,
+                        grpc_schedule_on_exec_ctx);
+      sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure,
+                    sp->server->user_data);
+      sp->orphan_notified = true;
     }
     gpr_mu_unlock(&s->mu);
   } else {
@@ -391,6 +423,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd,
     sp->read_cb = read_cb;
     sp->write_cb = write_cb;
     sp->orphan_cb = orphan_cb;
+    sp->orphan_notified = false;
     GPR_ASSERT(sp->emfd);
     gpr_mu_unlock(&s->mu);
     gpr_free(name);

data/src/core/lib/iomgr/udp_server.h

@@ -55,7 +55,9 @@ typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
 
 /* Called when the grpc_fd is about to be orphaned (and the FD closed). */
 typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx,
-                                          grpc_fd *emfd, void *user_data);
+                                          grpc_fd *emfd,
+                                          grpc_closure *shutdown_fd_callback,
+                                          void *user_data);
 
 /* Create a server, initially not bound to any ports */
 grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args);