wearefair-grpc 1.3.1.pre.c → 1.4.0.fair

data/src/core/lib/iomgr/exec_ctx.c

@@ -62,6 +62,11 @@ bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) {
   return true;
 }
 
+bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) {
+  return exec_ctx->active_combiner != NULL ||
+         !grpc_closure_list_empty(exec_ctx->closure_list);
+}
+
 bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
   bool did_something = 0;
   GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);

data/src/core/lib/iomgr/exec_ctx.h

@@ -93,6 +93,8 @@ struct grpc_exec_ctx {
 
 extern grpc_closure_scheduler *grpc_schedule_on_exec_ctx;
 
+bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx);
+
 /** Flush any work that has been enqueued onto this grpc_exec_ctx.
  *  Caller must guarantee that no interfering locks are held.
  *  Returns true if work was performed, false otherwise. */

data/src/core/lib/iomgr/iomgr.c

@@ -47,6 +47,7 @@
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 
@@ -67,6 +68,8 @@ void grpc_iomgr_init(void) {
   grpc_iomgr_platform_init();
 }
 
+void grpc_iomgr_start(void) { grpc_timer_manager_init(); }
+
 static size_t count_objects(void) {
   grpc_iomgr_object *obj;
   size_t n = 0;
@@ -88,6 +91,7 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) {
       gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN));
   gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME);
 
+  grpc_timer_manager_shutdown();
   grpc_iomgr_platform_flush();
 
   gpr_mu_lock(&g_mu);

data/src/core/lib/iomgr/iomgr.h

@@ -40,6 +40,9 @@
 /** Initializes the iomgr. */
 void grpc_iomgr_init(void);
 
+/** Starts any background threads for iomgr. */
+void grpc_iomgr_start(void);
+
 /** Signals the intention to shutdown the iomgr. Expects to be able to flush
  * exec_ctx. */
 void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx);

data/src/core/lib/iomgr/is_epollexclusive_available.c

@@ -0,0 +1,116 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/port.h"
+
+#include "src/core/lib/iomgr/is_epollexclusive_available.h"
+
+#ifdef GRPC_LINUX_EPOLL
+
+#include <grpc/support/log.h>
+
+#include <errno.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
+
+#include "src/core/lib/iomgr/sys_epoll_wrapper.h"
+
+/* This polling engine is only relevant on linux kernels supporting epoll() */
+bool grpc_is_epollexclusive_available(void) {
+  static bool logged_why_not = false;
+
+  int fd = epoll_create1(EPOLL_CLOEXEC);
+  if (fd < 0) {
+    if (!logged_why_not) {
+      gpr_log(GPR_ERROR,
+              "epoll_create1 failed with error: %d. Not using epollex polling "
+              "engine.",
+              fd);
+      logged_why_not = true;
+    }
+    return false;
+  }
+  int evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+  if (evfd < 0) {
+    if (!logged_why_not) {
+      gpr_log(GPR_ERROR,
+              "eventfd failed with error: %d. Not using epollex polling "
+              "engine.",
+              fd);
+      logged_why_not = true;
+    }
+    close(fd);
+    return false;
+  }
+  struct epoll_event ev = {
+      /* choose events that should cause an error on
+         EPOLLEXCLUSIVE enabled kernels - specifically the combination of
+         EPOLLONESHOT and EPOLLEXCLUSIVE */
+      .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT),
+      .data.ptr = NULL};
+  if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) {
+    if (errno != EINVAL) {
+      if (!logged_why_not) {
+        gpr_log(
+            GPR_ERROR,
+            "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: "
+            "%d. Not using epollex polling engine.",
+            errno);
+        logged_why_not = true;
+      }
+      close(fd);
+      close(evfd);
+      return false;
+    }
+  } else {
+    if (!logged_why_not) {
+      gpr_log(GPR_ERROR,
+              "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is "
+              "evidence of no EPOLLEXCLUSIVE support. Not using "
+              "epollex polling engine.");
+      logged_why_not = true;
+    }
+    close(fd);
+    close(evfd);
+    return false;
+  }
+  close(evfd);
+  close(fd);
+  return true;
+}
+
+#else
+
+bool grpc_is_epollexclusive_available(void) { return false; }
+
+#endif

data/src/core/lib/iomgr/is_epollexclusive_available.h

@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H
+#define GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H
+
+#include <stdbool.h>
+
+bool grpc_is_epollexclusive_available(void);
+
+#endif /* GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H */

data/src/core/lib/iomgr/lockfree_event.c

@@ -35,6 +35,10 @@
 
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/trace.h"
+
+extern grpc_tracer_flag grpc_polling_trace;
+
 /* 'state' holds the to call when the fd is readable or writable respectively.
    It can contain one of the following values:
      CLOSURE_READY     : The fd has an I/O event of interest but there is no
@@ -93,6 +97,10 @@ void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state,
                          grpc_closure *closure) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state,
+              (void *)curr, closure);
+    }
     switch (curr) {
       case CLOSURE_NOT_READY: {
         /* CLOSURE_NOT_READY -> <closure>.
@@ -155,6 +163,10 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
 
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state,
+              (void *)curr, grpc_error_string(shutdown_err));
+    }
     switch (curr) {
       case CLOSURE_READY:
       case CLOSURE_NOT_READY:
@@ -200,6 +212,10 @@ void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
 
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr);
+    }
+
     switch (curr) {
       case CLOSURE_READY: {
         /* Already ready. We are done here */

data/src/core/lib/iomgr/pollset.h

@@ -40,8 +40,6 @@
 
 #include "src/core/lib/iomgr/exec_ctx.h"
 
-#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
-
 /* A grpc_pollset is a set of file descriptors that a higher level item is
    interested in. For example:
     - a server will typically keep a pollset containing all connected channels,
@@ -59,7 +57,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu);
  * pollset's mutex must be held */
 void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_closure *closure);
-void grpc_pollset_destroy(grpc_pollset *pollset);
+void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset);
 
 /* Do some work on a pollset.
    May involve invoking asynchronous callbacks, or actually polling file
@@ -88,8 +86,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               gpr_timespec deadline) GRPC_MUST_USE_RESULT;
 
 /* Break one polling thread out of polling work for this pollset.
-   If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers.
-   Otherwise, if specific_worker is non-NULL, then kick that worker. */
+   If specific_worker is non-NULL, then kick that worker. */
 grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
                               grpc_pollset_worker *specific_worker)
     GRPC_MUST_USE_RESULT;

data/src/core/lib/iomgr/pollset_uv.c

@@ -106,7 +106,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE);
 }
 
-void grpc_pollset_destroy(grpc_pollset *pollset) {
+void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
   uv_close((uv_handle_t *)&pollset->timer, timer_close_cb);
   // timer.data is a boolean indicating that the timer has finished closing
   pollset->timer.data = (void *)0;

data/src/core/lib/iomgr/pollset_windows.c

@@ -43,6 +43,8 @@
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_windows.h"
 
+#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
+
 gpr_mu grpc_polling_mu;
 static grpc_pollset_worker *g_active_poller;
 static grpc_pollset_worker g_global_root_worker;
@@ -114,7 +116,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   }
 }
 
-void grpc_pollset_destroy(grpc_pollset *pollset) {}
+void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {}
 
 grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker **worker_hdl,
@@ -227,6 +229,4 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p,
   return GRPC_ERROR_NONE;
 }
 
-void grpc_kick_poller(void) { grpc_iocp_kick(); }
-
 #endif /* GRPC_WINSOCK_SOCKET */

data/src/core/lib/iomgr/resource_quota.c

@@ -44,7 +44,7 @@
 
 #include "src/core/lib/iomgr/combiner.h"
 
-int grpc_resource_quota_trace = 0;
+grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false);
 
 #define MEMORY_USAGE_ESTIMATION_MAX 65536
 
@@ -307,13 +307,14 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx,
       resource_user->free_pool = 0;
       resource_quota->free_pool -= amt;
       rq_update_estimate(resource_quota);
-      if (grpc_resource_quota_trace) {
+      if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
         gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64
                            " bytes; rq_free_pool -> %" PRId64,
                 resource_quota->name, resource_user->name, amt,
                 resource_quota->free_pool);
       }
-    } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) {
+    } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) &&
+               resource_user->free_pool >= 0) {
       gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request",
               resource_quota->name, resource_user->name);
     }
@@ -342,7 +343,7 @@ static bool rq_reclaim_from_per_user_free_pool(
       resource_user->free_pool = 0;
       resource_quota->free_pool += amt;
       rq_update_estimate(resource_quota);
-      if (grpc_resource_quota_trace) {
+      if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
         gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64
                            " bytes; rq_free_pool -> %" PRId64,
                 resource_quota->name, resource_user->name, amt,
@@ -365,7 +366,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx,
                                  : GRPC_RULIST_RECLAIMER_BENIGN;
   grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list);
   if (resource_user == NULL) return false;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation",
             resource_quota->name, resource_user->name,
             destructive ? "destructive" : "benign");
@@ -786,7 +787,7 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&resource_user->mu);
   ru_ref_by(resource_user, (gpr_atm)size);
   resource_user->free_pool -= (int64_t)size;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64,
             resource_user->resource_quota->name, resource_user->name, size,
             resource_user->free_pool);
@@ -810,7 +811,7 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&resource_user->mu);
   bool was_zero_or_negative = resource_user->free_pool <= 0;
   resource_user->free_pool += (int64_t)size;
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64,
             resource_user->resource_quota->name, resource_user->name, size,
             resource_user->free_pool);
@@ -839,7 +840,7 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx,
 
 void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx,
                                            grpc_resource_user *resource_user) {
-  if (grpc_resource_quota_trace) {
+  if (GRPC_TRACER_ON(grpc_resource_quota_trace)) {
     gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete",
             resource_user->resource_quota->name, resource_user->name);
   }

data/src/core/lib/iomgr/resource_quota.h

@@ -36,6 +36,7 @@
 
 #include <grpc/grpc.h>
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 /** \file Tracks resource usage against a pool.
@@ -75,7 +76,7 @@
     maintain lists of users (which users arrange to leave before they are
     destroyed) */
 
-extern int grpc_resource_quota_trace;
+extern grpc_tracer_flag grpc_resource_quota_trace;
 
 grpc_resource_quota *grpc_resource_quota_ref_internal(
     grpc_resource_quota *resource_quota);

data/src/core/lib/iomgr/sockaddr_utils.h

@@ -50,7 +50,7 @@ int grpc_sockaddr_to_v4mapped(const grpc_resolved_address *addr,
                               grpc_resolved_address *addr6_out);
 
 /* If addr is ::, 0.0.0.0, or ::ffff:0.0.0.0, writes the port number to
-   *port_out (if not NULL) and returns true, otherwise returns false. */
+ *port_out (if not NULL) and returns true, otherwise returns false. */
 int grpc_sockaddr_is_wildcard(const grpc_resolved_address *addr, int *port_out);
 
 /* Writes 0.0.0.0:port and [::]:port to separate sockaddrs. */

data/src/core/lib/iomgr/socket_mutator.h

@@ -37,6 +37,8 @@
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpc/support/sync.h>
 
+#include <stdbool.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif

data/src/core/lib/iomgr/sys_epoll_wrapper.h

@@ -0,0 +1,43 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H
+#define GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H
+
+#include <sys/epoll.h>
+
+#ifndef EPOLLEXCLUSIVE
+#define EPOLLEXCLUSIVE (1 << 28)
+#endif
+
+#endif /* GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H */