grpc 1.7.3 → 1.8.0

data/src/core/lib/http/parser.h

@@ -27,10 +27,14 @@
 /* Maximum length of a header string of the form 'Key: Value\r\n' */
 #define GRPC_HTTP_PARSER_MAX_HEADER_LENGTH 4096
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* A single header to be passed in a request */
 typedef struct grpc_http_header {
-  char *key;
-  char *value;
+  char* key;
+  char* value;
 } grpc_http_header;
 
 typedef enum {
@@ -53,17 +57,17 @@ typedef enum {
 /* A request */
 typedef struct grpc_http_request {
   /* Method of the request (e.g. GET, POST) */
-  char *method;
+  char* method;
   /* The path of the resource to fetch */
-  char *path;
+  char* path;
   /* HTTP version to use */
   grpc_http_version version;
   /* Headers attached to the request */
   size_t hdr_count;
-  grpc_http_header *hdrs;
+  grpc_http_header* hdrs;
   /* Body: length and contents; contents are NOT null-terminated */
   size_t body_length;
-  char *body;
+  char* body;
 } grpc_http_request;
 
 /* A response */
@@ -72,10 +76,10 @@ typedef struct grpc_http_response {
   int status;
   /* Headers: count and key/values */
   size_t hdr_count;
-  grpc_http_header *hdrs;
+  grpc_http_header* hdrs;
   /* Body: length and contents; contents are NOT null-terminated */
   size_t body_length;
-  char *body;
+  char* body;
 } grpc_http_response;
 
 typedef struct {
@@ -83,9 +87,9 @@ typedef struct {
   grpc_http_type type;
 
   union {
-    grpc_http_response *response;
-    grpc_http_request *request;
-    void *request_or_response;
+    grpc_http_response* response;
+    grpc_http_request* request;
+    void* request_or_response;
   } http;
   size_t body_capacity;
   size_t hdr_capacity;
@@ -95,18 +99,22 @@ typedef struct {
   size_t cur_line_end_length;
 } grpc_http_parser;
 
-void grpc_http_parser_init(grpc_http_parser *parser, grpc_http_type type,
-                           void *request_or_response);
-void grpc_http_parser_destroy(grpc_http_parser *parser);
+void grpc_http_parser_init(grpc_http_parser* parser, grpc_http_type type,
+                           void* request_or_response);
+void grpc_http_parser_destroy(grpc_http_parser* parser);
 
 /* Sets \a start_of_body to the offset in \a slice of the start of the body. */
-grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, grpc_slice slice,
-                                   size_t *start_of_body);
-grpc_error *grpc_http_parser_eof(grpc_http_parser *parser);
+grpc_error* grpc_http_parser_parse(grpc_http_parser* parser, grpc_slice slice,
+                                   size_t* start_of_body);
+grpc_error* grpc_http_parser_eof(grpc_http_parser* parser);
+
+void grpc_http_request_destroy(grpc_http_request* request);
+void grpc_http_response_destroy(grpc_http_response* response);
 
-void grpc_http_request_destroy(grpc_http_request *request);
-void grpc_http_response_destroy(grpc_http_response *response);
+extern grpc_core::TraceFlag grpc_http1_trace;
 
-extern grpc_tracer_flag grpc_http1_trace;
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* GRPC_CORE_LIB_HTTP_PARSER_H */

data/src/core/lib/{support → iomgr}/block_annotate.h

@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
-#define GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H
+#ifndef GRPC_CORE_LIB_IOMGR_BLOCK_ANNOTATE_H
+#define GRPC_CORE_LIB_IOMGR_BLOCK_ANNOTATE_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,17 +39,26 @@ void gpr_thd_end_blocking_region();
   do {                                        \
     gpr_thd_start_blocking_region();          \
   } while (0)
-#define GRPC_SCHEDULING_END_BLOCKING_REGION \
-  do {                                      \
-    gpr_thd_end_blocking_region();          \
+#define GRPC_SCHEDULING_END_BLOCKING_REGION_NO_EXEC_CTX \
+  do {                                                  \
+    gpr_thd_end_blocking_region();                      \
+  } while (0)
+#define GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(ec) \
+  do {                                                        \
+    gpr_thd_end_blocking_region();                            \
+    grpc_exec_ctx_invalidate_now((ec));                       \
   } while (0)
 #else
 #define GRPC_SCHEDULING_START_BLOCKING_REGION \
   do {                                        \
   } while (0)
-#define GRPC_SCHEDULING_END_BLOCKING_REGION \
-  do {                                      \
+#define GRPC_SCHEDULING_END_BLOCKING_REGION_NO_EXEC_CTX \
+  do {                                                  \
+  } while (0)
+#define GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(ec) \
+  do {                                                        \
+    grpc_exec_ctx_invalidate_now((ec));                       \
   } while (0)
 #endif
 
-#endif /* GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H */
+#endif /* GRPC_CORE_LIB_IOMGR_BLOCK_ANNOTATE_H */

data/src/core/lib/iomgr/{call_combiner.c → call_combiner.cc}

@@ -18,10 +18,13 @@
 
 #include "src/core/lib/iomgr/call_combiner.h"
 
+#include <inttypes.h>
+
 #include <grpc/support/log.h>
+#include "src/core/lib/debug/stats.h"
+#include "src/core/lib/profiling/timers.h"
 
-grpc_tracer_flag grpc_call_combiner_trace =
-    GRPC_TRACER_INITIALIZER(false, "call_combiner");
+grpc_core::TraceFlag grpc_call_combiner_trace(false, "call_combiner");
 
 static grpc_error* decode_cancel_state_error(gpr_atm cancel_state) {
   if (cancel_state & 1) {
@@ -58,7 +61,8 @@ void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
                               grpc_closure* closure,
                               grpc_error* error DEBUG_ARGS,
                               const char* reason) {
-  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+  GPR_TIMER_BEGIN("call_combiner_start", 0);
+  if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_DEBUG,
             "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR
             "%s] error=%s",
@@ -67,72 +71,79 @@ void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
   }
   size_t prev_size =
       (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)1);
-  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+  if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
             prev_size + 1);
   }
+  GRPC_STATS_INC_CALL_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx);
   if (prev_size == 0) {
-    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    GRPC_STATS_INC_CALL_COMBINER_LOCKS_INITIATED(exec_ctx);
+    GPR_TIMER_MARK("call_combiner_initiate", 0);
+    if (grpc_call_combiner_trace.enabled()) {
       gpr_log(GPR_DEBUG, "  EXECUTING IMMEDIATELY");
     }
     // Queue was empty, so execute this closure immediately.
     GRPC_CLOSURE_SCHED(exec_ctx, closure, error);
   } else {
-    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    if (grpc_call_combiner_trace.enabled()) {
       gpr_log(GPR_INFO, "  QUEUING");
     }
     // Queue was not empty, so add closure to queue.
     closure->error_data.error = error;
     gpr_mpscq_push(&call_combiner->queue, (gpr_mpscq_node*)closure);
   }
+  GPR_TIMER_END("call_combiner_start", 0);
 }
 
 void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
                              grpc_call_combiner* call_combiner DEBUG_ARGS,
                              const char* reason) {
-  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+  GPR_TIMER_BEGIN("call_combiner_stop", 0);
+  if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_DEBUG,
             "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]",
             call_combiner DEBUG_FMT_ARGS, reason);
   }
   size_t prev_size =
       (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)-1);
-  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+  if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
             prev_size - 1);
   }
   GPR_ASSERT(prev_size >= 1);
   if (prev_size > 1) {
     while (true) {
-      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      if (grpc_call_combiner_trace.enabled()) {
         gpr_log(GPR_DEBUG, "  checking queue");
       }
       bool empty;
       grpc_closure* closure = (grpc_closure*)gpr_mpscq_pop_and_check_end(
           &call_combiner->queue, &empty);
-      if (closure == NULL) {
+      if (closure == nullptr) {
         // This can happen either due to a race condition within the mpscq
         // code or because of a race with grpc_call_combiner_start().
-        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_DEBUG, "  queue returned no result; checking again");
         }
         continue;
       }
-      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      if (grpc_call_combiner_trace.enabled()) {
         gpr_log(GPR_DEBUG, "  EXECUTING FROM QUEUE: closure=%p error=%s",
                 closure, grpc_error_string(closure->error_data.error));
       }
       GRPC_CLOSURE_SCHED(exec_ctx, closure, closure->error_data.error);
       break;
     }
-  } else if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+  } else if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_DEBUG, "  queue empty");
   }
+  GPR_TIMER_END("call_combiner_stop", 0);
 }
 
 void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
                                              grpc_call_combiner* call_combiner,
                                              grpc_closure* closure) {
+  GRPC_STATS_INC_CALL_COMBINER_SET_NOTIFY_ON_CANCEL(exec_ctx);
   while (true) {
     // Decode original state.
     gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
@@ -140,7 +151,7 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
     // If error is set, invoke the cancellation closure immediately.
     // Otherwise, store the new closure.
     if (original_error != GRPC_ERROR_NONE) {
-      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      if (grpc_call_combiner_trace.enabled()) {
         gpr_log(GPR_DEBUG,
                 "call_combiner=%p: scheduling notify_on_cancel callback=%p "
                 "for pre-existing cancellation",
@@ -151,7 +162,7 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
     } else {
       if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
                            (gpr_atm)closure)) {
-        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_DEBUG, "call_combiner=%p: setting notify_on_cancel=%p",
                   call_combiner, closure);
         }
@@ -160,7 +171,7 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
         // up any resources they may be holding for the callback.
         if (original_state != 0) {
           closure = (grpc_closure*)original_state;
-          if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          if (grpc_call_combiner_trace.enabled()) {
             gpr_log(GPR_DEBUG,
                     "call_combiner=%p: scheduling old cancel callback=%p",
                     call_combiner, closure);
@@ -177,6 +188,7 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
 void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
                                grpc_call_combiner* call_combiner,
                                grpc_error* error) {
+  GRPC_STATS_INC_CALL_COMBINER_CANCELLED(exec_ctx);
   while (true) {
     gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
     grpc_error* original_error = decode_cancel_state_error(original_state);
@@ -188,7 +200,7 @@ void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
                          encode_cancel_state_error(error))) {
       if (original_state != 0) {
         grpc_closure* notify_on_cancel = (grpc_closure*)original_state;
-        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_DEBUG,
                   "call_combiner=%p: scheduling notify_on_cancel callback=%p",
                   call_combiner, notify_on_cancel);

data/src/core/lib/iomgr/call_combiner.h

@@ -27,6 +27,10 @@
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/mpscq.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // A simple, lock-free mechanism for serializing activity related to a
 // single call.  This is similar to a combiner but is more lightweight.
 //
@@ -36,7 +40,7 @@
 // when it is done with the action that was kicked off by the original
 // callback.
 
-extern grpc_tracer_flag grpc_call_combiner_trace;
+extern grpc_core::TraceFlag grpc_call_combiner_trace;
 
 typedef struct {
   gpr_atm size;  // size_t, num closures in queue or currently executing
@@ -118,4 +122,8 @@ void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
                                grpc_call_combiner* call_combiner,
                                grpc_error* error);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H */

data/src/core/lib/iomgr/closure.h

@@ -21,25 +21,23 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <assert.h>
 #include <grpc/impl/codegen/exec_ctx_fwd.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include <stdbool.h>
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/mpscq.h"
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 struct grpc_closure;
 typedef struct grpc_closure grpc_closure;
 
-#ifndef NDEBUG
-extern grpc_tracer_flag grpc_trace_closure;
-#endif
+extern grpc_core::DebugOnlyTraceFlag grpc_trace_closure;
 
 typedef struct grpc_closure_list {
-  grpc_closure *head;
-  grpc_closure *tail;
+  grpc_closure* head;
+  grpc_closure* tail;
 } grpc_closure_list;
 
 /** gRPC Callback definition.
@@ -49,24 +47,24 @@ typedef struct grpc_closure_list {
  *              describing what went wrong.
  *              Error contract: it is not the cb's job to unref this error;
  *              the closure scheduler will do that after the cb returns */
-typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg,
-                                   grpc_error *error);
+typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx* exec_ctx, void* arg,
+                                   grpc_error* error);
 
 typedef struct grpc_closure_scheduler grpc_closure_scheduler;
 
 typedef struct grpc_closure_scheduler_vtable {
   /* NOTE: for all these functions, closure->scheduler == the scheduler that was
            used to find this vtable */
-  void (*run)(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-              grpc_error *error);
-  void (*sched)(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                grpc_error *error);
-  const char *name;
+  void (*run)(grpc_exec_ctx* exec_ctx, grpc_closure* closure,
+              grpc_error* error);
+  void (*sched)(grpc_exec_ctx* exec_ctx, grpc_closure* closure,
+                grpc_error* error);
+  const char* name;
 } grpc_closure_scheduler_vtable;
 
 /** Abstract type that can schedule closures for execution */
 struct grpc_closure_scheduler {
-  const grpc_closure_scheduler_vtable *vtable;
+  const grpc_closure_scheduler_vtable* vtable;
 };
 
 /** A closure over a grpc_iomgr_cb_func. */
@@ -74,7 +72,7 @@ struct grpc_closure {
   /** Once queued, next indicates the next queued closure; before then, scratch
    *  space */
   union {
-    grpc_closure *next;
+    grpc_closure* next;
     gpr_mpscq_node atm_next;
     uintptr_t scratch;
   } next_data;
@@ -83,15 +81,15 @@ struct grpc_closure {
   grpc_iomgr_cb_func cb;
 
   /** Arguments to be passed to "cb". */
-  void *cb_arg;
+  void* cb_arg;
 
-  /** Scheduler to schedule against: NULL to schedule against current execution
-      context */
-  grpc_closure_scheduler *scheduler;
+  /** Scheduler to schedule against: nullptr to schedule against current
+     execution context */
+  grpc_closure_scheduler* scheduler;
 
   /** Once queued, the result of the closure. Before then: scratch space */
   union {
-    grpc_error *error;
+    grpc_error* error;
     uintptr_t scratch;
   } error_data;
 
@@ -100,109 +98,269 @@ struct grpc_closure {
 #ifndef NDEBUG
   bool scheduled;
   bool run;  // true = run, false = scheduled
-  const char *file_created;
+  const char* file_created;
   int line_created;
-  const char *file_initiated;
+  const char* file_initiated;
   int line_initiated;
 #endif
 };
 
+#ifndef NDEBUG
+inline grpc_closure* grpc_closure_init(const char* file, int line,
+                                       grpc_closure* closure,
+                                       grpc_iomgr_cb_func cb, void* cb_arg,
+                                       grpc_closure_scheduler* scheduler) {
+#else
+inline grpc_closure* grpc_closure_init(grpc_closure* closure,
+                                       grpc_iomgr_cb_func cb, void* cb_arg,
+                                       grpc_closure_scheduler* scheduler) {
+#endif
+  closure->cb = cb;
+  closure->cb_arg = cb_arg;
+  closure->scheduler = scheduler;
+#ifndef NDEBUG
+  closure->scheduled = false;
+  closure->file_initiated = nullptr;
+  closure->line_initiated = 0;
+  closure->run = false;
+  closure->file_created = file;
+  closure->line_created = line;
+#endif
+  return closure;
+}
+
 /** Initializes \a closure with \a cb and \a cb_arg. Returns \a closure. */
 #ifndef NDEBUG
-grpc_closure *grpc_closure_init(const char *file, int line,
-                                grpc_closure *closure, grpc_iomgr_cb_func cb,
-                                void *cb_arg,
-                                grpc_closure_scheduler *scheduler);
 #define GRPC_CLOSURE_INIT(closure, cb, cb_arg, scheduler) \
   grpc_closure_init(__FILE__, __LINE__, closure, cb, cb_arg, scheduler)
 #else
-grpc_closure *grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
-                                void *cb_arg,
-                                grpc_closure_scheduler *scheduler);
 #define GRPC_CLOSURE_INIT(closure, cb, cb_arg, scheduler) \
   grpc_closure_init(closure, cb, cb_arg, scheduler)
 #endif
 
+namespace closure_impl {
+
+typedef struct {
+  grpc_iomgr_cb_func cb;
+  void* cb_arg;
+  grpc_closure wrapper;
+} wrapped_closure;
+
+inline void closure_wrapper(grpc_exec_ctx* exec_ctx, void* arg,
+                            grpc_error* error) {
+  wrapped_closure* wc = (wrapped_closure*)arg;
+  grpc_iomgr_cb_func cb = wc->cb;
+  void* cb_arg = wc->cb_arg;
+  gpr_free(wc);
+  cb(exec_ctx, cb_arg, error);
+}
+
+}  // namespace closure_impl
+
+#ifndef NDEBUG
+inline grpc_closure* grpc_closure_create(const char* file, int line,
+                                         grpc_iomgr_cb_func cb, void* cb_arg,
+                                         grpc_closure_scheduler* scheduler) {
+#else
+inline grpc_closure* grpc_closure_create(grpc_iomgr_cb_func cb, void* cb_arg,
+                                         grpc_closure_scheduler* scheduler) {
+#endif
+  closure_impl::wrapped_closure* wc =
+      (closure_impl::wrapped_closure*)gpr_malloc(sizeof(*wc));
+  wc->cb = cb;
+  wc->cb_arg = cb_arg;
+#ifndef NDEBUG
+  grpc_closure_init(file, line, &wc->wrapper, closure_impl::closure_wrapper, wc,
+                    scheduler);
+#else
+  grpc_closure_init(&wc->wrapper, closure_impl::closure_wrapper, wc, scheduler);
+#endif
+  return &wc->wrapper;
+}
+
 /* Create a heap allocated closure: try to avoid except for very rare events */
 #ifndef NDEBUG
-grpc_closure *grpc_closure_create(const char *file, int line,
-                                  grpc_iomgr_cb_func cb, void *cb_arg,
-                                  grpc_closure_scheduler *scheduler);
 #define GRPC_CLOSURE_CREATE(cb, cb_arg, scheduler) \
   grpc_closure_create(__FILE__, __LINE__, cb, cb_arg, scheduler)
 #else
-grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg,
-                                  grpc_closure_scheduler *scheduler);
 #define GRPC_CLOSURE_CREATE(cb, cb_arg, scheduler) \
   grpc_closure_create(cb, cb_arg, scheduler)
 #endif
 
 #define GRPC_CLOSURE_LIST_INIT \
-  { NULL, NULL }
+  { nullptr, nullptr }
 
-void grpc_closure_list_init(grpc_closure_list *list);
+inline void grpc_closure_list_init(grpc_closure_list* closure_list) {
+  closure_list->head = closure_list->tail = nullptr;
+}
 
 /** add \a closure to the end of \a list
     and set \a closure's result to \a error
     Returns true if \a list becomes non-empty */
-bool grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure,
-                              grpc_error *error);
+inline bool grpc_closure_list_append(grpc_closure_list* closure_list,
+                                     grpc_closure* closure, grpc_error* error) {
+  if (closure == nullptr) {
+    GRPC_ERROR_UNREF(error);
+    return false;
+  }
+  closure->error_data.error = error;
+  closure->next_data.next = nullptr;
+  bool was_empty = (closure_list->head == nullptr);
+  if (was_empty) {
+    closure_list->head = closure;
+  } else {
+    closure_list->tail->next_data.next = closure;
+  }
+  closure_list->tail = closure;
+  return was_empty;
+}
 
 /** force all success bits in \a list to false */
-void grpc_closure_list_fail_all(grpc_closure_list *list,
-                                grpc_error *forced_failure);
+inline void grpc_closure_list_fail_all(grpc_closure_list* list,
+                                       grpc_error* forced_failure) {
+  for (grpc_closure* c = list->head; c != nullptr; c = c->next_data.next) {
+    if (c->error_data.error == GRPC_ERROR_NONE) {
+      c->error_data.error = GRPC_ERROR_REF(forced_failure);
+    }
+  }
+  GRPC_ERROR_UNREF(forced_failure);
+}
 
 /** append all closures from \a src to \a dst and empty \a src. */
-void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst);
+inline void grpc_closure_list_move(grpc_closure_list* src,
+                                   grpc_closure_list* dst) {
+  if (src->head == nullptr) {
+    return;
+  }
+  if (dst->head == nullptr) {
+    *dst = *src;
+  } else {
+    dst->tail->next_data.next = src->head;
+    dst->tail = src->tail;
+  }
+  src->head = src->tail = nullptr;
+}
 
 /** return whether \a list is empty. */
-bool grpc_closure_list_empty(grpc_closure_list list);
+inline bool grpc_closure_list_empty(grpc_closure_list closure_list) {
+  return closure_list.head == nullptr;
+}
+
+#ifndef NDEBUG
+inline void grpc_closure_run(const char* file, int line,
+                             grpc_exec_ctx* exec_ctx, grpc_closure* c,
+                             grpc_error* error) {
+#else
+inline void grpc_closure_run(grpc_exec_ctx* exec_ctx, grpc_closure* c,
+                             grpc_error* error) {
+#endif
+  GPR_TIMER_BEGIN("grpc_closure_run", 0);
+  if (c != nullptr) {
+#ifndef NDEBUG
+    c->file_initiated = file;
+    c->line_initiated = line;
+    c->run = true;
+#endif
+    assert(c->cb);
+    c->scheduler->vtable->run(exec_ctx, c, error);
+  } else {
+    GRPC_ERROR_UNREF(error);
+  }
+  GPR_TIMER_END("grpc_closure_run", 0);
+}
 
 /** Run a closure directly. Caller ensures that no locks are being held above.
  *  Note that calling this at the end of a closure callback function itself is
  *  by definition safe. */
 #ifndef NDEBUG
-void grpc_closure_run(const char *file, int line, grpc_exec_ctx *exec_ctx,
-                      grpc_closure *closure, grpc_error *error);
 #define GRPC_CLOSURE_RUN(exec_ctx, closure, error) \
   grpc_closure_run(__FILE__, __LINE__, exec_ctx, closure, error)
 #else
-void grpc_closure_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                      grpc_error *error);
 #define GRPC_CLOSURE_RUN(exec_ctx, closure, error) \
   grpc_closure_run(exec_ctx, closure, error)
 #endif
 
+#ifndef NDEBUG
+inline void grpc_closure_sched(const char* file, int line,
+                               grpc_exec_ctx* exec_ctx, grpc_closure* c,
+                               grpc_error* error) {
+#else
+inline void grpc_closure_sched(grpc_exec_ctx* exec_ctx, grpc_closure* c,
+                               grpc_error* error) {
+#endif
+  GPR_TIMER_BEGIN("grpc_closure_sched", 0);
+  if (c != nullptr) {
+#ifndef NDEBUG
+    if (c->scheduled) {
+      gpr_log(GPR_ERROR,
+              "Closure already scheduled. (closure: %p, created: [%s:%d], "
+              "previously scheduled at: [%s: %d] run?: %s",
+              c, c->file_created, c->line_created, c->file_initiated,
+              c->line_initiated, c->run ? "true" : "false");
+      abort();
+    }
+    c->scheduled = true;
+    c->file_initiated = file;
+    c->line_initiated = line;
+    c->run = false;
+#endif
+    assert(c->cb);
+    c->scheduler->vtable->sched(exec_ctx, c, error);
+  } else {
+    GRPC_ERROR_UNREF(error);
+  }
+  GPR_TIMER_END("grpc_closure_sched", 0);
+}
+
 /** Schedule a closure to be run. Does not need to be run from a safe point. */
 #ifndef NDEBUG
-void grpc_closure_sched(const char *file, int line, grpc_exec_ctx *exec_ctx,
-                        grpc_closure *closure, grpc_error *error);
 #define GRPC_CLOSURE_SCHED(exec_ctx, closure, error) \
   grpc_closure_sched(__FILE__, __LINE__, exec_ctx, closure, error)
 #else
-void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                        grpc_error *error);
 #define GRPC_CLOSURE_SCHED(exec_ctx, closure, error) \
   grpc_closure_sched(exec_ctx, closure, error)
 #endif
 
+#ifndef NDEBUG
+inline void grpc_closure_list_sched(const char* file, int line,
+                                    grpc_exec_ctx* exec_ctx,
+                                    grpc_closure_list* list) {
+#else
+inline void grpc_closure_list_sched(grpc_exec_ctx* exec_ctx,
+                                    grpc_closure_list* list) {
+#endif
+  grpc_closure* c = list->head;
+  while (c != nullptr) {
+    grpc_closure* next = c->next_data.next;
+#ifndef NDEBUG
+    if (c->scheduled) {
+      gpr_log(GPR_ERROR,
+              "Closure already scheduled. (closure: %p, created: [%s:%d], "
+              "previously scheduled at: [%s: %d] run?: %s",
+              c, c->file_created, c->line_created, c->file_initiated,
+              c->line_initiated, c->run ? "true" : "false");
+      abort();
+    }
+    c->scheduled = true;
+    c->file_initiated = file;
+    c->line_initiated = line;
+    c->run = false;
+#endif
+    assert(c->cb);
+    c->scheduler->vtable->sched(exec_ctx, c, c->error_data.error);
+    c = next;
+  }
+  list->head = list->tail = nullptr;
+}
+
 /** Schedule all closures in a list to be run. Does not need to be run from a
  * safe point. */
 #ifndef NDEBUG
-void grpc_closure_list_sched(const char *file, int line,
-                             grpc_exec_ctx *exec_ctx,
-                             grpc_closure_list *closure_list);
 #define GRPC_CLOSURE_LIST_SCHED(exec_ctx, closure_list) \
   grpc_closure_list_sched(__FILE__, __LINE__, exec_ctx, closure_list)
 #else
-void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx,
-                             grpc_closure_list *closure_list);
 #define GRPC_CLOSURE_LIST_SCHED(exec_ctx, closure_list) \
   grpc_closure_list_sched(exec_ctx, closure_list)
 #endif
 
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* GRPC_CORE_LIB_IOMGR_CLOSURE_H */