grpc 1.30.2

data/src/core/lib/iomgr/buffer_list.h

@@ -0,0 +1,165 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H
+#define GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/types/optional.h"
+
+#include "src/core/lib/iomgr/port.h"
+
+#include <grpc/support/time.h>
+
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/internal_errqueue.h"
+
+namespace grpc_core {
+
+struct ConnectionMetrics {
+  /* Delivery rate in Bytes/s. */
+  absl::optional<uint64_t> delivery_rate;
+  /* If the delivery rate is limited by the application, this is set to true. */
+  absl::optional<bool> is_delivery_rate_app_limited;
+  /* Total packets retransmitted. */
+  absl::optional<uint32_t> packet_retx;
+  /* Total packets retransmitted spuriously. This metric is smaller than or
+  equal to packet_retx. */
+  absl::optional<uint32_t> packet_spurious_retx;
+  /* Total packets sent. */
+  absl::optional<uint32_t> packet_sent;
+  /* Total packets delivered. */
+  absl::optional<uint32_t> packet_delivered;
+  /* Total packets delivered with ECE marked. This metric is smaller than or
+  equal to packet_delivered. */
+  absl::optional<uint32_t> packet_delivered_ce;
+  /* Total bytes lost so far. */
+  absl::optional<uint64_t> data_retx;
+  /* Total bytes sent so far. */
+  absl::optional<uint64_t> data_sent;
+  /* Total bytes in write queue but not sent. */
+  absl::optional<uint64_t> data_notsent;
+  /* Pacing rate of the connection in Bps */
+  absl::optional<uint64_t> pacing_rate;
+  /* Minimum RTT observed in usec. */
+  absl::optional<uint32_t> min_rtt;
+  /* Smoothed RTT in usec */
+  absl::optional<uint32_t> srtt;
+  /* Send congestion window. */
+  absl::optional<uint32_t> congestion_window;
+  /* Slow start threshold in packets. */
+  absl::optional<uint32_t> snd_ssthresh;
+  /* Maximum degree of reordering (i.e., maximum number of packets reodered)
+   on the connection. */
+  absl::optional<uint32_t> reordering;
+  /* Represents the number of recurring retransmissions of the first sequence
+  that is not acknowledged yet. */
+  absl::optional<uint8_t> recurring_retrans;
+  /* The cumulative time (in usec) that the transport protocol was busy
+   sending data. */
+  absl::optional<uint64_t> busy_usec;
+  /* The cumulative time (in usec) that the transport protocol was limited by
+   the receive window size. */
+  absl::optional<uint64_t> rwnd_limited_usec;
+  /* The cumulative time (in usec) that the transport protocol was limited by
+   the send buffer size. */
+  absl::optional<uint64_t> sndbuf_limited_usec;
+};
+
+struct Timestamp {
+  gpr_timespec time;
+  ConnectionMetrics metrics; /* Metrics collected with this timestamp */
+};
+
+struct Timestamps {
+  Timestamp sendmsg_time;
+  Timestamp scheduled_time;
+  Timestamp sent_time;
+  Timestamp acked_time;
+
+  uint32_t byte_offset; /* byte offset relative to the start of the RPC */
+
+#ifdef GRPC_LINUX_ERRQUEUE
+  grpc_core::tcp_info info; /* tcp_info collected on sendmsg */
+#endif                      /* GRPC_LINUX_ERRQUEUE */
+};
+
+/** TracedBuffer is a class to keep track of timestamps for a specific buffer in
+ * the TCP layer. We are only tracking timestamps for Linux kernels and hence
+ * this class would only be used by Linux platforms. For all other platforms,
+ * TracedBuffer would be an empty class.
+ *
+ * The timestamps collected are according to grpc_core::Timestamps declared
+ * above.
+ *
+ * A TracedBuffer list is kept track of using the head element of the list. If
+ * the head element of the list is nullptr, then the list is empty.
+ */
+#ifdef GRPC_LINUX_ERRQUEUE
+class TracedBuffer {
+ public:
+  /** Use AddNewEntry function instead of using this directly. */
+  TracedBuffer(uint32_t seq_no, void* arg)
+      : seq_no_(seq_no), arg_(arg), next_(nullptr) {}
+
+  /** Add a new entry in the TracedBuffer list pointed to by head. Also saves
+   * sendmsg_time with the current timestamp. */
+  static void AddNewEntry(grpc_core::TracedBuffer** head, uint32_t seq_no,
+                          int fd, void* arg);
+
+  /** Processes a received timestamp based on sock_extended_err and
+   * scm_timestamping structures. It will invoke the timestamps callback if the
+   * timestamp type is SCM_TSTAMP_ACK. */
+  static void ProcessTimestamp(grpc_core::TracedBuffer** head,
+                               struct sock_extended_err* serr,
+                               struct cmsghdr* opt_stats,
+                               struct scm_timestamping* tss);
+
+  /** Cleans the list by calling the callback for each traced buffer in the list
+   * with timestamps that it has. */
+  static void Shutdown(grpc_core::TracedBuffer** head, void* remaining,
+                       grpc_error* shutdown_err);
+
+ private:
+  uint32_t seq_no_; /* The sequence number for the last byte in the buffer */
+  void* arg_;       /* The arg to pass to timestamps_callback */
+  grpc_core::Timestamps ts_; /* The timestamps corresponding to this buffer */
+  grpc_core::TracedBuffer* next_; /* The next TracedBuffer in the list */
+};
+#else  /* GRPC_LINUX_ERRQUEUE */
+class TracedBuffer {
+ public:
+  /* Dummy shutdown function */
+  static void Shutdown(grpc_core::TracedBuffer** /*head*/, void* /*remaining*/,
+                       grpc_error* shutdown_err) {
+    GRPC_ERROR_UNREF(shutdown_err);
+  }
+};
+#endif /* GRPC_LINUX_ERRQUEUE */
+
+/** Sets the callback function to call when timestamps for a write are
+ *  collected. The callback does not own a reference to error. */
+void grpc_tcp_set_write_timestamps_callback(void (*fn)(void*,
+                                                       grpc_core::Timestamps*,
+                                                       grpc_error* error));
+
+} /* namespace grpc_core */
+
+#endif /* GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H */

data/src/core/lib/iomgr/call_combiner.cc

@@ -0,0 +1,256 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/call_combiner.h"
+
+#include <inttypes.h>
+
+#include <grpc/support/log.h>
+#include "src/core/lib/debug/stats.h"
+#include "src/core/lib/profiling/timers.h"
+
+namespace grpc_core {
+
+DebugOnlyTraceFlag grpc_call_combiner_trace(false, "call_combiner");
+
+namespace {
+
+grpc_error* DecodeCancelStateError(gpr_atm cancel_state) {
+  if (cancel_state & 1) {
+    return (grpc_error*)(cancel_state & ~static_cast<gpr_atm>(1));
+  }
+  return GRPC_ERROR_NONE;
+}
+
+gpr_atm EncodeCancelStateError(grpc_error* error) {
+  return static_cast<gpr_atm>(1) | (gpr_atm)error;
+}
+
+}  // namespace
+
+CallCombiner::CallCombiner() {
+  gpr_atm_no_barrier_store(&cancel_state_, 0);
+  gpr_atm_no_barrier_store(&size_, 0);
+#ifdef GRPC_TSAN_ENABLED
+  GRPC_CLOSURE_INIT(&tsan_closure_, TsanClosure, this,
+                    grpc_schedule_on_exec_ctx);
+#endif
+}
+
+CallCombiner::~CallCombiner() {
+  GRPC_ERROR_UNREF(DecodeCancelStateError(cancel_state_));
+}
+
+#ifdef GRPC_TSAN_ENABLED
+void CallCombiner::TsanClosure(void* arg, grpc_error* error) {
+  CallCombiner* self = static_cast<CallCombiner*>(arg);
+  // We ref-count the lock, and check if it's already taken.
+  // If it was taken, we should do nothing. Otherwise, we will mark it as
+  // locked. Note that if two different threads try to do this, only one of
+  // them will be able to mark the lock as acquired, while they both run their
+  // callbacks. In such cases (which should never happen for call_combiner),
+  // TSAN will correctly produce an error.
+  //
+  // TODO(soheil): This only covers the callbacks scheduled by
+  //               CallCombiner::Start() and CallCombiner::Stop().
+  //               If in the future, a callback gets scheduled using other
+  //               mechanisms, we will need to add APIs to externally lock
+  //               call combiners.
+  RefCountedPtr<TsanLock> lock = self->tsan_lock_;
+  bool prev = false;
+  if (lock->taken.compare_exchange_strong(prev, true)) {
+    TSAN_ANNOTATE_RWLOCK_ACQUIRED(&lock->taken, true);
+  } else {
+    lock.reset();
+  }
+  grpc_core::Closure::Run(DEBUG_LOCATION, self->original_closure_,
+                          GRPC_ERROR_REF(error));
+  if (lock != nullptr) {
+    TSAN_ANNOTATE_RWLOCK_RELEASED(&lock->taken, true);
+    bool prev = true;
+    GPR_ASSERT(lock->taken.compare_exchange_strong(prev, false));
+  }
+}
+#endif
+
+void CallCombiner::ScheduleClosure(grpc_closure* closure, grpc_error* error) {
+#ifdef GRPC_TSAN_ENABLED
+  original_closure_ = closure;
+  ExecCtx::Run(DEBUG_LOCATION, &tsan_closure_, error);
+#else
+  ExecCtx::Run(DEBUG_LOCATION, closure, error);
+#endif
+}
+
+#ifndef NDEBUG
+#define DEBUG_ARGS const char *file, int line,
+#define DEBUG_FMT_STR "%s:%d: "
+#define DEBUG_FMT_ARGS , file, line
+#else
+#define DEBUG_ARGS
+#define DEBUG_FMT_STR
+#define DEBUG_FMT_ARGS
+#endif
+
+void CallCombiner::Start(grpc_closure* closure, grpc_error* error,
+                         DEBUG_ARGS const char* reason) {
+  GPR_TIMER_SCOPE("CallCombiner::Start", 0);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+    gpr_log(GPR_INFO,
+            "==> CallCombiner::Start() [%p] closure=%p [" DEBUG_FMT_STR
+            "%s] error=%s",
+            this, closure DEBUG_FMT_ARGS, reason, grpc_error_string(error));
+  }
+  size_t prev_size =
+      static_cast<size_t>(gpr_atm_full_fetch_add(&size_, (gpr_atm)1));
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+    gpr_log(GPR_INFO, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size + 1);
+  }
+  GRPC_STATS_INC_CALL_COMBINER_LOCKS_SCHEDULED_ITEMS();
+  if (prev_size == 0) {
+    GRPC_STATS_INC_CALL_COMBINER_LOCKS_INITIATED();
+    GPR_TIMER_MARK("call_combiner_initiate", 0);
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+      gpr_log(GPR_INFO, "  EXECUTING IMMEDIATELY");
+    }
+    // Queue was empty, so execute this closure immediately.
+    ScheduleClosure(closure, error);
+  } else {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+      gpr_log(GPR_INFO, "  QUEUING");
+    }
+    // Queue was not empty, so add closure to queue.
+    closure->error_data.error = error;
+    queue_.Push(
+        reinterpret_cast<MultiProducerSingleConsumerQueue::Node*>(closure));
+  }
+}
+
+void CallCombiner::Stop(DEBUG_ARGS const char* reason) {
+  GPR_TIMER_SCOPE("CallCombiner::Stop", 0);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+    gpr_log(GPR_INFO, "==> CallCombiner::Stop() [%p] [" DEBUG_FMT_STR "%s]",
+            this DEBUG_FMT_ARGS, reason);
+  }
+  size_t prev_size =
+      static_cast<size_t>(gpr_atm_full_fetch_add(&size_, (gpr_atm)-1));
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+    gpr_log(GPR_INFO, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size - 1);
+  }
+  GPR_ASSERT(prev_size >= 1);
+  if (prev_size > 1) {
+    while (true) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+        gpr_log(GPR_INFO, "  checking queue");
+      }
+      bool empty;
+      grpc_closure* closure =
+          reinterpret_cast<grpc_closure*>(queue_.PopAndCheckEnd(&empty));
+      if (closure == nullptr) {
+        // This can happen either due to a race condition within the mpscq
+        // code or because of a race with Start().
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+          gpr_log(GPR_INFO, "  queue returned no result; checking again");
+        }
+        continue;
+      }
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+        gpr_log(GPR_INFO, "  EXECUTING FROM QUEUE: closure=%p error=%s",
+                closure, grpc_error_string(closure->error_data.error));
+      }
+      ScheduleClosure(closure, closure->error_data.error);
+      break;
+    }
+  } else if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+    gpr_log(GPR_INFO, "  queue empty");
+  }
+}
+
+void CallCombiner::SetNotifyOnCancel(grpc_closure* closure) {
+  GRPC_STATS_INC_CALL_COMBINER_SET_NOTIFY_ON_CANCEL();
+  while (true) {
+    // Decode original state.
+    gpr_atm original_state = gpr_atm_acq_load(&cancel_state_);
+    grpc_error* original_error = DecodeCancelStateError(original_state);
+    // If error is set, invoke the cancellation closure immediately.
+    // Otherwise, store the new closure.
+    if (original_error != GRPC_ERROR_NONE) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+        gpr_log(GPR_INFO,
+                "call_combiner=%p: scheduling notify_on_cancel callback=%p "
+                "for pre-existing cancellation",
+                this, closure);
+      }
+      ExecCtx::Run(DEBUG_LOCATION, closure, GRPC_ERROR_REF(original_error));
+      break;
+    } else {
+      if (gpr_atm_full_cas(&cancel_state_, original_state, (gpr_atm)closure)) {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+          gpr_log(GPR_INFO, "call_combiner=%p: setting notify_on_cancel=%p",
+                  this, closure);
+        }
+        // If we replaced an earlier closure, invoke the original
+        // closure with GRPC_ERROR_NONE.  This allows callers to clean
+        // up any resources they may be holding for the callback.
+        if (original_state != 0) {
+          closure = (grpc_closure*)original_state;
+          if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+            gpr_log(GPR_INFO,
+                    "call_combiner=%p: scheduling old cancel callback=%p", this,
+                    closure);
+          }
+          ExecCtx::Run(DEBUG_LOCATION, closure, GRPC_ERROR_NONE);
+        }
+        break;
+      }
+    }
+    // cas failed, try again.
+  }
+}
+
+void CallCombiner::Cancel(grpc_error* error) {
+  GRPC_STATS_INC_CALL_COMBINER_CANCELLED();
+  while (true) {
+    gpr_atm original_state = gpr_atm_acq_load(&cancel_state_);
+    grpc_error* original_error = DecodeCancelStateError(original_state);
+    if (original_error != GRPC_ERROR_NONE) {
+      GRPC_ERROR_UNREF(error);
+      break;
+    }
+    if (gpr_atm_full_cas(&cancel_state_, original_state,
+                         EncodeCancelStateError(error))) {
+      if (original_state != 0) {
+        grpc_closure* notify_on_cancel = (grpc_closure*)original_state;
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+          gpr_log(GPR_INFO,
+                  "call_combiner=%p: scheduling notify_on_cancel callback=%p",
+                  this, notify_on_cancel);
+        }
+        ExecCtx::Run(DEBUG_LOCATION, notify_on_cancel, GRPC_ERROR_REF(error));
+      }
+      break;
+    }
+    // cas failed, try again.
+  }
+}
+
+}  // namespace grpc_core

data/src/core/lib/iomgr/call_combiner.h

@@ -0,0 +1,217 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+#define GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stddef.h>
+
+#include "absl/container/inlined_vector.h"
+
+#include <grpc/support/atm.h>
+
+#include "src/core/lib/gprpp/mpscq.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/dynamic_annotations.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+
+// A simple, lock-free mechanism for serializing activity related to a
+// single call.  This is similar to a combiner but is more lightweight.
+//
+// It requires the callback (or, in the common case where the callback
+// actually kicks off a chain of callbacks, the last callback in that
+// chain) to explicitly indicate (by calling GRPC_CALL_COMBINER_STOP())
+// when it is done with the action that was kicked off by the original
+// callback.
+
+namespace grpc_core {
+
+extern DebugOnlyTraceFlag grpc_call_combiner_trace;
+
+class CallCombiner {
+ public:
+  CallCombiner();
+  ~CallCombiner();
+
+#ifndef NDEBUG
+#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason) \
+  (call_combiner)->Start((closure), (error), __FILE__, __LINE__, (reason))
+#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
+  (call_combiner)->Stop(__FILE__, __LINE__, (reason))
+  /// Starts processing \a closure.
+  void Start(grpc_closure* closure, grpc_error* error, const char* file,
+             int line, const char* reason);
+  /// Yields the call combiner to the next closure in the queue, if any.
+  void Stop(const char* file, int line, const char* reason);
+#else
+#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason) \
+  (call_combiner)->Start((closure), (error), (reason))
+#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
+  (call_combiner)->Stop((reason))
+  /// Starts processing \a closure.
+  void Start(grpc_closure* closure, grpc_error* error, const char* reason);
+  /// Yields the call combiner to the next closure in the queue, if any.
+  void Stop(const char* reason);
+#endif
+
+  /// Registers \a closure to be invoked when Cancel() is called.
+  ///
+  /// Once a closure is registered, it will always be scheduled exactly
+  /// once; this allows the closure to hold references that will be freed
+  /// regardless of whether or not the call was cancelled.  If a cancellation
+  /// does occur, the closure will be scheduled with the cancellation error;
+  /// otherwise, it will be scheduled with GRPC_ERROR_NONE.
+  ///
+  /// The closure will be scheduled in the following cases:
+  /// - If Cancel() was called prior to registering the closure, it will be
+  ///   scheduled immediately with the cancelation error.
+  /// - If Cancel() is called after registering the closure, the closure will
+  ///   be scheduled with the cancellation error.
+  /// - If SetNotifyOnCancel() is called again to register a new cancellation
+  ///   closure, the previous cancellation closure will be scheduled with
+  ///   GRPC_ERROR_NONE.
+  ///
+  /// If \a closure is NULL, then no closure will be invoked on
+  /// cancellation; this effectively unregisters the previously set closure.
+  /// However, most filters will not need to explicitly unregister their
+  /// callbacks, as this is done automatically when the call is destroyed.
+  /// Filters that schedule the cancellation closure on ExecCtx do not need
+  /// to take a ref on the call stack to guarantee closure liveness. This is
+  /// done by explicitly flushing ExecCtx after the unregistration during
+  /// call destruction.
+  void SetNotifyOnCancel(grpc_closure* closure);
+
+  /// Indicates that the call has been cancelled.
+  void Cancel(grpc_error* error);
+
+ private:
+  void ScheduleClosure(grpc_closure* closure, grpc_error* error);
+#ifdef GRPC_TSAN_ENABLED
+  static void TsanClosure(void* arg, grpc_error* error);
+#endif
+
+  gpr_atm size_ = 0;  // size_t, num closures in queue or currently executing
+  MultiProducerSingleConsumerQueue queue_;
+  // Either 0 (if not cancelled and no cancellation closure set),
+  // a grpc_closure* (if the lowest bit is 0),
+  // or a grpc_error* (if the lowest bit is 1).
+  gpr_atm cancel_state_ = 0;
+#ifdef GRPC_TSAN_ENABLED
+  // A fake ref-counted lock that is kept alive after the destruction of
+  // grpc_call_combiner, when we are running the original closure.
+  //
+  // Ideally we want to lock and unlock the call combiner as a pointer, when the
+  // callback is called. However, original_closure is free to trigger
+  // anything on the call combiner (including destruction of grpc_call).
+  // Thus, we need a ref-counted structure that can outlive the call combiner.
+  struct TsanLock : public RefCounted<TsanLock, NonPolymorphicRefCount> {
+    TsanLock() { TSAN_ANNOTATE_RWLOCK_CREATE(&taken); }
+    ~TsanLock() { TSAN_ANNOTATE_RWLOCK_DESTROY(&taken); }
+    // To avoid double-locking by the same thread, we should acquire/release
+    // the lock only when taken is false. On each acquire taken must be set to
+    // true.
+    std::atomic<bool> taken{false};
+  };
+  RefCountedPtr<TsanLock> tsan_lock_ = MakeRefCounted<TsanLock>();
+  grpc_closure tsan_closure_;
+  grpc_closure* original_closure_;
+#endif
+};
+
+// Helper for running a list of closures in a call combiner.
+//
+// Each callback running in the call combiner will eventually be
+// returned to the surface, at which point the surface will yield the
+// call combiner.  So when we are running in the call combiner and have
+// more than one callback to return to the surface, we need to re-enter
+// the call combiner for all but one of those callbacks.
+class CallCombinerClosureList {
+ public:
+  CallCombinerClosureList() {}
+
+  // Adds a closure to the list.  The closure must eventually result in
+  // the call combiner being yielded.
+  void Add(grpc_closure* closure, grpc_error* error, const char* reason) {
+    closures_.emplace_back(closure, error, reason);
+  }
+
+  // Runs all closures in the call combiner and yields the call combiner.
+  //
+  // All but one of the closures in the list will be scheduled via
+  // GRPC_CALL_COMBINER_START(), and the remaining closure will be
+  // scheduled via ExecCtx::Run(), which will eventually result
+  // in yielding the call combiner.  If the list is empty, then the call
+  // combiner will be yielded immediately.
+  void RunClosures(CallCombiner* call_combiner) {
+    if (closures_.empty()) {
+      GRPC_CALL_COMBINER_STOP(call_combiner, "no closures to schedule");
+      return;
+    }
+    for (size_t i = 1; i < closures_.size(); ++i) {
+      auto& closure = closures_[i];
+      GRPC_CALL_COMBINER_START(call_combiner, closure.closure, closure.error,
+                               closure.reason);
+    }
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) {
+      gpr_log(GPR_INFO,
+              "CallCombinerClosureList executing closure while already "
+              "holding call_combiner %p: closure=%p error=%s reason=%s",
+              call_combiner, closures_[0].closure,
+              grpc_error_string(closures_[0].error), closures_[0].reason);
+    }
+    // This will release the call combiner.
+    ExecCtx::Run(DEBUG_LOCATION, closures_[0].closure, closures_[0].error);
+    closures_.clear();
+  }
+
+  // Runs all closures in the call combiner, but does NOT yield the call
+  // combiner.  All closures will be scheduled via GRPC_CALL_COMBINER_START().
+  void RunClosuresWithoutYielding(CallCombiner* call_combiner) {
+    for (size_t i = 0; i < closures_.size(); ++i) {
+      auto& closure = closures_[i];
+      GRPC_CALL_COMBINER_START(call_combiner, closure.closure, closure.error,
+                               closure.reason);
+    }
+    closures_.clear();
+  }
+
+  size_t size() const { return closures_.size(); }
+
+ private:
+  struct CallCombinerClosure {
+    grpc_closure* closure;
+    grpc_error* error;
+    const char* reason;
+
+    CallCombinerClosure(grpc_closure* closure, grpc_error* error,
+                        const char* reason)
+        : closure(closure), error(error), reason(reason) {}
+  };
+
+  // There are generally a maximum of 6 closures to run in the call
+  // combiner, one for each pending op.
+  absl::InlinedVector<CallCombinerClosure, 6> closures_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H */