grpc 1.15.0 → 1.30.0

data/src/core/lib/surface/channel.h

@@ -21,15 +21,24 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <map>
+
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/channel/channelz.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/transport/metadata.h"
 
 grpc_channel* grpc_channel_create(const char* target,
                                   const grpc_channel_args* args,
                                   grpc_channel_stack_type channel_stack_type,
-                                  grpc_transport* optional_transport);
+                                  grpc_transport* optional_transport,
+                                  grpc_resource_user* resource_user = nullptr);
+
+/** The same as grpc_channel_destroy, but doesn't create an ExecCtx, and so
+ * is safe to use from within core. */
+void grpc_channel_destroy_internal(grpc_channel* channel);
 
 grpc_channel* grpc_channel_create_with_builder(
     grpc_channel_stack_builder* builder,
@@ -45,8 +54,8 @@ grpc_channel* grpc_channel_create_with_builder(
     value of \a propagation_mask (see propagation_bits.h for possible values) */
 grpc_call* grpc_channel_create_pollset_set_call(
     grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask,
-    grpc_pollset_set* pollset_set, grpc_slice method, const grpc_slice* host,
-    grpc_millis deadline, void* reserved);
+    grpc_pollset_set* pollset_set, const grpc_slice& method,
+    const grpc_slice* host, grpc_millis deadline, void* reserved);
 
 /** Get a (borrowed) pointer to this channels underlying channel stack */
 grpc_channel_stack* grpc_channel_get_channel_stack(grpc_channel* channel);
@@ -54,26 +63,88 @@ grpc_channel_stack* grpc_channel_get_channel_stack(grpc_channel* channel);
 grpc_core::channelz::ChannelNode* grpc_channel_get_channelz_node(
     grpc_channel* channel);
 
-/** Get a grpc_mdelem of grpc-status: X where X is the numeric value of
-    status_code.
-
-    The returned elem is owned by the caller. */
-grpc_mdelem grpc_channel_get_reffed_status_elem(grpc_channel* channel,
-                                                int status_code);
-
 size_t grpc_channel_get_call_size_estimate(grpc_channel* channel);
 void grpc_channel_update_call_size_estimate(grpc_channel* channel, size_t size);
 
+namespace grpc_core {
+
+struct RegisteredCall {
+  grpc_mdelem path;
+  grpc_mdelem authority;
+
+  explicit RegisteredCall(const char* method, const char* host);
+  // TODO(vjpai): delete copy constructor once all supported compilers allow
+  //              std::map value_type to be MoveConstructible.
+  RegisteredCall(const RegisteredCall& other);
+  RegisteredCall(RegisteredCall&& other) noexcept;
+
+  ~RegisteredCall();
+};
+
+struct CallRegistrationTable {
+  grpc_core::Mutex mu;
+  std::map<std::pair<const char*, const char*>, RegisteredCall>
+      map /* GUARDED_BY(mu) */;
+  int method_registration_attempts /* GUARDED_BY(mu) */ = 0;
+};
+
+}  // namespace grpc_core
+
+struct grpc_channel {
+  int is_client;
+  grpc_compression_options compression_options;
+
+  gpr_atm call_size_estimate;
+  grpc_resource_user* resource_user;
+
+  // TODO(vjpai): Once the grpc_channel is allocated via new rather than malloc,
+  //              expand the members of the CallRegistrationTable directly into
+  //              the grpc_channel. For now it is kept separate so that all the
+  //              manual constructing can be done with a single call rather than
+  //              a separate manual construction for each field.
+  grpc_core::ManualConstructor<grpc_core::CallRegistrationTable>
+      registration_table;
+  grpc_core::RefCountedPtr<grpc_core::channelz::ChannelNode> channelz_node;
+
+  char* target;
+};
+#define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack*)((c) + 1))
+
+inline grpc_compression_options grpc_channel_compression_options(
+    const grpc_channel* channel) {
+  return channel->compression_options;
+}
+
+inline grpc_channel_stack* grpc_channel_get_channel_stack(
+    grpc_channel* channel) {
+  return CHANNEL_STACK_FROM_CHANNEL(channel);
+}
+
+inline grpc_core::channelz::ChannelNode* grpc_channel_get_channelz_node(
+    grpc_channel* channel) {
+  return channel->channelz_node.get();
+}
+
 #ifndef NDEBUG
-void grpc_channel_internal_ref(grpc_channel* channel, const char* reason);
-void grpc_channel_internal_unref(grpc_channel* channel, const char* reason);
+inline void grpc_channel_internal_ref(grpc_channel* channel,
+                                      const char* reason) {
+  GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CHANNEL(channel), reason);
+}
+inline void grpc_channel_internal_unref(grpc_channel* channel,
+                                        const char* reason) {
+  GRPC_CHANNEL_STACK_UNREF(CHANNEL_STACK_FROM_CHANNEL(channel), reason);
+}
 #define GRPC_CHANNEL_INTERNAL_REF(channel, reason) \
   grpc_channel_internal_ref(channel, reason)
 #define GRPC_CHANNEL_INTERNAL_UNREF(channel, reason) \
   grpc_channel_internal_unref(channel, reason)
 #else
-void grpc_channel_internal_ref(grpc_channel* channel);
-void grpc_channel_internal_unref(grpc_channel* channel);
+inline void grpc_channel_internal_ref(grpc_channel* channel) {
+  GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CHANNEL(channel), "unused");
+}
+inline void grpc_channel_internal_unref(grpc_channel* channel) {
+  GRPC_CHANNEL_STACK_UNREF(CHANNEL_STACK_FROM_CHANNEL(channel), "unused");
+}
 #define GRPC_CHANNEL_INTERNAL_REF(channel, reason) \
   grpc_channel_internal_ref(channel)
 #define GRPC_CHANNEL_INTERNAL_UNREF(channel, reason) \

data/src/core/lib/surface/channel_init.h

@@ -45,6 +45,11 @@ void grpc_channel_init_init(void);
 /// registration order (in the case of a tie).
 /// Stages are registered against one of the pre-determined channel stack
 /// types.
+/// If the channel stack type is GRPC_CLIENT_SUBCHANNEL, the caller should
+/// ensure that subchannels with different filter lists will always have
+/// different channel args. This requires setting a channel arg in case the
+/// registration function relies on some condition other than channel args to
+/// decide whether to add a filter or not.
 void grpc_channel_init_register_stage(grpc_channel_stack_type type,
                                       int priority,
                                       grpc_channel_init_stage stage_fn,

data/src/core/lib/surface/channel_ping.cc

@@ -28,14 +28,13 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/completion_queue.h"
 
-typedef struct {
+struct ping_result {
   grpc_closure closure;
   void* tag;
   grpc_completion_queue* cq;
   grpc_cq_completion completion_storage;
-} ping_result;
-
-static void ping_destroy(void* arg, grpc_cq_completion* storage) {
+};
+static void ping_destroy(void* arg, grpc_cq_completion* /*storage*/) {
   gpr_free(arg);
 }
 

data/src/core/lib/surface/completion_queue.cc

@@ -23,6 +23,11 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
@@ -33,6 +38,8 @@
 #include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gpr/tls.h"
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
@@ -44,6 +51,8 @@ grpc_core::TraceFlag grpc_trace_operation_failures(false, "op_failure");
 grpc_core::DebugOnlyTraceFlag grpc_trace_pending_tags(false, "pending_tags");
 grpc_core::DebugOnlyTraceFlag grpc_trace_cq_refcount(false, "cq_refcount");
 
+namespace {
+
 // Specifies a cq thread local cache.
 // The first event that occurs on a thread
 // with a cq cache will go into that cache, and
@@ -52,12 +61,11 @@ grpc_core::DebugOnlyTraceFlag grpc_trace_cq_refcount(false, "cq_refcount");
 GPR_TLS_DECL(g_cached_event);
 GPR_TLS_DECL(g_cached_cq);
 
-typedef struct {
+struct plucker {
   grpc_pollset_worker** worker;
   void* tag;
-} plucker;
-
-typedef struct {
+};
+struct cq_poller_vtable {
   bool can_get_pollset;
   bool can_listen;
   size_t (*size)(void);
@@ -68,8 +76,7 @@ typedef struct {
                       grpc_millis deadline);
   void (*shutdown)(grpc_pollset* pollset, grpc_closure* closure);
   void (*destroy)(grpc_pollset* pollset);
-} cq_poller_vtable;
-
+};
 typedef struct non_polling_worker {
   gpr_cv cv;
   bool kicked;
@@ -77,32 +84,34 @@ typedef struct non_polling_worker {
   struct non_polling_worker* prev;
 } non_polling_worker;
 
-typedef struct {
+struct non_polling_poller {
   gpr_mu mu;
+  bool kicked_without_poller;
   non_polling_worker* root;
   grpc_closure* shutdown;
-} non_polling_poller;
-
-static size_t non_polling_poller_size(void) {
-  return sizeof(non_polling_poller);
-}
+};
+size_t non_polling_poller_size(void) { return sizeof(non_polling_poller); }
 
-static void non_polling_poller_init(grpc_pollset* pollset, gpr_mu** mu) {
+void non_polling_poller_init(grpc_pollset* pollset, gpr_mu** mu) {
   non_polling_poller* npp = reinterpret_cast<non_polling_poller*>(pollset);
   gpr_mu_init(&npp->mu);
   *mu = &npp->mu;
 }
 
-static void non_polling_poller_destroy(grpc_pollset* pollset) {
+void non_polling_poller_destroy(grpc_pollset* pollset) {
   non_polling_poller* npp = reinterpret_cast<non_polling_poller*>(pollset);
   gpr_mu_destroy(&npp->mu);
 }
 
-static grpc_error* non_polling_poller_work(grpc_pollset* pollset,
-                                           grpc_pollset_worker** worker,
-                                           grpc_millis deadline) {
+grpc_error* non_polling_poller_work(grpc_pollset* pollset,
+                                    grpc_pollset_worker** worker,
+                                    grpc_millis deadline) {
   non_polling_poller* npp = reinterpret_cast<non_polling_poller*>(pollset);
   if (npp->shutdown) return GRPC_ERROR_NONE;
+  if (npp->kicked_without_poller) {
+    npp->kicked_without_poller = false;
+    return GRPC_ERROR_NONE;
+  }
   non_polling_worker w;
   gpr_cv_init(&w.cv);
   if (worker != nullptr) *worker = reinterpret_cast<grpc_pollset_worker*>(&w);
@@ -124,7 +133,7 @@ static grpc_error* non_polling_poller_work(grpc_pollset* pollset,
     npp->root = w.next;
     if (&w == npp->root) {
       if (npp->shutdown) {
-        GRPC_CLOSURE_SCHED(npp->shutdown, GRPC_ERROR_NONE);
+        grpc_core::ExecCtx::Run(DEBUG_LOCATION, npp->shutdown, GRPC_ERROR_NONE);
       }
       npp->root = nullptr;
     }
@@ -136,8 +145,8 @@ static grpc_error* non_polling_poller_work(grpc_pollset* pollset,
   return GRPC_ERROR_NONE;
 }
 
-static grpc_error* non_polling_poller_kick(
-    grpc_pollset* pollset, grpc_pollset_worker* specific_worker) {
+grpc_error* non_polling_poller_kick(grpc_pollset* pollset,
+                                    grpc_pollset_worker* specific_worker) {
   non_polling_poller* p = reinterpret_cast<non_polling_poller*>(pollset);
   if (specific_worker == nullptr)
     specific_worker = reinterpret_cast<grpc_pollset_worker*>(p->root);
@@ -148,17 +157,18 @@ static grpc_error* non_polling_poller_kick(
       w->kicked = true;
       gpr_cv_signal(&w->cv);
     }
+  } else {
+    p->kicked_without_poller = true;
   }
   return GRPC_ERROR_NONE;
 }
 
-static void non_polling_poller_shutdown(grpc_pollset* pollset,
-                                        grpc_closure* closure) {
+void non_polling_poller_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
   non_polling_poller* p = reinterpret_cast<non_polling_poller*>(pollset);
   GPR_ASSERT(closure != nullptr);
   p->shutdown = closure;
   if (p->root == nullptr) {
-    GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_NONE);
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure, GRPC_ERROR_NONE);
   } else {
     non_polling_worker* w = p->root;
     do {
@@ -168,7 +178,7 @@ static void non_polling_poller_shutdown(grpc_pollset* pollset,
   }
 }
 
-static const cq_poller_vtable g_poller_vtable_by_poller_type[] = {
+const cq_poller_vtable g_poller_vtable_by_poller_type[] = {
     /* GRPC_CQ_DEFAULT_POLLING */
     {true, true, grpc_pollset_size, grpc_pollset_init, grpc_pollset_kick,
      grpc_pollset_work, grpc_pollset_shutdown, grpc_pollset_destroy},
@@ -181,99 +191,159 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = {
      non_polling_poller_shutdown, non_polling_poller_destroy},
 };
 
-typedef struct cq_vtable {
+}  // namespace
+
+struct cq_vtable {
   grpc_cq_completion_type cq_completion_type;
   size_t data_size;
-  void (*init)(void* data, grpc_core::CQCallbackInterface* shutdown_callback);
+  void (*init)(void* data,
+               grpc_experimental_completion_queue_functor* shutdown_callback);
   void (*shutdown)(grpc_completion_queue* cq);
   void (*destroy)(void* data);
   bool (*begin_op)(grpc_completion_queue* cq, void* tag);
   void (*end_op)(grpc_completion_queue* cq, void* tag, grpc_error* error,
                  void (*done)(void* done_arg, grpc_cq_completion* storage),
-                 void* done_arg, grpc_cq_completion* storage);
+                 void* done_arg, grpc_cq_completion* storage, bool internal);
   grpc_event (*next)(grpc_completion_queue* cq, gpr_timespec deadline,
                      void* reserved);
   grpc_event (*pluck)(grpc_completion_queue* cq, void* tag,
                       gpr_timespec deadline, void* reserved);
-} cq_vtable;
+};
+
+namespace {
 
-/* Queue that holds the cq_completion_events. Internally uses gpr_mpscq queue
- * (a lockfree multiproducer single consumer queue). It uses a queue_lock
- * to support multiple consumers.
+/* Queue that holds the cq_completion_events. Internally uses
+ * MultiProducerSingleConsumerQueue (a lockfree multiproducer single consumer
+ * queue). It uses a queue_lock to support multiple consumers.
  * Only used in completion queues whose completion_type is GRPC_CQ_NEXT */
-typedef struct grpc_cq_event_queue {
+class CqEventQueue {
+ public:
+  CqEventQueue() = default;
+  ~CqEventQueue() = default;
+
+  /* Note: The counter is not incremented/decremented atomically with push/pop.
+   * The count is only eventually consistent */
+  intptr_t num_items() const {
+    return num_queue_items_.Load(grpc_core::MemoryOrder::RELAXED);
+  }
+
+  bool Push(grpc_cq_completion* c);
+  grpc_cq_completion* Pop();
+
+ private:
   /* Spinlock to serialize consumers i.e pop() operations */
-  gpr_spinlock queue_lock;
+  gpr_spinlock queue_lock_ = GPR_SPINLOCK_INITIALIZER;
 
-  gpr_mpscq queue;
+  grpc_core::MultiProducerSingleConsumerQueue queue_;
 
   /* A lazy counter of number of items in the queue. This is NOT atomically
      incremented/decremented along with push/pop operations and hence is only
      eventually consistent */
-  gpr_atm num_queue_items;
-} grpc_cq_event_queue;
+  grpc_core::Atomic<intptr_t> num_queue_items_{0};
+};
+
+struct cq_next_data {
+  ~cq_next_data() {
+    GPR_ASSERT(queue.num_items() == 0);
+#ifndef NDEBUG
+    if (pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) != 0) {
+      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+    }
+#endif
+  }
 
-typedef struct cq_next_data {
   /** Completed events for completion-queues of type GRPC_CQ_NEXT */
-  grpc_cq_event_queue queue;
+  CqEventQueue queue;
 
   /** Counter of how many things have ever been queued on this completion queue
       useful for avoiding locks to check the queue */
-  gpr_atm things_queued_ever;
+  grpc_core::Atomic<intptr_t> things_queued_ever{0};
 
-  /* Number of outstanding events (+1 if not shut down) */
-  gpr_atm pending_events;
+  /** Number of outstanding events (+1 if not shut down)
+      Initial count is dropped by grpc_completion_queue_shutdown */
+  grpc_core::Atomic<intptr_t> pending_events{1};
 
   /** 0 initially. 1 once we initiated shutdown */
-  bool shutdown_called;
-} cq_next_data;
+  bool shutdown_called = false;
+};
+
+struct cq_pluck_data {
+  cq_pluck_data() {
+    completed_tail = &completed_head;
+    completed_head.next = reinterpret_cast<uintptr_t>(completed_tail);
+  }
+
+  ~cq_pluck_data() {
+    GPR_ASSERT(completed_head.next ==
+               reinterpret_cast<uintptr_t>(&completed_head));
+#ifndef NDEBUG
+    if (pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) != 0) {
+      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+    }
+#endif
+  }
 
-typedef struct cq_pluck_data {
   /** Completed events for completion-queues of type GRPC_CQ_PLUCK */
   grpc_cq_completion completed_head;
   grpc_cq_completion* completed_tail;
 
-  /** Number of pending events (+1 if we're not shutdown) */
-  gpr_atm pending_events;
+  /** Number of pending events (+1 if we're not shutdown).
+      Initial count is dropped by grpc_completion_queue_shutdown. */
+  grpc_core::Atomic<intptr_t> pending_events{1};
 
   /** Counter of how many things have ever been queued on this completion queue
       useful for avoiding locks to check the queue */
-  gpr_atm things_queued_ever;
+  grpc_core::Atomic<intptr_t> things_queued_ever{0};
 
   /** 0 initially. 1 once we completed shutting */
   /* TODO: (sreek) This is not needed since (shutdown == 1) if and only if
    * (pending_events == 0). So consider removing this in future and use
    * pending_events */
-  gpr_atm shutdown;
+  grpc_core::Atomic<bool> shutdown{false};
 
   /** 0 initially. 1 once we initiated shutdown */
-  bool shutdown_called;
+  bool shutdown_called = false;
 
-  int num_pluckers;
+  int num_pluckers = 0;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
-} cq_pluck_data;
+};
+
+struct cq_callback_data {
+  cq_callback_data(
+      grpc_experimental_completion_queue_functor* shutdown_callback)
+      : shutdown_callback(shutdown_callback) {}
+
+  ~cq_callback_data() {
+#ifndef NDEBUG
+    if (pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) != 0) {
+      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+    }
+#endif
+  }
 
-typedef struct cq_callback_data {
   /** No actual completed events queue, unlike other types */
 
-  /** Number of pending events (+1 if we're not shutdown) */
-  gpr_atm pending_events;
+  /** Number of pending events (+1 if we're not shutdown).
+      Initial count is dropped by grpc_completion_queue_shutdown. */
+  grpc_core::Atomic<intptr_t> pending_events{1};
 
   /** Counter of how many things have ever been queued on this completion queue
       useful for avoiding locks to check the queue */
-  gpr_atm things_queued_ever;
+  grpc_core::Atomic<intptr_t> things_queued_ever{0};
 
   /** 0 initially. 1 once we initiated shutdown */
-  bool shutdown_called;
+  bool shutdown_called = false;
 
   /** A callback that gets invoked when the CQ completes shutdown */
-  grpc_core::CQCallbackInterface* shutdown_callback;
-} cq_callback_data;
+  grpc_experimental_completion_queue_functor* shutdown_callback;
+};
+
+}  // namespace
 
 /* Completion queue structure */
 struct grpc_completion_queue {
   /** Once owning_refs drops to zero, we will destroy the cq */
-  gpr_refcount owning_refs;
+  grpc_core::RefCount owning_refs;
 
   gpr_mu* mu;
 
@@ -308,23 +378,20 @@ static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* tag);
 // queue. The done argument is a callback that will be invoked when it is
 // safe to free up that storage. The storage MUST NOT be freed until the
 // done callback is invoked.
-static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
-                               grpc_error* error,
-                               void (*done)(void* done_arg,
-                                            grpc_cq_completion* storage),
-                               void* done_arg, grpc_cq_completion* storage);
-
-static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag,
-                                grpc_error* error,
-                                void (*done)(void* done_arg,
-                                             grpc_cq_completion* storage),
-                                void* done_arg, grpc_cq_completion* storage);
-
-static void cq_end_op_for_callback(grpc_completion_queue* cq, void* tag,
-                                   grpc_error* error,
-                                   void (*done)(void* done_arg,
-                                                grpc_cq_completion* storage),
-                                   void* done_arg, grpc_cq_completion* storage);
+static void cq_end_op_for_next(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool internal);
+
+static void cq_end_op_for_pluck(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool internal);
+
+static void cq_end_op_for_callback(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool internal);
 
 static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
                           void* reserved);
@@ -333,12 +400,12 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
                            gpr_timespec deadline, void* reserved);
 
 // Note that cq_init_next and cq_init_pluck do not use the shutdown_callback
-static void cq_init_next(void* data,
-                         grpc_core::CQCallbackInterface* shutdown_callback);
-static void cq_init_pluck(void* data,
-                          grpc_core::CQCallbackInterface* shutdown_callback);
-static void cq_init_callback(void* data,
-                             grpc_core::CQCallbackInterface* shutdown_callback);
+static void cq_init_next(
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
+static void cq_init_pluck(
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
+static void cq_init_callback(
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
 static void cq_destroy_next(void* data);
 static void cq_destroy_pluck(void* data);
 static void cq_destroy_callback(void* data);
@@ -365,13 +432,15 @@ static const cq_vtable g_cq_vtable[] = {
 
 grpc_core::TraceFlag grpc_cq_pluck_trace(false, "queue_pluck");
 
-#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)                       \
-  if (grpc_api_trace.enabled() && (grpc_cq_pluck_trace.enabled() ||        \
-                                   (event)->type != GRPC_QUEUE_TIMEOUT)) { \
-    char* _ev = grpc_event_string(event);                                  \
-    gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev);                    \
-    gpr_free(_ev);                                                         \
-  }
+#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)     \
+  do {                                                   \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) &&       \
+        (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace) || \
+         (event)->type != GRPC_QUEUE_TIMEOUT)) {         \
+      gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq,      \
+              grpc_event_string(event).c_str());         \
+    }                                                    \
+  } while (0)
 
 static void on_pollset_shutdown_done(void* cq, grpc_error* error);
 
@@ -400,7 +469,7 @@ int grpc_completion_queue_thread_local_cache_flush(grpc_completion_queue* cq,
     storage->done(storage->done_arg, storage);
     ret = 1;
     cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
-    if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+    if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
       GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
       gpr_mu_lock(cq->mu);
       cq_finish_shutdown_next(cq);
@@ -414,31 +483,21 @@ int grpc_completion_queue_thread_local_cache_flush(grpc_completion_queue* cq,
   return ret;
 }
 
-static void cq_event_queue_init(grpc_cq_event_queue* q) {
-  gpr_mpscq_init(&q->queue);
-  q->queue_lock = GPR_SPINLOCK_INITIALIZER;
-  gpr_atm_no_barrier_store(&q->num_queue_items, 0);
+bool CqEventQueue::Push(grpc_cq_completion* c) {
+  queue_.Push(
+      reinterpret_cast<grpc_core::MultiProducerSingleConsumerQueue::Node*>(c));
+  return num_queue_items_.FetchAdd(1, grpc_core::MemoryOrder::RELAXED) == 0;
 }
 
-static void cq_event_queue_destroy(grpc_cq_event_queue* q) {
-  gpr_mpscq_destroy(&q->queue);
-}
-
-static bool cq_event_queue_push(grpc_cq_event_queue* q, grpc_cq_completion* c) {
-  gpr_mpscq_push(&q->queue, reinterpret_cast<gpr_mpscq_node*>(c));
-  return gpr_atm_no_barrier_fetch_add(&q->num_queue_items, 1) == 0;
-}
-
-static grpc_cq_completion* cq_event_queue_pop(grpc_cq_event_queue* q) {
+grpc_cq_completion* CqEventQueue::Pop() {
   grpc_cq_completion* c = nullptr;
 
-  if (gpr_spinlock_trylock(&q->queue_lock)) {
+  if (gpr_spinlock_trylock(&queue_lock_)) {
     GRPC_STATS_INC_CQ_EV_QUEUE_TRYLOCK_SUCCESSES();
 
     bool is_empty = false;
-    c = reinterpret_cast<grpc_cq_completion*>(
-        gpr_mpscq_pop_and_check_end(&q->queue, &is_empty));
-    gpr_spinlock_unlock(&q->queue_lock);
+    c = reinterpret_cast<grpc_cq_completion*>(queue_.PopAndCheckEnd(&is_empty));
+    gpr_spinlock_unlock(&queue_lock_);
 
     if (c == nullptr && !is_empty) {
       GRPC_STATS_INC_CQ_EV_QUEUE_TRANSIENT_POP_FAILURES();
@@ -448,21 +507,15 @@ static grpc_cq_completion* cq_event_queue_pop(grpc_cq_event_queue* q) {
   }
 
   if (c) {
-    gpr_atm_no_barrier_fetch_add(&q->num_queue_items, -1);
+    num_queue_items_.FetchSub(1, grpc_core::MemoryOrder::RELAXED);
   }
 
   return c;
 }
 
-/* Note: The counter is not incremented/decremented atomically with push/pop.
- * The count is only eventually consistent */
-static long cq_event_queue_num_items(grpc_cq_event_queue* q) {
-  return static_cast<long>(gpr_atm_no_barrier_load(&q->num_queue_items));
-}
-
 grpc_completion_queue* grpc_completion_queue_create_internal(
     grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type,
-    grpc_core::CQCallbackInterface* shutdown_callback) {
+    grpc_experimental_completion_queue_functor* shutdown_callback) {
   GPR_TIMER_SCOPE("grpc_completion_queue_create_internal", 0);
 
   grpc_completion_queue* cq;
@@ -487,7 +540,7 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
   cq->poller_vtable = poller_vtable;
 
   /* One for destroy(), one for pollset_shutdown */
-  gpr_ref_init(&cq->owning_refs, 2);
+  new (&cq->owning_refs) grpc_core::RefCount(2);
 
   poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu);
   vtable->init(DATA_FROM_CQ(cq), shutdown_callback);
@@ -497,51 +550,37 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
   return cq;
 }
 
-static void cq_init_next(void* data,
-                         grpc_core::CQCallbackInterface* shutdown_callback) {
-  cq_next_data* cqd = static_cast<cq_next_data*>(data);
-  /* Initial count is dropped by grpc_completion_queue_shutdown */
-  gpr_atm_no_barrier_store(&cqd->pending_events, 1);
-  cqd->shutdown_called = false;
-  gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
-  cq_event_queue_init(&cqd->queue);
+static void cq_init_next(
+    void* data,
+    grpc_experimental_completion_queue_functor* /*shutdown_callback*/) {
+  new (data) cq_next_data();
 }
 
 static void cq_destroy_next(void* data) {
   cq_next_data* cqd = static_cast<cq_next_data*>(data);
-  GPR_ASSERT(cq_event_queue_num_items(&cqd->queue) == 0);
-  cq_event_queue_destroy(&cqd->queue);
+  cqd->~cq_next_data();
 }
 
-static void cq_init_pluck(void* data,
-                          grpc_core::CQCallbackInterface* shutdown_callback) {
-  cq_pluck_data* cqd = static_cast<cq_pluck_data*>(data);
-  /* Initial count is dropped by grpc_completion_queue_shutdown */
-  gpr_atm_no_barrier_store(&cqd->pending_events, 1);
-  cqd->completed_tail = &cqd->completed_head;
-  cqd->completed_head.next = (uintptr_t)cqd->completed_tail;
-  gpr_atm_no_barrier_store(&cqd->shutdown, 0);
-  cqd->shutdown_called = false;
-  cqd->num_pluckers = 0;
-  gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
+static void cq_init_pluck(
+    void* data,
+    grpc_experimental_completion_queue_functor* /*shutdown_callback*/) {
+  new (data) cq_pluck_data();
 }
 
 static void cq_destroy_pluck(void* data) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*>(data);
-  GPR_ASSERT(cqd->completed_head.next == (uintptr_t)&cqd->completed_head);
+  cqd->~cq_pluck_data();
 }
 
 static void cq_init_callback(
-    void* data, grpc_core::CQCallbackInterface* shutdown_callback) {
-  cq_callback_data* cqd = static_cast<cq_callback_data*>(data);
-  /* Initial count is dropped by grpc_completion_queue_shutdown */
-  gpr_atm_no_barrier_store(&cqd->pending_events, 1);
-  cqd->shutdown_called = false;
-  gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
-  cqd->shutdown_callback = shutdown_callback;
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback) {
+  new (data) cq_callback_data(shutdown_callback);
 }
 
-static void cq_destroy_callback(void* data) {}
+static void cq_destroy_callback(void* data) {
+  cq_callback_data* cqd = static_cast<cq_callback_data*>(data);
+  cqd->~cq_callback_data();
+}
 
 grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue* cq) {
   return cq->vtable->cq_completion_type;
@@ -558,19 +597,16 @@ int grpc_get_cq_poll_num(grpc_completion_queue* cq) {
 #ifndef NDEBUG
 void grpc_cq_internal_ref(grpc_completion_queue* cq, const char* reason,
                           const char* file, int line) {
-  if (grpc_trace_cq_refcount.enabled()) {
-    gpr_atm val = gpr_atm_no_barrier_load(&cq->owning_refs.count);
-    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "CQ:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", cq, val, val + 1,
-            reason);
-  }
+  grpc_core::DebugLocation debug_location(file, line);
 #else
 void grpc_cq_internal_ref(grpc_completion_queue* cq) {
+  grpc_core::DebugLocation debug_location;
+  const char* reason = nullptr;
 #endif
-  gpr_ref(&cq->owning_refs);
+  cq->owning_refs.Ref(debug_location, reason);
 }
 
-static void on_pollset_shutdown_done(void* arg, grpc_error* error) {
+static void on_pollset_shutdown_done(void* arg, grpc_error* /*error*/) {
   grpc_completion_queue* cq = static_cast<grpc_completion_queue*>(arg);
   GRPC_CQ_INTERNAL_UNREF(cq, "pollset_destroy");
 }
@@ -578,16 +614,13 @@ static void on_pollset_shutdown_done(void* arg, grpc_error* error) {
 #ifndef NDEBUG
 void grpc_cq_internal_unref(grpc_completion_queue* cq, const char* reason,
                             const char* file, int line) {
-  if (grpc_trace_cq_refcount.enabled()) {
-    gpr_atm val = gpr_atm_no_barrier_load(&cq->owning_refs.count);
-    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "CQ:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", cq, val, val - 1,
-            reason);
-  }
+  grpc_core::DebugLocation debug_location(file, line);
 #else
 void grpc_cq_internal_unref(grpc_completion_queue* cq) {
+  grpc_core::DebugLocation debug_location;
+  const char* reason = nullptr;
 #endif
-  if (gpr_unref(&cq->owning_refs)) {
+  if (GPR_UNLIKELY(cq->owning_refs.Unref(debug_location, reason))) {
     cq->vtable->destroy(DATA_FROM_CQ(cq));
     cq->poller_vtable->destroy(POLLSET_FROM_CQ(cq));
 #ifndef NDEBUG
@@ -621,40 +654,23 @@ static void cq_check_tag(grpc_completion_queue* cq, void* tag, bool lock_cq) {
   GPR_ASSERT(found);
 }
 #else
-static void cq_check_tag(grpc_completion_queue* cq, void* tag, bool lock_cq) {}
+static void cq_check_tag(grpc_completion_queue* /*cq*/, void* /*tag*/,
+                         bool /*lock_cq*/) {}
 #endif
 
-/* Atomically increments a counter only if the counter is not zero. Returns
- * true if the increment was successful; false if the counter is zero */
-static bool atm_inc_if_nonzero(gpr_atm* counter) {
-  while (true) {
-    gpr_atm count = gpr_atm_acq_load(counter);
-    /* If zero, we are done. If not, we must to a CAS (instead of an atomic
-     * increment) to maintain the contract: do not increment the counter if it
-     * is zero. */
-    if (count == 0) {
-      return false;
-    } else if (gpr_atm_full_cas(counter, count, count + 1)) {
-      break;
-    }
-  }
-
-  return true;
-}
-
-static bool cq_begin_op_for_next(grpc_completion_queue* cq, void* tag) {
+static bool cq_begin_op_for_next(grpc_completion_queue* cq, void* /*tag*/) {
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
-  return atm_inc_if_nonzero(&cqd->pending_events);
+  return cqd->pending_events.IncrementIfNonzero();
 }
 
-static bool cq_begin_op_for_pluck(grpc_completion_queue* cq, void* tag) {
+static bool cq_begin_op_for_pluck(grpc_completion_queue* cq, void* /*tag*/) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
-  return atm_inc_if_nonzero(&cqd->pending_events);
+  return cqd->pending_events.IncrementIfNonzero();
 }
 
-static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* tag) {
+static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* /*tag*/) {
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
-  return atm_inc_if_nonzero(&cqd->pending_events);
+  return cqd->pending_events.IncrementIfNonzero();
 }
 
 bool grpc_cq_begin_op(grpc_completion_queue* cq, void* tag) {
@@ -675,21 +691,22 @@ bool grpc_cq_begin_op(grpc_completion_queue* cq, void* tag) {
 /* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
  * completion
  * type of GRPC_CQ_NEXT) */
-static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
-                               grpc_error* error,
-                               void (*done)(void* done_arg,
-                                            grpc_cq_completion* storage),
-                               void* done_arg, grpc_cq_completion* storage) {
+static void cq_end_op_for_next(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool /*internal*/) {
   GPR_TIMER_SCOPE("cq_end_op_for_next", 0);
 
-  if (grpc_api_trace.enabled() ||
-      (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+       error != GRPC_ERROR_NONE)) {
     const char* errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
         "cq_end_op_for_next(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg, done, done_arg, storage));
-    if (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+        error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
     }
   }
@@ -708,17 +725,14 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
     gpr_tls_set(&g_cached_event, (intptr_t)storage);
   } else {
     /* Add the completion to the queue */
-    bool is_first = cq_event_queue_push(&cqd->queue, storage);
-    gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1);
-
+    bool is_first = cqd->queue.Push(storage);
+    cqd->things_queued_ever.FetchAdd(1, grpc_core::MemoryOrder::RELAXED);
     /* Since we do not hold the cq lock here, it is important to do an 'acquire'
        load here (instead of a 'no_barrier' load) to match with the release
        store
-       (done via gpr_atm_full_fetch_add(pending_events, -1)) in cq_shutdown_next
+       (done via pending_events.FetchSub(1, ACQ_REL)) in cq_shutdown_next
        */
-    bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1;
-
-    if (!will_definitely_shutdown) {
+    if (cqd->pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) != 1) {
       /* Only kick if this is the first item queued */
       if (is_first) {
         gpr_mu_lock(cq->mu);
@@ -732,7 +746,8 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
           GRPC_ERROR_UNREF(kick_error);
         }
       }
-      if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+      if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) ==
+          1) {
         GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
         gpr_mu_lock(cq->mu);
         cq_finish_shutdown_next(cq);
@@ -741,7 +756,7 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
       }
     } else {
       GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
-      gpr_atm_rel_store(&cqd->pending_events, 0);
+      cqd->pending_events.Store(0, grpc_core::MemoryOrder::RELEASE);
       gpr_mu_lock(cq->mu);
       cq_finish_shutdown_next(cq);
       gpr_mu_unlock(cq->mu);
@@ -755,24 +770,25 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag,
 /* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
  * completion
  * type of GRPC_CQ_PLUCK) */
-static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag,
-                                grpc_error* error,
-                                void (*done)(void* done_arg,
-                                             grpc_cq_completion* storage),
-                                void* done_arg, grpc_cq_completion* storage) {
+static void cq_end_op_for_pluck(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool /*internal*/) {
   GPR_TIMER_SCOPE("cq_end_op_for_pluck", 0);
 
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
   int is_success = (error == GRPC_ERROR_NONE);
 
-  if (grpc_api_trace.enabled() ||
-      (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+       error != GRPC_ERROR_NONE)) {
     const char* errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
         "cq_end_op_for_pluck(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg, done, done_arg, storage));
-    if (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+        error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
     }
   }
@@ -787,12 +803,12 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag,
   cq_check_tag(cq, tag, false); /* Used in debug builds only */
 
   /* Add to the list of completions */
-  gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1);
+  cqd->things_queued_ever.FetchAdd(1, grpc_core::MemoryOrder::RELAXED);
   cqd->completed_tail->next =
       ((uintptr_t)storage) | (1u & cqd->completed_tail->next);
   cqd->completed_tail = storage;
 
-  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
     cq_finish_shutdown_pluck(cq);
     gpr_mu_unlock(cq->mu);
   } else {
@@ -820,24 +836,30 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag,
   GRPC_ERROR_UNREF(error);
 }
 
+static void functor_callback(void* arg, grpc_error* error) {
+  auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(arg);
+  functor->functor_run(functor, error == GRPC_ERROR_NONE);
+}
+
 /* Complete an event on a completion queue of type GRPC_CQ_CALLBACK */
 static void cq_end_op_for_callback(
     grpc_completion_queue* cq, void* tag, grpc_error* error,
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
-    grpc_cq_completion* storage) {
+    grpc_cq_completion* storage, bool internal) {
   GPR_TIMER_SCOPE("cq_end_op_for_callback", 0);
 
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
-  bool is_success = (error == GRPC_ERROR_NONE);
 
-  if (grpc_api_trace.enabled() ||
-      (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+       error != GRPC_ERROR_NONE)) {
     const char* errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
         "cq_end_op_for_callback(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg, done, done_arg, storage));
-    if (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+        error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
     }
   }
@@ -846,37 +868,50 @@ static void cq_end_op_for_callback(
   // for reserved storage. Invoke the done callback right away to release it.
   done(done_arg, storage);
 
-  gpr_mu_lock(cq->mu);
-  cq_check_tag(cq, tag, false); /* Used in debug builds only */
+  cq_check_tag(cq, tag, true); /* Used in debug builds only */
 
-  gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1);
-  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+  cqd->things_queued_ever.FetchAdd(1, grpc_core::MemoryOrder::RELAXED);
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
     cq_finish_shutdown_callback(cq);
-    gpr_mu_unlock(cq->mu);
-  } else {
-    gpr_mu_unlock(cq->mu);
   }
 
-  GRPC_ERROR_UNREF(error);
+  // If possible, schedule the callback onto an existing thread-local
+  // ApplicationCallbackExecCtx, which is a work queue. This is possible for:
+  // 1. The callback is internally-generated and there is an ACEC available
+  // 2. The callback is marked inlineable and there is an ACEC available
+  // 3. We are already running in a background poller thread (which always has
+  //    an ACEC available at the base of the stack).
+  auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(tag);
+  if (((internal || functor->inlineable) &&
+       grpc_core::ApplicationCallbackExecCtx::Available()) ||
+      grpc_iomgr_is_any_background_poller_thread()) {
+    grpc_core::ApplicationCallbackExecCtx::Enqueue(functor,
+                                                   (error == GRPC_ERROR_NONE));
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
 
-  (static_cast<grpc_core::CQCallbackInterface*>(tag))->Run(is_success);
+  // Schedule the callback on a closure if not internal or triggered
+  // from a background poller thread.
+  grpc_core::Executor::Run(
+      GRPC_CLOSURE_CREATE(functor_callback, functor, nullptr), error);
 }
 
 void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error,
                     void (*done)(void* done_arg, grpc_cq_completion* storage),
-                    void* done_arg, grpc_cq_completion* storage) {
-  cq->vtable->end_op(cq, tag, error, done, done_arg, storage);
+                    void* done_arg, grpc_cq_completion* storage,
+                    bool internal) {
+  cq->vtable->end_op(cq, tag, error, done, done_arg, storage, internal);
 }
 
-typedef struct {
+struct cq_is_finished_arg {
   gpr_atm last_seen_things_queued_ever;
   grpc_completion_queue* cq;
   grpc_millis deadline;
   grpc_cq_completion* stolen_completion;
   void* tag; /* for pluck */
   bool first_loop;
-} cq_is_finished_arg;
-
+};
 class ExecCtxNext : public grpc_core::ExecCtx {
  public:
   ExecCtxNext(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
@@ -888,20 +923,20 @@ class ExecCtxNext : public grpc_core::ExecCtx {
     cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
     GPR_ASSERT(a->stolen_completion == nullptr);
 
-    gpr_atm current_last_seen_things_queued_ever =
-        gpr_atm_no_barrier_load(&cqd->things_queued_ever);
+    intptr_t current_last_seen_things_queued_ever =
+        cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED);
 
     if (current_last_seen_things_queued_ever !=
         a->last_seen_things_queued_ever) {
       a->last_seen_things_queued_ever =
-          gpr_atm_no_barrier_load(&cqd->things_queued_ever);
+          cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED);
 
       /* Pop a cq_completion from the queue. Returns NULL if the queue is empty
        * might return NULL in some cases even if the queue is not empty; but
        * that
        * is ok and doesn't affect correctness. Might effect the tail latencies a
        * bit) */
-      a->stolen_completion = cq_event_queue_pop(&cqd->queue);
+      a->stolen_completion = cqd->queue.Pop();
       if (a->stolen_completion != nullptr) {
         return true;
       }
@@ -915,25 +950,18 @@ class ExecCtxNext : public grpc_core::ExecCtx {
 
 #ifndef NDEBUG
 static void dump_pending_tags(grpc_completion_queue* cq) {
-  if (!grpc_trace_pending_tags.enabled()) return;
-
-  gpr_strvec v;
-  gpr_strvec_init(&v);
-  gpr_strvec_add(&v, gpr_strdup("PENDING TAGS:"));
+  if (!GRPC_TRACE_FLAG_ENABLED(grpc_trace_pending_tags)) return;
+  std::vector<std::string> parts;
+  parts.push_back("PENDING TAGS:");
   gpr_mu_lock(cq->mu);
   for (size_t i = 0; i < cq->outstanding_tag_count; i++) {
-    char* s;
-    gpr_asprintf(&s, " %p", cq->outstanding_tags[i]);
-    gpr_strvec_add(&v, s);
+    parts.push_back(absl::StrFormat(" %p", cq->outstanding_tags[i]));
   }
   gpr_mu_unlock(cq->mu);
-  char* out = gpr_strvec_flatten(&v, nullptr);
-  gpr_strvec_destroy(&v);
-  gpr_log(GPR_DEBUG, "%s", out);
-  gpr_free(out);
+  gpr_log(GPR_DEBUG, "%s", absl::StrJoin(parts, "").c_str());
 }
 #else
-static void dump_pending_tags(grpc_completion_queue* cq) {}
+static void dump_pending_tags(grpc_completion_queue* /*cq*/) {}
 #endif
 
 static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
@@ -960,7 +988,7 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
 
   grpc_millis deadline_millis = grpc_timespec_to_millis_round_up(deadline);
   cq_is_finished_arg is_finished_arg = {
-      gpr_atm_no_barrier_load(&cqd->things_queued_ever),
+      cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED),
       cq,
       deadline_millis,
       nullptr,
@@ -980,7 +1008,7 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
       break;
     }
 
-    grpc_cq_completion* c = cq_event_queue_pop(&cqd->queue);
+    grpc_cq_completion* c = cqd->queue.Pop();
 
     if (c != nullptr) {
       ret.type = GRPC_OP_COMPLETE;
@@ -994,16 +1022,17 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
          so that the thread comes back quickly from poll to make a second
          attempt at popping. Not doing this can potentially deadlock this
          thread forever (if the deadline is infinity) */
-      if (cq_event_queue_num_items(&cqd->queue) > 0) {
+      if (cqd->queue.num_items() > 0) {
         iteration_deadline = 0;
       }
     }
 
-    if (gpr_atm_acq_load(&cqd->pending_events) == 0) {
+    if (cqd->pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) == 0) {
       /* Before returning, check if the queue has any items left over (since
-         gpr_mpscq_pop() can sometimes return NULL even if the queue is not
-         empty. If so, keep retrying but do not return GRPC_QUEUE_SHUTDOWN */
-      if (cq_event_queue_num_items(&cqd->queue) > 0) {
+         MultiProducerSingleConsumerQueue::Pop() can sometimes return NULL
+         even if the queue is not empty. If so, keep retrying but do not
+         return GRPC_QUEUE_SHUTDOWN */
+      if (cqd->queue.num_items() > 0) {
         /* Go to the beginning of the loop. No point doing a poll because
            (cq->shutdown == true) is only possible when there is no pending
            work (i.e cq->pending_events == 0) and any outstanding completion
@@ -1011,15 +1040,15 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
         continue;
       }
 
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_SHUTDOWN;
+      ret.success = 0;
       break;
     }
 
     if (!is_finished_arg.first_loop &&
         grpc_core::ExecCtx::Get()->Now() >= deadline_millis) {
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
+      ret.success = 0;
       dump_pending_tags(cq);
       break;
     }
@@ -1036,16 +1065,16 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
       gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
 
       GRPC_ERROR_UNREF(err);
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
+      ret.success = 0;
       dump_pending_tags(cq);
       break;
     }
     is_finished_arg.first_loop = false;
   }
 
-  if (cq_event_queue_num_items(&cqd->queue) > 0 &&
-      gpr_atm_acq_load(&cqd->pending_events) > 0) {
+  if (cqd->queue.num_items() > 0 &&
+      cqd->pending_events.Load(grpc_core::MemoryOrder::ACQUIRE) > 0) {
     gpr_mu_lock(cq->mu);
     cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), nullptr);
     gpr_mu_unlock(cq->mu);
@@ -1069,7 +1098,7 @@ static void cq_finish_shutdown_next(grpc_completion_queue* cq) {
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
 
   GPR_ASSERT(cqd->shutdown_called);
-  GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0);
+  GPR_ASSERT(cqd->pending_events.Load(grpc_core::MemoryOrder::RELAXED) == 0);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
 }
@@ -1091,10 +1120,10 @@ static void cq_shutdown_next(grpc_completion_queue* cq) {
     return;
   }
   cqd->shutdown_called = true;
-  /* Doing a full_fetch_add (i.e acq/release) here to match with
-   * cq_begin_op_for_next and and cq_end_op_for_next functions which read/write
+  /* Doing acq/release FetchSub here to match with
+   * cq_begin_op_for_next and cq_end_op_for_next functions which read/write
    * on this counter without necessarily holding a lock on cq */
-  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
     cq_finish_shutdown_next(cq);
   }
   gpr_mu_unlock(cq->mu);
@@ -1143,12 +1172,12 @@ class ExecCtxPluck : public grpc_core::ExecCtx {
 
     GPR_ASSERT(a->stolen_completion == nullptr);
     gpr_atm current_last_seen_things_queued_ever =
-        gpr_atm_no_barrier_load(&cqd->things_queued_ever);
+        cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED);
     if (current_last_seen_things_queued_ever !=
         a->last_seen_things_queued_ever) {
       gpr_mu_lock(cq->mu);
       a->last_seen_things_queued_ever =
-          gpr_atm_no_barrier_load(&cqd->things_queued_ever);
+          cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED);
       grpc_cq_completion* c;
       grpc_cq_completion* prev = &cqd->completed_head;
       while ((c = (grpc_cq_completion*)(prev->next &
@@ -1185,7 +1214,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
   grpc_pollset_worker* worker = nullptr;
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
-  if (grpc_cq_pluck_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace)) {
     GRPC_API_TRACE(
         "grpc_completion_queue_pluck("
         "cq=%p, tag=%p, "
@@ -1204,7 +1233,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
   gpr_mu_lock(cq->mu);
   grpc_millis deadline_millis = grpc_timespec_to_millis_round_up(deadline);
   cq_is_finished_arg is_finished_arg = {
-      gpr_atm_no_barrier_load(&cqd->things_queued_ever),
+      cqd->things_queued_ever.Load(grpc_core::MemoryOrder::RELAXED),
       cq,
       deadline_millis,
       nullptr,
@@ -1241,10 +1270,10 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
       }
       prev = c;
     }
-    if (gpr_atm_no_barrier_load(&cqd->shutdown)) {
+    if (cqd->shutdown.Load(grpc_core::MemoryOrder::RELAXED)) {
       gpr_mu_unlock(cq->mu);
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_SHUTDOWN;
+      ret.success = 0;
       break;
     }
     if (!add_plucker(cq, tag, &worker)) {
@@ -1253,9 +1282,9 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
               "is %d",
               GRPC_MAX_COMPLETION_QUEUE_PLUCKERS);
       gpr_mu_unlock(cq->mu);
-      memset(&ret, 0, sizeof(ret));
       /* TODO(ctiller): should we use a different result here */
       ret.type = GRPC_QUEUE_TIMEOUT;
+      ret.success = 0;
       dump_pending_tags(cq);
       break;
     }
@@ -1263,8 +1292,8 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
         grpc_core::ExecCtx::Get()->Now() >= deadline_millis) {
       del_plucker(cq, tag, &worker);
       gpr_mu_unlock(cq->mu);
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
+      ret.success = 0;
       dump_pending_tags(cq);
       break;
     }
@@ -1278,8 +1307,8 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
       gpr_log(GPR_ERROR, "Completion queue pluck failed: %s", msg);
 
       GRPC_ERROR_UNREF(err);
-      memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
+      ret.success = 0;
       dump_pending_tags(cq);
       break;
     }
@@ -1304,8 +1333,8 @@ static void cq_finish_shutdown_pluck(grpc_completion_queue* cq) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
   GPR_ASSERT(cqd->shutdown_called);
-  GPR_ASSERT(!gpr_atm_no_barrier_load(&cqd->shutdown));
-  gpr_atm_no_barrier_store(&cqd->shutdown, 1);
+  GPR_ASSERT(!cqd->shutdown.Load(grpc_core::MemoryOrder::RELAXED));
+  cqd->shutdown.Store(1, grpc_core::MemoryOrder::RELAXED);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
 }
@@ -1329,7 +1358,7 @@ static void cq_shutdown_pluck(grpc_completion_queue* cq) {
     return;
   }
   cqd->shutdown_called = true;
-  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
     cq_finish_shutdown_pluck(cq);
   }
   gpr_mu_unlock(cq->mu);
@@ -1343,7 +1372,16 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) {
   GPR_ASSERT(cqd->shutdown_called);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
-  callback->Run(true);
+  if (grpc_iomgr_is_any_background_poller_thread()) {
+    grpc_core::ApplicationCallbackExecCtx::Enqueue(callback, true);
+    return;
+  }
+
+  // Schedule the callback on a closure if not internal or triggered
+  // from a background poller thread.
+  grpc_core::Executor::Run(
+      GRPC_CLOSURE_CREATE(functor_callback, callback, nullptr),
+      GRPC_ERROR_NONE);
 }
 
 static void cq_shutdown_callback(grpc_completion_queue* cq) {
@@ -1363,10 +1401,12 @@ static void cq_shutdown_callback(grpc_completion_queue* cq) {
     return;
   }
   cqd->shutdown_called = true;
-  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
+    gpr_mu_unlock(cq->mu);
     cq_finish_shutdown_callback(cq);
+  } else {
+    gpr_mu_unlock(cq->mu);
   }
-  gpr_mu_unlock(cq->mu);
   GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)");
 }
 
@@ -1374,6 +1414,7 @@ static void cq_shutdown_callback(grpc_completion_queue* cq) {
    to zero here, then enter shutdown mode and wake up any waiters */
 void grpc_completion_queue_shutdown(grpc_completion_queue* cq) {
   GPR_TIMER_SCOPE("grpc_completion_queue_shutdown", 0);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_completion_queue_shutdown(cq=%p)", 1, (cq));
   cq->vtable->shutdown(cq);