grpc 1.28.0.pre2 → 1.31.0.pre1

data/src/core/lib/surface/call_log_batch.cc

@@ -22,98 +22,90 @@
 
 #include <inttypes.h>
 
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 
-static void add_metadata(gpr_strvec* b, const grpc_metadata* md, size_t count) {
-  size_t i;
+static void add_metadata(const grpc_metadata* md, size_t count,
+                         std::vector<std::string>* b) {
   if (md == nullptr) {
-    gpr_strvec_add(b, gpr_strdup("(nil)"));
+    b->push_back("(nil)");
     return;
   }
-  for (i = 0; i < count; i++) {
-    gpr_strvec_add(b, gpr_strdup("\nkey="));
-    gpr_strvec_add(b, grpc_slice_to_c_string(md[i].key));
-
-    gpr_strvec_add(b, gpr_strdup(" value="));
-    gpr_strvec_add(b,
-                   grpc_dump_slice(md[i].value, GPR_DUMP_HEX | GPR_DUMP_ASCII));
+  for (size_t i = 0; i < count; i++) {
+    b->push_back("\nkey=");
+    b->push_back(std::string(grpc_core::StringViewFromSlice(md[i].key)));
+    b->push_back(" value=");
+    char* dump = grpc_dump_slice(md[i].value, GPR_DUMP_HEX | GPR_DUMP_ASCII);
+    b->push_back(dump);
+    gpr_free(dump);
   }
 }
 
-char* grpc_op_string(const grpc_op* op) {
-  char* tmp;
-  char* out;
-
-  gpr_strvec b;
-  gpr_strvec_init(&b);
-
+static std::string grpc_op_string(const grpc_op* op) {
+  std::vector<std::string> parts;
   switch (op->op) {
     case GRPC_OP_SEND_INITIAL_METADATA:
-      gpr_strvec_add(&b, gpr_strdup("SEND_INITIAL_METADATA"));
-      add_metadata(&b, op->data.send_initial_metadata.metadata,
-                   op->data.send_initial_metadata.count);
+      parts.push_back("SEND_INITIAL_METADATA");
+      add_metadata(op->data.send_initial_metadata.metadata,
+                   op->data.send_initial_metadata.count, &parts);
       break;
     case GRPC_OP_SEND_MESSAGE:
-      gpr_asprintf(&tmp, "SEND_MESSAGE ptr=%p",
-                   op->data.send_message.send_message);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(absl::StrFormat("SEND_MESSAGE ptr=%p",
+                                      op->data.send_message.send_message));
       break;
     case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
-      gpr_strvec_add(&b, gpr_strdup("SEND_CLOSE_FROM_CLIENT"));
+      parts.push_back("SEND_CLOSE_FROM_CLIENT");
       break;
     case GRPC_OP_SEND_STATUS_FROM_SERVER:
-      gpr_asprintf(&tmp, "SEND_STATUS_FROM_SERVER status=%d details=",
-                   op->data.send_status_from_server.status);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(
+          absl::StrFormat("SEND_STATUS_FROM_SERVER status=%d details=",
+                          op->data.send_status_from_server.status));
       if (op->data.send_status_from_server.status_details != nullptr) {
-        gpr_strvec_add(&b, grpc_dump_slice(
-                               *op->data.send_status_from_server.status_details,
-                               GPR_DUMP_ASCII));
+        char* dump = grpc_dump_slice(
+            *op->data.send_status_from_server.status_details, GPR_DUMP_ASCII);
+        parts.push_back(dump);
+        gpr_free(dump);
       } else {
-        gpr_strvec_add(&b, gpr_strdup("(null)"));
+        parts.push_back("(null)");
       }
-      add_metadata(&b, op->data.send_status_from_server.trailing_metadata,
-                   op->data.send_status_from_server.trailing_metadata_count);
+      add_metadata(op->data.send_status_from_server.trailing_metadata,
+                   op->data.send_status_from_server.trailing_metadata_count,
+                   &parts);
       break;
     case GRPC_OP_RECV_INITIAL_METADATA:
-      gpr_asprintf(&tmp, "RECV_INITIAL_METADATA ptr=%p",
-                   op->data.recv_initial_metadata.recv_initial_metadata);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(absl::StrFormat(
+          "RECV_INITIAL_METADATA ptr=%p",
+          op->data.recv_initial_metadata.recv_initial_metadata));
       break;
     case GRPC_OP_RECV_MESSAGE:
-      gpr_asprintf(&tmp, "RECV_MESSAGE ptr=%p",
-                   op->data.recv_message.recv_message);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(absl::StrFormat("RECV_MESSAGE ptr=%p",
+                                      op->data.recv_message.recv_message));
       break;
     case GRPC_OP_RECV_STATUS_ON_CLIENT:
-      gpr_asprintf(&tmp,
-                   "RECV_STATUS_ON_CLIENT metadata=%p status=%p details=%p",
-                   op->data.recv_status_on_client.trailing_metadata,
-                   op->data.recv_status_on_client.status,
-                   op->data.recv_status_on_client.status_details);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(absl::StrFormat(
+          "RECV_STATUS_ON_CLIENT metadata=%p status=%p details=%p",
+          op->data.recv_status_on_client.trailing_metadata,
+          op->data.recv_status_on_client.status,
+          op->data.recv_status_on_client.status_details));
       break;
     case GRPC_OP_RECV_CLOSE_ON_SERVER:
-      gpr_asprintf(&tmp, "RECV_CLOSE_ON_SERVER cancelled=%p",
-                   op->data.recv_close_on_server.cancelled);
-      gpr_strvec_add(&b, tmp);
+      parts.push_back(absl::StrFormat("RECV_CLOSE_ON_SERVER cancelled=%p",
+                                      op->data.recv_close_on_server.cancelled));
   }
-  out = gpr_strvec_flatten(&b, nullptr);
-  gpr_strvec_destroy(&b);
-
-  return out;
+  return absl::StrJoin(parts, "");
 }
 
 void grpc_call_log_batch(const char* file, int line, gpr_log_severity severity,
                          const grpc_op* ops, size_t nops) {
-  char* tmp;
-  size_t i;
-  for (i = 0; i < nops; i++) {
-    tmp = grpc_op_string(&ops[i]);
-    gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i, tmp);
-    gpr_free(tmp);
+  for (size_t i = 0; i < nops; i++) {
+    gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i,
+            grpc_op_string(&ops[i]).c_str());
   }
 }

data/src/core/lib/surface/channel.cc

@@ -54,12 +54,6 @@
  *  (OK, Cancelled, Unknown). */
 #define NUM_CACHED_STATUS_ELEMS 3
 
-typedef struct registered_call {
-  grpc_mdelem path;
-  grpc_mdelem authority;
-  struct registered_call* next;
-} registered_call;
-
 static void destroy_channel(void* arg, grpc_error* error);
 
 grpc_channel* grpc_channel_create_with_builder(
@@ -90,8 +84,7 @@ grpc_channel* grpc_channel_create_with_builder(
   channel->target = target;
   channel->resource_user = resource_user;
   channel->is_client = grpc_channel_stack_type_is_client(channel_stack_type);
-  gpr_mu_init(&channel->registered_call_mu);
-  channel->registered_calls = nullptr;
+  channel->registration_table.Init();
 
   gpr_atm_no_barrier_store(
       &channel->call_size_estimate,
@@ -336,6 +329,7 @@ char* grpc_channel_get_target(grpc_channel* channel) {
 
 void grpc_channel_get_info(grpc_channel* channel,
                            const grpc_channel_info* channel_info) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   grpc_channel_element* elem =
       grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0);
@@ -343,6 +337,7 @@ void grpc_channel_get_info(grpc_channel* channel,
 }
 
 void grpc_channel_reset_connect_backoff(grpc_channel* channel) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1,
                  (channel));
@@ -393,6 +388,7 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel,
                                     grpc_slice method, const grpc_slice* host,
                                     gpr_timespec deadline, void* reserved) {
   GPR_ASSERT(!reserved);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   grpc_call* call = grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, cq, nullptr,
@@ -417,35 +413,66 @@ grpc_call* grpc_channel_create_pollset_set_call(
       deadline);
 }
 
+namespace grpc_core {
+
+RegisteredCall::RegisteredCall(const char* method, const char* host) {
+  path = grpc_mdelem_from_slices(GRPC_MDSTR_PATH,
+                                 grpc_core::ExternallyManagedSlice(method));
+  authority =
+      host ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
+                                     grpc_core::ExternallyManagedSlice(host))
+           : GRPC_MDNULL;
+}
+
+// TODO(vjpai): Delete copy-constructor when allowed by all supported compilers.
+RegisteredCall::RegisteredCall(const RegisteredCall& other) {
+  path = other.path;
+  authority = other.authority;
+  GRPC_MDELEM_REF(path);
+  GRPC_MDELEM_REF(authority);
+}
+
+RegisteredCall::RegisteredCall(RegisteredCall&& other) noexcept {
+  path = other.path;
+  authority = other.authority;
+  other.path = GRPC_MDNULL;
+  other.authority = GRPC_MDNULL;
+}
+
+RegisteredCall::~RegisteredCall() {
+  GRPC_MDELEM_UNREF(path);
+  GRPC_MDELEM_UNREF(authority);
+}
+
+}  // namespace grpc_core
+
 void* grpc_channel_register_call(grpc_channel* channel, const char* method,
                                  const char* host, void* reserved) {
-  registered_call* rc =
-      static_cast<registered_call*>(gpr_malloc(sizeof(registered_call)));
   GRPC_API_TRACE(
       "grpc_channel_register_call(channel=%p, method=%s, host=%s, reserved=%p)",
       4, (channel, method, host, reserved));
   GPR_ASSERT(!reserved);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
 
-  rc->path = grpc_mdelem_from_slices(GRPC_MDSTR_PATH,
-                                     grpc_core::ExternallyManagedSlice(method));
-  rc->authority =
-      host ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
-                                     grpc_core::ExternallyManagedSlice(host))
-           : GRPC_MDNULL;
-  gpr_mu_lock(&channel->registered_call_mu);
-  rc->next = channel->registered_calls;
-  channel->registered_calls = rc;
-  gpr_mu_unlock(&channel->registered_call_mu);
-
-  return rc;
+  grpc_core::MutexLock lock(&channel->registration_table->mu);
+  channel->registration_table->method_registration_attempts++;
+  auto key = std::make_pair(host, method);
+  auto rc_posn = channel->registration_table->map.find(key);
+  if (rc_posn != channel->registration_table->map.end()) {
+    return &rc_posn->second;
+  }
+  auto insertion_result = channel->registration_table->map.insert(
+      {key, grpc_core::RegisteredCall(method, host)});
+  return &insertion_result.first->second;
 }
 
 grpc_call* grpc_channel_create_registered_call(
     grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask,
     grpc_completion_queue* completion_queue, void* registered_call_handle,
     gpr_timespec deadline, void* reserved) {
-  registered_call* rc = static_cast<registered_call*>(registered_call_handle);
+  grpc_core::RegisteredCall* rc =
+      static_cast<grpc_core::RegisteredCall*>(registered_call_handle);
   GRPC_API_TRACE(
       "grpc_channel_create_registered_call("
       "channel=%p, parent_call=%p, propagation_mask=%x, completion_queue=%p, "
@@ -458,6 +485,7 @@ grpc_call* grpc_channel_create_registered_call(
        registered_call_handle, deadline.tv_sec, deadline.tv_nsec,
        (int)deadline.clock_type, reserved));
   GPR_ASSERT(!reserved);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   grpc_call* call = grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, completion_queue, nullptr,
@@ -486,18 +514,11 @@ static void destroy_channel(void* arg, grpc_error* /*error*/) {
     channel->channelz_node.reset();
   }
   grpc_channel_stack_destroy(CHANNEL_STACK_FROM_CHANNEL(channel));
-  while (channel->registered_calls) {
-    registered_call* rc = channel->registered_calls;
-    channel->registered_calls = rc->next;
-    GRPC_MDELEM_UNREF(rc->path);
-    GRPC_MDELEM_UNREF(rc->authority);
-    gpr_free(rc);
-  }
+  channel->registration_table.Destroy();
   if (channel->resource_user != nullptr) {
     grpc_resource_user_free(channel->resource_user,
                             GRPC_RESOURCE_QUOTA_CHANNEL_SIZE);
   }
-  gpr_mu_destroy(&channel->registered_call_mu);
   gpr_free(channel->target);
   gpr_free(channel);
   // See comment in grpc_channel_create() for why we do this.
@@ -516,6 +537,7 @@ void grpc_channel_destroy_internal(grpc_channel* channel) {
 }
 
 void grpc_channel_destroy(grpc_channel* channel) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   grpc_channel_destroy_internal(channel);
 }

data/src/core/lib/surface/channel.h

@@ -21,10 +21,14 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <map>
+
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/channel/channelz.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/transport/metadata.h"
 
 grpc_channel* grpc_channel_create(const char* target,
                                   const grpc_channel_args* args,
@@ -62,7 +66,30 @@ grpc_core::channelz::ChannelNode* grpc_channel_get_channelz_node(
 size_t grpc_channel_get_call_size_estimate(grpc_channel* channel);
 void grpc_channel_update_call_size_estimate(grpc_channel* channel, size_t size);
 
-struct registered_call;
+namespace grpc_core {
+
+struct RegisteredCall {
+  grpc_mdelem path;
+  grpc_mdelem authority;
+
+  explicit RegisteredCall(const char* method, const char* host);
+  // TODO(vjpai): delete copy constructor once all supported compilers allow
+  //              std::map value_type to be MoveConstructible.
+  RegisteredCall(const RegisteredCall& other);
+  RegisteredCall(RegisteredCall&& other) noexcept;
+
+  ~RegisteredCall();
+};
+
+struct CallRegistrationTable {
+  grpc_core::Mutex mu;
+  std::map<std::pair<const char*, const char*>, RegisteredCall>
+      map /* GUARDED_BY(mu) */;
+  int method_registration_attempts /* GUARDED_BY(mu) */ = 0;
+};
+
+}  // namespace grpc_core
+
 struct grpc_channel {
   int is_client;
   grpc_compression_options compression_options;
@@ -70,9 +97,13 @@ struct grpc_channel {
   gpr_atm call_size_estimate;
   grpc_resource_user* resource_user;
 
-  gpr_mu registered_call_mu;
-  registered_call* registered_calls;
-
+  // TODO(vjpai): Once the grpc_channel is allocated via new rather than malloc,
+  //              expand the members of the CallRegistrationTable directly into
+  //              the grpc_channel. For now it is kept separate so that all the
+  //              manual constructing can be done with a single call rather than
+  //              a separate manual construction for each field.
+  grpc_core::ManualConstructor<grpc_core::CallRegistrationTable>
+      registration_table;
   grpc_core::RefCountedPtr<grpc_core::channelz::ChannelNode> channelz_node;
 
   char* target;

data/src/core/lib/surface/channel_ping.cc

@@ -28,13 +28,12 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/completion_queue.h"
 
-typedef struct {
+struct ping_result {
   grpc_closure closure;
   void* tag;
   grpc_completion_queue* cq;
   grpc_cq_completion completion_storage;
-} ping_result;
-
+};
 static void ping_destroy(void* arg, grpc_cq_completion* /*storage*/) {
   gpr_free(arg);
 }

data/src/core/lib/surface/completion_queue.cc

@@ -23,6 +23,11 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
@@ -34,6 +39,7 @@
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -56,12 +62,11 @@ namespace {
 GPR_TLS_DECL(g_cached_event);
 GPR_TLS_DECL(g_cached_cq);
 
-typedef struct {
+struct plucker {
   grpc_pollset_worker** worker;
   void* tag;
-} plucker;
-
-typedef struct {
+};
+struct cq_poller_vtable {
   bool can_get_pollset;
   bool can_listen;
   size_t (*size)(void);
@@ -72,8 +77,7 @@ typedef struct {
                       grpc_millis deadline);
   void (*shutdown)(grpc_pollset* pollset, grpc_closure* closure);
   void (*destroy)(grpc_pollset* pollset);
-} cq_poller_vtable;
-
+};
 typedef struct non_polling_worker {
   gpr_cv cv;
   bool kicked;
@@ -81,13 +85,12 @@ typedef struct non_polling_worker {
   struct non_polling_worker* prev;
 } non_polling_worker;
 
-typedef struct {
+struct non_polling_poller {
   gpr_mu mu;
   bool kicked_without_poller;
   non_polling_worker* root;
   grpc_closure* shutdown;
-} non_polling_poller;
-
+};
 size_t non_polling_poller_size(void) { return sizeof(non_polling_poller); }
 
 void non_polling_poller_init(grpc_pollset* pollset, gpr_mu** mu) {
@@ -206,6 +209,9 @@ struct cq_vtable {
                      void* reserved);
   grpc_event (*pluck)(grpc_completion_queue* cq, void* tag,
                       gpr_timespec deadline, void* reserved);
+  // TODO(vjpai): Remove proxy_pollset once callback_alternative no longer
+  // needed.
+  grpc_pollset* (*proxy_pollset)(grpc_completion_queue* cq);
 };
 
 namespace {
@@ -307,7 +313,7 @@ struct cq_pluck_data {
 };
 
 struct cq_callback_data {
-  cq_callback_data(
+  explicit cq_callback_data(
       grpc_experimental_completion_queue_functor* shutdown_callback)
       : shutdown_callback(shutdown_callback) {}
 
@@ -325,17 +331,88 @@ struct cq_callback_data {
       Initial count is dropped by grpc_completion_queue_shutdown. */
   grpc_core::Atomic<intptr_t> pending_events{1};
 
-  /** Counter of how many things have ever been queued on this completion queue
-      useful for avoiding locks to check the queue */
-  grpc_core::Atomic<intptr_t> things_queued_ever{0};
+  /** 0 initially. 1 once we initiated shutdown */
+  bool shutdown_called = false;
+
+  /** A callback that gets invoked when the CQ completes shutdown */
+  grpc_experimental_completion_queue_functor* shutdown_callback;
+};
+
+// TODO(vjpai): Remove all callback_alternative variants when event manager is
+// the only supported poller.
+struct cq_callback_alternative_data {
+  explicit cq_callback_alternative_data(
+      grpc_experimental_completion_queue_functor* shutdown_callback)
+      : implementation(SharedNextableCQ()),
+        shutdown_callback(shutdown_callback) {}
+
+  /* This just points to a single shared nextable CQ */
+  grpc_completion_queue* const implementation;
+
+  /** Number of outstanding events (+1 if not shut down)
+      Initial count is dropped by grpc_completion_queue_shutdown */
+  grpc_core::Atomic<intptr_t> pending_events{1};
 
   /** 0 initially. 1 once we initiated shutdown */
   bool shutdown_called = false;
 
   /** A callback that gets invoked when the CQ completes shutdown */
   grpc_experimental_completion_queue_functor* shutdown_callback;
+
+  static grpc_completion_queue* SharedNextableCQ() {
+    grpc_core::MutexLock lock(&*shared_cq_next_mu);
+
+    if (shared_cq_next == nullptr) {
+      shared_cq_next = grpc_completion_queue_create_for_next(nullptr);
+      int num_nexting_threads = GPR_CLAMP(gpr_cpu_num_cores(), 1, 32);
+      threads_remaining.Store(num_nexting_threads,
+                              grpc_core::MemoryOrder::RELEASE);
+      for (int i = 0; i < num_nexting_threads; i++) {
+        grpc_core::Executor::Run(
+            GRPC_CLOSURE_CREATE(
+                [](void* arg, grpc_error* /*error*/) {
+                  grpc_completion_queue* cq =
+                      static_cast<grpc_completion_queue*>(arg);
+                  while (true) {
+                    grpc_event event = grpc_completion_queue_next(
+                        cq, gpr_inf_future(GPR_CLOCK_REALTIME), nullptr);
+                    if (event.type == GRPC_QUEUE_SHUTDOWN) {
+                      break;
+                    }
+                    GPR_DEBUG_ASSERT(event.type == GRPC_OP_COMPLETE);
+                    // We can always execute the callback inline rather than
+                    // pushing it to another Executor thread because this
+                    // thread is definitely running on an executor, does not
+                    // hold any application locks before executing the callback,
+                    // and cannot be entered recursively.
+                    auto* functor = static_cast<
+                        grpc_experimental_completion_queue_functor*>(event.tag);
+                    functor->functor_run(functor, event.success);
+                  }
+                  if (threads_remaining.FetchSub(
+                          1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
+                    grpc_completion_queue_destroy(cq);
+                  }
+                },
+                shared_cq_next, nullptr),
+            GRPC_ERROR_NONE, grpc_core::ExecutorType::DEFAULT,
+            grpc_core::ExecutorJobType::LONG);
+      }
+    }
+    return shared_cq_next;
+  }
+  // Use manually-constructed Mutex to avoid static construction issues
+  static grpc_core::ManualConstructor<grpc_core::Mutex> shared_cq_next_mu;
+  static grpc_completion_queue*
+      shared_cq_next;  // GUARDED_BY(shared_cq_next_mu)
+  static grpc_core::Atomic<int> threads_remaining;
 };
 
+grpc_core::ManualConstructor<grpc_core::Mutex>
+    cq_callback_alternative_data::shared_cq_next_mu;
+grpc_completion_queue* cq_callback_alternative_data::shared_cq_next = nullptr;
+grpc_core::Atomic<int> cq_callback_alternative_data::threads_remaining{0};
+
 }  // namespace
 
 /* Completion queue structure */
@@ -348,6 +425,12 @@ struct grpc_completion_queue {
   const cq_vtable* vtable;
   const cq_poller_vtable* poller_vtable;
 
+  // The pollset entry is allowed to enable proxy CQs like the
+  // callback_alternative.
+  // TODO(vjpai): Consider removing pollset and reverting to previous
+  // calculation of pollset once callback_alternative is no longer needed.
+  grpc_pollset* pollset;
+
 #ifndef NDEBUG
   void** outstanding_tags;
   size_t outstanding_tag_count;
@@ -362,13 +445,17 @@ struct grpc_completion_queue {
 static void cq_finish_shutdown_next(grpc_completion_queue* cq);
 static void cq_finish_shutdown_pluck(grpc_completion_queue* cq);
 static void cq_finish_shutdown_callback(grpc_completion_queue* cq);
+static void cq_finish_shutdown_callback_alternative(grpc_completion_queue* cq);
 static void cq_shutdown_next(grpc_completion_queue* cq);
 static void cq_shutdown_pluck(grpc_completion_queue* cq);
 static void cq_shutdown_callback(grpc_completion_queue* cq);
+static void cq_shutdown_callback_alternative(grpc_completion_queue* cq);
 
 static bool cq_begin_op_for_next(grpc_completion_queue* cq, void* tag);
 static bool cq_begin_op_for_pluck(grpc_completion_queue* cq, void* tag);
 static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* tag);
+static bool cq_begin_op_for_callback_alternative(grpc_completion_queue* cq,
+                                                 void* tag);
 
 // A cq_end_op function is called when an operation on a given CQ with
 // a given tag has completed. The storage argument is a reference to the
@@ -391,12 +478,20 @@ static void cq_end_op_for_callback(
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
     grpc_cq_completion* storage, bool internal);
 
+static void cq_end_op_for_callback_alternative(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool internal);
+
 static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
                           void* reserved);
 
 static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
                            gpr_timespec deadline, void* reserved);
 
+static grpc_pollset* cq_proxy_pollset_for_callback_alternative(
+    grpc_completion_queue* cq);
+
 // Note that cq_init_next and cq_init_pluck do not use the shutdown_callback
 static void cq_init_next(
     void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
@@ -404,41 +499,50 @@ static void cq_init_pluck(
     void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
 static void cq_init_callback(
     void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
+// poller becomes only option.
+static void cq_init_callback_alternative(
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback);
 static void cq_destroy_next(void* data);
 static void cq_destroy_pluck(void* data);
 static void cq_destroy_callback(void* data);
+static void cq_destroy_callback_alternative(void* data);
 
 /* Completion queue vtables based on the completion-type */
-static const cq_vtable g_cq_vtable[] = {
+// TODO(vjpai): Make this const again once we stop needing callback_alternative
+static cq_vtable g_polling_cq_vtable[] = {
     /* GRPC_CQ_NEXT */
     {GRPC_CQ_NEXT, sizeof(cq_next_data), cq_init_next, cq_shutdown_next,
      cq_destroy_next, cq_begin_op_for_next, cq_end_op_for_next, cq_next,
-     nullptr},
+     nullptr, nullptr},
     /* GRPC_CQ_PLUCK */
     {GRPC_CQ_PLUCK, sizeof(cq_pluck_data), cq_init_pluck, cq_shutdown_pluck,
      cq_destroy_pluck, cq_begin_op_for_pluck, cq_end_op_for_pluck, nullptr,
-     cq_pluck},
+     cq_pluck, nullptr},
     /* GRPC_CQ_CALLBACK */
     {GRPC_CQ_CALLBACK, sizeof(cq_callback_data), cq_init_callback,
      cq_shutdown_callback, cq_destroy_callback, cq_begin_op_for_callback,
-     cq_end_op_for_callback, nullptr, nullptr},
+     cq_end_op_for_callback, nullptr, nullptr, nullptr},
 };
 
+// Separate vtable for non-polling cqs, assign at init
+static cq_vtable g_nonpolling_cq_vtable[sizeof(g_polling_cq_vtable) /
+                                        sizeof(g_polling_cq_vtable[0])];
+
 #define DATA_FROM_CQ(cq) ((void*)(cq + 1))
-#define POLLSET_FROM_CQ(cq) \
+#define INLINE_POLLSET_FROM_CQ(cq) \
   ((grpc_pollset*)(cq->vtable->data_size + (char*)DATA_FROM_CQ(cq)))
+#define POLLSET_FROM_CQ(cq) (cq->pollset)
 
 grpc_core::TraceFlag grpc_cq_pluck_trace(false, "queue_pluck");
 
-#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)      \
-  do {                                                    \
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) &&        \
-        (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace) ||  \
-         (event)->type != GRPC_QUEUE_TIMEOUT)) {          \
-      char* _ev = grpc_event_string(event);               \
-      gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \
-      gpr_free(_ev);                                      \
-    }                                                     \
+#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)     \
+  do {                                                   \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) &&       \
+        (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace) || \
+         (event)->type != GRPC_QUEUE_TIMEOUT)) {         \
+      gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq,      \
+              grpc_event_string(event).c_str());         \
+    }                                                    \
   } while (0)
 
 static void on_pollset_shutdown_done(void* cq, grpc_error* error);
@@ -446,6 +550,46 @@ static void on_pollset_shutdown_done(void* cq, grpc_error* error);
 void grpc_cq_global_init() {
   gpr_tls_init(&g_cached_event);
   gpr_tls_init(&g_cached_cq);
+  g_nonpolling_cq_vtable[GRPC_CQ_NEXT] = g_polling_cq_vtable[GRPC_CQ_NEXT];
+  g_nonpolling_cq_vtable[GRPC_CQ_PLUCK] = g_polling_cq_vtable[GRPC_CQ_PLUCK];
+  g_nonpolling_cq_vtable[GRPC_CQ_CALLBACK] =
+      g_polling_cq_vtable[GRPC_CQ_CALLBACK];
+}
+
+// TODO(vjpai): Remove when callback_alternative is no longer needed
+void grpc_cq_init() {
+  // If the iomgr runs in the background, we can use the preferred callback CQ.
+  // If the iomgr is non-polling, we cannot use the alternative callback CQ.
+  if (!grpc_iomgr_run_in_background() && !grpc_iomgr_non_polling()) {
+    cq_callback_alternative_data::shared_cq_next_mu.Init();
+    g_polling_cq_vtable[GRPC_CQ_CALLBACK] = {
+        GRPC_CQ_CALLBACK,
+        sizeof(cq_callback_alternative_data),
+        cq_init_callback_alternative,
+        cq_shutdown_callback_alternative,
+        cq_destroy_callback_alternative,
+        cq_begin_op_for_callback_alternative,
+        cq_end_op_for_callback_alternative,
+        nullptr,
+        nullptr,
+        cq_proxy_pollset_for_callback_alternative};
+  }
+}
+
+// TODO(vjpai): Remove when callback_alternative is no longer needed
+void grpc_cq_shutdown() {
+  if (!grpc_iomgr_run_in_background() && !grpc_iomgr_non_polling()) {
+    {
+      grpc_core::MutexLock lock(
+          &*cq_callback_alternative_data::shared_cq_next_mu);
+      if (cq_callback_alternative_data::shared_cq_next != nullptr) {
+        grpc_completion_queue_shutdown(
+            cq_callback_alternative_data::shared_cq_next);
+      }
+      cq_callback_alternative_data::shared_cq_next = nullptr;
+    }
+    cq_callback_alternative_data::shared_cq_next_mu.Destroy();
+  }
 }
 
 void grpc_completion_queue_thread_local_cache_init(grpc_completion_queue* cq) {
@@ -524,7 +668,9 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
       "polling_type=%d)",
       2, (completion_type, polling_type));
 
-  const cq_vtable* vtable = &g_cq_vtable[completion_type];
+  const cq_vtable* vtable = (polling_type == GRPC_CQ_NON_POLLING)
+                                ? &g_nonpolling_cq_vtable[completion_type]
+                                : &g_polling_cq_vtable[completion_type];
   const cq_poller_vtable* poller_vtable =
       &g_poller_vtable_by_poller_type[polling_type];
 
@@ -541,9 +687,18 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
   /* One for destroy(), one for pollset_shutdown */
   new (&cq->owning_refs) grpc_core::RefCount(2);
 
-  poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu);
   vtable->init(DATA_FROM_CQ(cq), shutdown_callback);
 
+  // TODO(vjpai): When callback_alternative is no longer needed, cq->pollset can
+  // be removed and the nullptr proxy_pollset value below can be the definition
+  // of POLLSET_FROM_CQ.
+  cq->pollset = cq->vtable->proxy_pollset == nullptr
+                    ? INLINE_POLLSET_FROM_CQ(cq)
+                    : cq->vtable->proxy_pollset(cq);
+  // Init the inline pollset. If a proxy CQ is used, the proxy pollset will be
+  // init'ed in its CQ init.
+  cq->poller_vtable->init(INLINE_POLLSET_FROM_CQ(cq), &cq->mu);
+
   GRPC_CLOSURE_INIT(&cq->pollset_shutdown_done, on_pollset_shutdown_done, cq,
                     grpc_schedule_on_exec_ctx);
   return cq;
@@ -581,6 +736,17 @@ static void cq_destroy_callback(void* data) {
   cqd->~cq_callback_data();
 }
 
+static void cq_init_callback_alternative(
+    void* data, grpc_experimental_completion_queue_functor* shutdown_callback) {
+  new (data) cq_callback_alternative_data(shutdown_callback);
+}
+
+static void cq_destroy_callback_alternative(void* data) {
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*>(data);
+  cqd->~cq_callback_alternative_data();
+}
+
 grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue* cq) {
   return cq->vtable->cq_completion_type;
 }
@@ -621,7 +787,9 @@ void grpc_cq_internal_unref(grpc_completion_queue* cq) {
 #endif
   if (GPR_UNLIKELY(cq->owning_refs.Unref(debug_location, reason))) {
     cq->vtable->destroy(DATA_FROM_CQ(cq));
-    cq->poller_vtable->destroy(POLLSET_FROM_CQ(cq));
+    // Only destroy the inlined pollset. If a proxy CQ is used, the proxy
+    // pollset will be destroyed by the proxy CQ.
+    cq->poller_vtable->destroy(INLINE_POLLSET_FROM_CQ(cq));
 #ifndef NDEBUG
     gpr_free(cq->outstanding_tags);
 #endif
@@ -672,6 +840,14 @@ static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* /*tag*/) {
   return cqd->pending_events.IncrementIfNonzero();
 }
 
+static bool cq_begin_op_for_callback_alternative(grpc_completion_queue* cq,
+                                                 void* tag) {
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*> DATA_FROM_CQ(cq);
+  return grpc_cq_begin_op(cqd->implementation, tag) &&
+         cqd->pending_events.IncrementIfNonzero();
+}
+
 bool grpc_cq_begin_op(grpc_completion_queue* cq, void* tag) {
 #ifndef NDEBUG
   gpr_mu_lock(cq->mu);
@@ -835,7 +1011,7 @@ static void cq_end_op_for_pluck(
   GRPC_ERROR_UNREF(error);
 }
 
-static void functor_callback(void* arg, grpc_error* error) {
+void functor_callback(void* arg, grpc_error* error) {
   auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(arg);
   functor->functor_run(functor, error == GRPC_ERROR_NONE);
 }
@@ -869,13 +1045,19 @@ static void cq_end_op_for_callback(
 
   cq_check_tag(cq, tag, true); /* Used in debug builds only */
 
-  cqd->things_queued_ever.FetchAdd(1, grpc_core::MemoryOrder::RELAXED);
   if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
     cq_finish_shutdown_callback(cq);
   }
 
+  // If possible, schedule the callback onto an existing thread-local
+  // ApplicationCallbackExecCtx, which is a work queue. This is possible for:
+  // 1. The callback is internally-generated and there is an ACEC available
+  // 2. The callback is marked inlineable and there is an ACEC available
+  // 3. We are already running in a background poller thread (which always has
+  //    an ACEC available at the base of the stack).
   auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(tag);
-  if (internal || functor->inlineable ||
+  if (((internal || functor->inlineable) &&
+       grpc_core::ApplicationCallbackExecCtx::Available()) ||
       grpc_iomgr_is_any_background_poller_thread()) {
     grpc_core::ApplicationCallbackExecCtx::Enqueue(functor,
                                                    (error == GRPC_ERROR_NONE));
@@ -889,6 +1071,40 @@ static void cq_end_op_for_callback(
       GRPC_CLOSURE_CREATE(functor_callback, functor, nullptr), error);
 }
 
+static void cq_end_op_for_callback_alternative(
+    grpc_completion_queue* cq, void* tag, grpc_error* error,
+    void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
+    grpc_cq_completion* storage, bool internal) {
+  GPR_TIMER_SCOPE("cq_end_op_for_callback_alternative", 0);
+
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*> DATA_FROM_CQ(cq);
+
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+       error != GRPC_ERROR_NONE)) {
+    const char* errmsg = grpc_error_string(error);
+    GRPC_API_TRACE(
+        "cq_end_op_for_callback_alternative(cq=%p, tag=%p, error=%s, "
+        "done=%p, done_arg=%p, storage=%p)",
+        6, (cq, tag, errmsg, done, done_arg, storage));
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
+        error != GRPC_ERROR_NONE) {
+      gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
+    }
+  }
+
+  // Pass through the actual work to the internal nextable CQ
+  grpc_cq_end_op(cqd->implementation, tag, error, done, done_arg, storage,
+                 internal);
+
+  cq_check_tag(cq, tag, true); /* Used in debug builds only */
+
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
+    cq_finish_shutdown_callback_alternative(cq);
+  }
+}
+
 void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error,
                     void (*done)(void* done_arg, grpc_cq_completion* storage),
                     void* done_arg, grpc_cq_completion* storage,
@@ -896,15 +1112,21 @@ void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error,
   cq->vtable->end_op(cq, tag, error, done, done_arg, storage, internal);
 }
 
-typedef struct {
+static grpc_pollset* cq_proxy_pollset_for_callback_alternative(
+    grpc_completion_queue* cq) {
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*>(DATA_FROM_CQ(cq));
+  return POLLSET_FROM_CQ(cqd->implementation);
+}
+
+struct cq_is_finished_arg {
   gpr_atm last_seen_things_queued_ever;
   grpc_completion_queue* cq;
   grpc_millis deadline;
   grpc_cq_completion* stolen_completion;
   void* tag; /* for pluck */
   bool first_loop;
-} cq_is_finished_arg;
-
+};
 class ExecCtxNext : public grpc_core::ExecCtx {
  public:
   ExecCtxNext(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
@@ -944,21 +1166,14 @@ class ExecCtxNext : public grpc_core::ExecCtx {
 #ifndef NDEBUG
 static void dump_pending_tags(grpc_completion_queue* cq) {
   if (!GRPC_TRACE_FLAG_ENABLED(grpc_trace_pending_tags)) return;
-
-  gpr_strvec v;
-  gpr_strvec_init(&v);
-  gpr_strvec_add(&v, gpr_strdup("PENDING TAGS:"));
+  std::vector<std::string> parts;
+  parts.push_back("PENDING TAGS:");
   gpr_mu_lock(cq->mu);
   for (size_t i = 0; i < cq->outstanding_tag_count; i++) {
-    char* s;
-    gpr_asprintf(&s, " %p", cq->outstanding_tags[i]);
-    gpr_strvec_add(&v, s);
+    parts.push_back(absl::StrFormat(" %p", cq->outstanding_tags[i]));
   }
   gpr_mu_unlock(cq->mu);
-  char* out = gpr_strvec_flatten(&v, nullptr);
-  gpr_strvec_destroy(&v);
-  gpr_log(GPR_DEBUG, "%s", out);
-  gpr_free(out);
+  gpr_log(GPR_DEBUG, "%s", absl::StrJoin(parts, "").c_str());
 }
 #else
 static void dump_pending_tags(grpc_completion_queue* /*cq*/) {}
@@ -1384,6 +1599,21 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) {
       GRPC_ERROR_NONE);
 }
 
+static void cq_finish_shutdown_callback_alternative(grpc_completion_queue* cq) {
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*> DATA_FROM_CQ(cq);
+  auto* callback = cqd->shutdown_callback;
+
+  GPR_ASSERT(cqd->shutdown_called);
+
+  // Shutdown the non-proxy pollset
+  cq->poller_vtable->shutdown(INLINE_POLLSET_FROM_CQ(cq),
+                              &cq->pollset_shutdown_done);
+  grpc_core::Executor::Run(
+      GRPC_CLOSURE_CREATE(functor_callback, callback, nullptr),
+      GRPC_ERROR_NONE);
+}
+
 static void cq_shutdown_callback(grpc_completion_queue* cq) {
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
 
@@ -1410,6 +1640,33 @@ static void cq_shutdown_callback(grpc_completion_queue* cq) {
   GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)");
 }
 
+static void cq_shutdown_callback_alternative(grpc_completion_queue* cq) {
+  cq_callback_alternative_data* cqd =
+      static_cast<cq_callback_alternative_data*> DATA_FROM_CQ(cq);
+
+  /* Need an extra ref for cq here because:
+   * We call cq_finish_shutdown_callback() below, which calls pollset shutdown.
+   * Pollset shutdown decrements the cq ref count which can potentially destroy
+   * the cq (if that happens to be the last ref).
+   * Creating an extra ref here prevents the cq from getting destroyed while
+   * this function is still active */
+  GRPC_CQ_INTERNAL_REF(cq, "shutting_down (callback cq)");
+  gpr_mu_lock(cq->mu);
+  if (cqd->shutdown_called) {
+    gpr_mu_unlock(cq->mu);
+    GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)");
+    return;
+  }
+  cqd->shutdown_called = true;
+  if (cqd->pending_events.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
+    gpr_mu_unlock(cq->mu);
+    cq_finish_shutdown_callback_alternative(cq);
+  } else {
+    gpr_mu_unlock(cq->mu);
+  }
+  GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)");
+}
+
 /* Shutdown simply drops a ref that we reserved at creation time; if we drop
    to zero here, then enter shutdown mode and wake up any waiters */
 void grpc_completion_queue_shutdown(grpc_completion_queue* cq) {