grpc 1.25.0 → 1.26.0.pre1

data/src/core/lib/surface/version.cc

@@ -23,6 +23,6 @@
 
 #include <grpc/grpc.h>
 
-const char* grpc_version_string(void) { return "8.0.0"; }
+const char* grpc_version_string(void) { return "9.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "game"; }
+const char* grpc_g_stands_for(void) { return "gon"; }

data/src/core/lib/transport/byte_stream.cc

@@ -53,8 +53,8 @@ void SliceBufferByteStream::Orphan() {
   // filter stack.
 }
 
-bool SliceBufferByteStream::Next(size_t max_size_hint,
-                                 grpc_closure* on_complete) {
+bool SliceBufferByteStream::Next(size_t /*max_size_hint*/,
+                                 grpc_closure* /*on_complete*/) {
   GPR_DEBUG_ASSERT(backing_buffer_.count > 0);
   return true;
 }

data/src/core/lib/transport/byte_stream.h

@@ -40,7 +40,8 @@ class ByteStream : public Orphanable {
   // Returns true if the bytes are available immediately (in which case
   // on_complete will not be called), or false if the bytes will be available
   // asynchronously (in which case on_complete will be called when they
-  // are available).
+  // are available). Should not be called if there is no data left on the
+  // stream.
   //
   // max_size_hint can be set as a hint as to the maximum number
   // of bytes that would be acceptable to read.

data/src/core/lib/transport/connectivity_state.cc

@@ -67,19 +67,19 @@ class AsyncConnectivityStateWatcherInterface::Notifier {
     } else {
       GRPC_CLOSURE_INIT(&closure_, SendNotification, this,
                         grpc_schedule_on_exec_ctx);
-      GRPC_CLOSURE_SCHED(&closure_, GRPC_ERROR_NONE);
+      ExecCtx::Run(DEBUG_LOCATION, &closure_, GRPC_ERROR_NONE);
     }
   }
 
  private:
-  static void SendNotification(void* arg, grpc_error* ignored) {
+  static void SendNotification(void* arg, grpc_error* /*ignored*/) {
     Notifier* self = static_cast<Notifier*>(arg);
     if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
       gpr_log(GPR_INFO, "watcher %p: delivering async notification for %s",
               self->watcher_.get(), ConnectivityStateName(self->state_));
     }
     self->watcher_->OnConnectivityStateChange(self->state_);
-    Delete(self);
+    delete self;
   }
 
   RefCountedPtr<AsyncConnectivityStateWatcherInterface> watcher_;
@@ -89,7 +89,7 @@ class AsyncConnectivityStateWatcherInterface::Notifier {
 
 void AsyncConnectivityStateWatcherInterface::Notify(
     grpc_connectivity_state state) {
-  New<Notifier>(Ref(), state, combiner_);  // Deletes itself when done.
+  new Notifier(Ref(), state, combiner_);  // Deletes itself when done.
 }
 
 //

data/src/core/lib/transport/connectivity_state.h

@@ -119,8 +119,8 @@ class ConnectivityStateTracker {
   Atomic<grpc_connectivity_state> state_;
   // TODO(roth): Once we can use C++-14 heterogeneous lookups, this can
   // be a set instead of a map.
-  Map<ConnectivityStateWatcherInterface*,
-      OrphanablePtr<ConnectivityStateWatcherInterface>>
+  std::map<ConnectivityStateWatcherInterface*,
+           OrphanablePtr<ConnectivityStateWatcherInterface>>
       watchers_;
 };
 

data/src/core/lib/transport/metadata.cc

@@ -207,7 +207,7 @@ size_t InternedMetadata::CleanupLinkedMetadata(
     next = md->link_.next;
     if (md->AllRefsDropped()) {
       prev_next->next = next;
-      grpc_core::Delete(md);
+      delete md;
       num_freed++;
     } else {
       prev_next = &md->link_;
@@ -373,14 +373,13 @@ static grpc_mdelem md_create(
       // We allocate backing store.
       return key_definitely_static
                  ? GRPC_MAKE_MDELEM(
-                       grpc_core::New<AllocatedMetadata>(
+                       new AllocatedMetadata(
                            key, value,
                            static_cast<const AllocatedMetadata::NoRefKey*>(
                                nullptr)),
                        GRPC_MDELEM_STORAGE_ALLOCATED)
-                 : GRPC_MAKE_MDELEM(
-                       grpc_core::New<AllocatedMetadata>(key, value),
-                       GRPC_MDELEM_STORAGE_ALLOCATED);
+                 : GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                                    GRPC_MDELEM_STORAGE_ALLOCATED);
     }
   }
   return md_create_maybe_static<key_definitely_static, value_definitely_static>(
@@ -456,11 +455,10 @@ static grpc_mdelem md_create_must_intern(const grpc_slice& key,
 
   /* not found: create a new pair */
   md = key_definitely_static
-           ? grpc_core::New<InternedMetadata>(
+           ? new InternedMetadata(
                  key, value, hash, shard->elems[idx].next,
                  static_cast<const InternedMetadata::NoRefKey*>(nullptr))
-           : grpc_core::New<InternedMetadata>(key, value, hash,
-                                              shard->elems[idx].next);
+           : new InternedMetadata(key, value, hash, shard->elems[idx].next);
   shard->elems[idx].next = md;
   shard->count++;
 
@@ -656,7 +654,7 @@ void grpc_mdelem_do_unref(grpc_mdelem gmd DEBUG_ARGS) {
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
       auto* md = reinterpret_cast<AllocatedMetadata*> GRPC_MDELEM_DATA(gmd);
       if (GPR_UNLIKELY(md->Unref(FWD_DEBUG_ARGS))) {
-        grpc_core::Delete(md);
+        delete md;
       }
       break;
     }
@@ -674,7 +672,7 @@ void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
       break;
     }
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
-      grpc_core::Delete(reinterpret_cast<AllocatedMetadata*>(ptr));
+      delete reinterpret_cast<AllocatedMetadata*>(ptr);
       break;
     }
   }

data/src/core/lib/transport/metadata.h

@@ -37,11 +37,8 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
 /* This file provides a mechanism for tracking metadata through the grpc stack.
    It's not intended for consumption outside of the library.
 
-   Metadata is tracked in the context of a grpc_mdctx. For the time being there
-   is one of these per-channel, avoiding cross channel interference with memory
-   use and lock contention.
-
-   The context tracks unique strings (grpc_mdstr) and pairs of strings
+   Metadata is tracked in the context of a sharded global grpc_mdctx. The
+   context tracks unique strings (grpc_mdstr) and pairs of strings
    (grpc_mdelem). Any of these objects can be checked for equality by comparing
    their pointers. These objects are reference counted.
 
@@ -426,7 +423,7 @@ inline grpc_mdelem grpc_mdelem_from_slices(
     const grpc_core::ManagedMemorySlice& key,
     const grpc_core::UnmanagedMemorySlice& value) {
   using grpc_core::AllocatedMetadata;
-  return GRPC_MAKE_MDELEM(grpc_core::New<AllocatedMetadata>(key, value),
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
                           GRPC_MDELEM_STORAGE_ALLOCATED);
 }
 
@@ -434,7 +431,7 @@ inline grpc_mdelem grpc_mdelem_from_slices(
     const grpc_core::ExternallyManagedSlice& key,
     const grpc_core::UnmanagedMemorySlice& value) {
   using grpc_core::AllocatedMetadata;
-  return GRPC_MAKE_MDELEM(grpc_core::New<AllocatedMetadata>(key, value),
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
                           GRPC_MDELEM_STORAGE_ALLOCATED);
 }
 
@@ -442,7 +439,7 @@ inline grpc_mdelem grpc_mdelem_from_slices(
     const grpc_core::StaticMetadataSlice& key,
     const grpc_core::UnmanagedMemorySlice& value) {
   using grpc_core::AllocatedMetadata;
-  return GRPC_MAKE_MDELEM(grpc_core::New<AllocatedMetadata>(key, value),
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
                           GRPC_MDELEM_STORAGE_ALLOCATED);
 }
 

data/src/core/lib/transport/metadata_batch.cc

@@ -50,6 +50,9 @@ static void assert_valid_list(grpc_mdelem_list* list) {
     verified_count++;
   }
   GPR_ASSERT(list->count == verified_count);
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)list;
 #endif /* NDEBUG */
 }
 
@@ -64,6 +67,9 @@ static void assert_valid_callouts(grpc_metadata_batch* batch) {
     }
     grpc_slice_unref_internal(key_interned);
   }
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)batch;
 #endif
 }
 

data/src/core/lib/transport/static_metadata.cc

@@ -1149,8 +1149,7 @@ uint64_t StaticMetadataInitCanary() {
 }  // namespace grpc_core
 
 void grpc_init_static_metadata_ctx(void) {
-  grpc_core::g_static_metadata_slice_ctx =
-      grpc_core::New<grpc_core::StaticMetadataCtx>();
+  grpc_core::g_static_metadata_slice_ctx = new grpc_core::StaticMetadataCtx();
   grpc_core::g_static_metadata_slice_table =
       grpc_core::g_static_metadata_slice_ctx->slices;
   grpc_core::g_static_metadata_slice_refcounts =
@@ -1162,8 +1161,7 @@ void grpc_init_static_metadata_ctx(void) {
 }
 
 void grpc_destroy_static_metadata_ctx(void) {
-  grpc_core::Delete<grpc_core::StaticMetadataCtx>(
-      grpc_core::g_static_metadata_slice_ctx);
+  delete grpc_core::g_static_metadata_slice_ctx;
   grpc_core::g_static_metadata_slice_ctx = nullptr;
   grpc_core::g_static_metadata_slice_table = nullptr;
   grpc_core::g_static_metadata_slice_refcounts = nullptr;

data/src/core/lib/transport/status_metadata.cc

@@ -52,3 +52,10 @@ grpc_status_code grpc_get_status_code_from_metadata(grpc_mdelem md) {
       md, destroy_status, (void*)static_cast<intptr_t>(status + STATUS_OFFSET));
   return static_cast<grpc_status_code>(status);
 }
+
+grpc_mdelem grpc_get_reffed_status_elem_slowpath(int status_code) {
+  char tmp[GPR_LTOA_MIN_BUFSIZE];
+  gpr_ltoa(status_code, tmp);
+  return grpc_mdelem_from_slices(GRPC_MDSTR_GRPC_STATUS,
+                                 grpc_core::UnmanagedMemorySlice(tmp));
+}

data/src/core/lib/transport/status_metadata.h

@@ -24,7 +24,25 @@
 #include <grpc/status.h>
 
 #include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 grpc_status_code grpc_get_status_code_from_metadata(grpc_mdelem md);
 
+/** Get a grpc_mdelem of grpc-status: X where X is the numeric value of
+    status_code.
+
+    The returned elem is owned by the caller. */
+grpc_mdelem grpc_get_reffed_status_elem_slowpath(int status_code);
+inline grpc_mdelem grpc_get_reffed_status_elem(int status_code) {
+  switch (status_code) {
+    case 0:
+      return GRPC_MDELEM_GRPC_STATUS_0;
+    case 1:
+      return GRPC_MDELEM_GRPC_STATUS_1;
+    case 2:
+      return GRPC_MDELEM_GRPC_STATUS_2;
+  }
+  return grpc_get_reffed_status_elem_slowpath(status_code);
+}
+
 #endif /* GRPC_CORE_LIB_TRANSPORT_STATUS_METADATA_H */

data/src/core/lib/transport/transport.cc

@@ -50,10 +50,11 @@ void grpc_stream_destroy(grpc_stream_refcount* refcount) {
        cope with.
        Throw this over to the executor (on a core-owned thread) and process it
        there. */
-    refcount->destroy.scheduler =
-        grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT);
+    grpc_core::Executor::Run(&refcount->destroy, GRPC_ERROR_NONE);
+  } else {
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, &refcount->destroy,
+                            GRPC_ERROR_NONE);
   }
-  GRPC_CLOSURE_SCHED(&refcount->destroy, GRPC_ERROR_NONE);
 }
 
 void slice_stream_destroy(void* arg) {
@@ -218,12 +219,13 @@ struct made_transport_op {
 
 static void destroy_made_transport_op(void* arg, grpc_error* error) {
   made_transport_op* op = static_cast<made_transport_op*>(arg);
-  GRPC_CLOSURE_SCHED(op->inner_on_complete, GRPC_ERROR_REF(error));
-  grpc_core::Delete<made_transport_op>(op);
+  grpc_core::ExecCtx::Run(DEBUG_LOCATION, op->inner_on_complete,
+                          GRPC_ERROR_REF(error));
+  delete op;
 }
 
 grpc_transport_op* grpc_make_transport_op(grpc_closure* on_complete) {
-  made_transport_op* op = grpc_core::New<made_transport_op>();
+  made_transport_op* op = new made_transport_op();
   GRPC_CLOSURE_INIT(&op->outer_on_complete, destroy_made_transport_op, op,
                     grpc_schedule_on_exec_ctx);
   op->inner_on_complete = on_complete;
@@ -242,7 +244,7 @@ static void destroy_made_transport_stream_op(void* arg, grpc_error* error) {
   made_transport_stream_op* op = static_cast<made_transport_stream_op*>(arg);
   grpc_closure* c = op->inner_on_complete;
   gpr_free(op);
-  GRPC_CLOSURE_RUN(c, GRPC_ERROR_REF(error));
+  grpc_core::Closure::Run(DEBUG_LOCATION, c, GRPC_ERROR_REF(error));
 }
 
 grpc_transport_stream_op_batch* grpc_make_transport_stream_op(

data/src/core/plugin_registry/grpc_plugin_registry.cc

@@ -34,6 +34,8 @@ void grpc_resolver_fake_init(void);
 void grpc_resolver_fake_shutdown(void);
 void grpc_lb_policy_grpclb_init(void);
 void grpc_lb_policy_grpclb_shutdown(void);
+void grpc_lb_policy_cds_init(void);
+void grpc_lb_policy_cds_shutdown(void);
 void grpc_lb_policy_xds_init(void);
 void grpc_lb_policy_xds_shutdown(void);
 void grpc_lb_policy_pick_first_init(void);
@@ -74,6 +76,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_resolver_fake_shutdown);
   grpc_register_plugin(grpc_lb_policy_grpclb_init,
                        grpc_lb_policy_grpclb_shutdown);
+  grpc_register_plugin(grpc_lb_policy_cds_init,
+                       grpc_lb_policy_cds_shutdown);
   grpc_register_plugin(grpc_lb_policy_xds_init,
                        grpc_lb_policy_xds_shutdown);
   grpc_register_plugin(grpc_lb_policy_pick_first_init,

data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc

@@ -18,12 +18,15 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <list>
+
 #include "src/core/tsi/alts/handshaker/alts_handshaker_client.h"
 
 #include <grpc/byte_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
@@ -39,8 +42,18 @@ struct alts_handshaker_client {
   const alts_handshaker_client_vtable* vtable;
 };
 
+struct recv_message_result {
+  tsi_result status;
+  const unsigned char* bytes_to_send;
+  size_t bytes_to_send_size;
+  tsi_handshaker_result* result;
+};
+
 typedef struct alts_grpc_handshaker_client {
   alts_handshaker_client base;
+  /* One ref is held by the entity that created this handshaker_client, and
+   * another ref is held by the pending RECEIVE_STATUS_ON_CLIENT op. */
+  gpr_refcount refs;
   alts_tsi_handshaker* handshaker;
   grpc_call* call;
   /* A pointer to a function handling the interaction with handshaker service.
@@ -49,12 +62,8 @@ typedef struct alts_grpc_handshaker_client {
    * that validates the data to be sent to handshaker service in a testing use
    * case. */
   alts_grpc_caller grpc_caller;
-  /* A callback function provided by gRPC to handle the response returned from
-   * handshaker service. It also serves to bring the control safely back to
-   * application when dedicated CQ and thread are used. */
-  grpc_iomgr_cb_func grpc_cb;
   /* A gRPC closure to be scheduled when the response from handshaker service
-   * is received. It will be initialized with grpc_cb. */
+   * is received. It will be initialized with the injected grpc RPC callback. */
   grpc_closure on_handshaker_service_resp_recv;
   /* Buffers containing information to be sent (or received) to (or from) the
    * handshaker service. */
@@ -81,6 +90,18 @@ typedef struct alts_grpc_handshaker_client {
   /* a buffer containing data to be sent to the grpc client or server's peer. */
   unsigned char* buffer;
   size_t buffer_size;
+  /** callback for receiving handshake call status */
+  grpc_closure on_status_received;
+  /** gRPC status code of handshake call */
+  grpc_status_code handshake_status_code;
+  /** gRPC status details of handshake call */
+  grpc_slice handshake_status_details;
+  /* mu synchronizes all fields below including their internal fields. */
+  gpr_mu mu;
+  /* indicates if the handshaker call's RECV_STATUS_ON_CLIENT op is done. */
+  bool receive_status_finished;
+  /* if non-null, contains arguments to complete a TSI next callback. */
+  recv_message_result* pending_recv_message_result;
 } alts_grpc_handshaker_client;
 
 static void handshaker_client_send_buffer_destroy(
@@ -98,6 +119,75 @@ static bool is_handshake_finished_properly(grpc_gcp_HandshakerResp* resp) {
   return false;
 }
 
+static void alts_grpc_handshaker_client_unref(
+    alts_grpc_handshaker_client* client) {
+  if (gpr_unref(&client->refs)) {
+    if (client->base.vtable != nullptr &&
+        client->base.vtable->destruct != nullptr) {
+      client->base.vtable->destruct(&client->base);
+    }
+    grpc_byte_buffer_destroy(client->send_buffer);
+    grpc_byte_buffer_destroy(client->recv_buffer);
+    client->send_buffer = nullptr;
+    client->recv_buffer = nullptr;
+    grpc_metadata_array_destroy(&client->recv_initial_metadata);
+    grpc_slice_unref_internal(client->recv_bytes);
+    grpc_slice_unref_internal(client->target_name);
+    grpc_alts_credentials_options_destroy(client->options);
+    gpr_free(client->buffer);
+    grpc_slice_unref_internal(client->handshake_status_details);
+    gpr_mu_destroy(&client->mu);
+    gpr_free(client);
+  }
+}
+
+static void maybe_complete_tsi_next(
+    alts_grpc_handshaker_client* client, bool receive_status_finished,
+    recv_message_result* pending_recv_message_result) {
+  recv_message_result* r;
+  {
+    grpc_core::MutexLock lock(&client->mu);
+    client->receive_status_finished |= receive_status_finished;
+    if (pending_recv_message_result != nullptr) {
+      GPR_ASSERT(client->pending_recv_message_result == nullptr);
+      client->pending_recv_message_result = pending_recv_message_result;
+    }
+    if (client->pending_recv_message_result == nullptr) {
+      return;
+    }
+    const bool have_final_result =
+        client->pending_recv_message_result->result != nullptr ||
+        client->pending_recv_message_result->status != TSI_OK;
+    if (have_final_result && !client->receive_status_finished) {
+      // If we've received the final message from the handshake
+      // server, or we're about to invoke the TSI next callback
+      // with a status other than TSI_OK (which terminates the
+      // handshake), then first wait for the RECV_STATUS op to complete.
+      return;
+    }
+    r = client->pending_recv_message_result;
+    client->pending_recv_message_result = nullptr;
+  }
+  client->cb(r->status, client->user_data, r->bytes_to_send,
+             r->bytes_to_send_size, r->result);
+  gpr_free(r);
+}
+
+static void handle_response_done(alts_grpc_handshaker_client* client,
+                                 tsi_result status,
+                                 const unsigned char* bytes_to_send,
+                                 size_t bytes_to_send_size,
+                                 tsi_handshaker_result* result) {
+  recv_message_result* p =
+      static_cast<recv_message_result*>(gpr_zalloc(sizeof(*p)));
+  p->status = status;
+  p->bytes_to_send = bytes_to_send;
+  p->bytes_to_send_size = bytes_to_send_size;
+  p->result = result;
+  maybe_complete_tsi_next(client, false /* receive_status_finished */,
+                          p /* pending_recv_message_result */);
+}
+
 void alts_handshaker_client_handle_response(alts_handshaker_client* c,
                                             bool is_ok) {
   GPR_ASSERT(c != nullptr);
@@ -105,38 +195,35 @@ void alts_handshaker_client_handle_response(alts_handshaker_client* c,
       reinterpret_cast<alts_grpc_handshaker_client*>(c);
   grpc_byte_buffer* recv_buffer = client->recv_buffer;
   grpc_status_code status = client->status;
-  tsi_handshaker_on_next_done_cb cb = client->cb;
-  void* user_data = client->user_data;
   alts_tsi_handshaker* handshaker = client->handshaker;
-
   /* Invalid input check. */
-  if (cb == nullptr) {
+  if (client->cb == nullptr) {
     gpr_log(GPR_ERROR,
-            "cb is nullptr in alts_tsi_handshaker_handle_response()");
+            "client->cb is nullptr in alts_tsi_handshaker_handle_response()");
     return;
   }
   if (handshaker == nullptr) {
     gpr_log(GPR_ERROR,
             "handshaker is nullptr in alts_tsi_handshaker_handle_response()");
-    cb(TSI_INTERNAL_ERROR, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_INTERNAL_ERROR, nullptr, 0, nullptr);
     return;
   }
   /* TSI handshake has been shutdown. */
   if (alts_tsi_handshaker_has_shutdown(handshaker)) {
     gpr_log(GPR_ERROR, "TSI handshake shutdown");
-    cb(TSI_HANDSHAKE_SHUTDOWN, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_HANDSHAKE_SHUTDOWN, nullptr, 0, nullptr);
     return;
   }
   /* Failed grpc call check. */
   if (!is_ok || status != GRPC_STATUS_OK) {
     gpr_log(GPR_ERROR, "grpc call made to handshaker service failed");
-    cb(TSI_INTERNAL_ERROR, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_INTERNAL_ERROR, nullptr, 0, nullptr);
     return;
   }
   if (recv_buffer == nullptr) {
     gpr_log(GPR_ERROR,
             "recv_buffer is nullptr in alts_tsi_handshaker_handle_response()");
-    cb(TSI_INTERNAL_ERROR, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_INTERNAL_ERROR, nullptr, 0, nullptr);
     return;
   }
   upb::Arena arena;
@@ -147,14 +234,14 @@ void alts_handshaker_client_handle_response(alts_handshaker_client* c,
   /* Invalid handshaker response check. */
   if (resp == nullptr) {
     gpr_log(GPR_ERROR, "alts_tsi_utils_deserialize_response() failed");
-    cb(TSI_DATA_CORRUPTED, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_DATA_CORRUPTED, nullptr, 0, nullptr);
     return;
   }
   const grpc_gcp_HandshakerStatus* resp_status =
       grpc_gcp_HandshakerResp_status(resp);
   if (resp_status == nullptr) {
     gpr_log(GPR_ERROR, "No status in HandshakerResp");
-    cb(TSI_DATA_CORRUPTED, user_data, nullptr, 0, nullptr);
+    handle_response_done(client, TSI_DATA_CORRUPTED, nullptr, 0, nullptr);
     return;
   }
   upb_strview out_frames = grpc_gcp_HandshakerResp_out_frames(resp);
@@ -188,22 +275,38 @@ void alts_handshaker_client_handle_response(alts_handshaker_client* c,
       gpr_free(error_details);
     }
   }
-  cb(alts_tsi_utils_convert_to_tsi_result(code), user_data, bytes_to_send,
-     bytes_to_send_size, result);
+  // TODO(apolcyn): consider short ciruiting handle_response_done and
+  // invoking the TSI callback directly if we aren't done yet, if
+  // handle_response_done's allocation per message received causes
+  // a performance issue.
+  handle_response_done(client, alts_tsi_utils_convert_to_tsi_result(code),
+                       bytes_to_send, bytes_to_send_size, result);
 }
 
-/**
- * Populate grpc operation data with the fields of ALTS handshaker client and
- * make a grpc call.
- */
-static tsi_result make_grpc_call(alts_handshaker_client* c, bool is_start) {
-  GPR_ASSERT(c != nullptr);
-  alts_grpc_handshaker_client* client =
-      reinterpret_cast<alts_grpc_handshaker_client*>(c);
+static tsi_result continue_make_grpc_call(alts_grpc_handshaker_client* client,
+                                          bool is_start) {
+  GPR_ASSERT(client != nullptr);
   grpc_op ops[kHandshakerClientOpNum];
   memset(ops, 0, sizeof(ops));
   grpc_op* op = ops;
   if (is_start) {
+    op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+    op->data.recv_status_on_client.trailing_metadata = nullptr;
+    op->data.recv_status_on_client.status = &client->handshake_status_code;
+    op->data.recv_status_on_client.status_details =
+        &client->handshake_status_details;
+    op->flags = 0;
+    op->reserved = nullptr;
+    op++;
+    GPR_ASSERT(op - ops <= kHandshakerClientOpNum);
+    gpr_ref(&client->refs);
+    grpc_call_error call_error =
+        client->grpc_caller(client->call, ops, static_cast<size_t>(op - ops),
+                            &client->on_status_received);
+    // TODO(apolcyn): return the error here instead, as done for other ops?
+    GPR_ASSERT(call_error == GRPC_CALL_OK);
+    memset(ops, 0, sizeof(ops));
+    op = ops;
     op->op = GRPC_OP_SEND_INITIAL_METADATA;
     op->data.send_initial_metadata.count = 0;
     op++;
@@ -232,6 +335,125 @@ static tsi_result make_grpc_call(alts_handshaker_client* c, bool is_start) {
   return TSI_OK;
 }
 
+// TODO(apolcyn): remove this global queue when we can safely rely
+// on a MAX_CONCURRENT_STREAMS setting in the ALTS handshake server to
+// limit the number of concurrent handshakes.
+namespace {
+
+class HandshakeQueue {
+ public:
+  explicit HandshakeQueue(size_t max_outstanding_handshakes)
+      : max_outstanding_handshakes_(max_outstanding_handshakes) {}
+
+  void RequestHandshake(alts_grpc_handshaker_client* client) {
+    {
+      grpc_core::MutexLock lock(&mu_);
+      if (outstanding_handshakes_ == max_outstanding_handshakes_) {
+        // Max number already running, add to queue.
+        queued_handshakes_.push_back(client);
+        return;
+      }
+      // Start the handshake immediately.
+      ++outstanding_handshakes_;
+    }
+    continue_make_grpc_call(client, true /* is_start */);
+  }
+
+  void HandshakeDone() {
+    alts_grpc_handshaker_client* client = nullptr;
+    {
+      grpc_core::MutexLock lock(&mu_);
+      if (queued_handshakes_.empty()) {
+        // Nothing more in queue.  Decrement count and return immediately.
+        --outstanding_handshakes_;
+        return;
+      }
+      // Remove next entry from queue and start the handshake.
+      client = queued_handshakes_.front();
+      queued_handshakes_.pop_front();
+    }
+    continue_make_grpc_call(client, true /* is_start */);
+  }
+
+ private:
+  grpc_core::Mutex mu_;
+  std::list<alts_grpc_handshaker_client*> queued_handshakes_;
+  size_t outstanding_handshakes_ = 0;
+  const size_t max_outstanding_handshakes_;
+};
+
+gpr_once g_queued_handshakes_init = GPR_ONCE_INIT;
+/* Using separate queues for client and server handshakes is a
+ * hack that's mainly intended to satisfy the alts_concurrent_connectivity_test,
+ * which runs many concurrent handshakes where both endpoints
+ * are in the same process; this situation is problematic with a
+ * single queue because we have a high chance of using up all outstanding
+ * slots in the queue, such that there aren't any
+ * mutual client/server handshakes outstanding at the same time and
+ * able to make progress. */
+HandshakeQueue* g_client_handshake_queue;
+HandshakeQueue* g_server_handshake_queue;
+
+void DoHandshakeQueuesInit(void) {
+  const size_t per_queue_max_outstanding_handshakes = 40;
+  g_client_handshake_queue =
+      new HandshakeQueue(per_queue_max_outstanding_handshakes);
+  g_server_handshake_queue =
+      new HandshakeQueue(per_queue_max_outstanding_handshakes);
+}
+
+void RequestHandshake(alts_grpc_handshaker_client* client, bool is_client) {
+  gpr_once_init(&g_queued_handshakes_init, DoHandshakeQueuesInit);
+  HandshakeQueue* queue =
+      is_client ? g_client_handshake_queue : g_server_handshake_queue;
+  queue->RequestHandshake(client);
+}
+
+void HandshakeDone(bool is_client) {
+  HandshakeQueue* queue =
+      is_client ? g_client_handshake_queue : g_server_handshake_queue;
+  queue->HandshakeDone();
+}
+
+};  // namespace
+
+/**
+ * Populate grpc operation data with the fields of ALTS handshaker client and
+ * make a grpc call.
+ */
+static tsi_result make_grpc_call(alts_handshaker_client* c, bool is_start) {
+  GPR_ASSERT(c != nullptr);
+  alts_grpc_handshaker_client* client =
+      reinterpret_cast<alts_grpc_handshaker_client*>(c);
+  if (is_start) {
+    RequestHandshake(client, client->is_client);
+    return TSI_OK;
+  } else {
+    return continue_make_grpc_call(client, is_start);
+  }
+}
+
+static void on_status_received(void* arg, grpc_error* error) {
+  alts_grpc_handshaker_client* client =
+      static_cast<alts_grpc_handshaker_client*>(arg);
+  if (client->handshake_status_code != GRPC_STATUS_OK) {
+    // TODO(apolcyn): consider overriding the handshake result's
+    // status from the final ALTS message with the status here.
+    char* status_details =
+        grpc_slice_to_c_string(client->handshake_status_details);
+    gpr_log(GPR_INFO,
+            "alts_grpc_handshaker_client:%p on_status_received "
+            "status:%d details:|%s| error:|%s|",
+            client, client->handshake_status_code, status_details,
+            grpc_error_string(error));
+    gpr_free(status_details);
+  }
+  maybe_complete_tsi_next(client, true /* receive_status_finished */,
+                          nullptr /* pending_recv_message_result */);
+  HandshakeDone(client->is_client);
+  alts_grpc_handshaker_client_unref(client);
+}
+
 /* Serializes a grpc_gcp_HandshakerReq message into a buffer and returns newly
  * grpc_byte_buffer holding it. */
 static grpc_byte_buffer* get_serialized_handshaker_req(
@@ -415,6 +637,11 @@ static void handshaker_client_shutdown(alts_handshaker_client* c) {
   }
 }
 
+static void handshaker_call_unref(void* arg, grpc_error* error) {
+  grpc_call* call = static_cast<grpc_call*>(arg);
+  grpc_call_unref(call);
+}
+
 static void handshaker_client_destruct(alts_handshaker_client* c) {
   if (c == nullptr) {
     return;
@@ -422,7 +649,17 @@ static void handshaker_client_destruct(alts_handshaker_client* c) {
   alts_grpc_handshaker_client* client =
       reinterpret_cast<alts_grpc_handshaker_client*>(c);
   if (client->call != nullptr) {
-    grpc_call_unref(client->call);
+    // Throw this grpc_call_unref over to the ExecCtx so that
+    // we invoke it at the bottom of the call stack and
+    // prevent lock inversion problems due to nested ExecCtx flushing.
+    // TODO(apolcyn): we could remove this indirection and call
+    // grpc_call_unref inline if there was an internal variant of
+    // grpc_call_unref that didn't need to flush an ExecCtx.
+    grpc_core::ExecCtx::Run(
+        DEBUG_LOCATION,
+        GRPC_CLOSURE_CREATE(handshaker_call_unref, client->call,
+                            grpc_schedule_on_exec_ctx),
+        GRPC_ERROR_NONE);
   }
 }
 
@@ -444,6 +681,8 @@ alts_handshaker_client* alts_grpc_handshaker_client_create(
   }
   alts_grpc_handshaker_client* client =
       static_cast<alts_grpc_handshaker_client*>(gpr_zalloc(sizeof(*client)));
+  gpr_mu_init(&client->mu);
+  gpr_ref_init(&client->refs, 1);
   client->grpc_caller = grpc_call_start_batch_and_execute;
   client->handshaker = handshaker;
   client->cb = cb;
@@ -454,7 +693,6 @@ alts_handshaker_client* alts_grpc_handshaker_client_create(
   client->target_name = grpc_slice_copy(target_name);
   client->recv_bytes = grpc_empty_slice();
   grpc_metadata_array_init(&client->recv_initial_metadata);
-  client->grpc_cb = grpc_cb;
   client->is_client = is_client;
   client->buffer_size = TSI_ALTS_INITIAL_BUFFER_SIZE;
   client->buffer = static_cast<unsigned char*>(gpr_zalloc(client->buffer_size));
@@ -469,8 +707,10 @@ alts_handshaker_client* alts_grpc_handshaker_client_create(
                 GRPC_MILLIS_INF_FUTURE, nullptr);
   client->base.vtable =
       vtable_for_testing == nullptr ? &vtable : vtable_for_testing;
-  GRPC_CLOSURE_INIT(&client->on_handshaker_service_resp_recv, client->grpc_cb,
-                    client, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&client->on_handshaker_service_resp_recv, grpc_cb, client,
+                    grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&client->on_status_received, on_status_received, client,
+                    grpc_schedule_on_exec_ctx);
   grpc_slice_unref_internal(slice);
   return &client->base;
 }
@@ -580,6 +820,27 @@ grpc_closure* alts_handshaker_client_get_closure_for_testing(
   return &client->on_handshaker_service_resp_recv;
 }
 
+void alts_handshaker_client_ref_for_testing(alts_handshaker_client* c) {
+  alts_grpc_handshaker_client* client =
+      reinterpret_cast<alts_grpc_handshaker_client*>(c);
+  gpr_ref(&client->refs);
+}
+
+void alts_handshaker_client_on_status_received_for_testing(
+    alts_handshaker_client* c, grpc_status_code status, grpc_error* error) {
+  // We first make sure that the handshake queue has been initialized
+  // here because there are tests that use this API that mock out
+  // other parts of the alts_handshaker_client in such a way that the
+  // code path that would normally ensure that the handshake queue
+  // has been initialized isn't taken.
+  gpr_once_init(&g_queued_handshakes_init, DoHandshakeQueuesInit);
+  alts_grpc_handshaker_client* client =
+      reinterpret_cast<alts_grpc_handshaker_client*>(c);
+  client->handshake_status_code = status;
+  client->handshake_status_details = grpc_empty_slice();
+  grpc_core::Closure::Run(DEBUG_LOCATION, &client->on_status_received, error);
+}
+
 }  // namespace internal
 }  // namespace grpc_core
 
@@ -624,20 +885,8 @@ void alts_handshaker_client_shutdown(alts_handshaker_client* client) {
 
 void alts_handshaker_client_destroy(alts_handshaker_client* c) {
   if (c != nullptr) {
-    if (c->vtable != nullptr && c->vtable->destruct != nullptr) {
-      c->vtable->destruct(c);
-    }
     alts_grpc_handshaker_client* client =
         reinterpret_cast<alts_grpc_handshaker_client*>(c);
-    grpc_byte_buffer_destroy(client->send_buffer);
-    grpc_byte_buffer_destroy(client->recv_buffer);
-    client->send_buffer = nullptr;
-    client->recv_buffer = nullptr;
-    grpc_metadata_array_destroy(&client->recv_initial_metadata);
-    grpc_slice_unref_internal(client->recv_bytes);
-    grpc_slice_unref_internal(client->target_name);
-    grpc_alts_credentials_options_destroy(client->options);
-    gpr_free(client->buffer);
-    gpr_free(client);
+    alts_grpc_handshaker_client_unref(client);
   }
 }