grpc 1.30.1 → 1.32.0.pre1

data/src/core/ext/filters/message_size/message_size_filter.h

@@ -40,6 +40,9 @@ class MessageSizeParsedConfig : public ServiceConfigParser::ParsedConfig {
 
   const message_size_limits& limits() const { return limits_; }
 
+  static const MessageSizeParsedConfig* GetFromCallContext(
+      const grpc_call_context_element* context);
+
  private:
   message_size_limits limits_;
 };
@@ -54,6 +57,9 @@ class MessageSizeParser : public ServiceConfigParser::Parser {
   static size_t ParserIndex();
 };
 
+int GetMaxRecvSizeFromChannelArgs(const grpc_channel_args* args);
+int GetMaxSendSizeFromChannelArgs(const grpc_channel_args* args);
+
 }  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_MESSAGE_SIZE_MESSAGE_SIZE_FILTER_H */

data/src/core/ext/transport/chttp2/client/chttp2_connector.cc

@@ -129,6 +129,15 @@ void Chttp2Connector::StartHandshakeLocked() {
   endpoint_ = nullptr;  // Endpoint handed off to handshake manager.
 }
 
+namespace {
+void NullThenSchedClosure(const DebugLocation& location, grpc_closure** closure,
+                          grpc_error* error) {
+  grpc_closure* c = *closure;
+  *closure = nullptr;
+  ExecCtx::Run(location, c, error);
+}
+}  // namespace
+
 void Chttp2Connector::OnHandshakeDone(void* arg, grpc_error* error) {
   auto* args = static_cast<HandshakerArgs*>(arg);
   Chttp2Connector* self = static_cast<Chttp2Connector*>(args->user_data);
@@ -154,53 +163,100 @@ void Chttp2Connector::OnHandshakeDone(void* arg, grpc_error* error) {
         error = GRPC_ERROR_REF(error);
       }
       self->result_->Reset();
+      NullThenSchedClosure(DEBUG_LOCATION, &self->notify_, error);
     } else if (args->endpoint != nullptr) {
-      grpc_endpoint_delete_from_pollset_set(args->endpoint,
-                                            self->args_.interested_parties);
       self->result_->transport =
           grpc_create_chttp2_transport(args->args, args->endpoint, true);
       self->result_->socket_node =
           grpc_chttp2_transport_get_socket_node(self->result_->transport);
+      self->result_->channel_args = args->args;
       GPR_ASSERT(self->result_->transport != nullptr);
-      // TODO(roth): We ideally want to wait until we receive HTTP/2
-      // settings from the server before we consider the connection
-      // established.  If that doesn't happen before the connection
-      // timeout expires, then we should consider the connection attempt a
-      // failure and feed that information back into the backoff code.
-      // We could pass a notify_on_receive_settings callback to
-      // grpc_chttp2_transport_start_reading() to let us know when
-      // settings are received, but we would need to figure out how to use
-      // that information here.
-      //
-      // Unfortunately, we don't currently have a way to split apart the two
-      // effects of scheduling c->notify: we start sending RPCs immediately
-      // (which we want to do) and we consider the connection attempt successful
-      // (which we don't want to do until we get the notify_on_receive_settings
-      // callback from the transport).  If we could split those things
-      // apart, then we could start sending RPCs but then wait for our
-      // timeout before deciding if the connection attempt is successful.
-      // If the attempt is not successful, then we would tear down the
-      // transport and feed the failure back into the backoff code.
-      //
-      // In addition, even if we did that, we would probably not want to do
-      // so until after transparent retries is implemented.  Otherwise, any
-      // RPC that we attempt to send on the connection before the timeout
-      // would fail instead of being retried on a subsequent attempt.
+      self->endpoint_ = args->endpoint;
+      self->Ref().release();  // Ref held by OnReceiveSettings()
+      GRPC_CLOSURE_INIT(&self->on_receive_settings_, OnReceiveSettings, self,
+                        grpc_schedule_on_exec_ctx);
+      self->Ref().release();  // Ref held by OnTimeout()
       grpc_chttp2_transport_start_reading(self->result_->transport,
-                                          args->read_buffer, nullptr);
-      self->result_->channel_args = args->args;
+                                          args->read_buffer,
+                                          &self->on_receive_settings_);
+      GRPC_CLOSURE_INIT(&self->on_timeout_, OnTimeout, self,
+                        grpc_schedule_on_exec_ctx);
+      grpc_timer_init(&self->timer_, self->args_.deadline, &self->on_timeout_);
     } else {
       // If the handshaking succeeded but there is no endpoint, then the
       // handshaker may have handed off the connection to some external
       // code. Just verify that exit_early flag is set.
       GPR_DEBUG_ASSERT(args->exit_early);
+      NullThenSchedClosure(DEBUG_LOCATION, &self->notify_, error);
     }
-    grpc_closure* notify = self->notify_;
-    self->notify_ = nullptr;
-    ExecCtx::Run(DEBUG_LOCATION, notify, error);
     self->handshake_mgr_.reset();
   }
   self->Unref();
 }
 
+void Chttp2Connector::OnReceiveSettings(void* arg, grpc_error* error) {
+  Chttp2Connector* self = static_cast<Chttp2Connector*>(arg);
+  {
+    MutexLock lock(&self->mu_);
+    if (!self->notify_error_.has_value()) {
+      grpc_endpoint_delete_from_pollset_set(self->endpoint_,
+                                            self->args_.interested_parties);
+      if (error != GRPC_ERROR_NONE) {
+        // Transport got an error while waiting on SETTINGS frame.
+        // TODO(yashykt): The following two lines should be moved to
+        // SubchannelConnector::Result::Reset()
+        grpc_transport_destroy(self->result_->transport);
+        grpc_channel_args_destroy(self->result_->channel_args);
+        self->result_->Reset();
+      }
+      self->MaybeNotify(GRPC_ERROR_REF(error));
+      grpc_timer_cancel(&self->timer_);
+    } else {
+      // OnTimeout() was already invoked. Call Notify() again so that notify_
+      // can be invoked.
+      self->MaybeNotify(GRPC_ERROR_NONE);
+    }
+  }
+  self->Unref();
+}
+
+void Chttp2Connector::OnTimeout(void* arg, grpc_error* error) {
+  Chttp2Connector* self = static_cast<Chttp2Connector*>(arg);
+  {
+    MutexLock lock(&self->mu_);
+    if (!self->notify_error_.has_value()) {
+      // The transport did not receive the settings frame in time. Destroy the
+      // transport.
+      grpc_endpoint_delete_from_pollset_set(self->endpoint_,
+                                            self->args_.interested_parties);
+      // TODO(yashykt): The following two lines should be moved to
+      // SubchannelConnector::Result::Reset()
+      grpc_transport_destroy(self->result_->transport);
+      grpc_channel_args_destroy(self->result_->channel_args);
+      self->result_->Reset();
+      self->MaybeNotify(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "connection attempt timed out before receiving SETTINGS frame"));
+    } else {
+      // OnReceiveSettings() was already invoked. Call Notify() again so that
+      // notify_ can be invoked.
+      self->MaybeNotify(GRPC_ERROR_NONE);
+    }
+  }
+  self->Unref();
+}
+
+void Chttp2Connector::MaybeNotify(grpc_error* error) {
+  if (notify_error_.has_value()) {
+    GRPC_ERROR_UNREF(error);
+    NullThenSchedClosure(DEBUG_LOCATION, &notify_, notify_error_.value());
+    // Clear state for a new Connect().
+    // Clear out the endpoint_, since it is the responsibility of
+    // the transport to shut it down.
+    endpoint_ = nullptr;
+    notify_error_.reset();
+  } else {
+    notify_error_ = error;
+  }
+}
+
 }  // namespace grpc_core

data/src/core/ext/transport/chttp2/client/chttp2_connector.h

@@ -39,6 +39,19 @@ class Chttp2Connector : public SubchannelConnector {
   static void Connected(void* arg, grpc_error* error);
   void StartHandshakeLocked();
   static void OnHandshakeDone(void* arg, grpc_error* error);
+  static void OnReceiveSettings(void* arg, grpc_error* error);
+  static void OnTimeout(void* arg, grpc_error* error);
+
+  // We cannot invoke notify_ until both OnTimeout() and OnReceiveSettings()
+  // have been called since that is an indicator to the upper layer that we are
+  // done with the connection attempt. So, the notification process is broken
+  // into two steps. 1) Either OnTimeout() or OnReceiveSettings() gets invoked
+  // first. Whichever gets invoked, calls MaybeNotify() to set the result and
+  // triggers the other callback to be invoked. 2) When the other callback is
+  // invoked, we call MaybeNotify() again to actually invoke the notify_
+  // callback. Note that this only happens if the handshake is done and the
+  // connector is waiting on the SETTINGS frame.
+  void MaybeNotify(grpc_error* error);
 
   Mutex mu_;
   Args args_;
@@ -47,9 +60,13 @@ class Chttp2Connector : public SubchannelConnector {
   bool shutdown_ = false;
   bool connecting_ = false;
   // Holds the endpoint when first created before being handed off to
-  // the handshake manager.
+  // the handshake manager, and then again after handshake is done.
   grpc_endpoint* endpoint_ = nullptr;
   grpc_closure connected_;
+  grpc_closure on_receive_settings_;
+  grpc_timer timer_;
+  grpc_closure on_timeout_;
+  absl::optional<grpc_error*> notify_error_;
   RefCountedPtr<HandshakeManager> handshake_mgr_;
 };
 

data/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc

@@ -33,11 +33,10 @@
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/security_connector/security_connector.h"
-#include "src/core/lib/security/transport/target_authority_table.h"
-#include "src/core/lib/slice/slice_hash_table.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/authority_override.h"
 #include "src/core/lib/uri/uri_parser.h"
 
 namespace grpc_core {
@@ -75,39 +74,16 @@ class Chttp2SecureClientChannelFactory : public ClientChannelFactory {
               "channel args.");
       return nullptr;
     }
-    // To which address are we connecting? By default, use the server URI.
-    const grpc_arg* server_uri_arg =
-        grpc_channel_args_find(args, GRPC_ARG_SERVER_URI);
-    const char* server_uri_str = grpc_channel_arg_get_string(server_uri_arg);
-    GPR_ASSERT(server_uri_str != nullptr);
-    grpc_uri* server_uri =
-        grpc_uri_parse(server_uri_str, true /* suppress errors */);
-    GPR_ASSERT(server_uri != nullptr);
-    const TargetAuthorityTable* target_authority_table =
-        FindTargetAuthorityTableInArgs(args);
-    grpc_core::UniquePtr<char> authority;
-    if (target_authority_table != nullptr) {
-      // Find the authority for the target.
-      const char* target_uri_str =
-          Subchannel::GetUriFromSubchannelAddressArg(args);
-      grpc_uri* target_uri =
-          grpc_uri_parse(target_uri_str, false /* suppress errors */);
-      GPR_ASSERT(target_uri != nullptr);
-      if (target_uri->path[0] != '\0') {  // "path" may be empty
-        const grpc_slice key = grpc_slice_from_static_string(
-            target_uri->path[0] == '/' ? target_uri->path + 1
-                                       : target_uri->path);
-        const grpc_core::UniquePtr<char>* value =
-            target_authority_table->Get(key);
-        if (value != nullptr) authority.reset(gpr_strdup(value->get()));
-        grpc_slice_unref_internal(key);
-      }
-      grpc_uri_destroy(target_uri);
-    }
-    // If the authority hasn't already been set (either because no target
-    // authority table was present or because the target was not present
-    // in the table), fall back to using the original server URI.
+    // Find the authority to use in the security connector.
+    // First, check the authority override channel arg.
+    // Otherwise, get it from the server name used to construct the
+    // channel.
+    grpc_core::UniquePtr<char> authority(
+        gpr_strdup(FindAuthorityOverrideInArgs(args)));
     if (authority == nullptr) {
+      const char* server_uri_str =
+          grpc_channel_args_find_string(args, GRPC_ARG_SERVER_URI);
+      GPR_ASSERT(server_uri_str != nullptr);
       authority = ResolverRegistry::GetDefaultAuthority(server_uri_str);
     }
     grpc_arg args_to_add[2];
@@ -120,7 +96,6 @@ class Chttp2SecureClientChannelFactory : public ClientChannelFactory {
     }
     grpc_channel_args* args_with_authority =
         grpc_channel_args_copy_and_add(args, args_to_add, num_args_to_add);
-    grpc_uri_destroy(server_uri);
     // Create the security connector using the credentials and target name.
     grpc_channel_args* new_args_from_connector = nullptr;
     RefCountedPtr<grpc_channel_security_connector>

data/src/core/ext/transport/chttp2/server/chttp2_server.cc

@@ -23,22 +23,25 @@
 #include <inttypes.h>
 #include <limits.h>
 #include <string.h>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
 
 #include <grpc/grpc.h>
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 
-#include "absl/strings/str_format.h"
-
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/channel/handshaker_registry.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/resource_quota.h"
@@ -47,405 +50,432 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-struct server_state {
-  grpc_server* server;
-  grpc_tcp_server* tcp_server;
-  grpc_channel_args* args;
-  gpr_mu mu;
-  bool shutdown;
-  grpc_closure tcp_server_shutdown_complete;
-  grpc_closure* server_destroy_listener_done;
-  grpc_core::HandshakeManager* pending_handshake_mgrs;
-  grpc_core::RefCountedPtr<grpc_core::channelz::ListenSocketNode>
-      channelz_listen_socket;
-};
+namespace grpc_core {
+namespace {
+
+class Chttp2ServerListener : public Server::ListenerInterface {
+ public:
+  static grpc_error* Create(Server* server, const char* addr,
+                            grpc_channel_args* args, int* port_num);
+
+  static grpc_error* CreateWithAcceptor(Server* server, const char* name,
+                                        grpc_channel_args* args);
+
+  // Do not instantiate directly.  Use one of the factory methods above.
+  Chttp2ServerListener(Server* server, grpc_channel_args* args);
+  ~Chttp2ServerListener();
+
+  void Start(Server* server,
+             const std::vector<grpc_pollset*>* pollsets) override;
+
+  channelz::ListenSocketNode* channelz_listen_socket_node() const override {
+    return channelz_listen_socket_.get();
+  }
+
+  void SetOnDestroyDone(grpc_closure* on_destroy_done) override;
 
-struct server_connection_state {
-  gpr_refcount refs;
-  server_state* svr_state;
-  grpc_pollset* accepting_pollset;
-  grpc_tcp_server_acceptor* acceptor;
-  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
-  // State for enforcing handshake timeout on receiving HTTP/2 settings.
-  grpc_chttp2_transport* transport;
-  grpc_millis deadline;
-  grpc_timer timer;
-  grpc_closure on_timeout;
-  grpc_closure on_receive_settings;
-  grpc_pollset_set* interested_parties;
+  void Orphan() override;
+
+ private:
+  class ConnectionState : public RefCounted<ConnectionState> {
+   public:
+    ConnectionState(Chttp2ServerListener* listener,
+                    grpc_pollset* accepting_pollset,
+                    grpc_tcp_server_acceptor* acceptor,
+                    RefCountedPtr<HandshakeManager> handshake_mgr,
+                    grpc_channel_args* args, grpc_endpoint* endpoint);
+
+    ~ConnectionState();
+
+   private:
+    static void OnTimeout(void* arg, grpc_error* error);
+    static void OnReceiveSettings(void* arg, grpc_error* error);
+    static void OnHandshakeDone(void* arg, grpc_error* error);
+
+    Chttp2ServerListener* const listener_;
+    grpc_pollset* const accepting_pollset_;
+    grpc_tcp_server_acceptor* const acceptor_;
+    RefCountedPtr<HandshakeManager> handshake_mgr_;
+    // State for enforcing handshake timeout on receiving HTTP/2 settings.
+    grpc_chttp2_transport* transport_ = nullptr;
+    grpc_millis deadline_;
+    grpc_timer timer_;
+    grpc_closure on_timeout_;
+    grpc_closure on_receive_settings_;
+    grpc_pollset_set* const interested_parties_;
+  };
+
+  static void OnAccept(void* arg, grpc_endpoint* tcp,
+                       grpc_pollset* accepting_pollset,
+                       grpc_tcp_server_acceptor* acceptor);
+
+  RefCountedPtr<HandshakeManager> CreateHandshakeManager();
+
+  static void TcpServerShutdownComplete(void* arg, grpc_error* error);
+
+  static void DestroyListener(Server* /*server*/, void* arg,
+                              grpc_closure* destroy_done);
+
+  Server* const server_;
+  grpc_channel_args* const args_;
+  grpc_tcp_server* tcp_server_;
+  Mutex mu_;
+  bool shutdown_ = true;
+  grpc_closure tcp_server_shutdown_complete_;
+  grpc_closure* on_destroy_done_ = nullptr;
+  HandshakeManager* pending_handshake_mgrs_ = nullptr;
+  RefCountedPtr<channelz::ListenSocketNode> channelz_listen_socket_;
 };
 
-static void server_connection_state_unref(
-    server_connection_state* connection_state) {
-  if (gpr_unref(&connection_state->refs)) {
-    if (connection_state->transport != nullptr) {
-      GRPC_CHTTP2_UNREF_TRANSPORT(connection_state->transport,
-                                  "receive settings timeout");
-    }
-    grpc_pollset_set_del_pollset(connection_state->interested_parties,
-                                 connection_state->accepting_pollset);
-    grpc_pollset_set_destroy(connection_state->interested_parties);
-    gpr_free(connection_state);
+//
+// Chttp2ServerListener::ConnectionState
+//
+
+grpc_millis GetConnectionDeadline(const grpc_channel_args* args) {
+  int timeout_ms =
+      grpc_channel_args_find_integer(args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS,
+                                     {120 * GPR_MS_PER_SEC, 1, INT_MAX});
+  return ExecCtx::Get()->Now() + timeout_ms;
+}
+
+Chttp2ServerListener::ConnectionState::ConnectionState(
+    Chttp2ServerListener* listener, grpc_pollset* accepting_pollset,
+    grpc_tcp_server_acceptor* acceptor,
+    RefCountedPtr<HandshakeManager> handshake_mgr, grpc_channel_args* args,
+    grpc_endpoint* endpoint)
+    : listener_(listener),
+      accepting_pollset_(accepting_pollset),
+      acceptor_(acceptor),
+      handshake_mgr_(std::move(handshake_mgr)),
+      deadline_(GetConnectionDeadline(args)),
+      interested_parties_(grpc_pollset_set_create()) {
+  grpc_pollset_set_add_pollset(interested_parties_, accepting_pollset_);
+  HandshakerRegistry::AddHandshakers(HANDSHAKER_SERVER, args,
+                                     interested_parties_, handshake_mgr_.get());
+  handshake_mgr_->DoHandshake(endpoint, args, deadline_, acceptor_,
+                              OnHandshakeDone, this);
+}
+
+Chttp2ServerListener::ConnectionState::~ConnectionState() {
+  if (transport_ != nullptr) {
+    GRPC_CHTTP2_UNREF_TRANSPORT(transport_, "receive settings timeout");
   }
+  grpc_pollset_set_del_pollset(interested_parties_, accepting_pollset_);
+  grpc_pollset_set_destroy(interested_parties_);
 }
 
-static void on_timeout(void* arg, grpc_error* error) {
-  server_connection_state* connection_state =
-      static_cast<server_connection_state*>(arg);
+void Chttp2ServerListener::ConnectionState::OnTimeout(void* arg,
+                                                      grpc_error* error) {
+  ConnectionState* self = static_cast<ConnectionState*>(arg);
   // Note that we may be called with GRPC_ERROR_NONE when the timer fires
   // or with an error indicating that the timer system is being shut down.
   if (error != GRPC_ERROR_CANCELLED) {
     grpc_transport_op* op = grpc_make_transport_op(nullptr);
     op->disconnect_with_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
         "Did not receive HTTP/2 settings before handshake timeout");
-    grpc_transport_perform_op(&connection_state->transport->base, op);
+    grpc_transport_perform_op(&self->transport_->base, op);
   }
-  server_connection_state_unref(connection_state);
+  self->Unref();
 }
 
-static void on_receive_settings(void* arg, grpc_error* error) {
-  server_connection_state* connection_state =
-      static_cast<server_connection_state*>(arg);
+void Chttp2ServerListener::ConnectionState::OnReceiveSettings(
+    void* arg, grpc_error* error) {
+  ConnectionState* self = static_cast<ConnectionState*>(arg);
   if (error == GRPC_ERROR_NONE) {
-    grpc_timer_cancel(&connection_state->timer);
+    grpc_timer_cancel(&self->timer_);
   }
-  server_connection_state_unref(connection_state);
+  self->Unref();
 }
 
-static void on_handshake_done(void* arg, grpc_error* error) {
-  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
-  server_connection_state* connection_state =
-      static_cast<server_connection_state*>(args->user_data);
-  gpr_mu_lock(&connection_state->svr_state->mu);
-  grpc_resource_user* resource_user = grpc_server_get_default_resource_user(
-      connection_state->svr_state->server);
-  if (error != GRPC_ERROR_NONE || connection_state->svr_state->shutdown) {
-    const char* error_str = grpc_error_string(error);
-    gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str);
-    grpc_resource_user* resource_user = grpc_server_get_default_resource_user(
-        connection_state->svr_state->server);
-    if (resource_user != nullptr) {
-      grpc_resource_user_free(resource_user, GRPC_RESOURCE_QUOTA_CHANNEL_SIZE);
-    }
-    if (error == GRPC_ERROR_NONE && args->endpoint != nullptr) {
-      // We were shut down after handshaking completed successfully, so
-      // destroy the endpoint here.
-      // TODO(ctiller): It is currently necessary to shutdown endpoints
-      // before destroying them, even if we know that there are no
-      // pending read/write callbacks.  This should be fixed, at which
-      // point this can be removed.
-      grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_NONE);
-      grpc_endpoint_destroy(args->endpoint);
-      grpc_channel_args_destroy(args->args);
-      grpc_slice_buffer_destroy_internal(args->read_buffer);
-      gpr_free(args->read_buffer);
-    }
-  } else {
-    // If the handshaking succeeded but there is no endpoint, then the
-    // handshaker may have handed off the connection to some external
-    // code, so we can just clean up here without creating a transport.
-    if (args->endpoint != nullptr) {
-      grpc_transport* transport = grpc_create_chttp2_transport(
-          args->args, args->endpoint, false, resource_user);
-      grpc_server_setup_transport(
-          connection_state->svr_state->server, transport,
-          connection_state->accepting_pollset, args->args,
-          grpc_chttp2_transport_get_socket_node(transport), resource_user);
-      // Use notify_on_receive_settings callback to enforce the
-      // handshake deadline.
-      connection_state->transport =
-          reinterpret_cast<grpc_chttp2_transport*>(transport);
-      gpr_ref(&connection_state->refs);
-      GRPC_CLOSURE_INIT(&connection_state->on_receive_settings,
-                        on_receive_settings, connection_state,
-                        grpc_schedule_on_exec_ctx);
-      grpc_chttp2_transport_start_reading(
-          transport, args->read_buffer, &connection_state->on_receive_settings);
-      grpc_channel_args_destroy(args->args);
-      gpr_ref(&connection_state->refs);
-      GRPC_CHTTP2_REF_TRANSPORT((grpc_chttp2_transport*)transport,
-                                "receive settings timeout");
-      GRPC_CLOSURE_INIT(&connection_state->on_timeout, on_timeout,
-                        connection_state, grpc_schedule_on_exec_ctx);
-      grpc_timer_init(&connection_state->timer, connection_state->deadline,
-                      &connection_state->on_timeout);
-    } else {
+void Chttp2ServerListener::ConnectionState::OnHandshakeDone(void* arg,
+                                                            grpc_error* error) {
+  auto* args = static_cast<HandshakerArgs*>(arg);
+  ConnectionState* self = static_cast<ConnectionState*>(args->user_data);
+  {
+    MutexLock lock(&self->listener_->mu_);
+    grpc_resource_user* resource_user =
+        self->listener_->server_->default_resource_user();
+    if (error != GRPC_ERROR_NONE || self->listener_->shutdown_) {
+      const char* error_str = grpc_error_string(error);
+      gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str);
       if (resource_user != nullptr) {
         grpc_resource_user_free(resource_user,
                                 GRPC_RESOURCE_QUOTA_CHANNEL_SIZE);
       }
+      if (error == GRPC_ERROR_NONE && args->endpoint != nullptr) {
+        // We were shut down after handshaking completed successfully, so
+        // destroy the endpoint here.
+        // TODO(ctiller): It is currently necessary to shutdown endpoints
+        // before destroying them, even if we know that there are no
+        // pending read/write callbacks.  This should be fixed, at which
+        // point this can be removed.
+        grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_NONE);
+        grpc_endpoint_destroy(args->endpoint);
+        grpc_channel_args_destroy(args->args);
+        grpc_slice_buffer_destroy_internal(args->read_buffer);
+        gpr_free(args->read_buffer);
+      }
+    } else {
+      // If the handshaking succeeded but there is no endpoint, then the
+      // handshaker may have handed off the connection to some external
+      // code, so we can just clean up here without creating a transport.
+      if (args->endpoint != nullptr) {
+        grpc_transport* transport = grpc_create_chttp2_transport(
+            args->args, args->endpoint, false, resource_user);
+        self->listener_->server_->SetupTransport(
+            transport, self->accepting_pollset_, args->args,
+            grpc_chttp2_transport_get_socket_node(transport), resource_user);
+        // Use notify_on_receive_settings callback to enforce the
+        // handshake deadline.
+        // Note: The reinterpret_cast<>s here are safe, because
+        // grpc_chttp2_transport is a C-style extension of
+        // grpc_transport, so this is morally equivalent of a
+        // static_cast<> to a derived class.
+        // TODO(roth): Change to static_cast<> when we C++-ify the
+        // transport API.
+        self->transport_ = reinterpret_cast<grpc_chttp2_transport*>(transport);
+        self->Ref().release();  // Held by OnReceiveSettings().
+        GRPC_CLOSURE_INIT(&self->on_receive_settings_, OnReceiveSettings, self,
+                          grpc_schedule_on_exec_ctx);
+        grpc_chttp2_transport_start_reading(transport, args->read_buffer,
+                                            &self->on_receive_settings_);
+        grpc_channel_args_destroy(args->args);
+        self->Ref().release();  // Held by OnTimeout().
+        GRPC_CHTTP2_REF_TRANSPORT(
+            reinterpret_cast<grpc_chttp2_transport*>(transport),
+            "receive settings timeout");
+        GRPC_CLOSURE_INIT(&self->on_timeout_, OnTimeout, self,
+                          grpc_schedule_on_exec_ctx);
+        grpc_timer_init(&self->timer_, self->deadline_, &self->on_timeout_);
+      } else {
+        if (resource_user != nullptr) {
+          grpc_resource_user_free(resource_user,
+                                  GRPC_RESOURCE_QUOTA_CHANNEL_SIZE);
+        }
+      }
     }
+    self->handshake_mgr_->RemoveFromPendingMgrList(
+        &self->listener_->pending_handshake_mgrs_);
   }
-  connection_state->handshake_mgr->RemoveFromPendingMgrList(
-      &connection_state->svr_state->pending_handshake_mgrs);
-  gpr_mu_unlock(&connection_state->svr_state->mu);
-  connection_state->handshake_mgr.reset();
-  gpr_free(connection_state->acceptor);
-  grpc_tcp_server_unref(connection_state->svr_state->tcp_server);
-  server_connection_state_unref(connection_state);
+  self->handshake_mgr_.reset();
+  gpr_free(self->acceptor_);
+  grpc_tcp_server_unref(self->listener_->tcp_server_);
+  self->Unref();
 }
 
-static void on_accept(void* arg, grpc_endpoint* tcp,
-                      grpc_pollset* accepting_pollset,
-                      grpc_tcp_server_acceptor* acceptor) {
-  server_state* state = static_cast<server_state*>(arg);
-  gpr_mu_lock(&state->mu);
-  if (state->shutdown) {
-    gpr_mu_unlock(&state->mu);
-    grpc_endpoint_shutdown(tcp, GRPC_ERROR_NONE);
-    grpc_endpoint_destroy(tcp);
-    gpr_free(acceptor);
-    return;
+//
+// Chttp2ServerListener
+//
+
+grpc_error* Chttp2ServerListener::Create(Server* server, const char* addr,
+                                         grpc_channel_args* args,
+                                         int* port_num) {
+  std::vector<grpc_error*> error_list;
+  grpc_resolved_addresses* resolved = nullptr;
+  Chttp2ServerListener* listener = nullptr;
+  // The bulk of this method is inside of a lambda to make cleanup
+  // easier without using goto.
+  grpc_error* error = [&]() {
+    *port_num = -1;
+    /* resolve address */
+    grpc_error* error = grpc_blocking_resolve_address(addr, "https", &resolved);
+    if (error != GRPC_ERROR_NONE) return error;
+    // Create Chttp2ServerListener.
+    listener = new Chttp2ServerListener(server, args);
+    error = grpc_tcp_server_create(&listener->tcp_server_shutdown_complete_,
+                                   args, &listener->tcp_server_);
+    if (error != GRPC_ERROR_NONE) return error;
+    for (size_t i = 0; i < resolved->naddrs; i++) {
+      int port_temp;
+      error = grpc_tcp_server_add_port(listener->tcp_server_,
+                                       &resolved->addrs[i], &port_temp);
+      if (error != GRPC_ERROR_NONE) {
+        error_list.push_back(error);
+      } else {
+        if (*port_num == -1) {
+          *port_num = port_temp;
+        } else {
+          GPR_ASSERT(*port_num == port_temp);
+        }
+      }
+    }
+    if (error_list.size() == resolved->naddrs) {
+      std::string msg =
+          absl::StrFormat("No address added out of total %" PRIuPTR " resolved",
+                          resolved->naddrs);
+      return GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(
+          msg.c_str(), error_list.data(), error_list.size());
+    } else if (!error_list.empty()) {
+      std::string msg = absl::StrFormat(
+          "Only %" PRIuPTR " addresses added out of total %" PRIuPTR
+          " resolved",
+          resolved->naddrs - error_list.size(), resolved->naddrs);
+      error = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(
+          msg.c_str(), error_list.data(), error_list.size());
+      gpr_log(GPR_INFO, "WARNING: %s", grpc_error_string(error));
+      GRPC_ERROR_UNREF(error);
+      /* we managed to bind some addresses: continue */
+    }
+    // Create channelz node.
+    if (grpc_channel_args_find_bool(args, GRPC_ARG_ENABLE_CHANNELZ,
+                                    GRPC_ENABLE_CHANNELZ_DEFAULT)) {
+      listener->channelz_listen_socket_ =
+          MakeRefCounted<channelz::ListenSocketNode>(
+              addr, absl::StrFormat("chttp2 listener %s", addr));
+    }
+    /* Register with the server only upon success */
+    server->AddListener(OrphanablePtr<Server::ListenerInterface>(listener));
+    return GRPC_ERROR_NONE;
+  }();
+  if (resolved != nullptr) {
+    grpc_resolved_addresses_destroy(resolved);
+  }
+  if (error != GRPC_ERROR_NONE) {
+    if (listener != nullptr) {
+      if (listener->tcp_server_ != nullptr) {
+        grpc_tcp_server_unref(listener->tcp_server_);
+      } else {
+        delete listener;
+      }
+    } else {
+      grpc_channel_args_destroy(args);
+    }
+    *port_num = 0;
   }
-  grpc_resource_user* resource_user =
-      grpc_server_get_default_resource_user(state->server);
+  for (grpc_error* error : error_list) {
+    GRPC_ERROR_UNREF(error);
+  }
+  return error;
+}
+
+grpc_error* Chttp2ServerListener::CreateWithAcceptor(Server* server,
+                                                     const char* name,
+                                                     grpc_channel_args* args) {
+  Chttp2ServerListener* listener = new Chttp2ServerListener(server, args);
+  grpc_error* error = grpc_tcp_server_create(
+      &listener->tcp_server_shutdown_complete_, args, &listener->tcp_server_);
+  if (error != GRPC_ERROR_NONE) {
+    delete listener;
+    return error;
+  }
+  // TODO(yangg) channelz
+  TcpServerFdHandler** arg_val =
+      grpc_channel_args_find_pointer<TcpServerFdHandler*>(args, name);
+  *arg_val = grpc_tcp_server_create_fd_handler(listener->tcp_server_);
+  server->AddListener(OrphanablePtr<Server::ListenerInterface>(listener));
+  return GRPC_ERROR_NONE;
+}
+
+Chttp2ServerListener::Chttp2ServerListener(Server* server,
+                                           grpc_channel_args* args)
+    : server_(server), args_(args) {
+  GRPC_CLOSURE_INIT(&tcp_server_shutdown_complete_, TcpServerShutdownComplete,
+                    this, grpc_schedule_on_exec_ctx);
+}
+
+Chttp2ServerListener::~Chttp2ServerListener() {
+  grpc_channel_args_destroy(args_);
+}
+
+/* Server callback: start listening on our ports */
+void Chttp2ServerListener::Start(Server* /*server*/,
+                                 const std::vector<grpc_pollset*>* pollsets) {
+  {
+    MutexLock lock(&mu_);
+    shutdown_ = false;
+  }
+  grpc_tcp_server_start(tcp_server_, pollsets, OnAccept, this);
+}
+
+void Chttp2ServerListener::SetOnDestroyDone(grpc_closure* on_destroy_done) {
+  MutexLock lock(&mu_);
+  on_destroy_done_ = on_destroy_done;
+}
+
+RefCountedPtr<HandshakeManager> Chttp2ServerListener::CreateHandshakeManager() {
+  MutexLock lock(&mu_);
+  if (shutdown_) return nullptr;
+  grpc_resource_user* resource_user = server_->default_resource_user();
   if (resource_user != nullptr &&
       !grpc_resource_user_safe_alloc(resource_user,
                                      GRPC_RESOURCE_QUOTA_CHANNEL_SIZE)) {
-    gpr_log(
-        GPR_ERROR,
-        "Memory quota exhausted, rejecting the connection, no handshaking.");
-    gpr_mu_unlock(&state->mu);
+    gpr_log(GPR_ERROR,
+            "Memory quota exhausted, rejecting connection, no handshaking.");
+    return nullptr;
+  }
+  auto handshake_mgr = MakeRefCounted<HandshakeManager>();
+  handshake_mgr->AddToPendingMgrList(&pending_handshake_mgrs_);
+  grpc_tcp_server_ref(tcp_server_);  // Ref held by ConnectionState.
+  return handshake_mgr;
+}
+
+void Chttp2ServerListener::OnAccept(void* arg, grpc_endpoint* tcp,
+                                    grpc_pollset* accepting_pollset,
+                                    grpc_tcp_server_acceptor* acceptor) {
+  Chttp2ServerListener* self = static_cast<Chttp2ServerListener*>(arg);
+  RefCountedPtr<HandshakeManager> handshake_mgr =
+      self->CreateHandshakeManager();
+  if (handshake_mgr == nullptr) {
     grpc_endpoint_shutdown(tcp, GRPC_ERROR_NONE);
     grpc_endpoint_destroy(tcp);
     gpr_free(acceptor);
     return;
   }
-  auto handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
-  handshake_mgr->AddToPendingMgrList(&state->pending_handshake_mgrs);
-  grpc_tcp_server_ref(state->tcp_server);
-  gpr_mu_unlock(&state->mu);
-  server_connection_state* connection_state =
-      static_cast<server_connection_state*>(
-          gpr_zalloc(sizeof(*connection_state)));
-  gpr_ref_init(&connection_state->refs, 1);
-  connection_state->svr_state = state;
-  connection_state->accepting_pollset = accepting_pollset;
-  connection_state->acceptor = acceptor;
-  connection_state->handshake_mgr = handshake_mgr;
-  connection_state->interested_parties = grpc_pollset_set_create();
-  grpc_pollset_set_add_pollset(connection_state->interested_parties,
-                               connection_state->accepting_pollset);
-  grpc_core::HandshakerRegistry::AddHandshakers(
-      grpc_core::HANDSHAKER_SERVER, state->args,
-      connection_state->interested_parties,
-      connection_state->handshake_mgr.get());
-  const grpc_arg* timeout_arg =
-      grpc_channel_args_find(state->args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS);
-  connection_state->deadline =
-      grpc_core::ExecCtx::Get()->Now() +
-      grpc_channel_arg_get_integer(timeout_arg,
-                                   {120 * GPR_MS_PER_SEC, 1, INT_MAX});
-  connection_state->handshake_mgr->DoHandshake(
-      tcp, state->args, connection_state->deadline, acceptor, on_handshake_done,
-      connection_state);
+  // Deletes itself when done.
+  new ConnectionState(self, accepting_pollset, acceptor,
+                      std::move(handshake_mgr), self->args_, tcp);
 }
 
-/* Server callback: start listening on our ports */
-static void server_start_listener(grpc_server* /*server*/, void* arg,
-                                  grpc_pollset** pollsets,
-                                  size_t pollset_count) {
-  server_state* state = static_cast<server_state*>(arg);
-  gpr_mu_lock(&state->mu);
-  state->shutdown = false;
-  gpr_mu_unlock(&state->mu);
-  grpc_tcp_server_start(state->tcp_server, pollsets, pollset_count, on_accept,
-                        state);
-}
-
-static void tcp_server_shutdown_complete(void* arg, grpc_error* error) {
-  server_state* state = static_cast<server_state*>(arg);
+void Chttp2ServerListener::TcpServerShutdownComplete(void* arg,
+                                                     grpc_error* error) {
+  Chttp2ServerListener* self = static_cast<Chttp2ServerListener*>(arg);
   /* ensure all threads have unlocked */
-  gpr_mu_lock(&state->mu);
-  grpc_closure* destroy_done = state->server_destroy_listener_done;
-  GPR_ASSERT(state->shutdown);
-  if (state->pending_handshake_mgrs != nullptr) {
-    state->pending_handshake_mgrs->ShutdownAllPending(GRPC_ERROR_REF(error));
+  grpc_closure* destroy_done = nullptr;
+  {
+    MutexLock lock(&self->mu_);
+    destroy_done = self->on_destroy_done_;
+    GPR_ASSERT(self->shutdown_);
+    if (self->pending_handshake_mgrs_ != nullptr) {
+      self->pending_handshake_mgrs_->ShutdownAllPending(GRPC_ERROR_REF(error));
+    }
+    self->channelz_listen_socket_.reset();
   }
-  state->channelz_listen_socket.reset();
-  gpr_mu_unlock(&state->mu);
   // Flush queued work before destroying handshaker factory, since that
   // may do a synchronous unref.
-  grpc_core::ExecCtx::Get()->Flush();
+  ExecCtx::Get()->Flush();
   if (destroy_done != nullptr) {
-    grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_done,
-                            GRPC_ERROR_REF(error));
-    grpc_core::ExecCtx::Get()->Flush();
+    ExecCtx::Run(DEBUG_LOCATION, destroy_done, GRPC_ERROR_REF(error));
+    ExecCtx::Get()->Flush();
   }
-  grpc_channel_args_destroy(state->args);
-  gpr_mu_destroy(&state->mu);
-  gpr_free(state);
+  delete self;
 }
 
 /* Server callback: destroy the tcp listener (so we don't generate further
    callbacks) */
-static void server_destroy_listener(grpc_server* /*server*/, void* arg,
-                                    grpc_closure* destroy_done) {
-  server_state* state = static_cast<server_state*>(arg);
-  gpr_mu_lock(&state->mu);
-  state->shutdown = true;
-  state->server_destroy_listener_done = destroy_done;
-  grpc_tcp_server* tcp_server = state->tcp_server;
-  gpr_mu_unlock(&state->mu);
+void Chttp2ServerListener::Orphan() {
+  grpc_tcp_server* tcp_server;
+  {
+    MutexLock lock(&mu_);
+    shutdown_ = true;
+    tcp_server = tcp_server_;
+  }
   grpc_tcp_server_shutdown_listeners(tcp_server);
   grpc_tcp_server_unref(tcp_server);
 }
 
-static grpc_error* chttp2_server_add_acceptor(grpc_server* server,
-                                              const char* name,
-                                              grpc_channel_args* args) {
-  grpc_tcp_server* tcp_server = nullptr;
-  grpc_error* err = GRPC_ERROR_NONE;
-  server_state* state = nullptr;
-  const grpc_arg* arg = nullptr;
-  grpc_core::TcpServerFdHandler** arg_val = nullptr;
-  state = static_cast<server_state*>(gpr_zalloc(sizeof(*state)));
-  GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete,
-                    tcp_server_shutdown_complete, state,
-                    grpc_schedule_on_exec_ctx);
-  err = grpc_tcp_server_create(&state->tcp_server_shutdown_complete, args,
-                               &tcp_server);
-  if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-  state->server = server;
-  state->tcp_server = tcp_server;
-  state->args = args;
-  state->shutdown = true;
-  gpr_mu_init(&state->mu);
-  // TODO(yangg) channelz
-  arg = grpc_channel_args_find(args, name);
-  GPR_ASSERT(arg->type == GRPC_ARG_POINTER);
-  arg_val = static_cast<grpc_core::TcpServerFdHandler**>(arg->value.pointer.p);
-  *arg_val = grpc_tcp_server_create_fd_handler(tcp_server);
-
-  grpc_server_add_listener(server, state, server_start_listener,
-                           server_destroy_listener, /* node */ nullptr);
-  return err;
-
-/* Error path: cleanup and return */
-error:
-  GPR_ASSERT(err != GRPC_ERROR_NONE);
-  if (tcp_server) {
-    grpc_tcp_server_unref(tcp_server);
-  } else {
-    grpc_channel_args_destroy(args);
-    gpr_free(state);
-  }
-  return err;
-}
+}  // namespace
 
-grpc_error* grpc_chttp2_server_add_port(grpc_server* server, const char* addr,
-                                        grpc_channel_args* args,
-                                        int* port_num) {
-  grpc_resolved_addresses* resolved = nullptr;
-  grpc_tcp_server* tcp_server = nullptr;
-  size_t i;
-  size_t count = 0;
-  int port_temp;
-  grpc_error* err = GRPC_ERROR_NONE;
-  server_state* state = nullptr;
-  grpc_error** errors = nullptr;
-  size_t naddrs = 0;
-  const grpc_arg* arg = nullptr;
-
-  *port_num = -1;
+//
+// Chttp2ServerAddPort()
+//
 
+grpc_error* Chttp2ServerAddPort(Server* server, const char* addr,
+                                grpc_channel_args* args, int* port_num) {
   if (strncmp(addr, "external:", 9) == 0) {
-    return chttp2_server_add_acceptor(server, addr, args);
-  }
-
-  /* resolve address */
-  err = grpc_blocking_resolve_address(addr, "https", &resolved);
-  if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-  state = static_cast<server_state*>(gpr_zalloc(sizeof(*state)));
-  GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete,
-                    tcp_server_shutdown_complete, state,
-                    grpc_schedule_on_exec_ctx);
-  err = grpc_tcp_server_create(&state->tcp_server_shutdown_complete, args,
-                               &tcp_server);
-  if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-
-  state->server = server;
-  state->tcp_server = tcp_server;
-  state->args = args;
-  state->shutdown = true;
-  gpr_mu_init(&state->mu);
-
-  naddrs = resolved->naddrs;
-  errors = static_cast<grpc_error**>(gpr_malloc(sizeof(*errors) * naddrs));
-  for (i = 0; i < naddrs; i++) {
-    errors[i] =
-        grpc_tcp_server_add_port(tcp_server, &resolved->addrs[i], &port_temp);
-    if (errors[i] == GRPC_ERROR_NONE) {
-      if (*port_num == -1) {
-        *port_num = port_temp;
-      } else {
-        GPR_ASSERT(*port_num == port_temp);
-      }
-      count++;
-    }
+    return grpc_core::Chttp2ServerListener::CreateWithAcceptor(server, addr,
+                                                               args);
   }
-  if (count == 0) {
-    char* msg;
-    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
-                 naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(msg, errors, naddrs);
-    gpr_free(msg);
-    goto error;
-  } else if (count != naddrs) {
-    char* msg;
-    gpr_asprintf(&msg,
-                 "Only %" PRIuPTR " addresses added out of total %" PRIuPTR
-                 " resolved",
-                 count, naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(msg, errors, naddrs);
-    gpr_free(msg);
-
-    const char* warning_message = grpc_error_string(err);
-    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
-
-    /* we managed to bind some addresses: continue */
-  }
-  grpc_resolved_addresses_destroy(resolved);
-
-  arg = grpc_channel_args_find(args, GRPC_ARG_ENABLE_CHANNELZ);
-  if (grpc_channel_arg_get_bool(arg, GRPC_ENABLE_CHANNELZ_DEFAULT)) {
-    state->channelz_listen_socket =
-        grpc_core::MakeRefCounted<grpc_core::channelz::ListenSocketNode>(
-            addr, absl::StrFormat("chttp2 listener %s", addr));
-  }
-
-  /* Register with the server only upon success */
-  grpc_server_add_listener(server, state, server_start_listener,
-                           server_destroy_listener,
-                           state->channelz_listen_socket);
-  goto done;
-
-/* Error path: cleanup and return */
-error:
-  GPR_ASSERT(err != GRPC_ERROR_NONE);
-  if (resolved) {
-    grpc_resolved_addresses_destroy(resolved);
-  }
-  if (tcp_server) {
-    grpc_tcp_server_unref(tcp_server);
-  } else {
-    grpc_channel_args_destroy(args);
-    gpr_free(state);
-  }
-  *port_num = 0;
-
-done:
-  if (errors != nullptr) {
-    for (i = 0; i < naddrs; i++) {
-      GRPC_ERROR_UNREF(errors[i]);
-    }
-    gpr_free(errors);
-  }
-  return err;
+  return grpc_core::Chttp2ServerListener::Create(server, addr, args, port_num);
 }
+
+}  // namespace grpc_core