grpc 1.40.0 → 1.46.3

data/src/core/lib/surface/server.cc

@@ -46,14 +46,13 @@
 #include "src/core/lib/gprpp/mpscq.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/resource_quota/api.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/init.h"
-#include "src/core/lib/transport/metadata.h"
-#include "src/core/lib/transport/static_metadata.h"
 
 namespace grpc_core {
 
@@ -480,7 +479,6 @@ class ChannelBroadcaster {
                   GRPC_ERROR_CREATE_FROM_STATIC_STRING("Server shutdown"),
                   GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_OK)
             : GRPC_ERROR_NONE;
-    op->set_accept_stream = true;
     sc->slice = grpc_slice_from_copied_string("Server shutdown");
     op->disconnect_with_error = send_disconnect;
     elem =
@@ -499,6 +497,7 @@ class ChannelBroadcaster {
 
 const grpc_channel_filter Server::kServerTopFilter = {
     Server::CallData::StartTransportStreamOpBatch,
+    nullptr,
     grpc_channel_next_op,
     sizeof(Server::CallData),
     Server::CallData::InitCallElement,
@@ -513,17 +512,6 @@ const grpc_channel_filter Server::kServerTopFilter = {
 
 namespace {
 
-grpc_resource_user* CreateDefaultResourceUser(const grpc_channel_args* args) {
-  if (args != nullptr) {
-    grpc_resource_quota* resource_quota =
-        grpc_resource_quota_from_channel_args(args, false /* create */);
-    if (resource_quota != nullptr) {
-      return grpc_resource_user_create(resource_quota, "default");
-    }
-  }
-  return nullptr;
-}
-
 RefCountedPtr<channelz::ServerNode> CreateChannelzNode(
     const grpc_channel_args* args) {
   RefCountedPtr<channelz::ServerNode> channelz_node;
@@ -545,7 +533,6 @@ RefCountedPtr<channelz::ServerNode> CreateChannelzNode(
 
 Server::Server(const grpc_channel_args* args)
     : channel_args_(grpc_channel_args_copy(args)),
-      default_resource_user_(CreateDefaultResourceUser(args)),
       channelz_node_(CreateChannelzNode(args)) {}
 
 Server::~Server() {
@@ -612,12 +599,11 @@ void Server::Start() {
 grpc_error_handle Server::SetupTransport(
     grpc_transport* transport, grpc_pollset* accepting_pollset,
     const grpc_channel_args* args,
-    const RefCountedPtr<grpc_core::channelz::SocketNode>& socket_node,
-    grpc_resource_user* resource_user) {
+    const RefCountedPtr<channelz::SocketNode>& socket_node) {
   // Create channel.
   grpc_error_handle error = GRPC_ERROR_NONE;
-  grpc_channel* channel = grpc_channel_create(
-      nullptr, args, GRPC_SERVER_CHANNEL, transport, resource_user, &error);
+  grpc_channel* channel = grpc_channel_create_internal(
+      nullptr, args, GRPC_SERVER_CHANNEL, transport, &error);
   if (channel == nullptr) {
     return error;
   }
@@ -698,7 +684,7 @@ Server::RegisteredMethod* Server::RegisterMethod(
       return nullptr;
     }
   }
-  if ((flags & ~GRPC_INITIAL_METADATA_USED_MASK) != 0) {
+  if (flags != 0) {
     gpr_log(GPR_ERROR, "grpc_server_register_method invalid flags 0x%08x",
             flags);
     return nullptr;
@@ -805,6 +791,7 @@ void DonePublishedShutdown(void* /*done_arg*/, grpc_cq_completion* storage) {
 //       connection is NOT closed until the server is done with all those calls.
 //    -- Once there are no more calls in progress, the channel is closed.
 void Server::ShutdownAndNotify(grpc_completion_queue* cq, void* tag) {
+  absl::Notification* await_requests = nullptr;
   ChannelBroadcaster broadcaster;
   {
     // Wait for startup to be finished.  Locks mu_global.
@@ -831,7 +818,12 @@ void Server::ShutdownAndNotify(grpc_completion_queue* cq, void* tag) {
       KillPendingWorkLocked(
           GRPC_ERROR_CREATE_FROM_STATIC_STRING("Server Shutdown"));
     }
-    ShutdownUnrefOnShutdownCall();
+    await_requests = ShutdownUnrefOnShutdownCall();
+  }
+  // We expect no new requests but there can still be requests in-flight.
+  // Wait for them to complete before proceeding.
+  if (await_requests != nullptr) {
+    await_requests->WaitForNotification();
   }
   // Shutdown listeners.
   for (auto& listener : listeners_) {
@@ -866,11 +858,6 @@ void Server::Orphan() {
     GPR_ASSERT(ShutdownCalled() || listeners_.empty());
     GPR_ASSERT(listeners_destroyed_ == listeners_.size());
   }
-  if (default_resource_user_ != nullptr) {
-    grpc_resource_quota_unref(grpc_resource_user_quota(default_resource_user_));
-    grpc_resource_user_shutdown(default_resource_user_);
-    grpc_resource_user_unref(default_resource_user_);
-  }
   Unref();
 }
 
@@ -995,19 +982,7 @@ class Server::ChannelData::ConnectivityWatcher
 //
 
 Server::ChannelData::~ChannelData() {
-  if (registered_methods_ != nullptr) {
-    for (const ChannelRegisteredMethod& crm : *registered_methods_) {
-      grpc_slice_unref_internal(crm.method);
-      GPR_DEBUG_ASSERT(crm.method.refcount == &kNoopRefcount ||
-                       crm.method.refcount == nullptr);
-      if (crm.has_host) {
-        grpc_slice_unref_internal(crm.host);
-        GPR_DEBUG_ASSERT(crm.host.refcount == &kNoopRefcount ||
-                         crm.host.refcount == nullptr);
-      }
-    }
-    registered_methods_.reset();
-  }
+  registered_methods_.reset();
   if (server_ != nullptr) {
     if (server_->channelz_node_ != nullptr && channelz_socket_uuid_ != 0) {
       server_->channelz_node_->RemoveChildSocket(channelz_socket_uuid_);
@@ -1040,14 +1015,13 @@ void Server::ChannelData::InitTransport(RefCountedPtr<Server> server,
     registered_methods_ =
         absl::make_unique<std::vector<ChannelRegisteredMethod>>(slots);
     for (std::unique_ptr<RegisteredMethod>& rm : server_->registered_methods_) {
-      ExternallyManagedSlice host;
-      ExternallyManagedSlice method(rm->method.c_str());
+      Slice host;
+      Slice method = Slice::FromExternalString(rm->method);
       const bool has_host = !rm->host.empty();
       if (has_host) {
-        host = ExternallyManagedSlice(rm->host.c_str());
+        host = Slice::FromExternalString(rm->host.c_str());
       }
-      uint32_t hash =
-          GRPC_MDSTR_KV_HASH(has_host ? host.Hash() : 0, method.Hash());
+      uint32_t hash = MixHash32(has_host ? host.Hash() : 0, method.Hash());
       uint32_t probes = 0;
       for (probes = 0; (*registered_methods_)[(hash + probes) % slots]
                            .server_registered_method != nullptr;
@@ -1060,9 +1034,9 @@ void Server::ChannelData::InitTransport(RefCountedPtr<Server> server,
       crm->flags = rm->flags;
       crm->has_host = has_host;
       if (has_host) {
-        crm->host = host;
+        crm->host = std::move(host);
       }
-      crm->method = method;
+      crm->method = std::move(method);
     }
     GPR_ASSERT(slots <= UINT32_MAX);
     registered_method_max_probes_ = max_probes;
@@ -1087,12 +1061,12 @@ void Server::ChannelData::InitTransport(RefCountedPtr<Server> server,
 }
 
 Server::ChannelRegisteredMethod* Server::ChannelData::GetRegisteredMethod(
-    const grpc_slice& host, const grpc_slice& path, bool is_idempotent) {
+    const grpc_slice& host, const grpc_slice& path) {
   if (registered_methods_ == nullptr) return nullptr;
   /* TODO(ctiller): unify these two searches */
   /* check for an exact match with host */
-  uint32_t hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash_internal(host),
-                                     grpc_slice_hash_internal(path));
+  uint32_t hash =
+      MixHash32(grpc_slice_hash_internal(host), grpc_slice_hash_internal(path));
   for (size_t i = 0; i <= registered_method_max_probes_; i++) {
     ChannelRegisteredMethod* rm =
         &(*registered_methods_)[(hash + i) % registered_methods_->size()];
@@ -1100,24 +1074,16 @@ Server::ChannelRegisteredMethod* Server::ChannelData::GetRegisteredMethod(
     if (!rm->has_host) continue;
     if (rm->host != host) continue;
     if (rm->method != path) continue;
-    if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
-        !is_idempotent) {
-      continue;
-    }
     return rm;
   }
   /* check for a wildcard method definition (no host set) */
-  hash = GRPC_MDSTR_KV_HASH(0, grpc_slice_hash_internal(path));
+  hash = MixHash32(0, grpc_slice_hash_internal(path));
   for (size_t i = 0; i <= registered_method_max_probes_; i++) {
     ChannelRegisteredMethod* rm =
         &(*registered_methods_)[(hash + i) % registered_methods_->size()];
     if (rm->server_registered_method == nullptr) break;
     if (rm->has_host) continue;
     if (rm->method != path) continue;
-    if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) &&
-        !is_idempotent) {
-      continue;
-    }
     return rm;
   }
   return nullptr;
@@ -1135,9 +1101,7 @@ void Server::ChannelData::AcceptStream(void* arg, grpc_transport* /*transport*/,
   args.cq = nullptr;
   args.pollset_set_alternative = nullptr;
   args.server_transport_data = transport_server_data;
-  args.add_initial_metadata = nullptr;
-  args.add_initial_metadata_count = 0;
-  args.send_deadline = GRPC_MILLIS_INF_FUTURE;
+  args.send_deadline = Timestamp::InfFuture();
   grpc_call* call;
   grpc_error_handle error = grpc_call_create(&args, &call);
   grpc_call_element* elem =
@@ -1209,39 +1173,33 @@ Server::CallData::CallData(grpc_call_element* elem,
 }
 
 Server::CallData::~CallData() {
-  GPR_ASSERT(state_.Load(MemoryOrder::RELAXED) != CallState::PENDING);
+  GPR_ASSERT(state_.load(std::memory_order_relaxed) != CallState::PENDING);
   GRPC_ERROR_UNREF(recv_initial_metadata_error_);
-  if (host_.has_value()) {
-    grpc_slice_unref_internal(*host_);
-  }
-  if (path_.has_value()) {
-    grpc_slice_unref_internal(*path_);
-  }
   grpc_metadata_array_destroy(&initial_metadata_);
   grpc_byte_buffer_destroy(payload_);
 }
 
 void Server::CallData::SetState(CallState state) {
-  state_.Store(state, MemoryOrder::RELAXED);
+  state_.store(state, std::memory_order_relaxed);
 }
 
 bool Server::CallData::MaybeActivate() {
   CallState expected = CallState::PENDING;
-  return state_.CompareExchangeStrong(&expected, CallState::ACTIVATED,
-                                      MemoryOrder::ACQ_REL,
-                                      MemoryOrder::RELAXED);
+  return state_.compare_exchange_strong(expected, CallState::ACTIVATED,
+                                        std::memory_order_acq_rel,
+                                        std::memory_order_relaxed);
 }
 
 void Server::CallData::FailCallCreation() {
   CallState expected_not_started = CallState::NOT_STARTED;
   CallState expected_pending = CallState::PENDING;
-  if (state_.CompareExchangeStrong(&expected_not_started, CallState::ZOMBIED,
-                                   MemoryOrder::ACQ_REL,
-                                   MemoryOrder::ACQUIRE)) {
+  if (state_.compare_exchange_strong(expected_not_started, CallState::ZOMBIED,
+                                     std::memory_order_acq_rel,
+                                     std::memory_order_acquire)) {
     KillZombie();
-  } else if (state_.CompareExchangeStrong(&expected_pending, CallState::ZOMBIED,
-                                          MemoryOrder::ACQ_REL,
-                                          MemoryOrder::RELAXED)) {
+  } else if (state_.compare_exchange_strong(
+                 expected_pending, CallState::ZOMBIED,
+                 std::memory_order_acq_rel, std::memory_order_relaxed)) {
     // Zombied call will be destroyed when it's removed from the pending
     // queue... later.
   }
@@ -1264,20 +1222,21 @@ void Server::CallData::Publish(size_t cq_idx, RequestedCall* rc) {
   grpc_call_set_completion_queue(call_, rc->cq_bound_to_call);
   *rc->call = call_;
   cq_new_ = server_->cqs_[cq_idx];
-  GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, initial_metadata_);
+  std::swap(*rc->initial_metadata, initial_metadata_);
   switch (rc->type) {
     case RequestedCall::Type::BATCH_CALL:
       GPR_ASSERT(host_.has_value());
       GPR_ASSERT(path_.has_value());
-      rc->data.batch.details->host = grpc_slice_ref_internal(*host_);
-      rc->data.batch.details->method = grpc_slice_ref_internal(*path_);
+      rc->data.batch.details->host = grpc_slice_ref_internal(host_->c_slice());
+      rc->data.batch.details->method =
+          grpc_slice_ref_internal(path_->c_slice());
       rc->data.batch.details->deadline =
-          grpc_millis_to_timespec(deadline_, GPR_CLOCK_MONOTONIC);
+          deadline_.as_timespec(GPR_CLOCK_MONOTONIC);
       rc->data.batch.details->flags = recv_initial_metadata_flags_;
       break;
     case RequestedCall::Type::REGISTERED_CALL:
       *rc->data.registered.deadline =
-          grpc_millis_to_timespec(deadline_, GPR_CLOCK_MONOTONIC);
+          deadline_.as_timespec(GPR_CLOCK_MONOTONIC);
       if (rc->data.registered.optional_payload != nullptr) {
         *rc->data.registered.optional_payload = payload_;
         payload_ = nullptr;
@@ -1297,7 +1256,7 @@ void Server::CallData::PublishNewRpc(void* arg, grpc_error_handle error) {
   RequestMatcherInterface* rm = calld->matcher_;
   Server* server = rm->server();
   if (error != GRPC_ERROR_NONE || server->ShutdownCalled()) {
-    calld->state_.Store(CallState::ZOMBIED, MemoryOrder::RELAXED);
+    calld->state_.store(CallState::ZOMBIED, std::memory_order_relaxed);
     calld->KillZombie();
     return;
   }
@@ -1321,7 +1280,7 @@ void Server::CallData::KillZombie() {
 void Server::CallData::StartNewRpc(grpc_call_element* elem) {
   auto* chand = static_cast<ChannelData*>(elem->channel_data);
   if (server_->ShutdownCalled()) {
-    state_.Store(CallState::ZOMBIED, MemoryOrder::RELAXED);
+    state_.store(CallState::ZOMBIED, std::memory_order_relaxed);
     KillZombie();
     return;
   }
@@ -1331,9 +1290,7 @@ void Server::CallData::StartNewRpc(grpc_call_element* elem) {
       GRPC_SRM_PAYLOAD_NONE;
   if (path_.has_value() && host_.has_value()) {
     ChannelRegisteredMethod* rm =
-        chand->GetRegisteredMethod(*host_, *path_,
-                                   (recv_initial_metadata_flags_ &
-                                    GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST));
+        chand->GetRegisteredMethod(host_->c_slice(), path_->c_slice());
     if (rm != nullptr) {
       matcher_ = rm->server_registered_method->matcher.get();
       payload_handling = rm->server_registered_method->payload_handling;
@@ -1363,6 +1320,8 @@ void Server::CallData::RecvInitialMetadataBatchComplete(
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   auto* calld = static_cast<Server::CallData*>(elem->call_data);
   if (error != GRPC_ERROR_NONE) {
+    gpr_log(GPR_DEBUG, "Failed call creation: %s",
+            grpc_error_std_string(error).c_str());
     calld->FailCallCreation();
     return;
   }
@@ -1395,24 +1354,15 @@ void Server::CallData::RecvInitialMetadataReady(void* arg,
                                                 grpc_error_handle error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  grpc_millis op_deadline;
   if (error == GRPC_ERROR_NONE) {
-    GPR_DEBUG_ASSERT(calld->recv_initial_metadata_->idx.named.path != nullptr);
-    GPR_DEBUG_ASSERT(calld->recv_initial_metadata_->idx.named.authority !=
-                     nullptr);
-    calld->path_.emplace(grpc_slice_ref_internal(
-        GRPC_MDVALUE(calld->recv_initial_metadata_->idx.named.path->md)));
-    calld->host_.emplace(grpc_slice_ref_internal(
-        GRPC_MDVALUE(calld->recv_initial_metadata_->idx.named.authority->md)));
-    grpc_metadata_batch_remove(calld->recv_initial_metadata_, GRPC_BATCH_PATH);
-    grpc_metadata_batch_remove(calld->recv_initial_metadata_,
-                               GRPC_BATCH_AUTHORITY);
+    calld->path_ = calld->recv_initial_metadata_->Take(HttpPathMetadata());
+    calld->host_ = calld->recv_initial_metadata_->Take(HttpAuthorityMetadata());
   } else {
-    GRPC_ERROR_REF(error);
+    (void)GRPC_ERROR_REF(error);
   }
-  op_deadline = calld->recv_initial_metadata_->deadline;
-  if (op_deadline != GRPC_MILLIS_INF_FUTURE) {
-    calld->deadline_ = op_deadline;
+  auto op_deadline = calld->recv_initial_metadata_->get(GrpcTimeoutMetadata());
+  if (op_deadline.has_value()) {
+    calld->deadline_ = *op_deadline;
   }
   if (calld->host_.has_value() && calld->path_.has_value()) {
     /* do nothing */
@@ -1486,9 +1436,12 @@ void Server::CallData::StartTransportStreamOpBatch(
 grpc_server* grpc_server_create(const grpc_channel_args* args, void* reserved) {
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved));
-  grpc_server* c_server = new grpc_server;
-  c_server->core_server = grpc_core::MakeOrphanable<grpc_core::Server>(args);
-  return c_server;
+  const grpc_channel_args* new_args = grpc_core::CoreConfiguration::Get()
+                                          .channel_args_preconditioning()
+                                          .PreconditionChannelArgs(args);
+  grpc_core::Server* server = new grpc_core::Server(new_args);
+  grpc_channel_args_destroy(new_args);
+  return server->c_ptr();
 }
 
 void grpc_server_register_completion_queue(grpc_server* server,
@@ -1507,7 +1460,7 @@ void grpc_server_register_completion_queue(grpc_server* server,
     /* Ideally we should log an error and abort but ruby-wrapped-language API
        calls grpc_completion_queue_pluck() on server completion queues */
   }
-  server->core_server->RegisterCompletionQueue(cq);
+  grpc_core::Server::FromC(server)->RegisterCompletionQueue(cq);
 }
 
 void* grpc_server_register_method(
@@ -1518,14 +1471,14 @@ void* grpc_server_register_method(
       "grpc_server_register_method(server=%p, method=%s, host=%s, "
       "flags=0x%08x)",
       4, (server, method, host, flags));
-  return server->core_server->RegisterMethod(method, host, payload_handling,
-                                             flags);
+  return grpc_core::Server::FromC(server)->RegisterMethod(
+      method, host, payload_handling, flags);
 }
 
 void grpc_server_start(grpc_server* server) {
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
-  server->core_server->Start();
+  grpc_core::Server::FromC(server)->Start();
 }
 
 void grpc_server_shutdown_and_notify(grpc_server* server,
@@ -1534,21 +1487,21 @@ void grpc_server_shutdown_and_notify(grpc_server* server,
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_shutdown_and_notify(server=%p, cq=%p, tag=%p)", 3,
                  (server, cq, tag));
-  server->core_server->ShutdownAndNotify(cq, tag);
+  grpc_core::Server::FromC(server)->ShutdownAndNotify(cq, tag);
 }
 
 void grpc_server_cancel_all_calls(grpc_server* server) {
   grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_cancel_all_calls(server=%p)", 1, (server));
-  server->core_server->CancelAllCalls();
+  grpc_core::Server::FromC(server)->CancelAllCalls();
 }
 
 void grpc_server_destroy(grpc_server* server) {
   grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_destroy(server=%p)", 1, (server));
-  delete server;
+  grpc_core::Server::FromC(server)->Orphan();
 }
 
 grpc_call_error grpc_server_request_call(
@@ -1566,9 +1519,9 @@ grpc_call_error grpc_server_request_call(
       7,
       (server, call, details, request_metadata, cq_bound_to_call,
        cq_for_notification, tag));
-  return server->core_server->RequestCall(call, details, request_metadata,
-                                          cq_bound_to_call, cq_for_notification,
-                                          tag);
+  return grpc_core::Server::FromC(server)->RequestCall(
+      call, details, request_metadata, cq_bound_to_call, cq_for_notification,
+      tag);
 }
 
 grpc_call_error grpc_server_request_registered_call(
@@ -1591,7 +1544,7 @@ grpc_call_error grpc_server_request_registered_call(
       9,
       (server, registered_method, call, deadline, request_metadata,
        optional_payload, cq_bound_to_call, cq_for_notification, tag_new));
-  return server->core_server->RequestRegisteredCall(
+  return grpc_core::Server::FromC(server)->RequestRegisteredCall(
       rm, call, deadline, request_metadata, optional_payload, cq_bound_to_call,
       cq_for_notification, tag_new);
 }
@@ -1602,7 +1555,7 @@ void grpc_server_set_config_fetcher(
   grpc_core::ExecCtx exec_ctx;
   GRPC_API_TRACE("grpc_server_set_config_fetcher(server=%p, config_fetcher=%p)",
                  2, (server, server_config_fetcher));
-  server->core_server->set_config_fetcher(
+  grpc_core::Server::FromC(server)->set_config_fetcher(
       std::unique_ptr<grpc_server_config_fetcher>(server_config_fetcher));
 }
 

data/src/core/lib/surface/server.h

@@ -19,10 +19,12 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <atomic>
 #include <list>
 #include <vector>
 
 #include "absl/status/statusor.h"
+#include "absl/synchronization/notification.h"
 #include "absl/types/optional.h"
 
 #include <grpc/grpc.h>
@@ -31,8 +33,10 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/cpp_impl_of.h"
+#include "src/core/lib/gprpp/dual_ref_counted.h"
 #include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/transport/transport.h"
 
@@ -40,7 +44,8 @@ namespace grpc_core {
 
 extern TraceFlag grpc_server_channel_trace;
 
-class Server : public InternallyRefCounted<Server> {
+class Server : public InternallyRefCounted<Server>,
+               public CppImplOf<Server, grpc_server> {
  public:
   // Filter vtable.
   static const grpc_channel_filter kServerTopFilter;
@@ -97,9 +102,6 @@ class Server : public InternallyRefCounted<Server> {
   void Orphan() ABSL_LOCKS_EXCLUDED(mu_global_) override;
 
   const grpc_channel_args* channel_args() const { return channel_args_; }
-  grpc_resource_user* default_resource_user() const {
-    return default_resource_user_;
-  }
   channelz::ServerNode* channelz_node() const { return channelz_node_.get(); }
 
   // Do not call this before Start(). Returns the pollsets. The
@@ -128,11 +130,11 @@ class Server : public InternallyRefCounted<Server> {
 
   // Sets up a transport.  Creates a channel stack and binds the transport to
   // the server.  Called from the listener when a new connection is accepted.
+  // Takes ownership of a ref on resource_user from the caller.
   grpc_error_handle SetupTransport(
       grpc_transport* transport, grpc_pollset* accepting_pollset,
       const grpc_channel_args* args,
-      const RefCountedPtr<channelz::SocketNode>& socket_node,
-      grpc_resource_user* resource_user = nullptr);
+      const RefCountedPtr<channelz::SocketNode>& socket_node);
 
   void RegisterCompletionQueue(grpc_completion_queue* cq);
 
@@ -174,8 +176,8 @@ class Server : public InternallyRefCounted<Server> {
     RegisteredMethod* server_registered_method = nullptr;
     uint32_t flags;
     bool has_host;
-    ExternallyManagedSlice method;
-    ExternallyManagedSlice host;
+    Slice method;
+    Slice host;
   };
 
   class RequestMatcherInterface;
@@ -198,8 +200,7 @@ class Server : public InternallyRefCounted<Server> {
     size_t cq_idx() const { return cq_idx_; }
 
     ChannelRegisteredMethod* GetRegisteredMethod(const grpc_slice& host,
-                                                 const grpc_slice& path,
-                                                 bool is_idempotent);
+                                                 const grpc_slice& path);
 
     // Filter vtable functions.
     static grpc_error_handle InitChannelElement(
@@ -289,11 +290,11 @@ class Server : public InternallyRefCounted<Server> {
 
     grpc_call* call_;
 
-    Atomic<CallState> state_{CallState::NOT_STARTED};
+    std::atomic<CallState> state_{CallState::NOT_STARTED};
 
-    absl::optional<grpc_slice> path_;
-    absl::optional<grpc_slice> host_;
-    grpc_millis deadline_ = GRPC_MILLIS_INF_FUTURE;
+    absl::optional<Slice> path_;
+    absl::optional<Slice> host_;
+    Timestamp deadline_ = Timestamp::InfFuture();
 
     grpc_completion_queue* cq_new_ = nullptr;
 
@@ -365,9 +366,9 @@ class Server : public InternallyRefCounted<Server> {
   std::vector<grpc_channel*> GetChannelsLocked() const;
 
   // Take a shutdown ref for a request (increment by 2) and return if shutdown
-  // has already been called.
+  // has not been called.
   bool ShutdownRefOnRequest() {
-    int old_value = shutdown_refs_.FetchAdd(2, MemoryOrder::ACQ_REL);
+    int old_value = shutdown_refs_.fetch_add(2, std::memory_order_acq_rel);
     return (old_value & 1) != 0;
   }
 
@@ -375,30 +376,41 @@ class Server : public InternallyRefCounted<Server> {
   // (for in-flight request) and possibly call MaybeFinishShutdown if
   // appropriate.
   void ShutdownUnrefOnRequest() ABSL_LOCKS_EXCLUDED(mu_global_) {
-    if (shutdown_refs_.FetchSub(2, MemoryOrder::ACQ_REL) == 2) {
+    if (shutdown_refs_.fetch_sub(2, std::memory_order_acq_rel) == 2) {
       MutexLock lock(&mu_global_);
       MaybeFinishShutdown();
+      // The last request in-flight during shutdown is now complete.
+      if (requests_complete_ != nullptr) {
+        GPR_ASSERT(!requests_complete_->HasBeenNotified());
+        requests_complete_->Notify();
+      }
     }
   }
-  void ShutdownUnrefOnShutdownCall() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_global_) {
-    if (shutdown_refs_.FetchSub(1, MemoryOrder::ACQ_REL) == 1) {
+  // Returns a notification pointer to wait on if there are requests in-flight,
+  // or null.
+  absl::Notification* ShutdownUnrefOnShutdownCall()
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_global_) GRPC_MUST_USE_RESULT {
+    if (shutdown_refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
+      // There is no request in-flight.
       MaybeFinishShutdown();
+      return nullptr;
     }
+    requests_complete_ = absl::make_unique<absl::Notification>();
+    return requests_complete_.get();
   }
 
   bool ShutdownCalled() const {
-    return (shutdown_refs_.Load(MemoryOrder::ACQUIRE) & 1) == 0;
+    return (shutdown_refs_.load(std::memory_order_acquire) & 1) == 0;
   }
 
   // Returns whether there are no more shutdown refs, which means that shutdown
   // has been called and all accepted requests have been published if using an
   // AllocatingRequestMatcher.
   bool ShutdownReady() const {
-    return shutdown_refs_.Load(MemoryOrder::ACQUIRE) == 0;
+    return shutdown_refs_.load(std::memory_order_acquire) == 0;
   }
 
   grpc_channel_args* const channel_args_;
-  grpc_resource_user* default_resource_user_ = nullptr;
   RefCountedPtr<channelz::ServerNode> channelz_node_;
   std::unique_ptr<grpc_server_config_fetcher> config_fetcher_;
 
@@ -432,9 +444,11 @@ class Server : public InternallyRefCounted<Server> {
   // the lowest bit will be 0 (defaults to 1) and the counter will be even. The
   // server should not notify on shutdown until the counter is 0 (shutdown is
   // called and there are no requests that are accepted but not started).
-  Atomic<int> shutdown_refs_{1};
+  std::atomic<int> shutdown_refs_{1};
   bool shutdown_published_ ABSL_GUARDED_BY(mu_global_) = false;
   std::vector<ShutdownTag> shutdown_tags_ ABSL_GUARDED_BY(mu_global_);
+  std::unique_ptr<absl::Notification> requests_complete_
+      ABSL_GUARDED_BY(mu_global_);
 
   std::list<ChannelData*> channels_;
 
@@ -447,19 +461,10 @@ class Server : public InternallyRefCounted<Server> {
 
 }  // namespace grpc_core
 
-struct grpc_server {
-  grpc_core::OrphanablePtr<grpc_core::Server> core_server;
-};
-
-// TODO(roth): Eventually, will need a way to modify configuration even after
-// a connection is established (e.g., to change things like L7 rate
-// limiting, RBAC, and fault injection configs).  One possible option
-// would be to do something like ServiceConfig and ConfigSelector, but
-// that might add unnecessary per-call overhead.  Need to consider other
-// approaches here.
 struct grpc_server_config_fetcher {
  public:
-  class ConnectionManager : public grpc_core::RefCounted<ConnectionManager> {
+  class ConnectionManager
+      : public grpc_core::DualRefCounted<ConnectionManager> {
    public:
     // Ownership of \a args is transfered.
     virtual absl::StatusOr<grpc_channel_args*> UpdateChannelArgsForConnection(
@@ -481,9 +486,7 @@ struct grpc_server_config_fetcher {
 
   virtual ~grpc_server_config_fetcher() = default;
 
-  // Ownership of \a args is transferred.
   virtual void StartWatch(std::string listening_address,
-                          grpc_channel_args* args,
                           std::unique_ptr<WatcherInterface> watcher) = 0;
   virtual void CancelWatch(WatcherInterface* watcher) = 0;
   virtual grpc_pollset_set* interested_parties() = 0;