grpc 1.56.0 → 1.61.0

data/src/core/lib/surface/server.cc

@@ -24,23 +24,24 @@
 
 #include <algorithm>
 #include <atomic>
-#include <initializer_list>
 #include <list>
+#include <memory>
 #include <new>
 #include <queue>
-#include <tuple>
 #include <type_traits>
 #include <utility>
 #include <vector>
 
 #include "absl/cleanup/cleanup.h"
+#include "absl/container/flat_hash_map.h"
 #include "absl/status/status.h"
 #include "absl/types/optional.h"
-#include "absl/types/variant.h"
 
 #include <grpc/byte_buffer.h>
 #include <grpc/grpc.h>
+#include <grpc/impl/channel_arg_names.h>
 #include <grpc/impl/connectivity_state.h>
+#include <grpc/slice.h>
 #include <grpc/status.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
@@ -50,23 +51,23 @@
 #include "src/core/lib/channel/channel_trace.h"
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/crash.h"
 #include "src/core/lib/gprpp/debug_location.h"
-#include "src/core/lib/gprpp/match.h"
 #include "src/core/lib/gprpp/mpscq.h"
 #include "src/core/lib/gprpp/status_helper.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/cancel_callback.h"
 #include "src/core/lib/promise/context.h"
-#include "src/core/lib/promise/detail/basic_join.h"
-#include "src/core/lib/promise/detail/basic_seq.h"
 #include "src/core/lib/promise/map.h"
 #include "src/core/lib/promise/pipe.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/promise/promise.h"
+#include "src/core/lib/promise/seq.h"
 #include "src/core/lib/promise/try_join.h"
 #include "src/core/lib/promise/try_seq.h"
 #include "src/core/lib/slice/slice_buffer.h"
@@ -76,6 +77,7 @@
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/channel_stack_type.h"
 #include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/wait_for_cq_end_op.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/error_utils.h"
 
@@ -83,57 +85,6 @@ namespace grpc_core {
 
 TraceFlag grpc_server_channel_trace(false, "server_channel");
 
-//
-// Server::RequestedCall
-//
-
-struct Server::RequestedCall {
-  enum class Type { BATCH_CALL, REGISTERED_CALL };
-
-  RequestedCall(void* tag_arg, grpc_completion_queue* call_cq,
-                grpc_call** call_arg, grpc_metadata_array* initial_md,
-                grpc_call_details* details)
-      : type(Type::BATCH_CALL),
-        tag(tag_arg),
-        cq_bound_to_call(call_cq),
-        call(call_arg),
-        initial_metadata(initial_md) {
-    data.batch.details = details;
-  }
-
-  RequestedCall(void* tag_arg, grpc_completion_queue* call_cq,
-                grpc_call** call_arg, grpc_metadata_array* initial_md,
-                RegisteredMethod* rm, gpr_timespec* deadline,
-                grpc_byte_buffer** optional_payload)
-      : type(Type::REGISTERED_CALL),
-        tag(tag_arg),
-        cq_bound_to_call(call_cq),
-        call(call_arg),
-        initial_metadata(initial_md) {
-    data.registered.method = rm;
-    data.registered.deadline = deadline;
-    data.registered.optional_payload = optional_payload;
-  }
-
-  MultiProducerSingleConsumerQueue::Node mpscq_node;
-  const Type type;
-  void* const tag;
-  grpc_completion_queue* const cq_bound_to_call;
-  grpc_call** const call;
-  grpc_cq_completion completion;
-  grpc_metadata_array* const initial_metadata;
-  union {
-    struct {
-      grpc_call_details* details;
-    } batch;
-    struct {
-      RegisteredMethod* method;
-      gpr_timespec* deadline;
-      grpc_byte_buffer** optional_payload;
-    } registered;
-  } data;
-};
-
 //
 // Server::RegisteredMethod
 //
@@ -249,33 +200,108 @@ class Server::RequestMatcherInterface {
   virtual Server* server() const = 0;
 };
 
+//
+// Server::RequestedCall
+//
+
+struct Server::RequestedCall {
+  enum class Type { BATCH_CALL, REGISTERED_CALL };
+
+  RequestedCall(void* tag_arg, grpc_completion_queue* call_cq,
+                grpc_call** call_arg, grpc_metadata_array* initial_md,
+                grpc_call_details* details)
+      : type(Type::BATCH_CALL),
+        tag(tag_arg),
+        cq_bound_to_call(call_cq),
+        call(call_arg),
+        initial_metadata(initial_md) {
+    data.batch.details = details;
+  }
+
+  RequestedCall(void* tag_arg, grpc_completion_queue* call_cq,
+                grpc_call** call_arg, grpc_metadata_array* initial_md,
+                RegisteredMethod* rm, gpr_timespec* deadline,
+                grpc_byte_buffer** optional_payload)
+      : type(Type::REGISTERED_CALL),
+        tag(tag_arg),
+        cq_bound_to_call(call_cq),
+        call(call_arg),
+        initial_metadata(initial_md) {
+    data.registered.method = rm;
+    data.registered.deadline = deadline;
+    data.registered.optional_payload = optional_payload;
+  }
+
+  void Complete(NextResult<MessageHandle> payload, ClientMetadata& md) {
+    Timestamp deadline = GetContext<CallContext>()->deadline();
+    switch (type) {
+      case RequestedCall::Type::BATCH_CALL:
+        GPR_ASSERT(!payload.has_value());
+        data.batch.details->host =
+            CSliceRef(md.get_pointer(HttpAuthorityMetadata())->c_slice());
+        data.batch.details->method =
+            CSliceRef(md.Take(HttpPathMetadata())->c_slice());
+        data.batch.details->deadline =
+            deadline.as_timespec(GPR_CLOCK_MONOTONIC);
+        break;
+      case RequestedCall::Type::REGISTERED_CALL:
+        md.Remove(HttpPathMetadata());
+        *data.registered.deadline = deadline.as_timespec(GPR_CLOCK_MONOTONIC);
+        if (data.registered.optional_payload != nullptr) {
+          if (payload.has_value()) {
+            auto* sb = payload.value()->payload()->c_slice_buffer();
+            *data.registered.optional_payload =
+                grpc_raw_byte_buffer_create(sb->slices, sb->count);
+          } else {
+            *data.registered.optional_payload = nullptr;
+          }
+        }
+        break;
+      default:
+        GPR_UNREACHABLE_CODE(abort());
+    }
+  }
+
+  MultiProducerSingleConsumerQueue::Node mpscq_node;
+  const Type type;
+  void* const tag;
+  grpc_completion_queue* const cq_bound_to_call;
+  grpc_call** const call;
+  grpc_cq_completion completion;
+  grpc_metadata_array* const initial_metadata;
+  union {
+    struct {
+      grpc_call_details* details;
+    } batch;
+    struct {
+      RegisteredMethod* method;
+      gpr_timespec* deadline;
+      grpc_byte_buffer** optional_payload;
+    } registered;
+  } data;
+};
+
 // The RealRequestMatcher is an implementation of RequestMatcherInterface that
 // actually uses all the features of RequestMatcherInterface: expecting the
 // application to explicitly request RPCs and then matching those to incoming
 // RPCs, along with a slow path by which incoming RPCs are put on a locked
 // pending list if they aren't able to be matched to an application request.
-class Server::RealRequestMatcher : public RequestMatcherInterface {
+class Server::RealRequestMatcherFilterStack : public RequestMatcherInterface {
  public:
-  explicit RealRequestMatcher(Server* server)
+  explicit RealRequestMatcherFilterStack(Server* server)
       : server_(server), requests_per_cq_(server->cqs_.size()) {}
 
-  ~RealRequestMatcher() override {
+  ~RealRequestMatcherFilterStack() override {
     for (LockedMultiProducerSingleConsumerQueue& queue : requests_per_cq_) {
       GPR_ASSERT(queue.Pop() == nullptr);
     }
+    GPR_ASSERT(pending_.empty());
   }
 
   void ZombifyPending() override {
     while (!pending_.empty()) {
-      Match(
-          pending_.front(),
-          [](CallData* calld) {
-            calld->SetState(CallData::CallState::ZOMBIED);
-            calld->KillZombie();
-          },
-          [](const std::shared_ptr<ActivityWaiter>& w) {
-            w->Finish(absl::InternalError("Server closed"));
-          });
+      pending_.front().calld->SetState(CallData::CallState::ZOMBIED);
+      pending_.front().calld->KillZombie();
       pending_.pop();
     }
   }
@@ -301,40 +327,36 @@ class Server::RealRequestMatcher : public RequestMatcherInterface {
       // matching calls
       struct NextPendingCall {
         RequestedCall* rc = nullptr;
-        PendingCall pending;
+        CallData* pending;
       };
-      auto pop_next_pending = [this, request_queue_index] {
+      while (true) {
         NextPendingCall pending_call;
         {
           MutexLock lock(&server_->mu_call_);
+          while (!pending_.empty() &&
+                 pending_.front().Age() > server_->max_time_in_pending_queue_) {
+            pending_.front().calld->SetState(CallData::CallState::ZOMBIED);
+            pending_.front().calld->KillZombie();
+            pending_.pop();
+          }
           if (!pending_.empty()) {
             pending_call.rc = reinterpret_cast<RequestedCall*>(
                 requests_per_cq_[request_queue_index].Pop());
             if (pending_call.rc != nullptr) {
-              pending_call.pending = std::move(pending_.front());
+              pending_call.pending = pending_.front().calld;
               pending_.pop();
             }
           }
         }
-        return pending_call;
-      };
-      while (true) {
-        NextPendingCall next_pending = pop_next_pending();
-        if (next_pending.rc == nullptr) break;
-        auto mr = MatchResult(server(), request_queue_index, next_pending.rc);
-        Match(
-            next_pending.pending,
-            [&mr](CallData* calld) {
-              if (!calld->MaybeActivate()) {
-                // Zombied Call
-                calld->KillZombie();
-              } else {
-                calld->Publish(mr.cq_idx(), mr.TakeCall());
-              }
-            },
-            [&mr](const std::shared_ptr<ActivityWaiter>& w) {
-              w->Finish(std::move(mr));
-            });
+        if (pending_call.rc == nullptr) break;
+        if (!pending_call.pending->MaybeActivate()) {
+          // Zombied Call
+          pending_call.pending->KillZombie();
+          requests_per_cq_[request_queue_index].Push(
+              &pending_call.rc->mpscq_node);
+        } else {
+          pending_call.pending->Publish(request_queue_index, pending_call.rc);
+        }
       }
     }
   }
@@ -371,7 +393,7 @@ class Server::RealRequestMatcher : public RequestMatcherInterface {
       }
       if (rc == nullptr) {
         calld->SetState(CallData::CallState::PENDING);
-        pending_.push(calld);
+        pending_.push(PendingCall{calld});
         return;
       }
     }
@@ -379,6 +401,91 @@ class Server::RealRequestMatcher : public RequestMatcherInterface {
     calld->Publish(cq_idx, rc);
   }
 
+  ArenaPromise<absl::StatusOr<MatchResult>> MatchRequest(size_t) override {
+    Crash("not implemented for filter stack request matcher");
+  }
+
+  Server* server() const final { return server_; }
+
+ private:
+  Server* const server_;
+  struct PendingCall {
+    CallData* calld;
+    Timestamp created = Timestamp::Now();
+    Duration Age() { return Timestamp::Now() - created; }
+  };
+  std::queue<PendingCall> pending_;
+  std::vector<LockedMultiProducerSingleConsumerQueue> requests_per_cq_;
+};
+
+class Server::RealRequestMatcherPromises : public RequestMatcherInterface {
+ public:
+  explicit RealRequestMatcherPromises(Server* server)
+      : server_(server), requests_per_cq_(server->cqs_.size()) {}
+
+  ~RealRequestMatcherPromises() override {
+    for (LockedMultiProducerSingleConsumerQueue& queue : requests_per_cq_) {
+      GPR_ASSERT(queue.Pop() == nullptr);
+    }
+  }
+
+  void ZombifyPending() override {
+    while (!pending_.empty()) {
+      pending_.front()->Finish(absl::InternalError("Server closed"));
+      pending_.pop();
+    }
+  }
+
+  void KillRequests(grpc_error_handle error) override {
+    for (size_t i = 0; i < requests_per_cq_.size(); i++) {
+      RequestedCall* rc;
+      while ((rc = reinterpret_cast<RequestedCall*>(
+                  requests_per_cq_[i].Pop())) != nullptr) {
+        server_->FailCall(i, rc, error);
+      }
+    }
+  }
+
+  size_t request_queue_count() const override {
+    return requests_per_cq_.size();
+  }
+
+  void RequestCallWithPossiblePublish(size_t request_queue_index,
+                                      RequestedCall* call) override {
+    if (requests_per_cq_[request_queue_index].Push(&call->mpscq_node)) {
+      // this was the first queued request: we need to lock and start
+      // matching calls
+      struct NextPendingCall {
+        RequestedCall* rc = nullptr;
+        PendingCall pending;
+      };
+      while (true) {
+        NextPendingCall pending_call;
+        {
+          MutexLock lock(&server_->mu_call_);
+          if (!pending_.empty()) {
+            pending_call.rc = reinterpret_cast<RequestedCall*>(
+                requests_per_cq_[request_queue_index].Pop());
+            if (pending_call.rc != nullptr) {
+              pending_call.pending = std::move(pending_.front());
+              pending_.pop();
+            }
+          }
+        }
+        if (pending_call.rc == nullptr) break;
+        if (!pending_call.pending->Finish(server(), request_queue_index,
+                                          pending_call.rc)) {
+          requests_per_cq_[request_queue_index].Push(
+              &pending_call.rc->mpscq_node);
+        }
+      }
+    }
+  }
+
+  void MatchOrQueue(size_t, CallData*) override {
+    Crash("not implemented for promises");
+  }
+
   ArenaPromise<absl::StatusOr<MatchResult>> MatchRequest(
       size_t start_request_queue_index) override {
     for (size_t i = 0; i < requests_per_cq_.size(); i++) {
@@ -398,25 +505,36 @@ class Server::RealRequestMatcher : public RequestMatcherInterface {
     size_t cq_idx = 0;
     size_t loop_count;
     {
+      std::vector<std::shared_ptr<ActivityWaiter>> removed_pending;
       MutexLock lock(&server_->mu_call_);
+      while (!pending_.empty() &&
+             pending_.front()->Age() > server_->max_time_in_pending_queue_) {
+        removed_pending.push_back(std::move(pending_.front()));
+        pending_.pop();
+      }
       for (loop_count = 0; loop_count < requests_per_cq_.size(); loop_count++) {
         cq_idx =
             (start_request_queue_index + loop_count) % requests_per_cq_.size();
         rc = reinterpret_cast<RequestedCall*>(requests_per_cq_[cq_idx].Pop());
-        if (rc != nullptr) {
-          break;
-        }
+        if (rc != nullptr) break;
       }
       if (rc == nullptr) {
+        if (server_->pending_backlog_protector_.Reject(pending_.size(),
+                                                       server_->bitgen_)) {
+          return Immediate(absl::ResourceExhaustedError(
+              "Too many pending requests for this server"));
+        }
         auto w = std::make_shared<ActivityWaiter>(
             Activity::current()->MakeOwningWaker());
         pending_.push(w);
-        return [w]() -> Poll<absl::StatusOr<MatchResult>> {
-          std::unique_ptr<absl::StatusOr<MatchResult>> r(
-              w->result.exchange(nullptr, std::memory_order_acq_rel));
-          if (r == nullptr) return Pending{};
-          return std::move(*r);
-        };
+        return OnCancel(
+            [w]() -> Poll<absl::StatusOr<MatchResult>> {
+              std::unique_ptr<absl::StatusOr<MatchResult>> r(
+                  w->result.exchange(nullptr, std::memory_order_acq_rel));
+              if (r == nullptr) return Pending{};
+              return std::move(*r);
+            },
+            [w]() { w->Expire(); });
       }
     }
     return Immediate(MatchResult(server(), cq_idx, rc));
@@ -427,17 +545,40 @@ class Server::RealRequestMatcher : public RequestMatcherInterface {
  private:
   Server* const server_;
   struct ActivityWaiter {
+    using ResultType = absl::StatusOr<MatchResult>;
     explicit ActivityWaiter(Waker waker) : waker(std::move(waker)) {}
     ~ActivityWaiter() { delete result.load(std::memory_order_acquire); }
-    void Finish(absl::StatusOr<MatchResult> r) {
-      result.store(new absl::StatusOr<MatchResult>(std::move(r)),
-                   std::memory_order_release);
-      waker.Wakeup();
+    void Finish(absl::Status status) {
+      delete result.exchange(new ResultType(std::move(status)),
+                             std::memory_order_acq_rel);
+      waker.WakeupAsync();
     }
+    // Returns true if requested_call consumed, false otherwise.
+    GRPC_MUST_USE_RESULT bool Finish(Server* server, size_t cq_idx,
+                                     RequestedCall* requested_call) {
+      ResultType* expected = nullptr;
+      ResultType* new_value =
+          new ResultType(MatchResult(server, cq_idx, requested_call));
+      if (!result.compare_exchange_strong(expected, new_value,
+                                          std::memory_order_acq_rel,
+                                          std::memory_order_acquire)) {
+        GPR_ASSERT(new_value->value().TakeCall() == requested_call);
+        delete new_value;
+        return false;
+      }
+      waker.WakeupAsync();
+      return true;
+    }
+    void Expire() {
+      delete result.exchange(new ResultType(absl::CancelledError()),
+                             std::memory_order_acq_rel);
+    }
+    Duration Age() { return Timestamp::Now() - created; }
     Waker waker;
-    std::atomic<absl::StatusOr<MatchResult>*> result{nullptr};
+    std::atomic<ResultType*> result{nullptr};
+    const Timestamp created = Timestamp::Now();
   };
-  using PendingCall = absl::variant<CallData*, std::shared_ptr<ActivityWaiter>>;
+  using PendingCall = std::shared_ptr<ActivityWaiter>;
   std::queue<PendingCall> pending_;
   std::vector<LockedMultiProducerSingleConsumerQueue> requests_per_cq_;
 };
@@ -519,21 +660,14 @@ class Server::AllocatingRequestMatcherBatch
 
   ArenaPromise<absl::StatusOr<MatchResult>> MatchRequest(
       size_t /*start_request_queue_index*/) override {
-    const bool still_running = server()->ShutdownRefOnRequest();
-    auto cleanup_ref =
-        absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); });
-    if (still_running) {
-      BatchCallAllocation call_info = allocator_();
-      GPR_ASSERT(server()->ValidateServerRequest(
-                     cq(), static_cast<void*>(call_info.tag), nullptr,
-                     nullptr) == GRPC_CALL_OK);
-      RequestedCall* rc = new RequestedCall(
-          static_cast<void*>(call_info.tag), call_info.cq, call_info.call,
-          call_info.initial_metadata, call_info.details);
-      return Immediate(MatchResult(server(), cq_idx(), rc));
-    } else {
-      return Immediate(absl::InternalError("Server shutdown"));
-    }
+    BatchCallAllocation call_info = allocator_();
+    GPR_ASSERT(server()->ValidateServerRequest(
+                   cq(), static_cast<void*>(call_info.tag), nullptr, nullptr) ==
+               GRPC_CALL_OK);
+    RequestedCall* rc = new RequestedCall(
+        static_cast<void*>(call_info.tag), call_info.cq, call_info.call,
+        call_info.initial_metadata, call_info.details);
+    return Immediate(MatchResult(server(), cq_idx(), rc));
   }
 
  private:
@@ -573,22 +707,14 @@ class Server::AllocatingRequestMatcherRegistered
 
   ArenaPromise<absl::StatusOr<MatchResult>> MatchRequest(
       size_t /*start_request_queue_index*/) override {
-    const bool still_running = server()->ShutdownRefOnRequest();
-    auto cleanup_ref =
-        absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); });
-    if (still_running) {
-      RegisteredCallAllocation call_info = allocator_();
-      GPR_ASSERT(server()->ValidateServerRequest(
-                     cq(), call_info.tag, call_info.optional_payload,
-                     registered_method_) == GRPC_CALL_OK);
-      RequestedCall* rc =
-          new RequestedCall(call_info.tag, call_info.cq, call_info.call,
-                            call_info.initial_metadata, registered_method_,
-                            call_info.deadline, call_info.optional_payload);
-      return Immediate(MatchResult(server(), cq_idx(), rc));
-    } else {
-      return Immediate(absl::InternalError("Server shutdown"));
-    }
+    RegisteredCallAllocation call_info = allocator_();
+    GPR_ASSERT(server()->ValidateServerRequest(
+                   cq(), call_info.tag, call_info.optional_payload,
+                   registered_method_) == GRPC_CALL_OK);
+    RequestedCall* rc = new RequestedCall(
+        call_info.tag, call_info.cq, call_info.call, call_info.initial_metadata,
+        registered_method_, call_info.deadline, call_info.optional_payload);
+    return Immediate(MatchResult(server(), cq_idx(), rc));
   }
 
  private:
@@ -664,6 +790,9 @@ class ChannelBroadcaster {
 const grpc_channel_filter Server::kServerTopFilter = {
     Server::CallData::StartTransportStreamOpBatch,
     Server::ChannelData::MakeCallPromise,
+    [](grpc_channel_element*, CallSpineInterface*) {
+      // TODO(ctiller): remove the server filter when call-v3 is finalized
+    },
     grpc_channel_next_op,
     sizeof(Server::CallData),
     Server::CallData::InitCallElement,
@@ -699,7 +828,13 @@ RefCountedPtr<channelz::ServerNode> CreateChannelzNode(
 }  // namespace
 
 Server::Server(const ChannelArgs& args)
-    : channel_args_(args), channelz_node_(CreateChannelzNode(args)) {}
+    : channel_args_(args),
+      channelz_node_(CreateChannelzNode(args)),
+      server_call_tracer_factory_(ServerCallTracerFactory::Get(args)),
+      max_time_in_pending_queue_(Duration::Seconds(
+          channel_args_
+              .GetInt(GRPC_ARG_SERVER_MAX_UNREQUESTED_TIME_IN_SERVER_SECONDS)
+              .value_or(30))) {}
 
 Server::~Server() {
   // Remove the cq pollsets from the config_fetcher.
@@ -719,12 +854,22 @@ void Server::AddListener(OrphanablePtr<ListenerInterface> listener) {
   channelz::ListenSocketNode* listen_socket_node =
       listener->channelz_listen_socket_node();
   if (listen_socket_node != nullptr && channelz_node_ != nullptr) {
-    channelz_node_->AddChildListenSocket(listen_socket_node->Ref());
+    channelz_node_->AddChildListenSocket(
+        listen_socket_node->RefAsSubclass<channelz::ListenSocketNode>());
   }
   listeners_.emplace_back(std::move(listener));
 }
 
 void Server::Start() {
+  auto make_real_request_matcher =
+      [this]() -> std::unique_ptr<RequestMatcherInterface> {
+    if (IsPromiseBasedServerCallEnabled()) {
+      return std::make_unique<RealRequestMatcherPromises>(this);
+    } else {
+      return std::make_unique<RealRequestMatcherFilterStack>(this);
+    }
+  };
+
   started_ = true;
   for (grpc_completion_queue* cq : cqs_) {
     if (grpc_cq_can_listen(cq)) {
@@ -732,11 +877,11 @@ void Server::Start() {
     }
   }
   if (unregistered_request_matcher_ == nullptr) {
-    unregistered_request_matcher_ = std::make_unique<RealRequestMatcher>(this);
+    unregistered_request_matcher_ = make_real_request_matcher();
   }
-  for (std::unique_ptr<RegisteredMethod>& rm : registered_methods_) {
-    if (rm->matcher == nullptr) {
-      rm->matcher = std::make_unique<RealRequestMatcher>(this);
+  for (auto& rm : registered_methods_) {
+    if (rm.second->matcher == nullptr) {
+      rm.second->matcher = make_real_request_matcher();
     }
   }
   {
@@ -762,7 +907,7 @@ void Server::Start() {
 }
 
 grpc_error_handle Server::SetupTransport(
-    grpc_transport* transport, grpc_pollset* accepting_pollset,
+    Transport* transport, grpc_pollset* accepting_pollset,
     const ChannelArgs& args,
     const RefCountedPtr<channelz::SocketNode>& socket_node) {
   // Create channel.
@@ -823,39 +968,34 @@ void Server::RegisterCompletionQueue(grpc_completion_queue* cq) {
   cqs_.push_back(cq);
 }
 
-namespace {
-
-bool streq(const std::string& a, const char* b) {
-  return (a.empty() && b == nullptr) ||
-         ((b != nullptr) && !strcmp(a.c_str(), b));
-}
-
-}  // namespace
-
 Server::RegisteredMethod* Server::RegisterMethod(
     const char* method, const char* host,
     grpc_server_register_method_payload_handling payload_handling,
     uint32_t flags) {
+  if (started_) {
+    Crash("Attempting to register method after server started");
+  }
+
   if (!method) {
     gpr_log(GPR_ERROR,
             "grpc_server_register_method method string cannot be NULL");
     return nullptr;
   }
-  for (std::unique_ptr<RegisteredMethod>& m : registered_methods_) {
-    if (streq(m->method, method) && streq(m->host, host)) {
-      gpr_log(GPR_ERROR, "duplicate registration for %s@%s", method,
-              host ? host : "*");
-      return nullptr;
-    }
+  auto key = std::make_pair(host ? host : "", method);
+  if (registered_methods_.find(key) != registered_methods_.end()) {
+    gpr_log(GPR_ERROR, "duplicate registration for %s@%s", method,
+            host ? host : "*");
+    return nullptr;
   }
   if (flags != 0) {
     gpr_log(GPR_ERROR, "grpc_server_register_method invalid flags 0x%08x",
             flags);
     return nullptr;
   }
-  registered_methods_.emplace_back(std::make_unique<RegisteredMethod>(
-      method, host, payload_handling, flags));
-  return registered_methods_.back().get();
+  auto it = registered_methods_.emplace(
+      key, std::make_unique<RegisteredMethod>(method, host, payload_handling,
+                                              flags));
+  return it.first->second.get();
 }
 
 void Server::DoneRequestEvent(void* req, grpc_cq_completion* /*c*/) {
@@ -906,9 +1046,9 @@ void Server::KillPendingWorkLocked(grpc_error_handle error) {
   if (started_) {
     unregistered_request_matcher_->KillRequests(error);
     unregistered_request_matcher_->ZombifyPending();
-    for (std::unique_ptr<RegisteredMethod>& rm : registered_methods_) {
-      rm->matcher->KillRequests(error);
-      rm->matcher->ZombifyPending();
+    for (auto& rm : registered_methods_) {
+      rm.second->matcher->KillRequests(error);
+      rm.second->matcher->ZombifyPending();
     }
   }
 }
@@ -952,7 +1092,6 @@ void DonePublishedShutdown(void* /*done_arg*/, grpc_cq_completion* storage) {
 //       connection is NOT closed until the server is done with all those calls.
 //    -- Once there are no more calls in progress, the channel is closed.
 void Server::ShutdownAndNotify(grpc_completion_queue* cq, void* tag) {
-  Notification* await_requests = nullptr;
   ChannelBroadcaster broadcaster;
   {
     // Wait for startup to be finished.  Locks mu_global.
@@ -978,12 +1117,7 @@ void Server::ShutdownAndNotify(grpc_completion_queue* cq, void* tag) {
       MutexLock lock(&mu_call_);
       KillPendingWorkLocked(GRPC_ERROR_CREATE("Server Shutdown"));
     }
-    await_requests = ShutdownUnrefOnShutdownCall();
-  }
-  // We expect no new requests but there can still be requests in-flight.
-  // Wait for them to complete before proceeding.
-  if (await_requests != nullptr) {
-    await_requests->WaitForNotification();
+    ShutdownUnrefOnShutdownCall();
   }
   StopListening();
   broadcaster.BroadcastShutdown(/*send_goaway=*/true, absl::OkStatus());
@@ -1149,7 +1283,6 @@ class Server::ChannelData::ConnectivityWatcher
 //
 
 Server::ChannelData::~ChannelData() {
-  registered_methods_.reset();
   if (server_ != nullptr) {
     if (server_->channelz_node_ != nullptr && channelz_socket_uuid_ != 0) {
       server_->channelz_node_->RemoveChildSocket(channelz_socket_uuid_);
@@ -1165,50 +1298,28 @@ Server::ChannelData::~ChannelData() {
   }
 }
 
+Arena* Server::ChannelData::CreateArena() {
+  const auto initial_size = channel_->CallSizeEstimate();
+  global_stats().IncrementCallInitialSize(initial_size);
+  return Arena::Create(initial_size, channel_->allocator());
+}
+
+absl::StatusOr<CallInitiator> Server::ChannelData::CreateCall(
+    ClientMetadata& client_initial_metadata, Arena* arena) {
+  SetRegisteredMethodOnMetadata(client_initial_metadata);
+  auto call = MakeServerCall(server_.get(), channel_.get(), arena);
+  InitCall(call);
+  return CallInitiator(std::move(call));
+}
+
 void Server::ChannelData::InitTransport(RefCountedPtr<Server> server,
                                         RefCountedPtr<Channel> channel,
-                                        size_t cq_idx,
-                                        grpc_transport* transport,
+                                        size_t cq_idx, Transport* transport,
                                         intptr_t channelz_socket_uuid) {
   server_ = std::move(server);
   channel_ = channel;
   cq_idx_ = cq_idx;
   channelz_socket_uuid_ = channelz_socket_uuid;
-  // Build a lookup table phrased in terms of mdstr's in this channels context
-  // to quickly find registered methods.
-  size_t num_registered_methods = server_->registered_methods_.size();
-  if (num_registered_methods > 0) {
-    uint32_t max_probes = 0;
-    size_t slots = 2 * num_registered_methods;
-    registered_methods_ =
-        std::make_unique<std::vector<ChannelRegisteredMethod>>(slots);
-    for (std::unique_ptr<RegisteredMethod>& rm : server_->registered_methods_) {
-      Slice host;
-      Slice method = Slice::FromExternalString(rm->method);
-      const bool has_host = !rm->host.empty();
-      if (has_host) {
-        host = Slice::FromExternalString(rm->host.c_str());
-      }
-      uint32_t hash = MixHash32(has_host ? host.Hash() : 0, method.Hash());
-      uint32_t probes = 0;
-      for (probes = 0; (*registered_methods_)[(hash + probes) % slots]
-                           .server_registered_method != nullptr;
-           probes++) {
-      }
-      if (probes > max_probes) max_probes = probes;
-      ChannelRegisteredMethod* crm =
-          &(*registered_methods_)[(hash + probes) % slots];
-      crm->server_registered_method = rm.get();
-      crm->flags = rm->flags;
-      crm->has_host = has_host;
-      if (has_host) {
-        crm->host = std::move(host);
-      }
-      crm->method = std::move(method);
-    }
-    GPR_ASSERT(slots <= UINT32_MAX);
-    registered_method_max_probes_ = max_probes;
-  }
   // Publish channel.
   {
     MutexLock lock(&server_->mu_global_);
@@ -1217,45 +1328,70 @@ void Server::ChannelData::InitTransport(RefCountedPtr<Server> server,
   }
   // Start accept_stream transport op.
   grpc_transport_op* op = grpc_make_transport_op(nullptr);
-  op->set_accept_stream = true;
-  op->set_accept_stream_fn = AcceptStream;
-  op->set_accept_stream_user_data = this;
+  int accept_stream_types = 0;
+  if (transport->filter_stack_transport() != nullptr) {
+    ++accept_stream_types;
+    op->set_accept_stream = true;
+    op->set_accept_stream_fn = AcceptStream;
+    if (IsRegisteredMethodLookupInTransportEnabled()) {
+      op->set_registered_method_matcher_fn = [](void* arg,
+                                                ClientMetadata* metadata) {
+        static_cast<ChannelData*>(arg)->SetRegisteredMethodOnMetadata(
+            *metadata);
+      };
+    }
+    op->set_accept_stream_user_data = this;
+  }
+  if (transport->server_transport() != nullptr) {
+    ++accept_stream_types;
+    transport->server_transport()->SetAcceptor(this);
+  }
+  GPR_ASSERT(accept_stream_types == 1);
   op->start_connectivity_watch = MakeOrphanable<ConnectivityWatcher>(this);
   if (server_->ShutdownCalled()) {
     op->disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
   }
-  grpc_transport_perform_op(transport, op);
+  transport->PerformOp(op);
 }
 
-Server::ChannelRegisteredMethod* Server::ChannelData::GetRegisteredMethod(
-    const grpc_slice& host, const grpc_slice& path) {
-  if (registered_methods_ == nullptr) return nullptr;
-  // TODO(ctiller): unify these two searches
+Server::RegisteredMethod* Server::ChannelData::GetRegisteredMethod(
+    const absl::string_view& host, const absl::string_view& path) {
+  if (server_->registered_methods_.empty()) return nullptr;
   // check for an exact match with host
-  uint32_t hash = MixHash32(grpc_slice_hash(host), grpc_slice_hash(path));
-  for (size_t i = 0; i <= registered_method_max_probes_; i++) {
-    ChannelRegisteredMethod* rm =
-        &(*registered_methods_)[(hash + i) % registered_methods_->size()];
-    if (rm->server_registered_method == nullptr) break;
-    if (!rm->has_host) continue;
-    if (rm->host != host) continue;
-    if (rm->method != path) continue;
-    return rm;
-  }
-  // check for a wildcard method definition (no host set)
-  hash = MixHash32(0, grpc_slice_hash(path));
-  for (size_t i = 0; i <= registered_method_max_probes_; i++) {
-    ChannelRegisteredMethod* rm =
-        &(*registered_methods_)[(hash + i) % registered_methods_->size()];
-    if (rm->server_registered_method == nullptr) break;
-    if (rm->has_host) continue;
-    if (rm->method != path) continue;
-    return rm;
+  auto it = server_->registered_methods_.find(std::make_pair(host, path));
+  if (it != server_->registered_methods_.end()) {
+    return it->second.get();
+  }
+  // check for wildcard method definition (no host set)
+  it = server_->registered_methods_.find(std::make_pair("", path));
+  if (it != server_->registered_methods_.end()) {
+    return it->second.get();
   }
   return nullptr;
 }
 
-void Server::ChannelData::AcceptStream(void* arg, grpc_transport* /*transport*/,
+void Server::ChannelData::SetRegisteredMethodOnMetadata(
+    ClientMetadata& metadata) {
+  auto* authority = metadata.get_pointer(HttpAuthorityMetadata());
+  if (authority == nullptr) {
+    authority = metadata.get_pointer(HostMetadata());
+    if (authority == nullptr) {
+      // Authority not being set is an RPC error.
+      return;
+    }
+  }
+  auto* path = metadata.get_pointer(HttpPathMetadata());
+  if (path == nullptr) {
+    // Path not being set would result in an RPC error.
+    return;
+  }
+  RegisteredMethod* method =
+      GetRegisteredMethod(authority->as_string_view(), path->as_string_view());
+  // insert in metadata
+  metadata.Set(GrpcRegisteredMethod(), method);
+}
+
+void Server::ChannelData::AcceptStream(void* arg, Transport* /*transport*/,
                                        const void* transport_server_data) {
   auto* chand = static_cast<Server::ChannelData*>(arg);
   // create a call
@@ -1286,18 +1422,97 @@ void Server::ChannelData::AcceptStream(void* arg, grpc_transport* /*transport*/,
   }
 }
 
+namespace {
+auto CancelledDueToServerShutdown() {
+  return [] {
+    return ServerMetadataFromStatus(absl::CancelledError("Server shutdown"));
+  };
+}
+}  // namespace
+
+void Server::ChannelData::InitCall(RefCountedPtr<CallSpineInterface> call) {
+  call->SpawnGuarded("request_matcher", [this, call]() {
+    return TrySeq(
+        // Wait for initial metadata to pass through all filters
+        Map(call->client_initial_metadata().receiver.Next(),
+            [](NextResult<ClientMetadataHandle> md)
+                -> absl::StatusOr<ClientMetadataHandle> {
+              if (!md.has_value()) {
+                return absl::InternalError("Missing metadata");
+              }
+              if (!md.value()->get_pointer(HttpPathMetadata())) {
+                return absl::InternalError("Missing :path header");
+              }
+              if (!md.value()->get_pointer(HttpAuthorityMetadata())) {
+                return absl::InternalError("Missing :authority header");
+              }
+              return std::move(*md);
+            }),
+        // Match request with requested call
+        [this, call](ClientMetadataHandle md) {
+          auto* registered_method = static_cast<RegisteredMethod*>(
+              md->get(GrpcRegisteredMethod()).value_or(nullptr));
+          RequestMatcherInterface* rm;
+          grpc_server_register_method_payload_handling payload_handling =
+              GRPC_SRM_PAYLOAD_NONE;
+          if (registered_method == nullptr) {
+            rm = server_->unregistered_request_matcher_.get();
+          } else {
+            rm = registered_method->matcher.get();
+          }
+          auto maybe_read_first_message = If(
+              payload_handling == GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER,
+              [call]() {
+                return call->client_to_server_messages().receiver.Next();
+              },
+              []() -> NextResult<MessageHandle> {
+                return NextResult<MessageHandle>();
+              });
+          return TryJoin<absl::StatusOr>(
+              Map(std::move(maybe_read_first_message),
+                  [](NextResult<MessageHandle> n) {
+                    return ValueOrFailure<NextResult<MessageHandle>>{
+                        std::move(n)};
+                  }),
+              rm->MatchRequest(cq_idx()), [md = std::move(md)]() mutable {
+                return ValueOrFailure<ClientMetadataHandle>(std::move(md));
+              });
+        },
+        // Publish call to cq
+        [](std::tuple<NextResult<MessageHandle>,
+                      RequestMatcherInterface::MatchResult,
+                      ClientMetadataHandle>
+               r) {
+          RequestMatcherInterface::MatchResult& mr = std::get<1>(r);
+          auto md = std::move(std::get<2>(r));
+          auto* rc = mr.TakeCall();
+          rc->Complete(std::move(std::get<0>(r)), *md);
+          auto* call_context = GetContext<CallContext>();
+          *rc->call = call_context->c_call();
+          grpc_call_set_completion_queue(call_context->c_call(),
+                                         rc->cq_bound_to_call);
+          call_context->server_call_context()->PublishInitialMetadata(
+              std::move(md), rc->initial_metadata);
+          // TODO(ctiller): publish metadata
+          return Map(WaitForCqEndOp(false, rc->tag, absl::OkStatus(), mr.cq()),
+                     [rc = std::unique_ptr<RequestedCall>(rc)](Empty) {
+                       return absl::OkStatus();
+                     });
+        });
+  });
+}
+
 ArenaPromise<ServerMetadataHandle> Server::ChannelData::MakeCallPromise(
     grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory) {
   auto* chand = static_cast<Server::ChannelData*>(elem->channel_data);
   auto* server = chand->server_.get();
-  if (server->ShutdownCalled()) {
-    return [] {
-      return ServerMetadataFromStatus(absl::InternalError("Server shutdown"));
-    };
-  }
-  absl::optional<Slice> path =
-      call_args.client_initial_metadata->Take(HttpPathMetadata());
-  if (!path.has_value()) {
+  if (server->ShutdownCalled()) return CancelledDueToServerShutdown();
+  auto cleanup_ref =
+      absl::MakeCleanup([server] { server->ShutdownUnrefOnRequest(); });
+  if (!server->ShutdownRefOnRequest()) return CancelledDueToServerShutdown();
+  auto path_ptr =
+      call_args.client_initial_metadata->get_pointer(HttpPathMetadata());
+  if (path_ptr == nullptr) {
     return [] {
       return ServerMetadataFromStatus(
           absl::InternalError("Missing :path header"));
@@ -1311,16 +1526,22 @@ ArenaPromise<ServerMetadataHandle> Server::ChannelData::MakeCallPromise(
           absl::InternalError("Missing :authority header"));
     };
   }
-  Timestamp deadline = GetContext<CallContext>()->deadline();
   // Find request matcher.
   RequestMatcherInterface* matcher;
-  ChannelRegisteredMethod* rm =
-      chand->GetRegisteredMethod(host_ptr->c_slice(), path->c_slice());
+  RegisteredMethod* rm = nullptr;
+  if (IsRegisteredMethodLookupInTransportEnabled()) {
+    rm = static_cast<RegisteredMethod*>(
+        call_args.client_initial_metadata->get(GrpcRegisteredMethod())
+            .value_or(nullptr));
+  } else {
+    rm = chand->GetRegisteredMethod(host_ptr->as_string_view(),
+                                    path_ptr->as_string_view());
+  }
   ArenaPromise<absl::StatusOr<NextResult<MessageHandle>>>
       maybe_read_first_message([] { return NextResult<MessageHandle>(); });
   if (rm != nullptr) {
-    matcher = rm->server_registered_method->matcher.get();
-    switch (rm->server_registered_method->payload_handling) {
+    matcher = rm->matcher.get();
+    switch (rm->payload_handling) {
       case GRPC_SRM_PAYLOAD_NONE:
         break;
       case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER:
@@ -1335,52 +1556,43 @@ ArenaPromise<ServerMetadataHandle> Server::ChannelData::MakeCallPromise(
     matcher = server->unregistered_request_matcher_.get();
   }
   return TrySeq(
-      TryJoin(matcher->MatchRequest(chand->cq_idx()),
-              std::move(maybe_read_first_message)),
-      [path = std::move(*path), host_ptr, deadline,
-       call_args = std::move(call_args)](
-          std::tuple<RequestMatcherInterface::MatchResult,
-                     NextResult<MessageHandle>>
-              match_result_and_payload) mutable {
-        auto& mr = std::get<0>(match_result_and_payload);
-        auto& payload = std::get<1>(match_result_and_payload);
+      std::move(maybe_read_first_message),
+      [cleanup_ref = std::move(cleanup_ref), matcher,
+       chand](NextResult<MessageHandle> payload) mutable {
+        return Map(
+            [cleanup_ref = std::move(cleanup_ref),
+             mr = matcher->MatchRequest(chand->cq_idx())]() mutable {
+              return mr();
+            },
+            [payload = std::move(payload)](
+                absl::StatusOr<RequestMatcherInterface::MatchResult> mr) mutable
+            -> absl::StatusOr<std::pair<RequestMatcherInterface::MatchResult,
+                                        NextResult<MessageHandle>>> {
+              if (!mr.ok()) return mr.status();
+              return std::make_pair(std::move(*mr), std::move(payload));
+            });
+      },
+      [call_args =
+           std::move(call_args)](std::pair<RequestMatcherInterface::MatchResult,
+                                           NextResult<MessageHandle>>
+                                     r) mutable {
+        auto& mr = r.first;
+        auto& payload = r.second;
         auto* rc = mr.TakeCall();
         auto* cq_for_new_request = mr.cq();
-        switch (rc->type) {
-          case RequestedCall::Type::BATCH_CALL:
-            GPR_ASSERT(!payload.has_value());
-            rc->data.batch.details->host = CSliceRef(host_ptr->c_slice());
-            rc->data.batch.details->method = CSliceRef(path.c_slice());
-            rc->data.batch.details->deadline =
-                deadline.as_timespec(GPR_CLOCK_MONOTONIC);
-            break;
-          case RequestedCall::Type::REGISTERED_CALL:
-            *rc->data.registered.deadline =
-                deadline.as_timespec(GPR_CLOCK_MONOTONIC);
-            if (rc->data.registered.optional_payload != nullptr) {
-              if (payload.has_value()) {
-                auto* sb = payload.value()->payload()->c_slice_buffer();
-                *rc->data.registered.optional_payload =
-                    grpc_raw_byte_buffer_create(sb->slices, sb->count);
-              } else {
-                *rc->data.registered.optional_payload = nullptr;
-              }
-            }
-            break;
-          default:
-            GPR_UNREACHABLE_CODE(abort());
-        }
-        return GetContext<CallContext>()
-            ->server_call_context()
-            ->MakeTopOfServerCallPromise(
-                std::move(call_args), rc->cq_bound_to_call,
-                rc->initial_metadata,
-                [rc, cq_for_new_request](grpc_call* call) {
-                  *rc->call = call;
-                  grpc_cq_end_op(cq_for_new_request, rc->tag, absl::OkStatus(),
-                                 Server::DoneRequestEvent, rc, &rc->completion,
-                                 true);
-                });
+        auto* server_call_context =
+            GetContext<CallContext>()->server_call_context();
+        rc->Complete(std::move(payload), *call_args.client_initial_metadata);
+        server_call_context->PublishInitialMetadata(
+            std::move(call_args.client_initial_metadata), rc->initial_metadata);
+        return server_call_context->MakeTopOfServerCallPromise(
+            std::move(call_args), rc->cq_bound_to_call,
+            [rc, cq_for_new_request](grpc_call* call) {
+              *rc->call = call;
+              grpc_cq_end_op(cq_for_new_request, rc->tag, absl::OkStatus(),
+                             Server::DoneRequestEvent, rc, &rc->completion,
+                             true);
+            });
       });
 }
 
@@ -1560,11 +1772,18 @@ void Server::CallData::StartNewRpc(grpc_call_element* elem) {
   grpc_server_register_method_payload_handling payload_handling =
       GRPC_SRM_PAYLOAD_NONE;
   if (path_.has_value() && host_.has_value()) {
-    ChannelRegisteredMethod* rm =
-        chand->GetRegisteredMethod(host_->c_slice(), path_->c_slice());
+    RegisteredMethod* rm;
+    if (IsRegisteredMethodLookupInTransportEnabled()) {
+      rm = static_cast<RegisteredMethod*>(
+          recv_initial_metadata_->get(GrpcRegisteredMethod())
+              .value_or(nullptr));
+    } else {
+      rm = chand->GetRegisteredMethod(host_->as_string_view(),
+                                      path_->as_string_view());
+    }
     if (rm != nullptr) {
-      matcher_ = rm->server_registered_method->matcher.get();
-      payload_handling = rm->server_registered_method->payload_handling;
+      matcher_ = rm->matcher.get();
+      payload_handling = rm->payload_handling;
     }
   }
   // Start recv_message op if needed.