grpc 1.63.0 → 1.71.0

data/src/core/lib/transport/call_filters.h

@@ -18,18 +18,27 @@
 #include <grpc/support/port_platform.h>
 
 #include <cstdint>
+#include <limits>
 #include <memory>
+#include <ostream>
 #include <type_traits>
 
-#include "src/core/lib/gprpp/ref_counted.h"
-#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "absl/log/check.h"
+#include "src/core/lib/promise/for_each.h"
+#include "src/core/lib/promise/if.h"
 #include "src/core/lib/promise/latch.h"
 #include "src/core/lib/promise/map.h"
 #include "src/core/lib/promise/promise.h"
+#include "src/core/lib/promise/seq.h"
 #include "src/core/lib/promise/status_flag.h"
+#include "src/core/lib/promise/try_seq.h"
 #include "src/core/lib/transport/call_final_info.h"
+#include "src/core/lib/transport/call_state.h"
 #include "src/core/lib/transport/message.h"
 #include "src/core/lib/transport/metadata.h"
+#include "src/core/util/dump_args.h"
+#include "src/core/util/ref_counted.h"
+#include "src/core/util/ref_counted_ptr.h"
 
 // CallFilters tracks a list of filters that are attached to a call.
 // At a high level, a filter (for the purposes of this module) is a class
@@ -41,6 +50,7 @@
 // - OnServerInitialMetadata  - $VALUE_TYPE = ServerMetadata
 // - OnServerToClientMessage  - $VALUE_TYPE = Message
 // - OnClientToServerMessage  - $VALUE_TYPE = Message
+// - OnClientToServerHalfClose - no value
 // - OnServerTrailingMetadata - $VALUE_TYPE = ServerMetadata
 // - OnFinalize               - special, see below
 // These members define an interception point for a particular event in
@@ -48,7 +58,7 @@
 //
 // The type of these members matters, and is selectable by the class
 // author. For $INTERCEPTOR_NAME in the above list:
-// - static const NoInterceptor $INTERCEPTOR_NAME:
+// - static inline const NoInterceptor $INTERCEPTOR_NAME:
 //   defines that this filter does not intercept this event.
 //   there is zero runtime cost added to handling that event by this filter.
 // - void $INTERCEPTOR_NAME($VALUE_TYPE&):
@@ -81,7 +91,7 @@
 //
 // Finally, OnFinalize is added to intecept call finalization.
 // It must have one of the signatures:
-// - static const NoInterceptor OnFinalize:
+// - static inline const NoInterceptor OnFinalize:
 //   the filter does not intercept call finalization.
 // - void OnFinalize(const grpc_call_final_info*):
 //   the filter intercepts call finalization.
@@ -101,17 +111,114 @@ namespace grpc_core {
 // Tag type to indicate no interception.
 // This is used to indicate that a filter does not intercept a particular
 // event.
-// In C++14 we declare these as (for example):
-//   static const NoInterceptor OnClientInitialMetadata;
-// and out-of-line provide the definition:
-//   const MyFilter::Call::NoInterceptor
-//   MyFilter::Call::OnClientInitialMetadata;
-// In C++17 and later we can use inline variables instead:
-//   inline static const NoInterceptor OnClientInitialMetadata;
+// We declare these as (for example):
+//   inline static inline const NoInterceptor OnClientInitialMetadata;
 struct NoInterceptor {};
 
 namespace filters_detail {
 
+// Flow control across pipe stages.
+// This ends up being exceedingly subtle - essentially we need to ensure that
+// across a series of pipes we have no more than one outstanding message at a
+// time - but those pipes are for the most part independent.
+// How we achieve this is that this NextMessage object holds both the message
+// and a completion token - the last owning NextMessage instance will call
+// the on_progress method on the referenced CallState - and at that point that
+// CallState will allow the next message to be sent through it.
+// Next, the ForEach promise combiner explicitly holds onto the wrapper object
+// owning the result (this object) and extracts the message from it, but doesn't
+// dispose that instance until the action promise for the ForEach iteration
+// completes, ensuring most callers need do nothing special to have the
+// flow control work correctly.
+template <void (CallState::*on_progress)()>
+class NextMessage {
+ public:
+  ~NextMessage() {
+    if (message_ != end_of_stream() && message_ != error() &&
+        message_ != taken()) {
+      delete message_;
+    }
+    if (call_state_ != nullptr) {
+      (call_state_->*on_progress)();
+    }
+  }
+
+  NextMessage() = default;
+  explicit NextMessage(Failure) : message_(error()), call_state_(nullptr) {}
+  NextMessage(MessageHandle message, CallState* call_state) {
+    DCHECK_NE(call_state, nullptr);
+    DCHECK_NE(message.get(), nullptr);
+    DCHECK(message.get_deleter().has_freelist());
+    message_ = message.release();
+    call_state_ = call_state;
+  }
+  NextMessage(const NextMessage& other) = delete;
+  NextMessage& operator=(const NextMessage& other) = delete;
+  NextMessage(NextMessage&& other) noexcept
+      : message_(std::exchange(other.message_, taken())),
+        call_state_(std::exchange(other.call_state_, nullptr)) {}
+  NextMessage& operator=(NextMessage&& other) noexcept {
+    if (message_ != end_of_stream() && message_ != error() &&
+        message_ != taken()) {
+      delete message_;
+    }
+    if (call_state_ != nullptr) {
+      (call_state_->*on_progress)();
+    }
+    message_ = std::exchange(other.message_, taken());
+    call_state_ = std::exchange(other.call_state_, nullptr);
+    return *this;
+  }
+
+  bool ok() const {
+    DCHECK_NE(message_, taken());
+    return message_ != error();
+  }
+  bool has_value() const {
+    DCHECK_NE(message_, taken());
+    DCHECK(ok());
+    return message_ != end_of_stream();
+  }
+  StatusFlag status() const { return StatusFlag(ok()); }
+  Message& value() {
+    DCHECK_NE(message_, taken());
+    DCHECK(ok());
+    DCHECK(has_value());
+    return *message_;
+  }
+  MessageHandle TakeValue() {
+    DCHECK_NE(message_, taken());
+    DCHECK(ok());
+    DCHECK(has_value());
+    return MessageHandle(std::exchange(message_, taken()),
+                         Arena::PooledDeleter());
+  }
+  bool progressed() const { return call_state_ == nullptr; }
+  void Progress() {
+    DCHECK(!progressed());
+    (call_state_->*on_progress)();
+    call_state_ = nullptr;
+  }
+
+ private:
+  static Message* end_of_stream() { return nullptr; }
+  static Message* error() { return reinterpret_cast<Message*>(1); }
+  static Message* taken() { return reinterpret_cast<Message*>(2); }
+  Message* message_ = end_of_stream();
+  CallState* call_state_ = nullptr;
+};
+
+template <typename T>
+struct ArgumentMustBeNextMessage;
+template <void (CallState::*on_progress)()>
+struct ArgumentMustBeNextMessage<NextMessage<on_progress>> {
+  static constexpr bool value() { return true; }
+};
+
+inline void* Offset(void* base, size_t amt) {
+  return static_cast<char*>(base) + amt;
+}
+
 // One call filter constructor
 // Contains enough information to allocate and initialize the
 // call data for one filter.
@@ -155,16 +262,19 @@ struct CallConstructor<FilterType,
 // Only one pointer can be set.
 template <typename T>
 struct ResultOr {
+  ResultOr(T ok, ServerMetadataHandle error)
+      : ok(std::move(ok)), error(std::move(error)) {
+    CHECK((this->ok == nullptr) ^ (this->error == nullptr));
+  }
   T ok;
   ServerMetadataHandle error;
 };
 
 // One filter operation metadata
-// Given a value of type V, produces a promise of type R.
-template <typename R, typename V>
+// Given a value of type T, produces a promise of type ResultOr<T>.
+template <typename T>
 struct Operator {
-  using Result = R;
-  using Arg = V;
+  using Arg = T;
   // Pointer to corresponding channel data for this filter
   void* channel_data;
   // Offset of the call data for this filter within the call data memory
@@ -172,13 +282,13 @@ struct Operator {
   // Initialize the promise data for this filter, and poll once.
   // Return the result of the poll.
   // If the promise finishes, also destroy the promise data!
-  Poll<R> (*promise_init)(void* promise_data, void* call_data,
-                          void* channel_data, V value);
+  Poll<ResultOr<T>> (*promise_init)(void* promise_data, void* call_data,
+                                    void* channel_data, T value);
   // Poll the promise data for this filter.
   // If the promise finishes, also destroy the promise data!
   // Note that if the promise always finishes on the first poll, then supplying
   // this method is unnecessary (as it will never be called).
-  Poll<R> (*poll)(void* promise_data);
+  Poll<ResultOr<T>> (*poll)(void* promise_data);
   // Destroy the promise data for this filter for an in-progress operation
   // before the promise finishes.
   // Note that if the promise always finishes on the first poll, then supplying
@@ -186,22 +296,39 @@ struct Operator {
   void (*early_destroy)(void* promise_data);
 };
 
-// We divide operations into fallible and infallible.
-// Fallible operations can fail, and that failure terminates the call.
-// Infallible operations cannot fail.
-// Fallible operations are used for client initial, and server initial metadata,
-// and messages.
-// Infallible operations are used for server trailing metadata.
-// (This is because server trailing metadata occurs when the call is finished -
-// and so we couldn't possibly become more finished - and also because it's the
-// preferred representation of failure anyway!)
-
-// An operation that could fail: takes a T argument, produces a ResultOr<T>
-template <typename T>
-using FallibleOperator = Operator<ResultOr<T>, T>;
-// And one that cannot: takes a T argument, produces a T
-template <typename T>
-using InfallibleOperator = Operator<T, T>;
+struct HalfCloseOperator {
+  // Pointer to corresponding channel data for this filter
+  void* channel_data;
+  // Offset of the call data for this filter within the call data memory
+  size_t call_offset;
+  void (*half_close)(void* call_data, void* channel_data);
+};
+
+struct ServerTrailingMetadataOperator {
+  // Pointer to corresponding channel data for this filter
+  void* channel_data;
+  // Offset of the call data for this filter within the call data memory
+  size_t call_offset;
+  ServerMetadataHandle (*server_trailing_metadata)(
+      void* call_data, void* channel_data, ServerMetadataHandle metadata);
+};
+
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline void RunHalfClose(
+    absl::Span<const HalfCloseOperator> ops, void* call_data) {
+  for (const auto& op : ops) {
+    op.half_close(Offset(call_data, op.call_offset), op.channel_data);
+  }
+}
+
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline ServerMetadataHandle
+RunServerTrailingMetadata(absl::Span<const ServerTrailingMetadataOperator> ops,
+                          void* call_data, ServerMetadataHandle md) {
+  for (auto& op : ops) {
+    md = op.server_trailing_metadata(Offset(call_data, op.call_offset),
+                                     op.channel_data, std::move(md));
+  }
+  return md;
+}
 
 // One call finalizer
 struct Finalizer {
@@ -213,19 +340,20 @@ struct Finalizer {
 
 // A layout of operations for a given filter stack
 // This includes which operations, how much memory is required, what alignment.
-template <typename Op>
+template <typename T>
 struct Layout {
   size_t promise_size = 0;
   size_t promise_alignment = 0;
-  std::vector<Op> ops;
+  std::vector<Operator<T>> ops;
 
-  void Add(size_t filter_promise_size, size_t filter_promise_alignment, Op op) {
+  void Add(size_t filter_promise_size, size_t filter_promise_alignment,
+           Operator<T> op) {
     promise_size = std::max(promise_size, filter_promise_size);
     promise_alignment = std::max(promise_alignment, filter_promise_alignment);
     ops.push_back(op);
   }
 
-  void Reverse() { std::reverse(ops.begin(), ops.end()); }
+  void Reverse() { absl::c_reverse(ops); }
 };
 
 // AddOp and friends
@@ -245,27 +373,104 @@ struct AddOpImpl;
 
 template <typename FunctionImpl, FunctionImpl impl, typename FilterType,
           typename T>
-void AddOp(FilterType* channel_data, size_t call_offset,
-           Layout<FallibleOperator<T>>& to) {
+void AddOp(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
   AddOpImpl<FilterType, T, FunctionImpl, impl>::Add(channel_data, call_offset,
                                                     to);
 }
 
-template <typename FunctionImpl, FunctionImpl impl, typename FilterType,
-          typename T>
-void AddOp(FilterType* channel_data, size_t call_offset,
-           Layout<InfallibleOperator<T>>& to) {
-  AddOpImpl<FilterType, T, FunctionImpl, impl>::Add(channel_data, call_offset,
-                                                    to);
+template <typename FilterType>
+void AddHalfClose(FilterType* channel_data, size_t call_offset,
+                  void (FilterType::Call::*)(),
+                  std::vector<HalfCloseOperator>& to) {
+  to.push_back(
+      HalfCloseOperator{channel_data, call_offset, [](void* call_data, void*) {
+                          static_cast<typename FilterType::Call*>(call_data)
+                              ->OnClientToServerHalfClose();
+                        }});
 }
 
+template <typename FilterType>
+void AddHalfClose(FilterType* channel_data, size_t call_offset,
+                  void (FilterType::Call::*)(FilterType*),
+                  std::vector<HalfCloseOperator>& to) {
+  to.push_back(HalfCloseOperator{
+      channel_data, call_offset, [](void* call_data, void* channel_data) {
+        static_cast<typename FilterType::Call*>(call_data)
+            ->OnClientToServerHalfClose(static_cast<FilterType*>(channel_data));
+      }});
+}
+
+template <typename FilterType>
+void AddHalfClose(FilterType*, size_t, const NoInterceptor*,
+                  std::vector<HalfCloseOperator>&) {}
+
+template <typename FilterType>
+void AddServerTrailingMetadata(
+    FilterType* channel_data, size_t call_offset,
+    void (FilterType::Call::*)(ServerMetadata&),
+    std::vector<ServerTrailingMetadataOperator>& to) {
+  to.push_back(ServerTrailingMetadataOperator{
+      channel_data, call_offset,
+      [](void* call_data, void*, ServerMetadataHandle metadata) {
+        static_cast<typename FilterType::Call*>(call_data)
+            ->OnServerTrailingMetadata(*metadata);
+        return metadata;
+      }});
+}
+
+template <typename FilterType>
+void AddServerTrailingMetadata(
+    FilterType* channel_data, size_t call_offset,
+    void (FilterType::Call::*)(ServerMetadata&, FilterType*),
+    std::vector<ServerTrailingMetadataOperator>& to) {
+  to.push_back(ServerTrailingMetadataOperator{
+      channel_data, call_offset,
+      [](void* call_data, void* channel_data, ServerMetadataHandle metadata) {
+        static_cast<typename FilterType::Call*>(call_data)
+            ->OnServerTrailingMetadata(*metadata,
+                                       static_cast<FilterType*>(channel_data));
+        return metadata;
+      }});
+}
+
+template <typename FilterType>
+void AddServerTrailingMetadata(
+    FilterType* channel_data, size_t call_offset,
+    absl::Status (FilterType::Call::*)(ServerMetadata&),
+    std::vector<ServerTrailingMetadataOperator>& to) {
+  to.push_back(ServerTrailingMetadataOperator{
+      channel_data, call_offset,
+      [](void* call_data, void*, ServerMetadataHandle metadata) {
+        auto r = static_cast<typename FilterType::Call*>(call_data)
+                     ->OnServerTrailingMetadata(*metadata);
+        if (r.ok()) return metadata;
+        return CancelledServerMetadataFromStatus(r);
+      }});
+}
+
+template <typename FilterType>
+void AddServerTrailingMetadata(
+    FilterType* channel_data, size_t call_offset,
+    ServerMetadataHandle (FilterType::Call::*)(ServerMetadataHandle),
+    std::vector<ServerTrailingMetadataOperator>& to) {
+  to.push_back(ServerTrailingMetadataOperator{
+      channel_data, call_offset,
+      [](void* call_data, void*, ServerMetadataHandle metadata) {
+        return static_cast<typename FilterType::Call*>(call_data)
+            ->OnServerTrailingMetadata(std::move(metadata));
+      }});
+}
+
+template <typename FilterType>
+void AddServerTrailingMetadata(FilterType*, size_t, const NoInterceptor*,
+                               std::vector<ServerTrailingMetadataOperator>&) {}
+
 // const NoInterceptor $EVENT
 // These do nothing, and specifically DO NOT add an operation to the layout.
 // Supported for fallible & infallible operations.
 template <typename FilterType, typename T, const NoInterceptor* which>
 struct AddOpImpl<FilterType, T, const NoInterceptor*, which> {
-  static void Add(FilterType*, size_t, Layout<FallibleOperator<T>>&) {}
-  static void Add(FilterType*, size_t, Layout<InfallibleOperator<T>>&) {}
+  static void Add(FilterType*, size_t, Layout<T>&) {}
 };
 
 // void $INTERCEPTOR_NAME($VALUE_TYPE&)
@@ -273,10 +478,9 @@ template <typename FilterType, typename T,
           void (FilterType::Call::*impl)(typename T::element_type&)>
 struct AddOpImpl<FilterType, T,
                  void (FilterType::Call::*)(typename T::element_type&), impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(0, 0,
-           FallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
                [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
@@ -288,16 +492,23 @@ struct AddOpImpl<FilterType, T,
                nullptr,
            });
   }
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<InfallibleOperator<T>>& to) {
+};
+
+// void $INTERCEPTOR_NAME(const $VALUE_TYPE&)
+template <typename FilterType, typename T,
+          void (FilterType::Call::*impl)(const typename T::element_type&)>
+struct AddOpImpl<FilterType, T,
+                 void (FilterType::Call::*)(const typename T::element_type&),
+                 impl> {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(0, 0,
-           InfallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
-               [](void*, void* call_data, void*, T value) -> Poll<T> {
+               [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
                  (static_cast<typename FilterType::Call*>(call_data)->*impl)(
                      *value);
-                 return std::move(value);
+                 return ResultOr<T>{std::move(value), nullptr};
                },
                nullptr,
                nullptr,
@@ -312,10 +523,9 @@ template <typename FilterType, typename T,
 struct AddOpImpl<
     FilterType, T,
     void (FilterType::Call::*)(typename T::element_type&, FilterType*), impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(0, 0,
-           FallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
                [](void*, void* call_data, void* channel_data,
@@ -328,21 +538,30 @@ struct AddOpImpl<
                nullptr,
            });
   }
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<InfallibleOperator<T>>& to) {
-    to.Add(
-        0, 0,
-        InfallibleOperator<T>{
-            channel_data,
-            call_offset,
-            [](void*, void* call_data, void* channel_data, T value) -> Poll<T> {
-              (static_cast<typename FilterType::Call*>(call_data)->*impl)(
-                  *value, static_cast<FilterType*>(channel_data));
-              return std::move(value);
-            },
-            nullptr,
-            nullptr,
-        });
+};
+
+// void $INTERCEPTOR_NAME(const $VALUE_TYPE&, FilterType*)
+template <typename FilterType, typename T,
+          void (FilterType::Call::*impl)(const typename T::element_type&,
+                                         FilterType*)>
+struct AddOpImpl<FilterType, T,
+                 void (FilterType::Call::*)(const typename T::element_type&,
+                                            FilterType*),
+                 impl> {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
+    to.Add(0, 0,
+           Operator<T>{
+               channel_data,
+               call_offset,
+               [](void*, void* call_data, void* channel_data,
+                  T value) -> Poll<ResultOr<T>> {
+                 (static_cast<typename FilterType::Call*>(call_data)->*impl)(
+                     *value, static_cast<FilterType*>(channel_data));
+                 return ResultOr<T>{std::move(value), nullptr};
+               },
+               nullptr,
+               nullptr,
+           });
   }
 };
 
@@ -350,11 +569,10 @@ struct AddOpImpl<
 template <typename FilterType, typename T,
           T (FilterType::Call::*impl)(T, FilterType*)>
 struct AddOpImpl<FilterType, T, T (FilterType::Call::*)(T, FilterType*), impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -368,24 +586,6 @@ struct AddOpImpl<FilterType, T, T (FilterType::Call::*)(T, FilterType*), impl> {
             nullptr,
         });
   }
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<InfallibleOperator<T>>& to) {
-    to.Add(
-        0, 0,
-        InfallibleOperator<T>{
-            channel_data,
-            call_offset,
-            [](void*, void* call_data, void* channel_data, T value) -> Poll<T> {
-              (static_cast<typename FilterType::Call*>(call_data)->*impl)(
-                  *value, static_cast<FilterType*>(channel_data));
-              return (
-                  static_cast<typename FilterType::Call*>(call_data)->*impl)(
-                  std::move(value), static_cast<FilterType*>(channel_data));
-            },
-            nullptr,
-            nullptr,
-        });
-  }
 };
 
 // absl::Status $INTERCEPTOR_NAME($VALUE_TYPE&)
@@ -394,11 +594,10 @@ template <typename FilterType, typename T,
 struct AddOpImpl<FilterType, T,
                  absl::Status (FilterType::Call::*)(typename T::element_type&),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
@@ -413,24 +612,6 @@ struct AddOpImpl<FilterType, T,
             nullptr,
         });
   }
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<InfallibleOperator<T>>& to) {
-    to.Add(
-        0, 0,
-        InfallibleOperator<T>{
-            channel_data,
-            call_offset,
-            [](void*, void* call_data, void*, T value) -> Poll<T> {
-              auto r =
-                  (static_cast<typename FilterType::Call*>(call_data)->*impl)(
-                      *value);
-              if (r.ok()) return std::move(value);
-              return StatusCast<ServerMetadataHandle>(std::move(r));
-            },
-            nullptr,
-            nullptr,
-        });
-  }
 };
 
 // absl::Status $INTERCEPTOR_NAME(const $VALUE_TYPE&)
@@ -440,11 +621,10 @@ template <typename FilterType, typename T,
 struct AddOpImpl<
     FilterType, T,
     absl::Status (FilterType::Call::*)(const typename T::element_type&), impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
@@ -469,11 +649,10 @@ struct AddOpImpl<FilterType, T,
                  absl::Status (FilterType::Call::*)(typename T::element_type&,
                                                     FilterType*),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -499,11 +678,10 @@ struct AddOpImpl<FilterType, T,
                  absl::Status (FilterType::Call::*)(
                      const typename T::element_type&, FilterType*),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -527,11 +705,10 @@ template <typename FilterType, typename T,
 struct AddOpImpl<FilterType, T,
                  absl::StatusOr<T> (FilterType::Call::*)(T, FilterType*),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -557,11 +734,10 @@ struct AddOpImpl<FilterType, T,
                  ServerMetadataHandle (FilterType::Call::*)(
                      typename T::element_type&),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
@@ -586,11 +762,10 @@ struct AddOpImpl<FilterType, T,
                  ServerMetadataHandle (FilterType::Call::*)(
                      const typename T::element_type&),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void*, T value) -> Poll<ResultOr<T>> {
@@ -615,11 +790,10 @@ struct AddOpImpl<FilterType, T,
                  ServerMetadataHandle (FilterType::Call::*)(
                      typename T::element_type&, FilterType*),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -645,11 +819,10 @@ struct AddOpImpl<FilterType, T,
                  ServerMetadataHandle (FilterType::Call::*)(
                      const typename T::element_type&, FilterType*),
                  impl> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     to.Add(
         0, 0,
-        FallibleOperator<T>{
+        Operator<T>{
             channel_data,
             call_offset,
             [](void*, void* call_data, void* channel_data,
@@ -673,8 +846,7 @@ template <typename FilterType, typename T, typename R,
 struct AddOpImpl<
     FilterType, T, R (FilterType::Call::*)(typename T::element_type&), impl,
     absl::enable_if_t<std::is_same<absl::Status, PromiseResult<R>>::value>> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     class Promise {
      public:
       Promise(T value, typename FilterType::Call* call_data, FilterType*)
@@ -689,7 +861,7 @@ struct AddOpImpl<
         if (r->ok()) {
           return ResultOr<T>{std::move(value), nullptr};
         }
-        return ResultOr<T>{nullptr, ServerMetadataFromStatus(*r)};
+        return ResultOr<T>{nullptr, CancelledServerMetadataFromStatus(*r)};
       }
 
      private:
@@ -697,7 +869,7 @@ struct AddOpImpl<
       GPR_NO_UNIQUE_ADDRESS R impl_;
     };
     to.Add(sizeof(Promise), alignof(Promise),
-           FallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
                [](void* promise_data, void* call_data, void* channel_data,
@@ -726,8 +898,7 @@ struct AddOpImpl<
     R (FilterType::Call::*)(typename T::element_type&, FilterType*), impl,
     absl::enable_if_t<!std::is_same<R, absl::Status>::value &&
                       std::is_same<absl::Status, PromiseResult<R>>::value>> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
     class Promise {
      public:
       Promise(T value, typename FilterType::Call* call_data,
@@ -744,7 +915,7 @@ struct AddOpImpl<
         if (r->ok()) {
           return ResultOr<T>{std::move(value), nullptr};
         }
-        return ResultOr<T>{nullptr, ServerMetadataFromStatus(*r)};
+        return ResultOr<T>{nullptr, CancelledServerMetadataFromStatus(*r)};
       }
 
      private:
@@ -752,7 +923,7 @@ struct AddOpImpl<
       GPR_NO_UNIQUE_ADDRESS R impl_;
     };
     to.Add(sizeof(Promise), alignof(Promise),
-           FallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
                [](void* promise_data, void* call_data, void* channel_data,
@@ -780,8 +951,31 @@ template <typename FilterType, typename T, typename R,
 struct AddOpImpl<FilterType, T, R (FilterType::Call::*)(T, FilterType*), impl,
                  absl::enable_if_t<std::is_same<absl::StatusOr<T>,
                                                 PromiseResult<R>>::value>> {
-  static void Add(FilterType* channel_data, size_t call_offset,
-                  Layout<FallibleOperator<T>>& to) {
+  static void Add(FilterType* channel_data, size_t call_offset, Layout<T>& to) {
+#if defined(__GNUC__) && __GNUC__ == 9
+    // Workaround for a bug in GNU C++ 9 compilers that fail to compile this
+    // class.
+    class Promise {
+     public:
+      Promise(T value, typename FilterType::Call* call_data,
+              FilterType* channel_data)
+          : impl_(std::make_unique<R>(
+                (call_data->*impl)(std::move(value), channel_data))) {}
+
+      Poll<ResultOr<T>> PollOnce() {
+        auto p = (*impl_)();
+        auto* r = p.value_if_ready();
+        if (r == nullptr) return Pending{};
+        this->~Promise();
+        if (r->ok()) return ResultOr<T>{std::move(**r), nullptr};
+        return ResultOr<T>{nullptr,
+                           CancelledServerMetadataFromStatus(r->status())};
+      }
+
+     private:
+      std::unique_ptr<R> impl_;
+    };
+#else
     class Promise {
      public:
       Promise(T value, typename FilterType::Call* call_data,
@@ -794,14 +988,16 @@ struct AddOpImpl<FilterType, T, R (FilterType::Call::*)(T, FilterType*), impl,
         if (r == nullptr) return Pending{};
         this->~Promise();
         if (r->ok()) return ResultOr<T>{std::move(**r), nullptr};
-        return ResultOr<T>{nullptr, ServerMetadataFromStatus(r->status())};
+        return ResultOr<T>{nullptr,
+                           CancelledServerMetadataFromStatus(r->status())};
       }
 
      private:
       GPR_NO_UNIQUE_ADDRESS R impl_;
     };
+#endif
     to.Add(sizeof(Promise), alignof(Promise),
-           FallibleOperator<T>{
+           Operator<T>{
                channel_data,
                call_offset,
                [](void* promise_data, void* call_data, void* channel_data,
@@ -843,11 +1039,12 @@ struct StackData {
   // For each kind of operation, a layout of the operations for this call.
   // (there's some duplicate data here, and that's ok: we want to avoid
   // pointer chasing as much as possible when executing a call)
-  Layout<FallibleOperator<ClientMetadataHandle>> client_initial_metadata;
-  Layout<FallibleOperator<ServerMetadataHandle>> server_initial_metadata;
-  Layout<FallibleOperator<MessageHandle>> client_to_server_messages;
-  Layout<FallibleOperator<MessageHandle>> server_to_client_messages;
-  Layout<InfallibleOperator<ServerMetadataHandle>> server_trailing_metadata;
+  Layout<ClientMetadataHandle> client_initial_metadata;
+  Layout<ServerMetadataHandle> server_initial_metadata;
+  Layout<MessageHandle> client_to_server_messages;
+  std::vector<HalfCloseOperator> client_to_server_half_close;
+  Layout<MessageHandle> server_to_client_messages;
+  std::vector<ServerTrailingMetadataOperator> server_trailing_metadata;
   // A list of finalizers for this call.
   // We use a bespoke data structure here because finalizers can never be
   // asynchronous.
@@ -856,6 +1053,17 @@ struct StackData {
   // (to capture ownership of channel data)
   std::vector<ChannelDataDestructor> channel_data_destructors;
 
+  bool empty() const {
+    return filter_constructor.empty() && filter_destructor.empty() &&
+           client_initial_metadata.ops.empty() &&
+           server_initial_metadata.ops.empty() &&
+           client_to_server_messages.ops.empty() &&
+           client_to_server_half_close.empty() &&
+           server_to_client_messages.ops.empty() &&
+           server_trailing_metadata.empty() && finalizers.empty() &&
+           channel_data_destructors.empty();
+  }
+
   // Add one filter to the list of filters, and update alignment.
   // Returns the offset of the call data for this filter.
   // Specifically does not update any of the layouts or finalizers.
@@ -922,7 +1130,7 @@ struct StackData {
     filter_destructor.push_back(FilterDestructor{
         call_offset,
         [](void* call_data) {
-          static_cast<typename FilterType::Call*>(call_data)->~Call();
+          Destruct(static_cast<typename FilterType::Call*>(call_data));
         },
     });
   }
@@ -966,6 +1174,14 @@ struct StackData {
         channel_data, call_offset, client_to_server_messages);
   }
 
+  template <typename FilterType>
+  void AddClientToServerHalfClose(FilterType* channel_data,
+                                  size_t call_offset) {
+    AddHalfClose(channel_data, call_offset,
+                 &FilterType::Call::OnClientToServerHalfClose,
+                 client_to_server_half_close);
+  }
+
   template <typename FilterType>
   void AddServerToClientMessageOp(FilterType* channel_data,
                                   size_t call_offset) {
@@ -977,22 +1193,22 @@ struct StackData {
   template <typename FilterType>
   void AddServerTrailingMetadataOp(FilterType* channel_data,
                                    size_t call_offset) {
-    AddOp<decltype(&FilterType::Call::OnServerTrailingMetadata),
-          &FilterType::Call::OnServerTrailingMetadata>(
-        channel_data, call_offset, server_trailing_metadata);
+    AddServerTrailingMetadata(channel_data, call_offset,
+                              &FilterType::Call::OnServerTrailingMetadata,
+                              server_trailing_metadata);
   }
 
   // Finalizer interception adders
 
   template <typename FilterType>
   void AddFinalizer(FilterType*, size_t, const NoInterceptor* p) {
-    GPR_DEBUG_ASSERT(p == &FilterType::Call::OnFinalize);
+    DCHECK(p == &FilterType::Call::OnFinalize);
   }
 
   template <typename FilterType>
   void AddFinalizer(FilterType* channel_data, size_t call_offset,
                     void (FilterType::Call::*p)(const grpc_call_final_info*)) {
-    GPR_DEBUG_ASSERT(p == &FilterType::Call::OnFinalize);
+    DCHECK(p == &FilterType::Call::OnFinalize);
     finalizers.push_back(Finalizer{
         channel_data,
         call_offset,
@@ -1007,7 +1223,7 @@ struct StackData {
   void AddFinalizer(FilterType* channel_data, size_t call_offset,
                     void (FilterType::Call::*p)(const grpc_call_final_info*,
                                                 FilterType*)) {
-    GPR_DEBUG_ASSERT(p == &FilterType::Call::OnFinalize);
+    DCHECK(p == &FilterType::Call::OnFinalize);
     finalizers.push_back(Finalizer{
         channel_data,
         call_offset,
@@ -1037,11 +1253,11 @@ class OperationExecutor {
   OperationExecutor(OperationExecutor&& other) noexcept
       : ops_(other.ops_), end_ops_(other.end_ops_) {
     // Movable iff we're not running.
-    GPR_DEBUG_ASSERT(other.promise_data_ == nullptr);
+    DCHECK_EQ(other.promise_data_, nullptr);
   }
   OperationExecutor& operator=(OperationExecutor&& other) noexcept {
-    GPR_DEBUG_ASSERT(other.promise_data_ == nullptr);
-    GPR_DEBUG_ASSERT(promise_data_ == nullptr);
+    DCHECK_EQ(other.promise_data_, nullptr);
+    DCHECK_EQ(promise_data_, nullptr);
     ops_ = other.ops_;
     end_ops_ = other.end_ops_;
     return *this;
@@ -1051,8 +1267,7 @@ class OperationExecutor {
   // Start executing a layout. May allocate space to store the relevant promise.
   // Returns the result of the first poll.
   // If the promise finishes, also destroy the promise data.
-  Poll<ResultOr<T>> Start(const Layout<FallibleOperator<T>>* layout, T input,
-                          void* call_data);
+  Poll<ResultOr<T>> Start(const Layout<T>* layout, T input, void* call_data);
   // Continue executing a layout. Returns the result of the next poll.
   // If the promise finishes, also destroy the promise data.
   Poll<ResultOr<T>> Step(void* call_data);
@@ -1074,145 +1289,91 @@ class OperationExecutor {
   Poll<ResultOr<T>> ContinueStep(void* call_data);
 
   void* promise_data_ = nullptr;
-  const FallibleOperator<T>* ops_;
-  const FallibleOperator<T>* end_ops_;
+  const Operator<T>* ops_;
+  const Operator<T>* end_ops_;
 };
 
-// Per OperationExecutor, but for infallible operation sequences.
 template <typename T>
-class InfallibleOperationExecutor {
- public:
-  InfallibleOperationExecutor() = default;
-  ~InfallibleOperationExecutor();
-  InfallibleOperationExecutor(const InfallibleOperationExecutor&) = delete;
-  InfallibleOperationExecutor& operator=(const InfallibleOperationExecutor&) =
-      delete;
-  InfallibleOperationExecutor(InfallibleOperationExecutor&& other) noexcept
-      : ops_(other.ops_), end_ops_(other.end_ops_) {
-    // Movable iff we're not running.
-    GPR_DEBUG_ASSERT(other.promise_data_ == nullptr);
-  }
-  InfallibleOperationExecutor& operator=(
-      InfallibleOperationExecutor&& other) noexcept {
-    GPR_DEBUG_ASSERT(other.promise_data_ == nullptr);
-    GPR_DEBUG_ASSERT(promise_data_ == nullptr);
-    ops_ = other.ops_;
-    end_ops_ = other.end_ops_;
-    return *this;
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline OperationExecutor<
+    T>::~OperationExecutor() {
+  if (promise_data_ != nullptr) {
+    ops_->early_destroy(promise_data_);
+    gpr_free_aligned(promise_data_);
   }
+}
 
-  // IsRunning() is true if we're currently executing a sequence of operations.
-  bool IsRunning() const { return promise_data_ != nullptr; }
-  // Start executing a layout. May allocate space to store the relevant promise.
-  // Returns the result of the first poll.
-  // If the promise finishes, also destroy the promise data.
-  Poll<T> Start(const Layout<InfallibleOperator<T>>* layout, T input,
-                void* call_data);
-  // Continue executing a layout. Returns the result of the next poll.
-  // If the promise finishes, also destroy the promise data.
-  Poll<T> Step(void* call_data);
-
- private:
-  // Start polling on the current step of the layout.
-  // `input` is the current value (either the input to the first step, or the
-  // so far transformed value)
-  // `call_data` is the call data for the filter stack.
-  // If this op finishes immediately then we iterative move to the next step.
-  // If we reach the end up the ops, we return the overall poll result,
-  // otherwise we return Pending.
-  Poll<T> InitStep(T input, void* call_data);
-  // Continue polling on the current step of the layout.
-  // Called on the next poll after InitStep returns pending.
-  // If the promise is still pending, returns this.
-  // If the promise completes we call into InitStep to continue execution
-  // through the filters.
-  Poll<T> ContinueStep(void* call_data);
-
-  void* promise_data_ = nullptr;
-  const InfallibleOperator<T>* ops_;
-  const InfallibleOperator<T>* end_ops_;
-};
+template <typename T>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline Poll<ResultOr<T>>
+OperationExecutor<T>::Start(const Layout<T>* layout, T input, void* call_data) {
+  ops_ = layout->ops.data();
+  end_ops_ = ops_ + layout->ops.size();
+  if (layout->promise_size == 0) {
+    // No call state ==> instantaneously ready
+    auto r = InitStep(std::move(input), call_data);
+    CHECK(r.ready());
+    return r;
+  }
+  promise_data_ =
+      gpr_malloc_aligned(layout->promise_size, layout->promise_alignment);
+  return InitStep(std::move(input), call_data);
+}
 
-// The current state of a pipe.
-// CallFilters expose a set of pipe like objects for client & server initial
-// metadata and for messages.
-// This class tracks the state of one of those pipes.
-// Size matters here: this state is kept for the lifetime of a call, and we keep
-// multiple of them.
-// This class encapsulates the untyped work of the state machine; there are
-// typed wrappers around this class as private members of CallFilters that
-// augment it to provide all the functionality that we must.
-class PipeState {
- public:
-  struct StartPushed {};
-  PipeState() = default;
-  explicit PipeState(StartPushed) : state_(ValueState::kQueued) {}
-  // Start the pipe: allows pulls to proceed
-  void Start();
-  // A push operation is beginning
-  void BeginPush();
-  // A previously started push operation has completed
-  void DropPush();
-  // Poll for push completion: occurs after the corresponding Pull()
-  Poll<StatusFlag> PollPush();
-  // Poll for pull completion; returns Failure{} if closed with error,
-  // true if a value is available, or false if the pipe was closed without
-  // error.
-  Poll<ValueOrFailure<bool>> PollPull();
-  // A pulled value has been consumed: we can unblock the push
-  void AckPull();
-  // A previously started pull operation has completed
-  void DropPull();
-  // Close sending
-  void CloseSending();
-  // Close sending with error
-  void CloseWithError();
-  // Poll for closedness - if true, closed with error
-  Poll<bool> PollClosed();
-
-  bool holds_error() const { return state_ == ValueState::kError; }
+template <typename T>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline Poll<ResultOr<T>>
+OperationExecutor<T>::InitStep(T input, void* call_data) {
+  CHECK(input != nullptr);
+  while (true) {
+    if (ops_ == end_ops_) {
+      return ResultOr<T>{std::move(input), nullptr};
+    }
+    auto p =
+        ops_->promise_init(promise_data_, Offset(call_data, ops_->call_offset),
+                           ops_->channel_data, std::move(input));
+    if (auto* r = p.value_if_ready()) {
+      if (r->ok == nullptr) return std::move(*r);
+      input = std::move(r->ok);
+      ++ops_;
+      continue;
+    }
+    return Pending{};
+  }
+}
 
-  std::string DebugString() const;
+template <typename T>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline Poll<ResultOr<T>>
+OperationExecutor<T>::Step(void* call_data) {
+  DCHECK_NE(promise_data_, nullptr);
+  auto p = ContinueStep(call_data);
+  if (p.ready()) {
+    gpr_free_aligned(promise_data_);
+    promise_data_ = nullptr;
+  }
+  return p;
+}
 
- private:
-  enum class ValueState : uint8_t {
-    // Nothing sending nor receiving
-    kIdle,
-    // Sent, but not yet received
-    kQueued,
-    // Trying to receive, but not yet sent
-    kWaiting,
-    // Ready to start processing, but not yet started
-    // (we have the value to send through the pipe, the reader is waiting,
-    // but it's not yet been polled)
-    kReady,
-    // Processing through filters
-    kProcessing,
-    // Closed sending
-    kClosed,
-    // Closed due to failure
-    kError
-  };
-  // Waiter for a promise blocked waiting to send.
-  IntraActivityWaiter wait_send_;
-  // Waiter for a promise blocked waiting to receive.
-  IntraActivityWaiter wait_recv_;
-  // Current state.
-  ValueState state_ = ValueState::kIdle;
-  // Has the pipe been started?
-  bool started_ = false;
-};
+template <typename T>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline Poll<ResultOr<T>>
+OperationExecutor<T>::ContinueStep(void* call_data) {
+  auto p = ops_->poll(promise_data_);
+  if (auto* r = p.value_if_ready()) {
+    if (r->ok == nullptr) return std::move(*r);
+    ++ops_;
+    return InitStep(std::move(r->ok), call_data);
+  }
+  return Pending{};
+}
 
 template <typename Fn>
 class ServerTrailingMetadataInterceptor {
  public:
   class Call {
    public:
-    static const NoInterceptor OnClientInitialMetadata;
-    static const NoInterceptor OnServerInitialMetadata;
-    static const NoInterceptor OnClientToServerMessage;
-    static const NoInterceptor OnServerToClientMessage;
-    static const NoInterceptor OnFinalize;
+    static const inline NoInterceptor OnClientInitialMetadata;
+    static const inline NoInterceptor OnServerInitialMetadata;
+    static const inline NoInterceptor OnClientToServerMessage;
+    static const inline NoInterceptor OnClientToServerHalfClose;
+    static const inline NoInterceptor OnServerToClientMessage;
+    static const inline NoInterceptor OnFinalize;
     void OnServerTrailingMetadata(ServerMetadata& md,
                                   ServerTrailingMetadataInterceptor* filter) {
       filter->fn_(md);
@@ -1224,23 +1385,136 @@ class ServerTrailingMetadataInterceptor {
  private:
   GPR_NO_UNIQUE_ADDRESS Fn fn_;
 };
+
 template <typename Fn>
-const NoInterceptor
-    ServerTrailingMetadataInterceptor<Fn>::Call::OnClientInitialMetadata;
-template <typename Fn>
-const NoInterceptor
-    ServerTrailingMetadataInterceptor<Fn>::Call::OnServerInitialMetadata;
-template <typename Fn>
-const NoInterceptor
-    ServerTrailingMetadataInterceptor<Fn>::Call::OnClientToServerMessage;
-template <typename Fn>
-const NoInterceptor
-    ServerTrailingMetadataInterceptor<Fn>::Call::OnServerToClientMessage;
-template <typename Fn>
-const NoInterceptor ServerTrailingMetadataInterceptor<Fn>::Call::OnFinalize;
+class ClientInitialMetadataInterceptor {
+ public:
+  class Call {
+   public:
+    auto OnClientInitialMetadata(ClientMetadata& md,
+                                 ClientInitialMetadataInterceptor* filter) {
+      return filter->fn_(md);
+    }
+    static const inline NoInterceptor OnServerInitialMetadata;
+    static const inline NoInterceptor OnClientToServerMessage;
+    static const inline NoInterceptor OnClientToServerHalfClose;
+    static const inline NoInterceptor OnServerToClientMessage;
+    static const inline NoInterceptor OnServerTrailingMetadata;
+    static const inline NoInterceptor OnFinalize;
+  };
+
+  explicit ClientInitialMetadataInterceptor(Fn fn) : fn_(std::move(fn)) {}
+
+ private:
+  GPR_NO_UNIQUE_ADDRESS Fn fn_;
+};
 
+// Determine if an interceptor needs to access the channel via one of its
+// arguments.
+
+template <typename T>
+constexpr bool MethodHasChannelAccess = false;
+
+template <typename T, typename R, typename A>
+constexpr bool MethodHasChannelAccess<R (T::*)(A)> = false;
+
+template <typename T, typename R>
+constexpr bool MethodHasChannelAccess<R (T::*)()> = false;
+
+template <typename T, typename R, typename A, typename C>
+constexpr bool MethodHasChannelAccess<R (T::*)(A, C)> = true;
+
+template <auto... Ts>
+constexpr bool AnyMethodHasChannelAccess =
+    (MethodHasChannelAccess<decltype(Ts)> || ...);
+
+// Composite for a given channel type to determine if any of its interceptors
+// fall into this category: later code should use this.
+template <typename Derived>
+inline constexpr bool CallHasChannelAccess() {
+  return AnyMethodHasChannelAccess<&Derived::Call::OnClientInitialMetadata,
+                                   &Derived::Call::OnClientToServerMessage,
+                                   &Derived::Call::OnServerInitialMetadata,
+                                   &Derived::Call::OnServerToClientMessage,
+                                   &Derived::Call::OnServerTrailingMetadata,
+                                   &Derived::Call::OnFinalize>;
+}
 }  // namespace filters_detail
 
+namespace for_each_detail {
+template <void (CallState::*on_progress)()>
+struct NextValueTraits<filters_detail::NextMessage<on_progress>> {
+  using NextMsg = filters_detail::NextMessage<on_progress>;
+  using Value = MessageHandle;
+
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static NextValueType Type(
+      const NextMsg& t) {
+    if (!t.ok()) return NextValueType::kError;
+    if (t.has_value()) return NextValueType::kValue;
+    return NextValueType::kEndOfStream;
+  }
+
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static MessageHandle TakeValue(
+      NextMsg& t) {
+    return t.TakeValue();
+  }
+};
+}  // namespace for_each_detail
+
+template <void (CallState::*on_progress)()>
+struct FailureStatusCastImpl<filters_detail::NextMessage<on_progress>,
+                             StatusFlag> {
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static filters_detail::NextMessage<
+      on_progress>
+  Cast(StatusFlag flag) {
+    DCHECK_EQ(flag, Failure{});
+    return filters_detail::NextMessage<on_progress>(Failure{});
+  }
+};
+
+namespace promise_detail {
+template <void (CallState::*on_progress)()>
+struct TrySeqTraitsWithSfinae<filters_detail::NextMessage<on_progress>> {
+  using UnwrappedType = MessageHandle;
+  using WrappedType = filters_detail::NextMessage<on_progress>;
+  template <typename Next>
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallFactory(
+      Next* next, WrappedType&& value) {
+    return next->Make(value.TakeValue());
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsOk(
+      const WrappedType& value) {
+    return value.ok();
+  }
+  static const char* ErrorString(const WrappedType& status) {
+    DCHECK(!status.ok());
+    return "failed";
+  }
+  template <typename R>
+  static R ReturnValue(WrappedType&& status) {
+    DCHECK(!status.ok());
+    return WrappedType(Failure{});
+  }
+  template <typename F, typename Elem>
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(
+      F& f, Elem&& elem, WrappedType value)
+      -> decltype(f(std::forward<Elem>(elem), std::declval<MessageHandle>())) {
+    return f(std::forward<Elem>(elem), value.TakeValue());
+  }
+  template <typename Result, typename RunNext>
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
+  CheckResultAndRunNext(WrappedType prior, RunNext run_next) {
+    if (!prior.ok()) return WrappedType(prior.status());
+    return run_next(std::move(prior));
+  }
+};
+}  // namespace promise_detail
+
+using ServerToClientNextMessage =
+    filters_detail::NextMessage<&CallState::FinishPullServerToClientMessage>;
+using ClientToServerNextMessage =
+    filters_detail::NextMessage<&CallState::FinishPullClientToServerMessage>;
+
 // Execution environment for a stack of filters.
 // This is a per-call object.
 class CallFilters {
@@ -1277,6 +1551,7 @@ class CallFilters {
       data_.AddClientInitialMetadataOp(filter, call_offset);
       data_.AddServerInitialMetadataOp(filter, call_offset);
       data_.AddClientToServerMessageOp(filter, call_offset);
+      data_.AddClientToServerHalfClose(filter, call_offset);
       data_.AddServerToClientMessageOp(filter, call_offset);
       data_.AddServerTrailingMetadataOp(filter, call_offset);
       data_.AddFinalizer(filter, call_offset, &FilterType::Call::OnFinalize);
@@ -1296,6 +1571,14 @@ class CallFilters {
       AddOwnedObject([](void* p) { delete static_cast<T*>(p); }, p.release());
     }
 
+    template <typename Fn>
+    void AddOnClientInitialMetadata(Fn fn) {
+      auto filter = std::make_unique<
+          filters_detail::ClientInitialMetadataInterceptor<Fn>>(std::move(fn));
+      Add(filter.get());
+      AddOwnedObject(std::move(filter));
+    }
+
     template <typename Fn>
     void AddOnServerTrailingMetadata(Fn fn) {
       auto filter = std::make_unique<
@@ -1310,489 +1593,355 @@ class CallFilters {
     filters_detail::StackData data_;
   };
 
-  class NextMessage {
-   public:
-    NextMessage() : has_value_(false), cancelled_(false) {}
-    explicit NextMessage(MessageHandle value)
-        : has_value_(true), value_(std::move(value)) {}
-    explicit NextMessage(bool cancelled)
-        : has_value_(false), cancelled_(cancelled) {}
-    NextMessage(const NextMessage&) = delete;
-    NextMessage& operator=(const NextMessage&) = delete;
-    NextMessage(NextMessage&& other) noexcept = default;
-    NextMessage& operator=(NextMessage&& other) = default;
-
-    using value_type = MessageHandle;
-
-    void reset() {
-      has_value_ = false;
-      cancelled_ = false;
-      value_.reset();
-    }
-    bool has_value() const { return has_value_; }
-    const MessageHandle& value() const {
-      GPR_DEBUG_ASSERT(has_value_);
-      return value_;
-    }
-    MessageHandle& value() {
-      GPR_DEBUG_ASSERT(has_value_);
-      return value_;
+  explicit CallFilters(ClientMetadataHandle client_initial_metadata)
+      : call_data_(nullptr),
+        push_client_initial_metadata_(std::move(client_initial_metadata)) {}
+  ~CallFilters() {
+    if (call_data_ != nullptr && call_data_ != &g_empty_call_data_) {
+      for (const auto& stack : stacks_) {
+        for (const auto& destructor : stack.stack->data_.filter_destructor) {
+          destructor.call_destroy(filters_detail::Offset(
+              call_data_, stack.call_data_offset + destructor.call_offset));
+        }
+      }
+      gpr_free_aligned(call_data_);
     }
-    const MessageHandle& operator*() const { return value(); }
-    MessageHandle& operator*() { return value(); }
-    bool cancelled() const { return !has_value_ && cancelled_; }
-
-   private:
-    bool has_value_;
-    bool cancelled_;
-    MessageHandle value_;
   };
 
-  explicit CallFilters(ClientMetadataHandle client_initial_metadata);
-  ~CallFilters();
-
   CallFilters(const CallFilters&) = delete;
   CallFilters& operator=(const CallFilters&) = delete;
   CallFilters(CallFilters&&) = delete;
   CallFilters& operator=(CallFilters&&) = delete;
 
-  void SetStack(RefCountedPtr<Stack> stack);
+  void AddStack(RefCountedPtr<Stack> stack) {
+    if (stack->data_.empty()) return;
+    stacks_.emplace_back(std::move(stack));
+  }
+  void Start();
 
   // Access client initial metadata before it's processed
   ClientMetadata* unprocessed_client_initial_metadata() {
-    return client_initial_metadata_.get();
+    return push_client_initial_metadata_.get();
   }
 
+ private:
+  template <typename Output, typename Input,
+            Input(CallFilters::* input_location),
+            filters_detail::Layout<Input>(filters_detail::StackData::* layout),
+            void (CallState::*on_done)(), typename StackIterator>
+  class MetadataExecutor {
+   public:
+    MetadataExecutor(CallFilters* filters, StackIterator stack_begin,
+                     StackIterator stack_end)
+        : stack_current_(stack_begin),
+          stack_end_(stack_end),
+          filters_(filters) {
+      DCHECK_NE((filters_->*input_location).get(), nullptr);
+    }
+
+    Poll<ValueOrFailure<Output>> operator()() {
+      if ((filters_->*input_location) != nullptr) {
+        if (stack_current_ == stack_end_) {
+          DCHECK_NE((filters_->*input_location).get(), nullptr);
+          (filters_->call_state_.*on_done)();
+          return Output(std::move(filters_->*input_location));
+        }
+        return FinishStep(executor_.Start(
+            &(stack_current_->stack->data_.*layout),
+            std::move(filters_->*input_location), filters_->call_data_));
+      } else {
+        return FinishStep(executor_.Step(filters_->call_data_));
+      }
+    }
+
+   private:
+    Poll<ValueOrFailure<Output>> FinishStep(
+        Poll<filters_detail::ResultOr<Input>> p) {
+      auto* r = p.value_if_ready();
+      if (r == nullptr) return Pending{};
+      if (r->ok != nullptr) {
+        ++stack_current_;
+        if (stack_current_ == stack_end_) {
+          (filters_->call_state_.*on_done)();
+          return ValueOrFailure<Output>{std::move(r->ok)};
+        }
+        return FinishStep(
+            executor_.Start(&(stack_current_->stack->data_.*layout),
+                            std::move(r->ok), filters_->call_data_));
+      }
+      (filters_->call_state_.*on_done)();
+      filters_->PushServerTrailingMetadata(std::move(r->error));
+      return Failure{};
+    }
+
+    StackIterator stack_current_;
+    StackIterator stack_end_;
+    CallFilters* filters_;
+    filters_detail::OperationExecutor<Input> executor_;
+  };
+
+  template <MessageHandle(CallFilters::* input_location),
+            filters_detail::Layout<MessageHandle>(
+                filters_detail::StackData::* layout),
+            void (CallState::*on_done)(), typename StackIterator>
+  class MessageExecutor {
+   public:
+    using NextMsg = filters_detail::NextMessage<on_done>;
+
+    MessageExecutor(CallFilters* filters, StackIterator stack_begin,
+                    StackIterator stack_end)
+        : stack_current_(stack_begin),
+          stack_end_(stack_end),
+          filters_(filters) {
+      DCHECK_NE((filters_->*input_location).get(), nullptr);
+    }
+
+    Poll<NextMsg> operator()() {
+      if ((filters_->*input_location) != nullptr) {
+        if (stack_current_ == stack_end_) {
+          DCHECK_NE((filters_->*input_location).get(), nullptr);
+          return NextMsg(std::move(filters_->*input_location),
+                         &filters_->call_state_);
+        }
+        return FinishStep(executor_.Start(
+            &(stack_current_->stack->data_.*layout),
+            std::move(filters_->*input_location), filters_->call_data_));
+      } else {
+        return FinishStep(executor_.Step(filters_->call_data_));
+      }
+    }
+
+   private:
+    Poll<NextMsg> FinishStep(Poll<filters_detail::ResultOr<MessageHandle>> p) {
+      auto* r = p.value_if_ready();
+      if (r == nullptr) return Pending{};
+      if (r->ok != nullptr) {
+        ++stack_current_;
+        if (stack_current_ == stack_end_) {
+          return NextMsg{std::move(r->ok), &filters_->call_state_};
+        }
+        return FinishStep(
+            executor_.Start(&(stack_current_->stack->data_.*layout),
+                            std::move(r->ok), filters_->call_data_));
+      }
+      (filters_->call_state_.*on_done)();
+      filters_->PushServerTrailingMetadata(std::move(r->error));
+      return Failure{};
+    }
+
+    StackIterator stack_current_;
+    StackIterator stack_end_;
+    CallFilters* filters_;
+    filters_detail::OperationExecutor<MessageHandle> executor_;
+  };
+
+ public:
   // Client: Fetch client initial metadata
   // Returns a promise that resolves to ValueOrFailure<ClientMetadataHandle>
-  GRPC_MUST_USE_RESULT auto PullClientInitialMetadata();
-  // Server: Indicate that no server initial metadata will be sent
-  void NoServerInitialMetadata() {
-    server_initial_metadata_state_.CloseSending();
+  GRPC_MUST_USE_RESULT auto PullClientInitialMetadata() {
+    call_state_.BeginPullClientInitialMetadata();
+    return MetadataExecutor<ClientMetadataHandle, ClientMetadataHandle,
+                            &CallFilters::push_client_initial_metadata_,
+                            &filters_detail::StackData::client_initial_metadata,
+                            &CallState::FinishPullClientInitialMetadata,
+                            StacksVector::const_iterator>(
+        this, stacks_.cbegin(), stacks_.cend());
   }
   // Server: Push server initial metadata
   // Returns a promise that resolves to a StatusFlag indicating success
-  GRPC_MUST_USE_RESULT auto PushServerInitialMetadata(ServerMetadataHandle md);
+  StatusFlag PushServerInitialMetadata(ServerMetadataHandle md) {
+    push_server_initial_metadata_ = std::move(md);
+    return call_state_.PushServerInitialMetadata();
+  }
   // Client: Fetch server initial metadata
   // Returns a promise that resolves to ValueOrFailure<ServerMetadataHandle>
-  GRPC_MUST_USE_RESULT auto PullServerInitialMetadata();
+  GRPC_MUST_USE_RESULT auto PullServerInitialMetadata() {
+    return Seq(
+        [this]() {
+          return call_state_.PollPullServerInitialMetadataAvailable();
+        },
+        [this](bool has_server_initial_metadata) {
+          return If(
+              has_server_initial_metadata,
+              [this]() {
+                return Map(
+                    MetadataExecutor<
+                        std::optional<ServerMetadataHandle>,
+                        ServerMetadataHandle,
+                        &CallFilters::push_server_initial_metadata_,
+                        &filters_detail::StackData::server_initial_metadata,
+                        &CallState::FinishPullServerInitialMetadata,
+                        StacksVector::const_reverse_iterator>(
+                        this, stacks_.crbegin(), stacks_.crend()),
+                    [](ValueOrFailure<std::optional<ServerMetadataHandle>> r) {
+                      if (r.ok()) return std::move(*r);
+                      return std::optional<ServerMetadataHandle>{};
+                    });
+              },
+              []() {
+                return Immediate(std::optional<ServerMetadataHandle>{});
+              });
+        });
+  }
   // Client: Push client to server message
   // Returns a promise that resolves to a StatusFlag indicating success
-  GRPC_MUST_USE_RESULT auto PushClientToServerMessage(MessageHandle message);
-  // Client: Indicate that no more messages will be sent
-  void FinishClientToServerSends() {
-    client_to_server_message_state_.CloseSending();
+  GRPC_MUST_USE_RESULT auto PushClientToServerMessage(MessageHandle message) {
+    call_state_.BeginPushClientToServerMessage();
+    DCHECK_NE(message.get(), nullptr);
+    DCHECK_EQ(push_client_to_server_message_.get(), nullptr);
+    push_client_to_server_message_ = std::move(message);
+    return [this]() { return call_state_.PollPushClientToServerMessage(); };
   }
+  // Client: Indicate that no more messages will be sent
+  void FinishClientToServerSends() { call_state_.ClientToServerHalfClose(); }
   // Server: Fetch client to server message
-  // Returns a promise that resolves to ValueOrFailure<MessageHandle>
-  GRPC_MUST_USE_RESULT auto PullClientToServerMessage();
+  // Returns a promise that resolves to ClientToServerNextMessage
+  GRPC_MUST_USE_RESULT auto PullClientToServerMessage() {
+    return TrySeq(
+        [this]() {
+          return call_state_.PollPullClientToServerMessageAvailable();
+        },
+        [this](bool message_available) {
+          return If(
+              message_available,
+              [this]() {
+                return MessageExecutor<
+                    &CallFilters::push_client_to_server_message_,
+                    &filters_detail::StackData::client_to_server_messages,
+                    &CallState::FinishPullClientToServerMessage,
+                    StacksVector::const_iterator>(this, stacks_.cbegin(),
+                                                  stacks_.cend());
+              },
+              []() -> ClientToServerNextMessage {
+                return ClientToServerNextMessage();
+              });
+        });
+  }
   // Server: Push server to client message
   // Returns a promise that resolves to a StatusFlag indicating success
-  GRPC_MUST_USE_RESULT auto PushServerToClientMessage(MessageHandle message);
+  GRPC_MUST_USE_RESULT auto PushServerToClientMessage(MessageHandle message) {
+    call_state_.BeginPushServerToClientMessage();
+    push_server_to_client_message_ = std::move(message);
+    return [this]() { return call_state_.PollPushServerToClientMessage(); };
+  }
   // Server: Fetch server to client message
-  // Returns a promise that resolves to ValueOrFailure<MessageHandle>
-  GRPC_MUST_USE_RESULT auto PullServerToClientMessage();
+  // Returns a promise that resolves to ServerToClientNextMessage
+  GRPC_MUST_USE_RESULT auto PullServerToClientMessage() {
+    return TrySeq(
+        [this]() {
+          return call_state_.PollPullServerToClientMessageAvailable();
+        },
+        [this](bool message_available) {
+          return If(
+              message_available,
+              [this]() {
+                return MessageExecutor<
+                    &CallFilters::push_server_to_client_message_,
+                    &filters_detail::StackData::server_to_client_messages,
+                    &CallState::FinishPullServerToClientMessage,
+                    StacksVector::const_reverse_iterator>(
+                    this, stacks_.crbegin(), stacks_.crend());
+              },
+              []() -> ServerToClientNextMessage {
+                return ServerToClientNextMessage();
+              });
+        });
+  }
   // Server: Indicate end of response
   // Closes the request entirely - no messages can be sent/received
   // If no server initial metadata has been sent, implies
   // NoServerInitialMetadata() called.
   void PushServerTrailingMetadata(ServerMetadataHandle md);
+  // Optimized path to push cancellation onto the call
+  void Cancel();
   // Client: Fetch server trailing metadata
   // Returns a promise that resolves to ServerMetadataHandle
-  GRPC_MUST_USE_RESULT auto PullServerTrailingMetadata();
+  GRPC_MUST_USE_RESULT auto PullServerTrailingMetadata() {
+    return Map(
+        [this]() { return call_state_.PollServerTrailingMetadataAvailable(); },
+        [this](Empty) {
+          auto value = std::move(push_server_trailing_metadata_);
+          if (call_data_ != nullptr) {
+            for (auto it = stacks_.crbegin(); it != stacks_.crend(); ++it) {
+              value = filters_detail::RunServerTrailingMetadata(
+                  it->stack->data_.server_trailing_metadata,
+                  filters_detail::Offset(call_data_, it->call_data_offset),
+                  std::move(value));
+            }
+          }
+          return value;
+        });
+  }
   // Server: Wait for server trailing metadata to have been sent
   // Returns a promise that resolves to a StatusFlag indicating whether the
   // request was cancelled or not -- failure to send trailing metadata is
   // considered a cancellation, as is actual cancellation -- but not application
   // errors.
-  GRPC_MUST_USE_RESULT auto WasCancelled();
+  GRPC_MUST_USE_RESULT auto WasCancelled() {
+    return [this]() { return call_state_.PollWasCancelled(); };
+  }
+  // Client & server: wait for server trailing metadata to be available, and
+  // resolve to empty when it is.
+  GRPC_MUST_USE_RESULT auto ServerTrailingMetadataWasPushed() {
+    return
+        [this]() { return call_state_.PollServerTrailingMetadataWasPushed(); };
+  }
+  // Client & server: returns true if server trailing metadata has been pushed
+  // *and* contained a cancellation, false otherwise.
+  GRPC_MUST_USE_RESULT bool WasCancelledPushed() const {
+    return call_state_.WasCancelledPushed();
+  }
+
+  // Returns true if server trailing metadata has been pulled
+  bool WasServerTrailingMetadataPulled() const {
+    return call_state_.WasServerTrailingMetadataPulled();
+  }
+
   // Client & server: fill in final_info with the final status of the call.
   void Finalize(const grpc_call_final_info* final_info);
 
   std::string DebugString() const;
 
  private:
-  template <filters_detail::PipeState(CallFilters::*state_ptr),
-            void*(CallFilters::*push_ptr), typename T,
-            filters_detail::Layout<filters_detail::FallibleOperator<T>>(
-                filters_detail::StackData::*layout_ptr)>
-  class PipePromise {
-   public:
-    class Push {
-     public:
-      Push(CallFilters* filters, T x)
-          : filters_(filters), value_(std::move(x)) {
-        state().BeginPush();
-        push_slot() = this;
-      }
-      ~Push() {
-        if (filters_ != nullptr) {
-          state().DropPush();
-          push_slot() = nullptr;
-        }
-      }
-
-      Push(const Push&) = delete;
-      Push& operator=(const Push&) = delete;
-      Push(Push&& other) noexcept
-          : filters_(std::exchange(other.filters_, nullptr)),
-            value_(std::move(other.value_)) {
-        if (filters_ != nullptr) {
-          GPR_DEBUG_ASSERT(push_slot() == &other);
-          push_slot() = this;
-        }
-      }
-
-      Push& operator=(Push&&) = delete;
-
-      Poll<StatusFlag> operator()() { return state().PollPush(); }
-
-      T TakeValue() { return std::move(value_); }
-
-      absl::string_view DebugString() const {
-        return value_ != nullptr ? " (not pulled)" : "";
-      }
-
-     private:
-      filters_detail::PipeState& state() { return filters_->*state_ptr; }
-      void*& push_slot() { return filters_->*push_ptr; }
-
-      CallFilters* filters_;
-      T value_;
-    };
-
-    static std::string DebugString(absl::string_view name,
-                                   const CallFilters* filters) {
-      auto* push = static_cast<Push*>(filters->*push_ptr);
-      return absl::StrCat(name, ":", (filters->*state_ptr).DebugString(),
-                          push == nullptr ? "" : push->DebugString());
-    }
-
-    class PullMaybe {
-     public:
-      explicit PullMaybe(CallFilters* filters) : filters_(filters) {}
-      ~PullMaybe() {
-        if (filters_ != nullptr) {
-          state().DropPull();
-        }
-      }
-
-      PullMaybe(const PullMaybe&) = delete;
-      PullMaybe& operator=(const PullMaybe&) = delete;
-      PullMaybe(PullMaybe&& other) noexcept
-          : filters_(std::exchange(other.filters_, nullptr)),
-            executor_(std::move(other.executor_)) {}
-      PullMaybe& operator=(PullMaybe&&) = delete;
-
-      Poll<ValueOrFailure<absl::optional<T>>> operator()() {
-        if (executor_.IsRunning()) {
-          auto c = state().PollClosed();
-          if (c.ready() && c.value()) {
-            filters_->CancelDueToFailedPipeOperation();
-            return Failure{};
-          }
-          return FinishOperationExecutor(executor_.Step(filters_->call_data_));
-        }
-        auto p = state().PollPull();
-        auto* r = p.value_if_ready();
-        if (r == nullptr) return Pending{};
-        if (!r->ok()) {
-          filters_->CancelDueToFailedPipeOperation();
-          return Failure{};
-        }
-        if (!**r) return absl::nullopt;
-        return FinishOperationExecutor(executor_.Start(
-            layout(), push()->TakeValue(), filters_->call_data_));
-      }
-
-     private:
-      filters_detail::PipeState& state() { return filters_->*state_ptr; }
-      Push* push() { return static_cast<Push*>(filters_->*push_ptr); }
-      const filters_detail::Layout<filters_detail::FallibleOperator<T>>*
-      layout() {
-        return &(filters_->stack_->data_.*layout_ptr);
-      }
-
-      Poll<ValueOrFailure<absl::optional<T>>> FinishOperationExecutor(
-          Poll<filters_detail::ResultOr<T>> p) {
-        auto* r = p.value_if_ready();
-        if (r == nullptr) return Pending{};
-        GPR_DEBUG_ASSERT(!executor_.IsRunning());
-        state().AckPull();
-        if (r->ok != nullptr) return std::move(r->ok);
-        filters_->PushServerTrailingMetadata(std::move(r->error));
-        return Failure{};
-      }
-
-      CallFilters* filters_;
-      filters_detail::OperationExecutor<T> executor_;
-    };
-
-    class PullMessage {
-     public:
-      explicit PullMessage(CallFilters* filters) : filters_(filters) {}
-      ~PullMessage() {
-        if (filters_ != nullptr) {
-          state().DropPull();
-        }
-      }
-
-      PullMessage(const PullMessage&) = delete;
-      PullMessage& operator=(const PullMessage&) = delete;
-      PullMessage(PullMessage&& other) noexcept
-          : filters_(std::exchange(other.filters_, nullptr)),
-            executor_(std::move(other.executor_)) {}
-      PullMessage& operator=(PullMessage&&) = delete;
-
-      Poll<NextMessage> operator()() {
-        if (executor_.IsRunning()) {
-          auto c = state().PollClosed();
-          if (c.ready() && c.value()) {
-            filters_->CancelDueToFailedPipeOperation();
-            return NextMessage(true);
-          }
-          return FinishOperationExecutor(executor_.Step(filters_->call_data_));
-        }
-        auto p = state().PollPull();
-        auto* r = p.value_if_ready();
-        if (r == nullptr) return Pending{};
-        if (!r->ok()) {
-          filters_->CancelDueToFailedPipeOperation();
-          return NextMessage(true);
-        }
-        if (!**r) return NextMessage(false);
-        return FinishOperationExecutor(executor_.Start(
-            layout(), push()->TakeValue(), filters_->call_data_));
-      }
-
-     private:
-      filters_detail::PipeState& state() { return filters_->*state_ptr; }
-      Push* push() { return static_cast<Push*>(filters_->*push_ptr); }
-      const filters_detail::Layout<filters_detail::FallibleOperator<T>>*
-      layout() {
-        return &(filters_->stack_->data_.*layout_ptr);
-      }
-
-      Poll<NextMessage> FinishOperationExecutor(
-          Poll<filters_detail::ResultOr<T>> p) {
-        auto* r = p.value_if_ready();
-        if (r == nullptr) return Pending{};
-        GPR_DEBUG_ASSERT(!executor_.IsRunning());
-        state().AckPull();
-        if (r->ok != nullptr) return NextMessage(std::move(r->ok));
-        filters_->PushServerTrailingMetadata(std::move(r->error));
-        return NextMessage(true);
-      }
-
-      CallFilters* filters_;
-      filters_detail::OperationExecutor<T> executor_;
-    };
-  };
-
-  class PullClientInitialMetadataPromise {
-   public:
-    explicit PullClientInitialMetadataPromise(CallFilters* filters)
-        : filters_(filters) {}
-
-    PullClientInitialMetadataPromise(const PullClientInitialMetadataPromise&) =
-        delete;
-    PullClientInitialMetadataPromise& operator=(
-        const PullClientInitialMetadataPromise&) = delete;
-    PullClientInitialMetadataPromise(
-        PullClientInitialMetadataPromise&& other) noexcept
-        : filters_(std::exchange(other.filters_, nullptr)),
-          executor_(std::move(other.executor_)) {}
-    PullClientInitialMetadataPromise& operator=(
-        PullClientInitialMetadataPromise&&) = delete;
-
-    Poll<ValueOrFailure<ClientMetadataHandle>> operator()() {
-      if (executor_.IsRunning()) {
-        return FinishOperationExecutor(executor_.Step(filters_->call_data_));
-      }
-      auto p = state().PollPull();
-      auto* r = p.value_if_ready();
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
-        gpr_log(GPR_INFO, "%s",
-                r == nullptr
-                    ? "PENDING"
-                    : (r->ok() ? (r->value() ? "TRUE" : "FALSE") : "FAILURE"));
-      }
-      if (r == nullptr) return Pending{};
-      if (!r->ok()) {
-        filters_->CancelDueToFailedPipeOperation();
-        return Failure{};
-      }
-      GPR_ASSERT(filters_->client_initial_metadata_ != nullptr);
-      return FinishOperationExecutor(executor_.Start(
-          &filters_->stack_->data_.client_initial_metadata,
-          std::move(filters_->client_initial_metadata_), filters_->call_data_));
-    }
-
-   private:
-    filters_detail::PipeState& state() {
-      return filters_->client_initial_metadata_state_;
-    }
-
-    Poll<ValueOrFailure<ClientMetadataHandle>> FinishOperationExecutor(
-        Poll<filters_detail::ResultOr<ClientMetadataHandle>> p) {
-      auto* r = p.value_if_ready();
-      if (r == nullptr) return Pending{};
-      GPR_DEBUG_ASSERT(!executor_.IsRunning());
-      state().AckPull();
-      if (r->ok != nullptr) return std::move(r->ok);
-      filters_->PushServerTrailingMetadata(std::move(r->error));
-      return Failure{};
-    }
-
-    CallFilters* filters_;
-    filters_detail::OperationExecutor<ClientMetadataHandle> executor_;
-  };
-
-  class PullServerTrailingMetadataPromise {
-   public:
-    explicit PullServerTrailingMetadataPromise(CallFilters* filters)
-        : filters_(filters) {}
-
-    PullServerTrailingMetadataPromise(
-        const PullServerTrailingMetadataPromise&) = delete;
-    PullServerTrailingMetadataPromise& operator=(
-        const PullServerTrailingMetadataPromise&) = delete;
-    PullServerTrailingMetadataPromise(
-        PullServerTrailingMetadataPromise&& other) noexcept
-        : filters_(std::exchange(other.filters_, nullptr)),
-          executor_(std::move(other.executor_)) {}
-    PullServerTrailingMetadataPromise& operator=(
-        PullServerTrailingMetadataPromise&&) = delete;
-
-    Poll<ServerMetadataHandle> operator()() {
-      if (executor_.IsRunning()) {
-        auto r = executor_.Step(filters_->call_data_);
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
-          if (r.pending()) {
-            gpr_log(GPR_INFO,
-                    "%s PullServerTrailingMetadata[%p]: Pending(but executing)",
-                    GetContext<Activity>()->DebugTag().c_str(), filters_);
-          } else {
-            gpr_log(GPR_INFO, "%s PullServerTrailingMetadata[%p]: Ready: %s",
-                    GetContext<Activity>()->DebugTag().c_str(), filters_,
-                    r.value()->DebugString().c_str());
-          }
-        }
-        return r;
-      }
-      if (filters_->server_trailing_metadata_ == nullptr) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
-          gpr_log(GPR_INFO,
-                  "%s PullServerTrailingMetadata[%p]: Pending(not pushed)",
-                  GetContext<Activity>()->DebugTag().c_str(), filters_);
-        }
-        return filters_->server_trailing_metadata_waiter_.pending();
-      }
-      // If no stack has been set, we can just return the result of the call
-      if (filters_->stack_ == nullptr) {
-        if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
-          gpr_log(GPR_INFO,
-                  "%s PullServerTrailingMetadata[%p]: Ready(no-stack): %s",
-                  GetContext<Activity>()->DebugTag().c_str(), filters_,
-                  filters_->server_trailing_metadata_->DebugString().c_str());
-        }
-        return std::move(filters_->server_trailing_metadata_);
-      }
-      // Otherwise we need to process it through all the filters.
-      return executor_.Start(&filters_->stack_->data_.server_trailing_metadata,
-                             std::move(filters_->server_trailing_metadata_),
-                             filters_->call_data_);
-    }
+  void CancelDueToFailedPipeOperation(SourceLocation but_where = {});
 
-   private:
-    CallFilters* filters_;
-    filters_detail::InfallibleOperationExecutor<ServerMetadataHandle> executor_;
+  struct AddedStack {
+    explicit AddedStack(RefCountedPtr<Stack> stack)
+        : call_data_offset(std::numeric_limits<size_t>::max()),
+          stack(std::move(stack)) {}
+    size_t call_data_offset;
+    RefCountedPtr<Stack> stack;
   };
 
-  void CancelDueToFailedPipeOperation(SourceLocation but_where = {});
+  using StacksVector = absl::InlinedVector<AddedStack, 2>;
 
-  RefCountedPtr<Stack> stack_;
+  StacksVector stacks_;
 
-  filters_detail::PipeState client_initial_metadata_state_{
-      filters_detail::PipeState::StartPushed{}};
-  filters_detail::PipeState server_initial_metadata_state_;
-  filters_detail::PipeState client_to_server_message_state_;
-  filters_detail::PipeState server_to_client_message_state_;
-  IntraActivityWaiter server_trailing_metadata_waiter_;
-  Latch<bool> cancelled_;
+  CallState call_state_;
 
   void* call_data_;
-  ClientMetadataHandle client_initial_metadata_;
-
-  // The following void*'s are pointers to a `Push` object (from above).
-  // They are used to track the current push operation for each pipe.
-  // It would be lovely for them to be typed pointers, but that would require
-  // a recursive type definition since the location of this field needs to be
-  // a template argument to the `Push` object itself.
-  void* server_initial_metadata_push_ = nullptr;
-  void* client_to_server_message_push_ = nullptr;
-  void* server_to_client_message_push_ = nullptr;
-
-  ServerMetadataHandle server_trailing_metadata_;
-
-  using ServerInitialMetadataPromises =
-      PipePromise<&CallFilters::server_initial_metadata_state_,
-                  &CallFilters::server_initial_metadata_push_,
-                  ServerMetadataHandle,
-                  &filters_detail::StackData::server_initial_metadata>;
-  using ClientToServerMessagePromises =
-      PipePromise<&CallFilters::client_to_server_message_state_,
-                  &CallFilters::client_to_server_message_push_, MessageHandle,
-                  &filters_detail::StackData::client_to_server_messages>;
-  using ServerToClientMessagePromises =
-      PipePromise<&CallFilters::server_to_client_message_state_,
-                  &CallFilters::server_to_client_message_push_, MessageHandle,
-                  &filters_detail::StackData::server_to_client_messages>;
-};
-
-inline auto CallFilters::PullClientInitialMetadata() {
-  return PullClientInitialMetadataPromise(this);
-}
-
-inline auto CallFilters::PushServerInitialMetadata(ServerMetadataHandle md) {
-  GPR_ASSERT(md != nullptr);
-  return [p = ServerInitialMetadataPromises::Push{
-              this, std::move(md)}]() mutable { return p(); };
-}
-
-inline auto CallFilters::PullServerInitialMetadata() {
-  return ServerInitialMetadataPromises::PullMaybe{this};
-}
-
-inline auto CallFilters::PushClientToServerMessage(MessageHandle message) {
-  GPR_ASSERT(message != nullptr);
-  return [p = ClientToServerMessagePromises::Push{
-              this, std::move(message)}]() mutable { return p(); };
-}
-
-inline auto CallFilters::PullClientToServerMessage() {
-  return ClientToServerMessagePromises::PullMessage{this};
-}
+  ClientMetadataHandle push_client_initial_metadata_;
+  ServerMetadataHandle push_server_initial_metadata_;
+  MessageHandle push_client_to_server_message_;
+  MessageHandle push_server_to_client_message_;
+  ServerMetadataHandle push_server_trailing_metadata_;
 
-inline auto CallFilters::PushServerToClientMessage(MessageHandle message) {
-  GPR_ASSERT(message != nullptr);
-  return [p = ServerToClientMessagePromises::Push{
-              this, std::move(message)}]() mutable { return p(); };
-}
-
-inline auto CallFilters::PullServerToClientMessage() {
-  return ServerToClientMessagePromises::PullMessage{this};
-}
-
-inline auto CallFilters::PullServerTrailingMetadata() {
-  return Map(PullServerTrailingMetadataPromise(this),
-             [this](ServerMetadataHandle h) {
-               cancelled_.Set(h->get(GrpcCallWasCancelled()).value_or(false));
-               return h;
-             });
-}
+  static char g_empty_call_data_;
+};
 
-inline auto CallFilters::WasCancelled() { return cancelled_.Wait(); }
+static_assert(
+    filters_detail::ArgumentMustBeNextMessage<
+        absl::remove_cvref_t<decltype(std::declval<CallFilters*>()
+                                          ->PullServerToClientMessage()()
+                                          .value())>>::value(),
+    "PullServerToClientMessage must return a NextMessage");
+
+static_assert(
+    filters_detail::ArgumentMustBeNextMessage<
+        absl::remove_cvref_t<decltype(std::declval<CallFilters*>()
+                                          ->PullClientToServerMessage()()
+                                          .value())>>::value(),
+    "PullServerToClientMessage must return a NextMessage");
 
 }  // namespace grpc_core