grpc 1.64.3 → 1.65.0

data/src/core/lib/surface/server_call.cc

@@ -0,0 +1,222 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/lib/surface/server_call.h"
+
+#include <inttypes.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/log/check.h"
+#include "absl/strings/string_view.h"
+
+#include <grpc/byte_buffer.h>
+#include <grpc/compression.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/call.h>
+#include <grpc/impl/propagation_bits.h>
+#include <grpc/slice.h>
+#include <grpc/slice_buffer.h>
+#include <grpc/status.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/lib/gprpp/bitset.h"
+#include "src/core/lib/promise/all_ok.h"
+#include "src/core/lib/promise/map.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/promise/status_flag.h"
+#include "src/core/lib/promise/try_seq.h"
+#include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/slice/slice_buffer.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/server/server_interface.h"
+
+namespace grpc_core {
+
+namespace {
+
+grpc_call_error ValidateServerBatch(const grpc_op* ops, size_t nops) {
+  BitSet<8> got_ops;
+  for (size_t op_idx = 0; op_idx < nops; op_idx++) {
+    const grpc_op& op = ops[op_idx];
+    switch (op.op) {
+      case GRPC_OP_SEND_INITIAL_METADATA:
+        if (!AreInitialMetadataFlagsValid(op.flags)) {
+          return GRPC_CALL_ERROR_INVALID_FLAGS;
+        }
+        if (!ValidateMetadata(op.data.send_initial_metadata.count,
+                              op.data.send_initial_metadata.metadata)) {
+          return GRPC_CALL_ERROR_INVALID_METADATA;
+        }
+        break;
+      case GRPC_OP_SEND_MESSAGE:
+        if (!AreWriteFlagsValid(op.flags)) {
+          return GRPC_CALL_ERROR_INVALID_FLAGS;
+        }
+        break;
+      case GRPC_OP_SEND_STATUS_FROM_SERVER:
+        if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
+        if (!ValidateMetadata(
+                op.data.send_status_from_server.trailing_metadata_count,
+                op.data.send_status_from_server.trailing_metadata)) {
+          return GRPC_CALL_ERROR_INVALID_METADATA;
+        }
+        break;
+      case GRPC_OP_RECV_MESSAGE:
+      case GRPC_OP_RECV_CLOSE_ON_SERVER:
+        if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
+        break;
+      case GRPC_OP_RECV_INITIAL_METADATA:
+      case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
+      case GRPC_OP_RECV_STATUS_ON_CLIENT:
+        return GRPC_CALL_ERROR_NOT_ON_SERVER;
+    }
+    if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
+    got_ops.set(op.op);
+  }
+  return GRPC_CALL_OK;
+}
+
+}  // namespace
+
+grpc_call_error ServerCall::StartBatch(const grpc_op* ops, size_t nops,
+                                       void* notify_tag,
+                                       bool is_notify_tag_closure) {
+  if (nops == 0) {
+    EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
+    return GRPC_CALL_OK;
+  }
+  const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
+  if (validation_result != GRPC_CALL_OK) {
+    return validation_result;
+  }
+  CommitBatch(ops, nops, notify_tag, is_notify_tag_closure);
+  return GRPC_CALL_OK;
+}
+
+void ServerCall::CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
+                             bool is_notify_tag_closure) {
+  BatchOpIndex op_index(ops, nops);
+  if (!is_notify_tag_closure) grpc_cq_begin_op(cq_, notify_tag);
+  auto send_initial_metadata =
+      op_index.OpHandler<GRPC_OP_SEND_INITIAL_METADATA>([this](
+                                                            const grpc_op& op) {
+        auto metadata = arena()->MakePooled<ServerMetadata>();
+        PrepareOutgoingInitialMetadata(op, *metadata);
+        CToMetadata(op.data.send_initial_metadata.metadata,
+                    op.data.send_initial_metadata.count, metadata.get());
+        GRPC_TRACE_LOG(call, INFO)
+            << DebugTag() << "[call] Send initial metadata";
+        return [this, metadata = std::move(metadata)]() mutable {
+          return call_handler_.PushServerInitialMetadata(std::move(metadata));
+        };
+      });
+  auto send_message =
+      op_index.OpHandler<GRPC_OP_SEND_MESSAGE>([this](const grpc_op& op) {
+        SliceBuffer send;
+        grpc_slice_buffer_swap(
+            &op.data.send_message.send_message->data.raw.slice_buffer,
+            send.c_slice_buffer());
+        auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
+        return [this, msg = std::move(msg)]() mutable {
+          return call_handler_.PushMessage(std::move(msg));
+        };
+      });
+  auto send_trailing_metadata =
+      op_index.OpHandler<GRPC_OP_SEND_STATUS_FROM_SERVER>(
+          [this](const grpc_op& op) {
+            auto metadata = arena()->MakePooled<ServerMetadata>();
+            CToMetadata(op.data.send_status_from_server.trailing_metadata,
+                        op.data.send_status_from_server.trailing_metadata_count,
+                        metadata.get());
+            metadata->Set(GrpcStatusMetadata(),
+                          op.data.send_status_from_server.status);
+            if (auto* details =
+                    op.data.send_status_from_server.status_details) {
+              // TODO(ctiller): this should not be a copy, but we have
+              // callers that allocate and pass in a slice created with
+              // grpc_slice_from_static_string and then delete the string
+              // after passing it in, which shouldn't be a supported API.
+              metadata->Set(GrpcMessageMetadata(),
+                            Slice(grpc_slice_copy(*details)));
+            }
+            CHECK(metadata != nullptr);
+            return [this, metadata = std::move(metadata)]() mutable {
+              CHECK(metadata != nullptr);
+              return [this, metadata = std::move(
+                                metadata)]() mutable -> Poll<Success> {
+                CHECK(metadata != nullptr);
+                call_handler_.PushServerTrailingMetadata(std::move(metadata));
+                return Success{};
+              };
+            };
+          });
+  auto recv_message =
+      op_index.OpHandler<GRPC_OP_RECV_MESSAGE>([this](const grpc_op& op) {
+        return message_receiver_.MakeBatchOp(op, &call_handler_);
+      });
+  auto primary_ops = AllOk<StatusFlag>(
+      TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
+                               std::move(send_message)),
+             std::move(send_trailing_metadata)),
+      std::move(recv_message));
+  if (auto* op = op_index.op(GRPC_OP_RECV_CLOSE_ON_SERVER)) {
+    auto recv_trailing_metadata = OpHandler<GRPC_OP_RECV_CLOSE_ON_SERVER>(
+        [this, cancelled = op->data.recv_close_on_server.cancelled]() {
+          return Map(call_handler_.WasCancelled(),
+                     [cancelled, this](bool result) -> Success {
+                       ResetDeadline();
+                       *cancelled = result ? 1 : 0;
+                       return Success{};
+                     });
+        });
+    call_handler_.SpawnInfallible(
+        "final-batch", InfallibleBatch(std::move(primary_ops),
+                                       std::move(recv_trailing_metadata),
+                                       is_notify_tag_closure, notify_tag, cq_));
+  } else {
+    call_handler_.SpawnInfallible(
+        "batch", FallibleBatch(std::move(primary_ops), is_notify_tag_closure,
+                               notify_tag, cq_));
+  }
+}
+
+grpc_call* MakeServerCall(CallHandler call_handler,
+                          ClientMetadataHandle client_initial_metadata,
+                          ServerInterface* server, grpc_completion_queue* cq,
+                          grpc_metadata_array* publish_initial_metadata) {
+  PublishMetadataArray(client_initial_metadata.get(), publish_initial_metadata,
+                       false);
+  // TODO(ctiller): ideally we'd put this in the arena with the CallHandler,
+  // but there's an ownership problem: CallHandler owns the arena, and so would
+  // get destroyed before the base class Call destructor runs, leading to
+  // UB/crash. Investigate another path.
+  return (new ServerCall(std::move(client_initial_metadata),
+                         std::move(call_handler), server, cq))
+      ->c_ptr();
+}
+
+}  // namespace grpc_core

data/src/core/lib/surface/server_call.h

@@ -0,0 +1,167 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_SURFACE_SERVER_CALL_H
+#define GRPC_SRC_CORE_LIB_SURFACE_SERVER_CALL_H
+
+#include <inttypes.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/log/check.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/string_view.h"
+
+#include <grpc/byte_buffer.h>
+#include <grpc/compression.h>
+#include <grpc/event_engine/event_engine.h>
+#include <grpc/grpc.h>
+#include <grpc/impl/call.h>
+#include <grpc/impl/propagation_bits.h>
+#include <grpc/slice.h>
+#include <grpc/slice_buffer.h>
+#include <grpc/status.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/lib/gprpp/crash.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/promise/poll.h"
+#include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/surface/call.h"
+#include "src/core/lib/surface/call_utils.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/server/server_interface.h"
+#include "src/core/telemetry/stats.h"
+#include "src/core/telemetry/stats_data.h"
+
+namespace grpc_core {
+
+class ServerCall final : public Call, public DualRefCounted<ServerCall> {
+ public:
+  ServerCall(ClientMetadataHandle client_initial_metadata,
+             CallHandler call_handler, ServerInterface* server,
+             grpc_completion_queue* cq)
+      : Call(false,
+             client_initial_metadata->get(GrpcTimeoutMetadata())
+                 .value_or(Timestamp::InfFuture()),
+             call_handler.arena()->Ref(), call_handler.event_engine()),
+        call_handler_(std::move(call_handler)),
+        client_initial_metadata_stored_(std::move(client_initial_metadata)),
+        cq_(cq),
+        server_(server) {
+    global_stats().IncrementServerCallsCreated();
+  }
+
+  void CancelWithError(grpc_error_handle error) override {
+    call_handler_.SpawnInfallible(
+        "CancelWithError",
+        [self = WeakRefAsSubclass<ServerCall>(), error = std::move(error)] {
+          auto status = ServerMetadataFromStatus(error);
+          status->Set(GrpcCallWasCancelled(), true);
+          self->call_handler_.PushServerTrailingMetadata(std::move(status));
+          return Empty{};
+        });
+  }
+  bool is_trailers_only() const override {
+    Crash("is_trailers_only not implemented for server calls");
+  }
+  absl::string_view GetServerAuthority() const override {
+    Crash("unimplemented");
+  }
+  grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
+                             bool is_notify_tag_closure) override;
+
+  void ExternalRef() override { Ref().release(); }
+  void ExternalUnref() override { Unref(); }
+  void InternalRef(const char*) override { WeakRef().release(); }
+  void InternalUnref(const char*) override { WeakUnref(); }
+
+  void Orphaned() override {
+    // TODO(ctiller): only when we're not already finished
+    CancelWithError(absl::CancelledError());
+  }
+
+  void SetCompletionQueue(grpc_completion_queue*) override {
+    Crash("unimplemented");
+  }
+
+  grpc_compression_options compression_options() override {
+    return server_->compression_options();
+  }
+
+  grpc_call_stack* call_stack() override { return nullptr; }
+
+  char* GetPeer() override {
+    Slice peer_slice = GetPeerString();
+    if (!peer_slice.empty()) {
+      absl::string_view peer_string_view = peer_slice.as_string_view();
+      char* peer_string =
+          static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
+      memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
+      peer_string[peer_string_view.size()] = '\0';
+      return peer_string;
+    }
+    return gpr_strdup("unknown");
+  }
+
+  bool Completed() final { Crash("unimplemented"); }
+  bool failed_before_recv_message() const final { Crash("unimplemented"); }
+
+  uint32_t test_only_message_flags() override {
+    return message_receiver_.last_message_flags();
+  }
+
+  grpc_compression_algorithm incoming_compression_algorithm() override {
+    return message_receiver_.incoming_compression_algorithm();
+  }
+
+  void SetIncomingCompressionAlgorithm(
+      grpc_compression_algorithm algorithm) override {
+    message_receiver_.SetIncomingCompressionAlgorithm(algorithm);
+  }
+
+ private:
+  void CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
+                   bool is_notify_tag_closure);
+
+  std::string DebugTag() { return absl::StrFormat("SERVER_CALL[%p]: ", this); }
+
+  CallHandler call_handler_;
+  MessageReceiver message_receiver_;
+  ClientMetadataHandle client_initial_metadata_stored_;
+  grpc_completion_queue* const cq_;
+  ServerInterface* const server_;
+};
+
+grpc_call* MakeServerCall(CallHandler call_handler,
+                          ClientMetadataHandle client_initial_metadata,
+                          ServerInterface* server, grpc_completion_queue* cq,
+                          grpc_metadata_array* publish_initial_metadata);
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_SURFACE_SERVER_CALL_H

data/src/core/lib/surface/version.cc

@@ -22,6 +22,6 @@
 #include <grpc/grpc.h>
 #include <grpc/support/port_platform.h>
 
-const char* grpc_version_string(void) { return "41.0.0"; }
+const char* grpc_version_string(void) { return "42.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "grateful"; }
+const char* grpc_g_stands_for(void) { return "gnarly"; }

data/src/core/lib/transport/bdp_estimator.cc

@@ -27,8 +27,6 @@
 
 #include <grpc/support/port_platform.h>
 
-grpc_core::TraceFlag grpc_bdp_estimator_trace(false, "bdp_estimator");
-
 namespace grpc_core {
 
 BdpEstimator::BdpEstimator(absl::string_view name)
@@ -48,7 +46,7 @@ Timestamp BdpEstimator::CompletePing() {
               1e-9 * static_cast<double>(dt_ts.tv_nsec);
   double bw = dt > 0 ? (static_cast<double>(accumulator_) / dt) : 0;
   Duration start_inter_ping_delay = inter_ping_delay_;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(bdp_estimator)) {
     gpr_log(GPR_INFO,
             "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64
             " dt=%lf bw=%lfMbs bw_est=%lfMbs",
@@ -59,7 +57,7 @@ Timestamp BdpEstimator::CompletePing() {
   if (accumulator_ > 2 * estimate_ / 3 && bw > bw_est_) {
     estimate_ = std::max(accumulator_, estimate_ * 2);
     bw_est_ = bw;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(bdp_estimator)) {
       gpr_log(GPR_INFO, "bdp[%s]: estimate increased to %" PRId64,
               std::string(name_).c_str(), estimate_);
     }
@@ -75,7 +73,7 @@ Timestamp BdpEstimator::CompletePing() {
   }
   if (start_inter_ping_delay != inter_ping_delay_) {
     stable_estimate_count_ = 0;
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(bdp_estimator)) {
       gpr_log(GPR_INFO, "bdp[%s]:update_inter_time to %" PRId64 "ms",
               std::string(name_).c_str(), inter_ping_delay_.millis());
     }

data/src/core/lib/transport/bdp_estimator.h

@@ -33,8 +33,6 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/time.h"
 
-extern grpc_core::TraceFlag grpc_bdp_estimator_trace;
-
 namespace grpc_core {
 
 class BdpEstimator {
@@ -51,7 +49,7 @@ class BdpEstimator {
   // grpc_bdp_estimator_add_incoming_bytes once a ping has been scheduled by a
   // transport (but not necessarily started)
   void SchedulePing() {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(bdp_estimator)) {
       gpr_log(GPR_INFO, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64,
               std::string(name_).c_str(), accumulator_, estimate_);
     }
@@ -64,7 +62,7 @@ class BdpEstimator {
   // once
   // the ping is on the wire
   void StartPing() {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(bdp_estimator)) {
       gpr_log(GPR_INFO, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64,
               std::string(name_).c_str(), accumulator_, estimate_);
     }

data/src/core/lib/transport/call_arena_allocator.h

@@ -28,7 +28,7 @@
 
 namespace grpc_core {
 
-class CallSizeEstimator {
+class CallSizeEstimator final {
  public:
   explicit CallSizeEstimator(size_t initial_estimate)
       : call_size_estimate_(initial_estimate) {}
@@ -52,19 +52,21 @@ class CallSizeEstimator {
   std::atomic<size_t> call_size_estimate_;
 };
 
-class CallArenaAllocator : public RefCounted<CallArenaAllocator> {
+class CallArenaAllocator final : public ArenaFactory {
  public:
   CallArenaAllocator(MemoryAllocator allocator, size_t initial_size)
-      : allocator_(std::move(allocator)), call_size_estimator_(initial_size) {}
+      : ArenaFactory(std::move(allocator)),
+        call_size_estimator_(initial_size) {}
 
-  Arena* MakeArena() {
-    return Arena::Create(call_size_estimator_.CallSizeEstimate(), &allocator_);
+  RefCountedPtr<Arena> MakeArena() override {
+    return Arena::Create(call_size_estimator_.CallSizeEstimate(), Ref());
   }
 
-  void Destroy(Arena* arena) { arena->Destroy(); }
+  void FinalizeArena(Arena* arena) override {
+    call_size_estimator_.UpdateCallSizeEstimate(arena->TotalUsedBytes());
+  }
 
  private:
-  MemoryAllocator allocator_;
   CallSizeEstimator call_size_estimator_;
 };
 

data/src/core/lib/transport/call_destination.h

@@ -0,0 +1,76 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_CALL_DESTINATION_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_CALL_DESTINATION_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/transport/call_spine.h"
+
+namespace grpc_core {
+
+// UnstartedCallDestination is responsible for starting an UnstartedCallHandler
+// and then processing operations on the resulting CallHandler.
+//
+// Examples of UnstartedCallDestinations include:
+// - a load-balanced call in the client channel
+// - a hijacking filter (see Interceptor)
+class UnstartedCallDestination
+    : public DualRefCounted<UnstartedCallDestination> {
+ public:
+  using DualRefCounted::DualRefCounted;
+
+  ~UnstartedCallDestination() override = default;
+  // Start a call. The UnstartedCallHandler will be consumed by the Destination
+  // and started.
+  // Must be called from the party owned by the call, eg the following must
+  // hold:
+  // CHECK(GetContext<Activity>() == unstarted_call_handler.party());
+  virtual void StartCall(UnstartedCallHandler unstarted_call_handler) = 0;
+};
+
+// CallDestination is responsible for handling processing of an already started
+// call.
+//
+// Examples of CallDestinations include:
+// - a client transport
+// - the server API
+class CallDestination : public DualRefCounted<CallDestination> {
+ public:
+  virtual void HandleCall(CallHandler unstarted_call_handler) = 0;
+};
+
+template <typename HC>
+auto MakeCallDestinationFromHandlerFunction(HC handle_call) {
+  class Impl : public CallDestination {
+   public:
+    explicit Impl(HC handle_call) : handle_call_(std::move(handle_call)) {}
+
+    void Orphaned() override {}
+
+    void HandleCall(CallHandler call_handler) override {
+      handle_call_(std::move(call_handler));
+    }
+
+   private:
+    HC handle_call_;
+  };
+  return MakeRefCounted<Impl>(std::move(handle_call));
+}
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_CALL_DESTINATION_H

data/src/core/lib/transport/call_filters.cc

@@ -29,6 +29,12 @@ void* Offset(void* base, size_t amt) { return static_cast<char*>(base) + amt; }
 
 namespace filters_detail {
 
+void RunHalfClose(absl::Span<const HalfCloseOperator> ops, void* call_data) {
+  for (const auto& op : ops) {
+    op.half_close(Offset(call_data, op.call_offset), op.channel_data);
+  }
+}
+
 template <typename T>
 OperationExecutor<T>::~OperationExecutor() {
   if (promise_data_ != nullptr) {
@@ -164,8 +170,15 @@ Poll<T> InfallibleOperationExecutor<T>::ContinueStep(void* call_data) {
 template class OperationExecutor<ClientMetadataHandle>;
 template class OperationExecutor<MessageHandle>;
 template class InfallibleOperationExecutor<ServerMetadataHandle>;
+
 }  // namespace filters_detail
 
+namespace {
+// Call data for those calls that don't have any call data
+// (we form pointers to this that aren't allowed to be nullptr)
+char g_empty_call_data;
+}  // namespace
+
 ///////////////////////////////////////////////////////////////////////////////
 // CallFilters
 
@@ -175,7 +188,7 @@ CallFilters::CallFilters(ClientMetadataHandle client_initial_metadata)
       client_initial_metadata_(std::move(client_initial_metadata)) {}
 
 CallFilters::~CallFilters() {
-  if (call_data_ != nullptr) {
+  if (call_data_ != nullptr && call_data_ != &g_empty_call_data) {
     for (const auto& destructor : stack_->data_.filter_destructor) {
       destructor.call_destroy(Offset(call_data_, destructor.call_offset));
     }
@@ -186,8 +199,12 @@ CallFilters::~CallFilters() {
 void CallFilters::SetStack(RefCountedPtr<Stack> stack) {
   CHECK_EQ(call_data_, nullptr);
   stack_ = std::move(stack);
-  call_data_ = gpr_malloc_aligned(stack_->data_.call_data_size,
-                                  stack_->data_.call_data_alignment);
+  if (stack_->data_.call_data_size != 0) {
+    call_data_ = gpr_malloc_aligned(stack_->data_.call_data_size,
+                                    stack_->data_.call_data_alignment);
+  } else {
+    call_data_ = &g_empty_call_data;
+  }
   for (const auto& constructor : stack_->data_.filter_constructor) {
     constructor.call_init(Offset(call_data_, constructor.call_offset),
                           constructor.channel_data);
@@ -208,25 +225,26 @@ void CallFilters::Finalize(const grpc_call_final_info* final_info) {
 void CallFilters::CancelDueToFailedPipeOperation(SourceLocation but_where) {
   // We expect something cancelled before now
   if (server_trailing_metadata_ == nullptr) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+  if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
     gpr_log(but_where.file(), but_where.line(), GPR_LOG_SEVERITY_DEBUG,
             "Cancelling due to failed pipe operation: %s",
             DebugString().c_str());
   }
-  server_trailing_metadata_ =
-      ServerMetadataFromStatus(absl::CancelledError("Failed pipe operation"));
+  PushServerTrailingMetadata(
+      ServerMetadataFromStatus(absl::CancelledError("Failed pipe operation")));
   server_trailing_metadata_waiter_.Wake();
 }
 
 void CallFilters::PushServerTrailingMetadata(ServerMetadataHandle md) {
   CHECK(md != nullptr);
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+  if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
     gpr_log(GPR_INFO, "%s PushServerTrailingMetadata[%p]: %s into %s",
             GetContext<Activity>()->DebugTag().c_str(), this,
             md->DebugString().c_str(), DebugString().c_str());
   }
   CHECK(md != nullptr);
-  if (server_trailing_metadata_ != nullptr) return;
+  if (cancelled_.is_set()) return;
+  cancelled_.Set(md->get(GrpcCallWasCancelled()).value_or(false));
   server_trailing_metadata_ = std::move(md);
   client_initial_metadata_state_.CloseWithError();
   server_initial_metadata_state_.CloseSending();
@@ -362,7 +380,7 @@ void filters_detail::PipeState::DropPush() {
     case ValueState::kReady:
     case ValueState::kProcessing:
     case ValueState::kWaiting:
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+      if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
         gpr_log(GPR_INFO, "%p drop push in state %s", this,
                 DebugString().c_str());
       }
@@ -382,7 +400,7 @@ void filters_detail::PipeState::DropPull() {
     case ValueState::kReady:
     case ValueState::kProcessing:
     case ValueState::kWaiting:
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_promise_primitives)) {
+      if (GRPC_TRACE_FLAG_ENABLED(promise_primitives)) {
         gpr_log(GPR_INFO, "%p drop pull in state %s", this,
                 DebugString().c_str());
       }