grpc 1.59.0 → 1.59.2

data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -24,7 +24,9 @@
 #include <string.h>
 
 #include <algorithm>
+#include <atomic>
 #include <initializer_list>
+#include <limits>
 #include <memory>
 #include <new>
 #include <string>
@@ -36,12 +38,14 @@
 #include "absl/container/flat_hash_map.h"
 #include "absl/hash/hash.h"
 #include "absl/meta/type_traits.h"
+#include "absl/random/random.h"
 #include "absl/status/status.h"
 #include "absl/strings/cord.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
+#include "absl/types/variant.h"
 
 #include <grpc/event_engine/event_engine.h>
 #include <grpc/grpc.h>
@@ -63,9 +67,12 @@
 #include "src/core/ext/transport/chttp2/transport/http_trace.h"
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/ext/transport/chttp2/transport/legacy_frame.h"
+#include "src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.h"
 #include "src/core/ext/transport/chttp2/transport/ping_abuse_policy.h"
+#include "src/core/ext/transport/chttp2/transport/ping_callbacks.h"
 #include "src/core/ext/transport/chttp2/transport/ping_rate_policy.h"
 #include "src/core/ext/transport/chttp2/transport/varint.h"
+#include "src/core/ext/transport/chttp2/transport/write_size_policy.h"
 #include "src/core/lib/channel/call_tracer.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/context.h"
@@ -116,6 +123,9 @@
 
 #define DEFAULT_MAX_PENDING_INDUCED_FRAMES 10000
 
+#define GRPC_ARG_HTTP2_PING_ON_RST_STREAM_PERCENT \
+  "grpc.http2.ping_on_rst_stream_percent"
+
 static grpc_core::Duration g_default_client_keepalive_time =
     grpc_core::Duration::Infinity();
 static grpc_core::Duration g_default_client_keepalive_timeout =
@@ -127,6 +137,11 @@ static grpc_core::Duration g_default_server_keepalive_timeout =
 static bool g_default_client_keepalive_permit_without_calls = false;
 static bool g_default_server_keepalive_permit_without_calls = false;
 
+// EXPERIMENTAL: control tarpitting in chttp2
+#define GRPC_ARG_HTTP_ALLOW_TARPIT "grpc.http.tarpit"
+#define GRPC_ARG_HTTP_TARPIT_MIN_DURATION_MS "grpc.http.tarpit_min_duration_ms"
+#define GRPC_ARG_HTTP_TARPIT_MAX_DURATION_MS "grpc.http.tarpit_max_duration_ms"
+
 #define MAX_CLIENT_STREAM_ID 0x7fffffffu
 grpc_core::TraceFlag grpc_keepalive_trace(false, "http_keepalive");
 grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount(false,
@@ -153,7 +168,7 @@ static void queue_setting_update(grpc_chttp2_transport* t,
                                  grpc_chttp2_setting_id id, uint32_t value);
 
 static void close_from_api(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
-                           grpc_error_handle error);
+                           grpc_error_handle error, bool tarpit);
 
 // Start new streams that have been created if we can
 static void maybe_start_some_streams(grpc_chttp2_transport* t);
@@ -202,21 +217,18 @@ static void init_keepalive_ping(
 static void init_keepalive_ping_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
     GRPC_UNUSED grpc_error_handle error);
-static void start_keepalive_ping(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t, grpc_error_handle error);
 static void finish_keepalive_ping(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t, grpc_error_handle error);
-static void start_keepalive_ping_locked(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t, grpc_error_handle error);
 static void finish_keepalive_ping_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t, grpc_error_handle error);
-static void keepalive_watchdog_fired(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t);
-static void keepalive_watchdog_fired_locked(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
-    GRPC_UNUSED grpc_error_handle error);
 static void maybe_reset_keepalive_ping_timer_locked(grpc_chttp2_transport* t);
 
+static void send_goaway(grpc_chttp2_transport* t, grpc_error_handle error,
+                        bool immediate_disconnect_hint);
+
+// Timeout for getting an ack back on settings changes
+#define GRPC_ARG_SETTINGS_TIMEOUT "grpc.http2.settings_timeout"
+
 namespace {
 grpc_core::CallTracerInterface* CallTracerIfEnabled(grpc_chttp2_stream* s) {
   if (s->context == nullptr || !grpc_core::IsTraceRecordCallopsEnabled()) {
@@ -317,6 +329,8 @@ void ForEachContextListEntryExecute(void* arg, Timestamps* ts,
 grpc_chttp2_transport::~grpc_chttp2_transport() {
   size_t i;
 
+  cancel_pings(this, GRPC_ERROR_CREATE("Transport destroyed"));
+
   event_engine.reset();
 
   if (channelz_socket != nullptr) {
@@ -327,8 +341,6 @@ grpc_chttp2_transport::~grpc_chttp2_transport() {
 
   grpc_slice_buffer_destroy(&qbuf);
 
-  grpc_slice_buffer_destroy(&outbuf);
-
   grpc_error_handle error = GRPC_ERROR_CREATE("Transport destroyed");
   // ContextList::Execute follows semantics of a callback function and does not
   // take a ref on error
@@ -348,8 +360,6 @@ grpc_chttp2_transport::~grpc_chttp2_transport() {
   GPR_ASSERT(stream_map.empty());
   GRPC_COMBINER_UNREF(combiner, "chttp2_transport");
 
-  cancel_pings(this, GRPC_ERROR_CREATE("Transport destroyed"));
-
   while (write_cb_pool) {
     grpc_chttp2_write_cb* next = write_cb_pool->next;
     gpr_free(write_cb_pool);
@@ -396,8 +406,16 @@ static void read_channel_args(grpc_chttp2_transport* t,
   t->keepalive_timeout = std::max(
       grpc_core::Duration::Zero(),
       channel_args.GetDurationFromIntMillis(GRPC_ARG_KEEPALIVE_TIMEOUT_MS)
-          .value_or(t->is_client ? g_default_client_keepalive_timeout
-                                 : g_default_server_keepalive_timeout));
+          .value_or(t->keepalive_time == grpc_core::Duration::Infinity()
+                        ? grpc_core::Duration::Infinity()
+                        : (t->is_client ? g_default_client_keepalive_timeout
+                                        : g_default_server_keepalive_timeout)));
+  t->ping_timeout = std::max(
+      grpc_core::Duration::Zero(),
+      channel_args.GetDurationFromIntMillis(GRPC_ARG_PING_TIMEOUT_MS)
+          .value_or(t->keepalive_time == grpc_core::Duration::Infinity()
+                        ? grpc_core::Duration::Infinity()
+                        : grpc_core::Duration::Minutes(1)));
   if (t->is_client) {
     t->keepalive_permit_without_calls =
         channel_args.GetBool(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)
@@ -412,6 +430,11 @@ static void read_channel_args(grpc_chttp2_transport* t,
                           : false);
   }
 
+  t->settings_timeout =
+      channel_args.GetDurationFromIntMillis(GRPC_ARG_SETTINGS_TIMEOUT)
+          .value_or(std::max(t->keepalive_timeout * 2,
+                             grpc_core::Duration::Minutes(1)));
+
   // Only send the prefered rx frame size http2 setting if we are instructed
   // to auto size the buffers allocated at tcp level and we also can adjust
   // sending frame size.
@@ -420,6 +443,17 @@ static void read_channel_args(grpc_chttp2_transport* t,
           .GetBool(GRPC_ARG_EXPERIMENTAL_HTTP2_PREFERRED_CRYPTO_FRAME_SIZE)
           .value_or(false);
 
+  const auto max_requests_per_read =
+      channel_args.GetInt("grpc.http2.max_requests_per_read");
+  if (max_requests_per_read.has_value()) {
+    t->max_requests_per_read =
+        grpc_core::Clamp(*max_requests_per_read, 1, 10000);
+  } else if (grpc_core::IsChttp2BatchRequestsEnabled()) {
+    t->max_requests_per_read = 32;
+  } else {
+    t->max_requests_per_read = std::numeric_limits<size_t>::max();
+  }
+
   if (channel_args.GetBool(GRPC_ARG_ENABLE_CHANNELZ)
           .value_or(GRPC_ENABLE_CHANNELZ_DEFAULT)) {
     t->channelz_socket =
@@ -434,6 +468,19 @@ static void read_channel_args(grpc_chttp2_transport* t,
 
   t->ack_pings = channel_args.GetBool("grpc.http2.ack_pings").value_or(true);
 
+  t->allow_tarpit = channel_args.GetBool(GRPC_ARG_HTTP_ALLOW_TARPIT)
+                        .value_or(grpc_core::IsTarpitEnabled());
+  t->min_tarpit_duration_ms =
+      channel_args
+          .GetDurationFromIntMillis(GRPC_ARG_HTTP_TARPIT_MIN_DURATION_MS)
+          .value_or(grpc_core::Duration::Milliseconds(100))
+          .millis();
+  t->max_tarpit_duration_ms =
+      channel_args
+          .GetDurationFromIntMillis(GRPC_ARG_HTTP_TARPIT_MAX_DURATION_MS)
+          .value_or(grpc_core::Duration::Seconds(1))
+          .millis();
+
   const int soft_limit =
       channel_args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
   if (soft_limit < 0) {
@@ -499,8 +546,12 @@ static void read_channel_args(grpc_chttp2_transport* t,
       const int value = channel_args.GetInt(setting.channel_arg_name)
                             .value_or(setting.default_value);
       if (value >= 0) {
-        queue_setting_update(t, setting.setting_id,
-                             grpc_core::Clamp(value, setting.min, setting.max));
+        const int clamped_value =
+            grpc_core::Clamp(value, setting.min, setting.max);
+        queue_setting_update(t, setting.setting_id, clamped_value);
+        if (setting.setting_id == GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS) {
+          t->max_concurrent_streams_policy.SetTarget(clamped_value);
+        }
       } else if (setting.setting_id ==
                  GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE) {
         // Set value to 1.25 * soft limit if this is larger than
@@ -533,6 +584,11 @@ static void read_channel_args(grpc_chttp2_transport* t,
         grpc_core::Clamp(INT_MAX, static_cast<int>(sp->min_value),
                          static_cast<int>(sp->max_value)));
   }
+
+  t->ping_on_rst_stream_percent = grpc_core::Clamp(
+      channel_args.GetInt(GRPC_ARG_HTTP2_PING_ON_RST_STREAM_PERCENT)
+          .value_or(1),
+      0, 100);
 }
 
 static void init_keepalive_pings_if_enabled_locked(
@@ -590,10 +646,10 @@ grpc_chttp2_transport::grpc_chttp2_transport(
   base.vtable = get_vtable();
 
   grpc_slice_buffer_init(&read_buffer);
-  grpc_slice_buffer_init(&outbuf);
   if (is_client) {
-    grpc_slice_buffer_add(&outbuf, grpc_slice_from_copied_string(
-                                       GRPC_CHTTP2_CLIENT_CONNECT_STRING));
+    grpc_slice_buffer_add(
+        outbuf.c_slice_buffer(),
+        grpc_slice_from_copied_string(GRPC_CHTTP2_CLIENT_CONNECT_STRING));
   }
   grpc_slice_buffer_init(&qbuf);
   // copy in initial settings to all setting sets
@@ -618,6 +674,12 @@ grpc_chttp2_transport::grpc_chttp2_transport(
 
   read_channel_args(this, channel_args, is_client);
 
+  // Initially allow *UP TO* MAX_CONCURRENT_STREAMS incoming before we start
+  // blanket cancelling them.
+  num_incoming_streams_before_settings_ack =
+      settings[GRPC_LOCAL_SETTINGS]
+              [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS];
+
   grpc_core::ExecCtx exec_ctx;
   combiner->Run(
       grpc_core::InitTransportClosure<init_keepalive_pings_if_enabled_locked>(
@@ -686,6 +748,18 @@ static void close_transport_locked(grpc_chttp2_transport* t,
     t->closed_with_error = error;
     connectivity_state_set(t, GRPC_CHANNEL_SHUTDOWN, absl::Status(),
                            "close_transport");
+    if (t->keepalive_ping_timeout_handle !=
+        grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid) {
+      t->event_engine->Cancel(std::exchange(
+          t->keepalive_ping_timeout_handle,
+          grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid));
+    }
+    if (t->settings_ack_watchdog !=
+        grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid) {
+      t->event_engine->Cancel(std::exchange(
+          t->settings_ack_watchdog,
+          grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid));
+    }
     if (t->delayed_ping_timer_handle.has_value()) {
       if (t->event_engine->Cancel(*t->delayed_ping_timer_handle)) {
         t->delayed_ping_timer_handle.reset();
@@ -710,11 +784,6 @@ static void close_transport_locked(grpc_chttp2_transport* t,
             t->keepalive_ping_timer_handle.reset();
           }
         }
-        if (t->keepalive_watchdog_timer_handle.has_value()) {
-          if (t->event_engine->Cancel(*t->keepalive_watchdog_timer_handle)) {
-            t->keepalive_watchdog_timer_handle.reset();
-          }
-        }
         break;
       case GRPC_CHTTP2_KEEPALIVE_STATE_DYING:
       case GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED:
@@ -777,6 +846,7 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t,
       initial_metadata_buffer(arena),
       trailing_metadata_buffer(arena),
       flow_control(&t->flow_control) {
+  t->streams_allocated.fetch_add(1, std::memory_order_relaxed);
   if (server_data) {
     id = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(server_data));
     if (grpc_http_trace.enabled()) {
@@ -793,6 +863,7 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t,
 }
 
 grpc_chttp2_stream::~grpc_chttp2_stream() {
+  t->streams_allocated.fetch_sub(1, std::memory_order_relaxed);
   grpc_chttp2_list_remove_stalled_by_stream(t.get(), this);
   grpc_chttp2_list_remove_stalled_by_transport(t.get(), this);
 
@@ -1016,7 +1087,12 @@ static void write_action(grpc_chttp2_transport* t) {
   if (max_frame_size == 0) {
     max_frame_size = INT_MAX;
   }
-  grpc_endpoint_write(t->ep, &t->outbuf,
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_ping_trace)) {
+    gpr_log(GPR_INFO, "%s[%p]: Write %" PRIdPTR " bytes",
+            t->is_client ? "CLIENT" : "SERVER", t, t->outbuf.Length());
+  }
+  t->write_size_policy.BeginWrite(t->outbuf.Length());
+  grpc_endpoint_write(t->ep, t->outbuf.c_slice_buffer(),
                       grpc_core::InitTransportClosure<write_action_end>(
                           t->Ref(), &t->write_action_end_locked),
                       cl, max_frame_size);
@@ -1025,6 +1101,10 @@ static void write_action(grpc_chttp2_transport* t) {
 static void write_action_end(grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
                              grpc_error_handle error) {
   auto* tp = t.get();
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_ping_trace)) {
+    gpr_log(GPR_INFO, "%s[%p]: Finish write",
+            t->is_client ? "CLIENT" : "SERVER", t.get());
+  }
   tp->combiner->Run(grpc_core::InitTransportClosure<write_action_end_locked>(
                         std::move(t), &tp->write_action_end_locked),
                     error);
@@ -1035,6 +1115,8 @@ static void write_action_end(grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
 static void write_action_end_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
     grpc_error_handle error) {
+  t->write_size_policy.EndWrite(error.ok());
+
   bool closed = false;
   if (!error.ok()) {
     close_transport_locked(t.get(), error);
@@ -1095,13 +1177,13 @@ static void queue_setting_update(grpc_chttp2_transport* t,
 
 // Cancel out streams that haven't yet started if we have received a GOAWAY
 static void cancel_unstarted_streams(grpc_chttp2_transport* t,
-                                     grpc_error_handle error) {
+                                     grpc_error_handle error, bool tarpit) {
   grpc_chttp2_stream* s;
   while (grpc_chttp2_list_pop_waiting_for_concurrency(t, &s)) {
     s->trailing_metadata_buffer.Set(
         grpc_core::GrpcStreamNetworkState(),
         grpc_core::GrpcStreamNetworkState::kNotSentOnWire);
-    grpc_chttp2_cancel_stream(t, s, error);
+    grpc_chttp2_cancel_stream(t, s, error, tarpit);
   }
 }
 
@@ -1134,7 +1216,7 @@ void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t,
             grpc_core::StatusToString(t->goaway_error).c_str());
   }
   if (t->is_client) {
-    cancel_unstarted_streams(t, t->goaway_error);
+    cancel_unstarted_streams(t, t->goaway_error, false);
     // Cancel all unseen streams
     std::vector<grpc_chttp2_stream*> to_cancel;
     for (auto id_stream : t->stream_map) {
@@ -1146,7 +1228,7 @@ void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t,
       s->trailing_metadata_buffer.Set(
           grpc_core::GrpcStreamNetworkState(),
           grpc_core::GrpcStreamNetworkState::kNotSeenByServer);
-      grpc_chttp2_cancel_stream(s->t.get(), s, s->t->goaway_error);
+      grpc_chttp2_cancel_stream(s->t.get(), s, s->t->goaway_error, false);
     }
   }
   absl::Status status = grpc_error_to_absl_status(t->goaway_error);
@@ -1185,7 +1267,7 @@ static void maybe_start_some_streams(grpc_chttp2_transport* t) {
   // maybe cancel out streams that haven't yet started if we have received a
   // GOAWAY
   if (!t->goaway_error.ok()) {
-    cancel_unstarted_streams(t, t->goaway_error);
+    cancel_unstarted_streams(t, t->goaway_error, false);
     return;
   }
   // start streams where we have free grpc_chttp2_stream ids and free
@@ -1227,7 +1309,8 @@ static void maybe_start_some_streams(grpc_chttp2_transport* t) {
           t, s,
           grpc_error_set_int(GRPC_ERROR_CREATE("Stream IDs exhausted"),
                              grpc_core::StatusIntProperty::kRpcStatus,
-                             GRPC_STATUS_UNAVAILABLE));
+                             GRPC_STATUS_UNAVAILABLE),
+          false);
     }
   }
 }
@@ -1372,7 +1455,8 @@ static void perform_stream_op_locked(void* stream_op,
   }
 
   if (op->cancel_stream) {
-    grpc_chttp2_cancel_stream(t, s, op_payload->cancel_stream.cancel_error);
+    grpc_chttp2_cancel_stream(t, s, op_payload->cancel_stream.cancel_error,
+                              op_payload->cancel_stream.tarpit);
   }
 
   if (op->send_initial_metadata) {
@@ -1410,7 +1494,8 @@ static void perform_stream_op_locked(void* stream_op,
                   GRPC_ERROR_CREATE_REFERENCING("Transport closed",
                                                 &t->closed_with_error, 1),
                   grpc_core::StatusIntProperty::kRpcStatus,
-                  GRPC_STATUS_UNAVAILABLE));
+                  GRPC_STATUS_UNAVAILABLE),
+              false);
         }
       } else {
         GPR_ASSERT(s->id != 0);
@@ -1618,14 +1703,38 @@ static void cancel_pings(grpc_chttp2_transport* t, grpc_error_handle error) {
                                  grpc_core::StatusToString(error).c_str()));
   // callback remaining pings: they're not allowed to call into the transport,
   //   and maybe they hold resources that need to be freed
-  grpc_chttp2_ping_queue* pq = &t->ping_queue;
-  GPR_ASSERT(!error.ok());
-  for (size_t j = 0; j < GRPC_CHTTP2_PCL_COUNT; j++) {
-    grpc_closure_list_fail_all(&pq->lists[j], error);
-    grpc_core::ExecCtx::RunList(DEBUG_LOCATION, &pq->lists[j]);
-  }
+  t->ping_callbacks.CancelAll(t->event_engine.get());
 }
 
+namespace {
+class PingClosureWrapper {
+ public:
+  explicit PingClosureWrapper(grpc_closure* closure) : closure_(closure) {}
+  PingClosureWrapper(const PingClosureWrapper&) = delete;
+  PingClosureWrapper& operator=(const PingClosureWrapper&) = delete;
+  PingClosureWrapper(PingClosureWrapper&& other) noexcept
+      : closure_(other.Take()) {}
+  PingClosureWrapper& operator=(PingClosureWrapper&& other) noexcept {
+    std::swap(closure_, other.closure_);
+    return *this;
+  }
+  ~PingClosureWrapper() {
+    if (closure_ != nullptr) {
+      grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure_, absl::CancelledError());
+    }
+  }
+
+  void operator()() {
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, Take(), absl::OkStatus());
+  }
+
+ private:
+  grpc_closure* Take() { return std::exchange(closure_, nullptr); }
+
+  grpc_closure* closure_ = nullptr;
+};
+}  // namespace
+
 static void send_ping_locked(grpc_chttp2_transport* t,
                              grpc_closure* on_initiate, grpc_closure* on_ack) {
   if (!t->closed_with_error.ok()) {
@@ -1633,11 +1742,8 @@ static void send_ping_locked(grpc_chttp2_transport* t,
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, on_ack, t->closed_with_error);
     return;
   }
-  grpc_chttp2_ping_queue* pq = &t->ping_queue;
-  grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_INITIATE], on_initiate,
-                           absl::OkStatus());
-  grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_NEXT], on_ack,
-                           absl::OkStatus());
+  t->ping_callbacks.OnPing(PingClosureWrapper(on_initiate),
+                           PingClosureWrapper(on_ack));
 }
 
 // Specialized form of send_ping_locked for keepalive ping. If there is already
@@ -1646,40 +1752,15 @@ static void send_ping_locked(grpc_chttp2_transport* t,
 static void send_keepalive_ping_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
   if (!t->closed_with_error.ok()) {
-    t->combiner->Run(
-        grpc_core::InitTransportClosure<start_keepalive_ping_locked>(
-            t->Ref(), &t->start_keepalive_ping_locked),
-        t->closed_with_error);
     t->combiner->Run(
         grpc_core::InitTransportClosure<finish_keepalive_ping_locked>(
             t->Ref(), &t->finish_keepalive_ping_locked),
         t->closed_with_error);
     return;
   }
-  grpc_chttp2_ping_queue* pq = &t->ping_queue;
-  if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) {
-    // There is a ping in flight. Add yourself to the inflight closure list.
-    t->combiner->Run(
-        grpc_core::InitTransportClosure<start_keepalive_ping_locked>(
-            t->Ref(), &t->start_keepalive_ping_locked),
-        t->closed_with_error);
-    grpc_closure_list_append(
-        &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT],
-        grpc_core::InitTransportClosure<finish_keepalive_ping>(
-            t->Ref(), &t->finish_keepalive_ping_locked),
-        absl::OkStatus());
-    return;
-  }
-  grpc_closure_list_append(
-      &pq->lists[GRPC_CHTTP2_PCL_INITIATE],
-      grpc_core::InitTransportClosure<start_keepalive_ping>(
-          t->Ref(), &t->start_keepalive_ping_locked),
-      absl::OkStatus());
-  grpc_closure_list_append(
-      &pq->lists[GRPC_CHTTP2_PCL_NEXT],
-      grpc_core::InitTransportClosure<finish_keepalive_ping>(
-          t->Ref(), &t->finish_keepalive_ping_locked),
-      absl::OkStatus());
+  t->ping_callbacks.OnPingAck(
+      PingClosureWrapper(grpc_core::InitTransportClosure<finish_keepalive_ping>(
+          t->Ref(), &t->finish_keepalive_ping_locked)));
 }
 
 void grpc_chttp2_retry_initiate_ping(
@@ -1701,19 +1782,79 @@ static void retry_initiate_ping_locked(
 }
 
 void grpc_chttp2_ack_ping(grpc_chttp2_transport* t, uint64_t id) {
-  grpc_chttp2_ping_queue* pq = &t->ping_queue;
-  if (pq->inflight_id != id) {
+  if (!t->ping_callbacks.AckPing(id, t->event_engine.get())) {
     gpr_log(GPR_DEBUG, "Unknown ping response from %s: %" PRIx64,
             std::string(t->peer_string.as_string_view()).c_str(), id);
     return;
   }
-  grpc_core::ExecCtx::RunList(DEBUG_LOCATION,
-                              &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]);
-  if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) {
+  if (t->ping_callbacks.ping_requested()) {
     grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS);
   }
 }
 
+void grpc_chttp2_keepalive_timeout(
+    grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
+  t->combiner->Run(
+      grpc_core::NewClosure([t](grpc_error_handle) {
+        gpr_log(GPR_INFO, "%s: Keepalive timeout. Closing transport.",
+                std::string(t->peer_string.as_string_view()).c_str());
+        send_goaway(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("keepalive_timeout"),
+                               grpc_core::StatusIntProperty::kHttp2Error,
+                               GRPC_HTTP2_ENHANCE_YOUR_CALM),
+            /*immediate_disconnect_hint=*/true);
+        close_transport_locked(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("keepalive timeout"),
+                               grpc_core::StatusIntProperty::kRpcStatus,
+                               GRPC_STATUS_UNAVAILABLE));
+      }),
+      absl::OkStatus());
+}
+
+void grpc_chttp2_ping_timeout(
+    grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
+  t->combiner->Run(
+      grpc_core::NewClosure([t](grpc_error_handle) {
+        gpr_log(GPR_INFO, "%s: Ping timeout. Closing transport.",
+                std::string(t->peer_string.as_string_view()).c_str());
+        send_goaway(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("ping_timeout"),
+                               grpc_core::StatusIntProperty::kHttp2Error,
+                               GRPC_HTTP2_ENHANCE_YOUR_CALM),
+            /*immediate_disconnect_hint=*/true);
+        close_transport_locked(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("ping timeout"),
+                               grpc_core::StatusIntProperty::kRpcStatus,
+                               GRPC_STATUS_UNAVAILABLE));
+      }),
+      absl::OkStatus());
+}
+
+void grpc_chttp2_settings_timeout(
+    grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
+  t->combiner->Run(
+      grpc_core::NewClosure([t](grpc_error_handle) {
+        gpr_log(GPR_INFO, "%s: Settings timeout. Closing transport.",
+                std::string(t->peer_string.as_string_view()).c_str());
+        send_goaway(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("settings_timeout"),
+                               grpc_core::StatusIntProperty::kHttp2Error,
+                               GRPC_HTTP2_SETTINGS_TIMEOUT),
+            /*immediate_disconnect_hint=*/true);
+        close_transport_locked(
+            t.get(),
+            grpc_error_set_int(GRPC_ERROR_CREATE("settings timeout"),
+                               grpc_core::StatusIntProperty::kRpcStatus,
+                               GRPC_STATUS_UNAVAILABLE));
+      }),
+      absl::OkStatus());
+}
+
 namespace {
 
 // Fire and forget (deletes itself on completion). Does a graceful shutdown by
@@ -1733,20 +1874,13 @@ class GracefulGoaway : public grpc_core::RefCounted<GracefulGoaway> {
   explicit GracefulGoaway(grpc_chttp2_transport* t) : t_(t->Ref()) {
     t->sent_goaway_state = GRPC_CHTTP2_GRACEFUL_GOAWAY;
     grpc_chttp2_goaway_append((1u << 31) - 1, 0, grpc_empty_slice(), &t->qbuf);
+    t->keepalive_timeout =
+        std::min(t->keepalive_timeout, grpc_core::Duration::Seconds(20));
+    t->ping_timeout =
+        std::min(t->ping_timeout, grpc_core::Duration::Seconds(20));
     send_ping_locked(
         t, nullptr, GRPC_CLOSURE_INIT(&on_ping_ack_, OnPingAck, this, nullptr));
     grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT);
-    timer_handle_ = t_->event_engine->RunAfter(
-        grpc_core::Duration::Seconds(20),
-        [self = Ref(DEBUG_LOCATION, "GoawayTimer")]() mutable {
-          grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
-          grpc_core::ExecCtx exec_ctx;
-          // The ref will be unreffed in the combiner.
-          auto* ptr = self.release();
-          ptr->t_->combiner->Run(
-              GRPC_CLOSURE_INIT(&ptr->on_timer_, OnTimerLocked, ptr, nullptr),
-              absl::OkStatus());
-        });
   }
 
   void MaybeSendFinalGoawayLocked() {
@@ -1787,26 +1921,12 @@ class GracefulGoaway : public grpc_core::RefCounted<GracefulGoaway> {
 
   static void OnPingAckLocked(void* arg, grpc_error_handle /* error */) {
     auto* self = static_cast<GracefulGoaway*>(arg);
-    if (self->timer_handle_ != TaskHandle::kInvalid) {
-      self->t_->event_engine->Cancel(
-          std::exchange(self->timer_handle_, TaskHandle::kInvalid));
-    }
-    self->MaybeSendFinalGoawayLocked();
-    self->Unref();
-  }
-
-  static void OnTimerLocked(void* arg, grpc_error_handle /* error */) {
-    auto* self = static_cast<GracefulGoaway*>(arg);
-    // Clearing the handle since the timer has fired and the handle is invalid.
-    self->timer_handle_ = TaskHandle::kInvalid;
     self->MaybeSendFinalGoawayLocked();
     self->Unref();
   }
 
   const grpc_core::RefCountedPtr<grpc_chttp2_transport> t_;
   grpc_closure on_ping_ack_;
-  TaskHandle timer_handle_ = TaskHandle::kInvalid;
-  grpc_closure on_timer_;
 };
 
 }  // namespace
@@ -2059,8 +2179,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t,
   }
 }
 
-static void remove_stream(grpc_chttp2_transport* t, uint32_t id,
-                          grpc_error_handle error) {
+static grpc_chttp2_transport::RemovedStreamHandle remove_stream(
+    grpc_chttp2_transport* t, uint32_t id, grpc_error_handle error) {
   grpc_chttp2_stream* s = t->stream_map.extract(id).mapped();
   GPR_DEBUG_ASSERT(s);
   if (t->incoming_stream == s) {
@@ -2083,29 +2203,74 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id,
   grpc_chttp2_list_remove_stalled_by_transport(t, s);
 
   maybe_start_some_streams(t);
+
+  if (t->is_client) return grpc_chttp2_transport::RemovedStreamHandle();
+  return grpc_chttp2_transport::RemovedStreamHandle(t->Ref());
 }
 
+namespace grpc_core {
+namespace {
+
+Duration TarpitDuration(grpc_chttp2_transport* t) {
+  return Duration::Milliseconds(absl::LogUniform<int>(
+      absl::BitGen(), t->min_tarpit_duration_ms, t->max_tarpit_duration_ms));
+}
+
+template <typename F>
+void MaybeTarpit(grpc_chttp2_transport* t, bool tarpit, F fn) {
+  if (!tarpit || !t->allow_tarpit || t->is_client) {
+    fn(t);
+    return;
+  }
+  const auto duration = TarpitDuration(t);
+  t->event_engine->RunAfter(
+      duration, [t = t->Ref(), fn = std::move(fn)]() mutable {
+        ApplicationCallbackExecCtx app_exec_ctx;
+        ExecCtx exec_ctx;
+        t->combiner->Run(
+            NewClosure([t, fn = std::move(fn)](grpc_error_handle) mutable {
+              // TODO(ctiller): this can result in not sending RST_STREAMS if a
+              // request gets tarpit behind a transport close.
+              if (!t->closed_with_error.ok()) return;
+              fn(t.get());
+            }),
+            absl::OkStatus());
+      });
+}
+
+}  // namespace
+}  // namespace grpc_core
+
 void grpc_chttp2_cancel_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
-                               grpc_error_handle due_to_error) {
+                               grpc_error_handle due_to_error, bool tarpit) {
   if (!t->is_client && !s->sent_trailing_metadata &&
-      grpc_error_has_clear_grpc_status(due_to_error)) {
-    close_from_api(t, s, due_to_error);
+      grpc_error_has_clear_grpc_status(due_to_error) &&
+      !(s->read_closed && s->write_closed)) {
+    close_from_api(t, s, due_to_error, tarpit);
     return;
   }
 
+  if (!due_to_error.ok() && !s->seen_error) {
+    s->seen_error = true;
+  }
   if (!s->read_closed || !s->write_closed) {
     if (s->id != 0) {
       grpc_http2_error_code http_error;
       grpc_error_get_status(due_to_error, s->deadline, nullptr, nullptr,
                             &http_error, nullptr);
-      grpc_chttp2_add_rst_stream_to_next_write(
-          t, s->id, static_cast<uint32_t>(http_error), &s->stats.outgoing);
-      grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM);
+      grpc_core::MaybeTarpit(
+          t, tarpit,
+          [id = s->id, http_error,
+           remove_stream_handle = grpc_chttp2_mark_stream_closed(
+               t, s, 1, 1, due_to_error)](grpc_chttp2_transport* t) {
+            grpc_chttp2_add_rst_stream_to_next_write(
+                t, id, static_cast<uint32_t>(http_error), nullptr);
+            grpc_chttp2_initiate_write(t,
+                                       GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM);
+          });
+      return;
     }
   }
-  if (!due_to_error.ok() && !s->seen_error) {
-    s->seen_error = true;
-  }
   grpc_chttp2_mark_stream_closed(t, s, 1, 1, due_to_error);
 }
 
@@ -2198,9 +2363,10 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t,
   flush_write_list(t, s, &s->on_flow_controlled_cbs, error);
 }
 
-void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t,
-                                    grpc_chttp2_stream* s, int close_reads,
-                                    int close_writes, grpc_error_handle error) {
+grpc_chttp2_transport::RemovedStreamHandle grpc_chttp2_mark_stream_closed(
+    grpc_chttp2_transport* t, grpc_chttp2_stream* s, int close_reads,
+    int close_writes, grpc_error_handle error) {
+  grpc_chttp2_transport::RemovedStreamHandle rsh;
   if (grpc_http_trace.enabled()) {
     gpr_log(
         GPR_DEBUG, "MARK_STREAM_CLOSED: t=%p s=%p(id=%d) %s [%s]", t, s, s->id,
@@ -2216,7 +2382,7 @@ void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t,
       grpc_chttp2_fake_status(t, s, overall_error);
     }
     grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s);
-    return;
+    return rsh;
   }
   bool closed_read = false;
   bool became_closed = false;
@@ -2234,7 +2400,7 @@ void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t,
     became_closed = true;
     grpc_error_handle overall_error = removal_error(error, s, "Stream removed");
     if (s->id != 0) {
-      remove_stream(t, s->id, overall_error);
+      rsh = remove_stream(t, s->id, overall_error);
     } else {
       // Purge streams waiting on concurrency still waiting for id assignment
       grpc_chttp2_list_remove_waiting_for_concurrency(t, s);
@@ -2258,17 +2424,11 @@ void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t,
     grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s);
     GRPC_CHTTP2_STREAM_UNREF(s, "chttp2");
   }
+  return rsh;
 }
 
 static void close_from_api(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
-                           grpc_error_handle error) {
-  grpc_slice hdr;
-  grpc_slice status_hdr;
-  grpc_slice http_status_hdr;
-  grpc_slice content_type_hdr;
-  grpc_slice message_pfx;
-  uint8_t* p;
-  uint32_t len = 0;
+                           grpc_error_handle error, bool tarpit) {
   grpc_status_code grpc_status;
   std::string message;
   grpc_error_get_status(error, s->deadline, &grpc_status, &message, nullptr,
@@ -2276,147 +2436,167 @@ static void close_from_api(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
 
   GPR_ASSERT(grpc_status >= 0 && (int)grpc_status < 100);
 
-  // Hand roll a header block.
-  //   This is unnecessarily ugly - at some point we should find a more
-  //   elegant solution.
-  //   It's complicated by the fact that our send machinery would be dead by
-  //   the time we got around to sending this, so instead we ignore HPACK
-  //   compression and just write the uncompressed bytes onto the wire.
-  if (!s->sent_initial_metadata) {
-    http_status_hdr = GRPC_SLICE_MALLOC(13);
-    p = GRPC_SLICE_START_PTR(http_status_hdr);
-    *p++ = 0x00;
-    *p++ = 7;
-    *p++ = ':';
-    *p++ = 's';
-    *p++ = 't';
-    *p++ = 'a';
-    *p++ = 't';
-    *p++ = 'u';
-    *p++ = 's';
-    *p++ = 3;
-    *p++ = '2';
-    *p++ = '0';
-    *p++ = '0';
-    GPR_ASSERT(p == GRPC_SLICE_END_PTR(http_status_hdr));
-    len += static_cast<uint32_t> GRPC_SLICE_LENGTH(http_status_hdr);
-
-    content_type_hdr = GRPC_SLICE_MALLOC(31);
-    p = GRPC_SLICE_START_PTR(content_type_hdr);
-    *p++ = 0x00;
-    *p++ = 12;
-    *p++ = 'c';
-    *p++ = 'o';
-    *p++ = 'n';
-    *p++ = 't';
-    *p++ = 'e';
-    *p++ = 'n';
-    *p++ = 't';
-    *p++ = '-';
-    *p++ = 't';
-    *p++ = 'y';
-    *p++ = 'p';
-    *p++ = 'e';
-    *p++ = 16;
-    *p++ = 'a';
-    *p++ = 'p';
-    *p++ = 'p';
-    *p++ = 'l';
-    *p++ = 'i';
-    *p++ = 'c';
-    *p++ = 'a';
-    *p++ = 't';
-    *p++ = 'i';
-    *p++ = 'o';
-    *p++ = 'n';
-    *p++ = '/';
-    *p++ = 'g';
-    *p++ = 'r';
-    *p++ = 'p';
-    *p++ = 'c';
-    GPR_ASSERT(p == GRPC_SLICE_END_PTR(content_type_hdr));
-    len += static_cast<uint32_t> GRPC_SLICE_LENGTH(content_type_hdr);
-  }
-
-  status_hdr = GRPC_SLICE_MALLOC(15 + (grpc_status >= 10));
-  p = GRPC_SLICE_START_PTR(status_hdr);
-  *p++ = 0x00;  // literal header, not indexed
-  *p++ = 11;    // len(grpc-status)
-  *p++ = 'g';
-  *p++ = 'r';
-  *p++ = 'p';
-  *p++ = 'c';
-  *p++ = '-';
-  *p++ = 's';
-  *p++ = 't';
-  *p++ = 'a';
-  *p++ = 't';
-  *p++ = 'u';
-  *p++ = 's';
-  if (grpc_status < 10) {
-    *p++ = 1;
-    *p++ = static_cast<uint8_t>('0' + grpc_status);
-  } else {
-    *p++ = 2;
-    *p++ = static_cast<uint8_t>('0' + (grpc_status / 10));
-    *p++ = static_cast<uint8_t>('0' + (grpc_status % 10));
-  }
-  GPR_ASSERT(p == GRPC_SLICE_END_PTR(status_hdr));
-  len += static_cast<uint32_t> GRPC_SLICE_LENGTH(status_hdr);
-
-  size_t msg_len = message.length();
-  GPR_ASSERT(msg_len <= UINT32_MAX);
-  grpc_core::VarintWriter<1> msg_len_writer(static_cast<uint32_t>(msg_len));
-  message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_writer.length());
-  p = GRPC_SLICE_START_PTR(message_pfx);
-  *p++ = 0x00;  // literal header, not indexed
-  *p++ = 12;    // len(grpc-message)
-  *p++ = 'g';
-  *p++ = 'r';
-  *p++ = 'p';
-  *p++ = 'c';
-  *p++ = '-';
-  *p++ = 'm';
-  *p++ = 'e';
-  *p++ = 's';
-  *p++ = 's';
-  *p++ = 'a';
-  *p++ = 'g';
-  *p++ = 'e';
-  msg_len_writer.Write(0, p);
-  p += msg_len_writer.length();
-  GPR_ASSERT(p == GRPC_SLICE_END_PTR(message_pfx));
-  len += static_cast<uint32_t> GRPC_SLICE_LENGTH(message_pfx);
-  len += static_cast<uint32_t>(msg_len);
-
-  hdr = GRPC_SLICE_MALLOC(9);
-  p = GRPC_SLICE_START_PTR(hdr);
-  *p++ = static_cast<uint8_t>(len >> 16);
-  *p++ = static_cast<uint8_t>(len >> 8);
-  *p++ = static_cast<uint8_t>(len);
-  *p++ = GRPC_CHTTP2_FRAME_HEADER;
-  *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
-  *p++ = static_cast<uint8_t>(s->id >> 24);
-  *p++ = static_cast<uint8_t>(s->id >> 16);
-  *p++ = static_cast<uint8_t>(s->id >> 8);
-  *p++ = static_cast<uint8_t>(s->id);
-  GPR_ASSERT(p == GRPC_SLICE_END_PTR(hdr));
-
-  grpc_slice_buffer_add(&t->qbuf, hdr);
-  if (!s->sent_initial_metadata) {
-    grpc_slice_buffer_add(&t->qbuf, http_status_hdr);
-    grpc_slice_buffer_add(&t->qbuf, content_type_hdr);
-  }
-  grpc_slice_buffer_add(&t->qbuf, status_hdr);
-  grpc_slice_buffer_add(&t->qbuf, message_pfx);
-  grpc_slice_buffer_add(&t->qbuf,
-                        grpc_slice_from_cpp_string(std::move(message)));
-  grpc_chttp2_reset_ping_clock(t);
-  grpc_chttp2_add_rst_stream_to_next_write(t, s->id, GRPC_HTTP2_NO_ERROR,
-                                           &s->stats.outgoing);
-
-  grpc_chttp2_mark_stream_closed(t, s, 1, 1, error);
-  grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API);
+  auto remove_stream_handle = grpc_chttp2_mark_stream_closed(t, s, 1, 1, error);
+  grpc_core::MaybeTarpit(
+      t, tarpit,
+      [error = std::move(error),
+       sent_initial_metadata = s->sent_initial_metadata, id = s->id,
+       grpc_status, message = std::move(message),
+       remove_stream_handle =
+           std::move(remove_stream_handle)](grpc_chttp2_transport* t) mutable {
+        grpc_slice hdr;
+        grpc_slice status_hdr;
+        grpc_slice http_status_hdr;
+        grpc_slice content_type_hdr;
+        grpc_slice message_pfx;
+        uint8_t* p;
+        uint32_t len = 0;
+
+        // Hand roll a header block.
+        //   This is unnecessarily ugly - at some point we should find a more
+        //   elegant solution.
+        //   It's complicated by the fact that our send machinery would be dead
+        //   by the time we got around to sending this, so instead we ignore
+        //   HPACK compression and just write the uncompressed bytes onto the
+        //   wire.
+        if (!sent_initial_metadata) {
+          http_status_hdr = GRPC_SLICE_MALLOC(13);
+          p = GRPC_SLICE_START_PTR(http_status_hdr);
+          *p++ = 0x00;
+          *p++ = 7;
+          *p++ = ':';
+          *p++ = 's';
+          *p++ = 't';
+          *p++ = 'a';
+          *p++ = 't';
+          *p++ = 'u';
+          *p++ = 's';
+          *p++ = 3;
+          *p++ = '2';
+          *p++ = '0';
+          *p++ = '0';
+          GPR_ASSERT(p == GRPC_SLICE_END_PTR(http_status_hdr));
+          len += static_cast<uint32_t> GRPC_SLICE_LENGTH(http_status_hdr);
+
+          content_type_hdr = GRPC_SLICE_MALLOC(31);
+          p = GRPC_SLICE_START_PTR(content_type_hdr);
+          *p++ = 0x00;
+          *p++ = 12;
+          *p++ = 'c';
+          *p++ = 'o';
+          *p++ = 'n';
+          *p++ = 't';
+          *p++ = 'e';
+          *p++ = 'n';
+          *p++ = 't';
+          *p++ = '-';
+          *p++ = 't';
+          *p++ = 'y';
+          *p++ = 'p';
+          *p++ = 'e';
+          *p++ = 16;
+          *p++ = 'a';
+          *p++ = 'p';
+          *p++ = 'p';
+          *p++ = 'l';
+          *p++ = 'i';
+          *p++ = 'c';
+          *p++ = 'a';
+          *p++ = 't';
+          *p++ = 'i';
+          *p++ = 'o';
+          *p++ = 'n';
+          *p++ = '/';
+          *p++ = 'g';
+          *p++ = 'r';
+          *p++ = 'p';
+          *p++ = 'c';
+          GPR_ASSERT(p == GRPC_SLICE_END_PTR(content_type_hdr));
+          len += static_cast<uint32_t> GRPC_SLICE_LENGTH(content_type_hdr);
+        }
+
+        status_hdr = GRPC_SLICE_MALLOC(15 + (grpc_status >= 10));
+        p = GRPC_SLICE_START_PTR(status_hdr);
+        *p++ = 0x00;  // literal header, not indexed
+        *p++ = 11;    // len(grpc-status)
+        *p++ = 'g';
+        *p++ = 'r';
+        *p++ = 'p';
+        *p++ = 'c';
+        *p++ = '-';
+        *p++ = 's';
+        *p++ = 't';
+        *p++ = 'a';
+        *p++ = 't';
+        *p++ = 'u';
+        *p++ = 's';
+        if (grpc_status < 10) {
+          *p++ = 1;
+          *p++ = static_cast<uint8_t>('0' + grpc_status);
+        } else {
+          *p++ = 2;
+          *p++ = static_cast<uint8_t>('0' + (grpc_status / 10));
+          *p++ = static_cast<uint8_t>('0' + (grpc_status % 10));
+        }
+        GPR_ASSERT(p == GRPC_SLICE_END_PTR(status_hdr));
+        len += static_cast<uint32_t> GRPC_SLICE_LENGTH(status_hdr);
+
+        size_t msg_len = message.length();
+        GPR_ASSERT(msg_len <= UINT32_MAX);
+        grpc_core::VarintWriter<1> msg_len_writer(
+            static_cast<uint32_t>(msg_len));
+        message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_writer.length());
+        p = GRPC_SLICE_START_PTR(message_pfx);
+        *p++ = 0x00;  // literal header, not indexed
+        *p++ = 12;    // len(grpc-message)
+        *p++ = 'g';
+        *p++ = 'r';
+        *p++ = 'p';
+        *p++ = 'c';
+        *p++ = '-';
+        *p++ = 'm';
+        *p++ = 'e';
+        *p++ = 's';
+        *p++ = 's';
+        *p++ = 'a';
+        *p++ = 'g';
+        *p++ = 'e';
+        msg_len_writer.Write(0, p);
+        p += msg_len_writer.length();
+        GPR_ASSERT(p == GRPC_SLICE_END_PTR(message_pfx));
+        len += static_cast<uint32_t> GRPC_SLICE_LENGTH(message_pfx);
+        len += static_cast<uint32_t>(msg_len);
+
+        hdr = GRPC_SLICE_MALLOC(9);
+        p = GRPC_SLICE_START_PTR(hdr);
+        *p++ = static_cast<uint8_t>(len >> 16);
+        *p++ = static_cast<uint8_t>(len >> 8);
+        *p++ = static_cast<uint8_t>(len);
+        *p++ = GRPC_CHTTP2_FRAME_HEADER;
+        *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM |
+               GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
+        *p++ = static_cast<uint8_t>(id >> 24);
+        *p++ = static_cast<uint8_t>(id >> 16);
+        *p++ = static_cast<uint8_t>(id >> 8);
+        *p++ = static_cast<uint8_t>(id);
+        GPR_ASSERT(p == GRPC_SLICE_END_PTR(hdr));
+
+        grpc_slice_buffer_add(&t->qbuf, hdr);
+        if (!sent_initial_metadata) {
+          grpc_slice_buffer_add(&t->qbuf, http_status_hdr);
+          grpc_slice_buffer_add(&t->qbuf, content_type_hdr);
+        }
+        grpc_slice_buffer_add(&t->qbuf, status_hdr);
+        grpc_slice_buffer_add(&t->qbuf, message_pfx);
+        grpc_slice_buffer_add(&t->qbuf,
+                              grpc_slice_from_cpp_string(std::move(message)));
+        grpc_chttp2_reset_ping_clock(t);
+        grpc_chttp2_add_rst_stream_to_next_write(t, id, GRPC_HTTP2_NO_ERROR,
+                                                 nullptr);
+
+        grpc_chttp2_initiate_write(t,
+                                   GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API);
+      });
 }
 
 static void end_all_the_calls(grpc_chttp2_transport* t,
@@ -2429,13 +2609,13 @@ static void end_all_the_calls(grpc_chttp2_transport* t,
     error = grpc_error_set_int(error, grpc_core::StatusIntProperty::kRpcStatus,
                                GRPC_STATUS_UNAVAILABLE);
   }
-  cancel_unstarted_streams(t, error);
+  cancel_unstarted_streams(t, error, false);
   std::vector<grpc_chttp2_stream*> to_cancel;
   for (auto id_stream : t->stream_map) {
     to_cancel.push_back(id_stream.second);
   }
   for (auto s : to_cancel) {
-    grpc_chttp2_cancel_stream(t, s, error);
+    grpc_chttp2_cancel_stream(t, s, error, false);
   }
 }
 
@@ -2528,21 +2708,34 @@ static void read_action(grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
                     error);
 }
 
-static void read_action_locked(
+static void read_action_parse_loop_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
     grpc_error_handle error) {
-  grpc_error_handle err = error;
-  if (!err.ok()) {
-    err = grpc_error_set_int(
-        GRPC_ERROR_CREATE_REFERENCING("Endpoint read failed", &err, 1),
-        grpc_core::StatusIntProperty::kOccurredDuringWrite, t->write_state);
-  }
-  std::swap(err, error);
   if (t->closed_with_error.ok()) {
-    size_t i = 0;
     grpc_error_handle errors[3] = {error, absl::OkStatus(), absl::OkStatus()};
-    for (; i < t->read_buffer.count && errors[1] == absl::OkStatus(); i++) {
-      errors[1] = grpc_chttp2_perform_read(t.get(), t->read_buffer.slices[i]);
+    size_t requests_started = 0;
+    for (size_t i = 0;
+         i < t->read_buffer.count && errors[1] == absl::OkStatus(); i++) {
+      auto r = grpc_chttp2_perform_read(t.get(), t->read_buffer.slices[i],
+                                        requests_started);
+      if (auto* partial_read_size = absl::get_if<size_t>(&r)) {
+        for (size_t j = 0; j < i; j++) {
+          grpc_core::CSliceUnref(grpc_slice_buffer_take_first(&t->read_buffer));
+        }
+        grpc_slice_buffer_sub_first(
+            &t->read_buffer, *partial_read_size,
+            GRPC_SLICE_LENGTH(t->read_buffer.slices[0]));
+        t->combiner->ForceOffload();
+        auto* tp = t.get();
+        tp->combiner->Run(
+            grpc_core::InitTransportClosure<read_action_parse_loop_locked>(
+                std::move(t), &tp->read_action_locked),
+            std::move(errors[0]));
+        // Early return: we queued to retry later.
+        return;
+      } else {
+        errors[1] = std::move(absl::get<absl::Status>(r));
+      }
     }
     if (errors[1] != absl::OkStatus()) {
       errors[2] = try_http_parsing(t.get());
@@ -2601,6 +2794,33 @@ static void read_action_locked(
   }
 }
 
+static void read_action_locked(
+    grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
+    grpc_error_handle error) {
+  // got an incoming read, cancel any pending keepalive timers
+  t->keepalive_incoming_data_wanted = false;
+  if (t->keepalive_ping_timeout_handle !=
+      grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_ping_trace) ||
+        GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) {
+      gpr_log(GPR_INFO,
+              "%s[%p]: Clear keepalive timer because data was received",
+              t->is_client ? "CLIENT" : "SERVER", t.get());
+    }
+    t->event_engine->Cancel(std::exchange(
+        t->keepalive_ping_timeout_handle,
+        grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid));
+  }
+  grpc_error_handle err = error;
+  if (!err.ok()) {
+    err = grpc_error_set_int(
+        GRPC_ERROR_CREATE_REFERENCING("Endpoint read failed", &err, 1),
+        grpc_core::StatusIntProperty::kOccurredDuringWrite, t->write_state);
+  }
+  std::swap(err, error);
+  read_action_parse_loop_locked(std::move(t), std::move(err));
+}
+
 static void continue_read_action_locked(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
   const bool urgent = !t->goaway_error.ok();
@@ -2793,39 +3013,6 @@ static void init_keepalive_ping_locked(
   }
 }
 
-static void start_keepalive_ping(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
-    grpc_error_handle error) {
-  auto* tp = t.get();
-  tp->combiner->Run(
-      grpc_core::InitTransportClosure<start_keepalive_ping_locked>(
-          std::move(t), &tp->start_keepalive_ping_locked),
-      error);
-}
-
-static void start_keepalive_ping_locked(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
-    grpc_error_handle error) {
-  if (!error.ok()) {
-    return;
-  }
-  if (t->channelz_socket != nullptr) {
-    t->channelz_socket->RecordKeepaliveSent();
-  }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
-      GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) {
-    gpr_log(GPR_INFO, "%s: Start keepalive ping",
-            std::string(t->peer_string.as_string_view()).c_str());
-  }
-  t->keepalive_watchdog_timer_handle =
-      t->event_engine->RunAfter(t->keepalive_timeout, [t]() mutable {
-        grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
-        grpc_core::ExecCtx exec_ctx;
-        keepalive_watchdog_fired(std::move(t));
-      });
-  t->keepalive_ping_started = true;
-}
-
 static void finish_keepalive_ping(
     grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
     grpc_error_handle error) {
@@ -2846,19 +3033,7 @@ static void finish_keepalive_ping_locked(
         gpr_log(GPR_INFO, "%s: Finish keepalive ping",
                 std::string(t->peer_string.as_string_view()).c_str());
       }
-      if (!t->keepalive_ping_started) {
-        // start_keepalive_ping_locked has not run yet. Reschedule
-        // finish_keepalive_ping_locked for it to be run later.
-        finish_keepalive_ping(std::move(t), std::move(error));
-        return;
-      }
-      t->keepalive_ping_started = false;
       t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING;
-      if (t->keepalive_watchdog_timer_handle.has_value()) {
-        if (t->event_engine->Cancel(*t->keepalive_watchdog_timer_handle)) {
-          t->keepalive_watchdog_timer_handle.reset();
-        }
-      }
       GPR_ASSERT(!t->keepalive_ping_timer_handle.has_value());
       t->keepalive_ping_timer_handle =
           t->event_engine->RunAfter(t->keepalive_time, [t] {
@@ -2870,39 +3045,6 @@ static void finish_keepalive_ping_locked(
   }
 }
 
-static void keepalive_watchdog_fired(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t) {
-  auto* tp = t.get();
-  tp->combiner->Run(
-      grpc_core::InitTransportClosure<keepalive_watchdog_fired_locked>(
-          std::move(t), &tp->keepalive_watchdog_fired_locked),
-      absl::OkStatus());
-}
-
-static void keepalive_watchdog_fired_locked(
-    grpc_core::RefCountedPtr<grpc_chttp2_transport> t,
-    GRPC_UNUSED grpc_error_handle error) {
-  GPR_DEBUG_ASSERT(error.ok());
-  GPR_ASSERT(t->keepalive_watchdog_timer_handle.has_value());
-  t->keepalive_watchdog_timer_handle.reset();
-  if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_PINGING) {
-    gpr_log(GPR_INFO, "%s: Keepalive watchdog fired. Closing transport.",
-            std::string(t->peer_string.as_string_view()).c_str());
-    t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DYING;
-    close_transport_locked(
-        t.get(),
-        grpc_error_set_int(GRPC_ERROR_CREATE("keepalive watchdog timeout"),
-                           grpc_core::StatusIntProperty::kRpcStatus,
-                           GRPC_STATUS_UNAVAILABLE));
-  } else {
-    // If keepalive_state is not PINGING, we consider it as an error. Maybe the
-    // cancellation failed in finish_keepalive_ping_locked. Users have seen
-    // other states: https://github.com/grpc/grpc/issues/32085.
-    gpr_log(GPR_ERROR, "keepalive_ping_end state error: %d (expect: %d)",
-            t->keepalive_state, GRPC_CHTTP2_KEEPALIVE_STATE_PINGING);
-  }
-}
-
 static void maybe_reset_keepalive_ping_timer_locked(grpc_chttp2_transport* t) {
   if (t->keepalive_ping_timer_handle.has_value()) {
     if (t->event_engine->Cancel(*t->keepalive_ping_timer_handle)) {
@@ -3038,7 +3180,8 @@ static void destructive_reclaimer_locked(
         t.get(), s,
         grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"),
                            grpc_core::StatusIntProperty::kHttp2Error,
-                           GRPC_HTTP2_ENHANCE_YOUR_CALM));
+                           GRPC_HTTP2_ENHANCE_YOUR_CALM),
+        false);
     if (!t->stream_map.empty()) {
       // Since we cancel one stream per destructive reclamation, if
       //   there are more streams left, we can immediately post a new