@agent-native/core 0.92.12 → 0.93.0

package/corpus/core/src/agent/run-manager.ts

@@ -18,6 +18,7 @@ import {
   cleanupOldRuns,
   updateRunHeartbeat,
   bumpRunProgress,
+  setRunInFlightMarker,
   reapIfStale,
   reapUnclaimedBackgroundRun,
   shouldRedispatchUnclaimedBackgroundRun,
@@ -627,6 +628,7 @@ export function startRun(
   let lastRealProgressAt = Date.now();
   let inFlightWorkCount = 0;
   const trackInFlightWork = (event: AgentChatEvent) => {
+    const wasIdle = inFlightWorkCount === 0;
     if (event.type === "tool_start") {
       inFlightWorkCount += 1;
     } else if (event.type === "tool_done") {
@@ -637,6 +639,23 @@ export function startRun(
       } else {
         inFlightWorkCount = Math.max(0, inFlightWorkCount - 1);
       }
+    } else {
+      return; // Not a work-tracking event — no transition possible.
+    }
+    // Mirror the 0<->N transition into SQL so a stale reaper running in a
+    // DIFFERENT isolate (a client's SQL-subscription poll, a sibling
+    // isolate's opportunistic cleanup, a fresh boot's startup sweep) can
+    // grant this demonstrably-alive run a bounded grace even when THIS
+    // isolate's own heartbeat write is failing (e.g. Neon pooler saturation)
+    // — `inFlightWorkCount` itself is in-memory and invisible to those other
+    // isolates. Fire-and-forget: never block event emission on this write,
+    // and a write failure here is no worse than today's behavior (the row
+    // just gets no grace). See `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS`
+    // in run-store.ts for the full reasoning and the bounded-grace derivation.
+    if (wasIdle && inFlightWorkCount > 0) {
+      setRunInFlightMarker(runId, true).catch(() => {});
+    } else if (!wasIdle && inFlightWorkCount === 0) {
+      setRunInFlightMarker(runId, false).catch(() => {});
     }
   };
   const checkNoProgressBackstop = () => {
@@ -1432,7 +1451,11 @@ export function getActiveRunForThread(threadId: string): ActiveRun | null {
  * dead even before the server-side stale reap has fired. Returns
  * `lastProgressAt` so the client-side stuck-detector can show a
  * user-visible "this chat looks stuck" affordance when a run is alive
- * (heartbeating) but not actually emitting events.
+ * (heartbeating) but not actually emitting events. Returns
+ * `awaitingRedispatch` so the client's background follow loop can tell a
+ * legitimately-deferred `chainServerDrivenContinuation` successor (recovery
+ * in progress server-side) apart from a genuinely dead run — see this
+ * field's own doc comment below.
  */
 export async function getActiveRunForThreadAsync(threadId: string): Promise<{
   runId: string;
@@ -1451,6 +1474,37 @@ export async function getActiveRunForThreadAsync(threadId: string): Promise<{
    * diagnosable from the client WITHOUT the unreadable bg-fn logs.
    */
   diagStage?: string | null;
+  /**
+   * True exactly when this run is a `chainServerDrivenContinuation` deferral
+   * (dispatch_mode === 'background', never claimed) still inside
+   * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — the same condition this
+   * function already uses below to skip its own `reapUnclaimedBackgroundRun`.
+   * Surfaced on `/runs/active` (agent-chat-plugin.ts) so
+   * `agent-chat-adapter.ts`'s follow loop can tell "silently deferred,
+   * server-side recovery in progress" apart from "dead" and stop counting the
+   * quiet gap against its idle timeout — see the THREE-SITE INVARIANT comment
+   * below and in agent-chat-plugin.ts / production-agent.ts. Always false for
+   * an in-memory run (that isolate IS the live producer) and for any run that
+   * isn't an unclaimed background dispatch.
+   */
+  awaitingRedispatch: boolean;
+  /**
+   * True exactly when this run's `in_flight_since` marker is set — a tool
+   * call or A2A `agent_call` delegation is open and has not yet resolved
+   * (see `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS` in
+   * run-store.ts). This is the SAME signal `reapIfStale` reads to grant its
+   * bounded stale-reap grace — computed here from the identical
+   * `in_flight_since` column via `getRunByThread`, never re-derived, so the
+   * client and the reaper cannot disagree about what "in flight" means.
+   *
+   * Surfaced on `/runs/active` (agent-chat-plugin.ts) as the
+   * server-authoritative alternative to the client-side proxy
+   * `RunStuckBanner` currently infers from unresolved `tool-call` content
+   * parts in the local message list — that proxy can go stale after a
+   * reconnect or reader-mode replay; this cannot, because it is read fresh
+   * from SQL on every poll.
+   */
+  hasInFlightWork: boolean;
 } | null> {
   // Check memory first — return both running AND recently-completed runs
   // that still have events in memory. This allows sub-agent tabs to replay
@@ -1483,6 +1537,18 @@ export async function getActiveRunForThreadAsync(threadId: string): Promise<{
       dispatchMode: sqlSnapshot?.dispatchMode ?? null,
       terminalReason: sqlSnapshot?.terminalReason ?? null,
       diagStage: sqlSnapshot?.diagStage ?? null,
+      // In-memory means this isolate is the live producer — never the
+      // "deferred, nobody producing" state this flag identifies.
+      awaitingRedispatch: false,
+      // Read from the SAME SQL snapshot the other fields above already read
+      // (`fetchRunThreadSnapshot` -> `getRunByThread`) rather than the
+      // producer's own in-memory `inFlightWorkCount` — this isolate IS the
+      // live producer, but there is no separate in-memory channel wired for
+      // that counter today, and the SQL marker is written on every 0<->N
+      // transition (see run-manager's `trackInFlightWork`), so it is at most
+      // one event behind here — the same tolerance `lastProgressAt` above
+      // already accepts.
+      hasInFlightWork: sqlSnapshot?.inFlightSince != null,
     };
   }
   // Fall back to SQL — also surface recently terminated runs so the client
@@ -1525,12 +1591,17 @@ export async function getActiveRunForThreadAsync(threadId: string): Promise<{
       // follow loop AUTO-continues — and once the redispatch bound is exceeded
       // this reap fires loudly as before. So recovery stays automatic in the
       // common case and loud failure is only moved later, never removed.
-      if (
-        sqlRun.dispatchMode === "background" &&
-        !shouldRedispatchUnclaimedBackgroundRun({
-          startedAt: sqlRun.startedAt,
-        })
-      ) {
+      //
+      // `isUnclaimedBackgroundDispatch` also becomes the `awaitingRedispatch`
+      // wire field below once the still-inside-the-bound check passes — see
+      // this function's doc comment and the THREE-SITE INVARIANT comment in
+      // agent-chat-plugin.ts / production-agent.ts.
+      const isUnclaimedBackgroundDispatch =
+        sqlRun.dispatchMode === "background";
+      const stillInsideRedispatchBound = shouldRedispatchUnclaimedBackgroundRun(
+        { startedAt: sqlRun.startedAt },
+      );
+      if (isUnclaimedBackgroundDispatch && !stillInsideRedispatchBound) {
         const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch(
           () => false,
         );
@@ -1551,6 +1622,12 @@ export async function getActiveRunForThreadAsync(threadId: string): Promise<{
         dispatchMode: sqlRun.dispatchMode,
         terminalReason: sqlRun.terminalReason,
         diagStage: sqlRun.diagStage,
+        awaitingRedispatch:
+          isUnclaimedBackgroundDispatch && stillInsideRedispatchBound,
+        // Same `in_flight_since` column `reapIfStale` (just called above,
+        // and it did NOT reap this row) reads for its own grace decision —
+        // one source of truth, not a second re-derived notion of "in flight".
+        hasInFlightWork: sqlRun.inFlightSince != null,
       };
     }
     if (sqlRun.status === "completed" || sqlRun.status === "errored") {
@@ -1579,6 +1656,10 @@ export async function getActiveRunForThreadAsync(threadId: string): Promise<{
         dispatchMode: sqlRun.dispatchMode,
         terminalReason: sqlRun.terminalReason,
         diagStage: sqlRun.diagStage,
+        // Terminal already — never the "still deferred, running" state.
+        awaitingRedispatch: false,
+        // Terminal already — no live work can still be in flight.
+        hasInFlightWork: false,
       };
     }
   } catch {
@@ -1606,15 +1687,54 @@ export function getRun(runId: string): ActiveRun | null {
   return activeRuns.get(runId) ?? null;
 }
 
-/** Explicitly abort a run (e.g. Stop button) */
-export function abortRun(runId: string, reason: string = "user"): boolean {
+function abortRunInMemory(runId: string, reason: string): boolean {
   const run = activeRuns.get(runId);
   if (run) {
     abortInMemoryRun(run, reason);
   }
+  return !!run;
+}
+
+/** Explicitly abort a run (e.g. Stop button). */
+export function abortRun(runId: string, reason: string = "user"): boolean {
+  const abortedInMemory = abortRunInMemory(runId, reason);
   // Also mark as aborted in SQL (for cross-isolate abort on Workers)
   markRunAborted(runId, reason).catch(() => {});
-  return !!run;
+  return abortedInMemory;
+}
+
+/**
+ * Abort a run and wait until the cross-isolate SQL state and terminal event
+ * are durable. Request handlers that start recovery immediately after aborting
+ * must use this path; otherwise the recovery POST can race the old row while
+ * it is still marked running.
+ */
+export async function abortRunDurably(
+  runId: string,
+  reason: string = "user",
+): Promise<boolean> {
+  const abortedInMemory = abortRunInMemory(runId, reason);
+  try {
+    await markRunAborted(runId, reason);
+  } catch (error) {
+    // The local run is already stopped. A transient durable cleanup failure
+    // must not turn the user's Stop/Retry request into a 500 after that
+    // irreversible in-memory outcome. Capture it for repair/reaping and let
+    // the request report the abort it did complete.
+    captureError(error, {
+      route: "/_agent-native/agent-chat/runs/:id/abort",
+      tags: {
+        source: "agent-run-manager",
+        phase: "abort-run",
+      },
+      extra: { runId, reason, abortedInMemory },
+    });
+    console.error(
+      "[run-manager] durable abort persistence failed:",
+      error instanceof Error ? error.message : error,
+    );
+  }
+  return abortedInMemory;
 }
 
 // Re-export so callers can avoid importing from run-store directly.

package/corpus/core/src/agent/run-store.ts

@@ -118,6 +118,84 @@ export const UNCLAIMED_BACKGROUND_RUN_GRACE_MS = 25_000;
  */
 export const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS = 5 * 60_000;
 
+/**
+ * Tick interval for the DEDICATED fast redispatch sweep in
+ * agent-chat-plugin.ts (distinct from that file's general-purpose 2-minute
+ * orphan/reap sweep). Only attempts redispatch for rows still inside
+ * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — it never reaps, so it
+ * cannot race the loud-failure fallback onto an earlier trigger.
+ *
+ * This constant exists because the general sweep's 2-minute cadence puts the
+ * FIRST redispatch attempt uncomfortably close to (and on a slow tick, past)
+ * `BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS` (150s, agent-chat-adapter.ts) — the
+ * client following a deferred successor would give up and report a fatal
+ * error for a turn the server was silently about to recover. The whole
+ * budget is a derived chain, each bound following from the one before it:
+ *
+ *   UNCLAIMED_BACKGROUND_RUN_GRACE_MS        (25s)  row must look abandoned
+ * + UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS   (20s)  worst-case tick latency
+ * = ~45s worst-case time-to-first-redispatch-attempt, ~65s to a second
+ *   attempt if the first fails — both comfortably under the client's 150s
+ *   idle timeout, which additionally no longer counts a known-deferred row
+ *   against its idle window at all (see `awaitingRedispatch` surfaced by
+ *   `/runs/active` and consumed by the client follow loop).
+ * < BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS       (150s) client's own backstop
+ * < UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS (300s) hard, unresettable
+ *   ceiling — untouched by this constant — past which the slow sweep's
+ *   existing loud reap (`background_worker_never_started`) still fires.
+ */
+export const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20_000;
+
+/**
+ * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
+ * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
+ * row that is marked in-flight (`in_flight_since`, see `setRunInFlightMarker`)
+ * even though its heartbeat/progress liveness basis
+ * (`livenessBasisSql`/`backgroundAwareStaleCutoffSql`) has gone stale.
+ *
+ * WHY a marker column at all: `inFlightWorkCount` in run-manager.ts (the
+ * no-progress backstop's guard) is in-memory, per-isolate — but all three
+ * reapers above can run in a DIFFERENT isolate than the one holding the
+ * producing run (a client's SQL-subscription poll, a sibling isolate's
+ * opportunistic `cleanupOldRuns` after ITS OWN run completes, or a fresh
+ * boot's `reapAllStaleRuns`). None of them can read another isolate's
+ * in-memory counter, so the counter's 0->1 / 1->0 transitions are mirrored
+ * into this column (`setRunInFlightMarker`, called from
+ * run-manager.ts's `trackInFlightWork`) so it is observable from SQL. This is
+ * exactly the gap that let a demonstrably-alive run holding a long tool call
+ * or A2A `call-agent` delegation get reaped: the heartbeat WRITE can fail
+ * silently (Neon pooler saturation) for the whole `BACKGROUND_RUN_STALE_MS`
+ * window while the run is provably still doing work.
+ *
+ * BOUNDED, not a silent hang — derived from two independent ceilings already
+ * in the codebase, not picked by feel:
+ *   - `DEFAULT_TOOL_TIMEOUT_MS` (12 min, production-agent.ts) is the longest
+ *     any SINGLE tool call or `agent_call` (A2A delegation) may legitimately
+ *     stay in flight — past that its own `AbortSignal.timeout` forces a
+ *     tool_done/error and clears the marker.
+ *   - `BACKGROUND_SOFT_TIMEOUT_CEILING_MS` (13 min, run-manager.ts) is the
+ *     background chunk's OWN soft-timeout ceiling. Unlike the no-progress
+ *     backstop, this timer is NOT gated on in-flight work (see the "secondary"
+ *     hazard documented next to the soft-timeout timer in run-manager.ts) —
+ *     it fires unconditionally and checkpoints/continues the run, so by 13
+ *     minutes the row leaves status='running' via that path regardless of
+ *     what the marker says.
+ * This grace is the LARGER of the two (13 min) plus one `BACKGROUND_RUN_STALE_MS`
+ * (90s) buffer for that checkpoint's own completion write to land under the
+ * same DB pressure that could have caused the heartbeat to lapse in the first
+ * place: 780_000 + 90_000 = 870_000ms (14.5 min). Past that, a "running" row
+ * that still shows in-flight work AND a stale liveness basis is not a slow
+ * producer anymore — every backstop that should have ended it has ALSO failed
+ * to write, and it is reaped loud like any other stale run.
+ *
+ * Never applied when a caller passes an explicit `maxStaleMs` override to
+ * `reapIfStale` — that escape hatch is an exact, caller-chosen window and
+ * stays exact. Never weakens the no-in-flight case: a row with no marker set
+ * evaluates this grace clause to a no-op and is reaped at the original
+ * `BACKGROUND_RUN_STALE_MS` / `RUN_STALE_MS` exactly as before.
+ */
+export const IN_FLIGHT_RUN_STALE_GRACE_MS = 14.5 * 60_000; // 870_000
+
 async function ensureRunTables(): Promise<void> {
   if (!_initPromise) {
     _initPromise = (async () => {
@@ -225,6 +303,14 @@ async function ensureRunTables(): Promise<void> {
           // rehydrates the body from this column via the marker's payloadRef.
           // Cleared on terminal status writes.
           ["dispatch_payload", "TEXT"],
+          // in_flight_since = ms epoch when run-manager's in-memory
+          // `inFlightWorkCount` last transitioned 0->1 (a tool call or nested
+          // `agent_call`/A2A delegation started), NULL once it drops back to 0.
+          // Lets the cross-isolate stale reapers grant a bounded grace to a
+          // demonstrably-alive run even when the SAME-isolate heartbeat write
+          // has failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and
+          // `setRunInFlightMarker`.
+          ["in_flight_since", intType()],
         ] as const) {
           await ensureColumnExists(
             "agent_runs",
@@ -250,6 +336,7 @@ async function ensureRunTables(): Promise<void> {
           "completed_at",
           "heartbeat_at",
           "last_progress_at",
+          "in_flight_since",
         ]);
         await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
         await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -286,6 +373,18 @@ async function ensureRunTables(): Promise<void> {
       } catch {
         // Column already exists — ignore
       }
+      // Backfill in_flight_since — ms epoch when run-manager's in-memory
+      // `inFlightWorkCount` last transitioned 0->1, NULL once back to 0. Lets
+      // the cross-isolate stale reapers grant a bounded grace to a
+      // demonstrably-alive run even when the heartbeat write itself has
+      // failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and `setRunInFlightMarker`.
+      try {
+        await client.execute(
+          `ALTER TABLE agent_runs ADD COLUMN in_flight_since ${intType()}`,
+        );
+      } catch {
+        // Column already exists — ignore
+      }
       // Backfill turn_id / error_code / error_detail.
       //   turn_id    = stable identity for one logical assistant turn that may
       //                span several continuation runs, so the durable record
@@ -339,6 +438,7 @@ async function ensureRunTables(): Promise<void> {
         "completed_at",
         "heartbeat_at",
         "last_progress_at",
+        "in_flight_since",
       ]);
       await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
       await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -529,6 +629,23 @@ function livenessBasisSql(): string {
   return `(CASE WHEN COALESCE(last_progress_at, started_at) > COALESCE(heartbeat_at, started_at) THEN COALESCE(last_progress_at, started_at) ELSE COALESCE(heartbeat_at, started_at) END)`;
 }
 
+/**
+ * Additive grace clause for the default (no explicit `maxStaleMs` override)
+ * heartbeat/liveness-based stale reap conditions — TRUE (row remains eligible
+ * for the surrounding staleness check) unless `in_flight_since` is set AND
+ * still inside `IN_FLIGHT_RUN_STALE_GRACE_MS`. A row with no marker set
+ * (`in_flight_since IS NULL`, the common case and every pre-existing row
+ * before this migration) always evaluates TRUE here, so this can only make
+ * reaping MORE conservative — the no-in-flight `BACKGROUND_RUN_STALE_MS` /
+ * `RUN_STALE_MS` behavior is unchanged. See `IN_FLIGHT_RUN_STALE_GRACE_MS`'s
+ * doc comment for why this is sound and bounded.
+ *
+ * Binds one param: the same `now` value the surrounding cutoff clause binds.
+ */
+function inFlightGraceSql(): string {
+  return `(in_flight_since IS NULL OR in_flight_since <= (CAST(? AS BIGINT) - ${IN_FLIGHT_RUN_STALE_GRACE_MS}))`;
+}
+
 /**
  * Atomically claim a background-dispatched run for processing. The foreground
  * POST inserts the run row with `dispatch_mode = 'background'`; the FIRST
@@ -1139,12 +1256,57 @@ export async function updateRunHeartbeat(runId: string): Promise<void> {
 export async function bumpRunProgress(runId: string): Promise<void> {
   await ensureRunTables();
   const client = getDbExec();
+  const now = Date.now();
   await client.execute({
-    sql: `UPDATE agent_runs SET last_progress_at = ? WHERE id = ?`,
-    args: [Date.now(), runId],
+    // Multiple event-persistence paths and serverless isolates can bump the
+    // same run concurrently. A slower, older write must never land after a
+    // newer one and move the user-visible no-progress clock backwards.
+    // CASE keeps this portable across SQLite and Postgres.
+    sql: `UPDATE agent_runs SET last_progress_at = CASE WHEN last_progress_at IS NULL OR last_progress_at < ? THEN ? ELSE last_progress_at END WHERE id = ? AND status = 'running'`,
+    args: [now, now, runId],
   });
 }
 
+/**
+ * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
+ * SQL so a stale reaper running in a DIFFERENT isolate can tell a
+ * demonstrably-alive run (holding a tool call or A2A `agent_call` delegation)
+ * apart from a genuinely dead one — see `IN_FLIGHT_RUN_STALE_GRACE_MS`'s doc
+ * comment for the full reasoning.
+ *
+ * `inFlight: true` only writes when the row is still `NULL` — a defense-in-
+ * depth belt-and-suspenders against a nested 1->2 transition clobbering the
+ * ORIGINAL start time with a later one (the caller's own counter already
+ * dedupes 0->1 transitions; this WHERE just makes the write itself
+ * idempotent/order-independent too). `inFlight: false` always clears
+ * unconditionally — if it races a fresh 0->1 write from a *different* tool
+ * finishing/starting back to back, worst case is losing a few seconds of
+ * grace, never gaining an incorrect one.
+ *
+ * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
+ * failure here never blocks event emission or aborts the run. If this write
+ * itself fails (the same DB pressure that could be starving the heartbeat),
+ * the row simply gets no grace — never worse than today's behavior.
+ */
+export async function setRunInFlightMarker(
+  runId: string,
+  inFlight: boolean,
+): Promise<void> {
+  await ensureRunTables();
+  const client = getDbExec();
+  if (inFlight) {
+    await client.execute({
+      sql: `UPDATE agent_runs SET in_flight_since = ? WHERE id = ? AND status = 'running' AND in_flight_since IS NULL`,
+      args: [Date.now(), runId],
+    });
+  } else {
+    await client.execute({
+      sql: `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ?`,
+      args: [runId],
+    });
+  }
+}
+
 /**
  * If the given run is marked "running" in SQL but its heartbeat is stale
  * (producer likely crashed), flip it to "errored" so watchers stop waiting.
@@ -1160,13 +1322,20 @@ export async function reapIfStale(
   const completedAt = Date.now();
   // Background-dispatched runs get the wider stale window so a slow cold-start
   // isn't reaped; foreground runs keep the tight 15s window. An explicit
-  // override forces a flat window for callers that want one.
+  // override forces a flat window for callers that want one — and, like that
+  // override already bypasses the background-aware widening, it also bypasses
+  // the in-flight grace below: an explicit `maxStaleMs` is an exact,
+  // caller-chosen window and stays exact.
   const staleClause =
     typeof maxStaleMs === "number"
       ? `${livenessBasisSql()} < ?`
-      : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`;
+      : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()}`;
   const staleArgs =
-    typeof maxStaleMs === "number" ? [completedAt - maxStaleMs] : [completedAt];
+    typeof maxStaleMs === "number"
+      ? [completedAt - maxStaleMs]
+      : // First `?` is backgroundAwareStaleCutoffSql's CAST param, second is
+        // inFlightGraceSql's — both bound to the same "now".
+        [completedAt, completedAt];
   const { rowsAffected } = await client.execute({
     sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1500,12 +1669,23 @@ export async function getRunByThread(
   dispatchMode: string | null;
   terminalReason: string | null;
   diagStage: string | null;
+  /**
+   * Raw `in_flight_since` marker (see `setRunInFlightMarker`) — non-null
+   * exactly when run-manager's in-memory `inFlightWorkCount` was last known
+   * (from THIS row's own producer) to be > 0: a tool call or A2A `agent_call`
+   * delegation is open and has not yet resolved. Callers that want the
+   * authoritative "does this run currently hold live work" signal (e.g. the
+   * `hasInFlightWork` wire field on `/runs/active`) should test this for
+   * non-null, not re-derive their own notion of in-flight — see
+   * `getActiveRunForThreadAsync` in run-manager.ts.
+   */
+  inFlightSince: number | null;
 } | null> {
   await ensureRunTables();
   const client = getDbExec();
   const sql = options?.includeTerminal
-    ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
-    : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
+    ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
+    : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
   const { rows } = await client.execute({ sql, args: [threadId] });
   if (rows.length === 0) return null;
   const r = rows[0] as {
@@ -1521,6 +1701,7 @@ export async function getRunByThread(
     terminal_reason?: string | null;
     diag_stage?: string | null;
     error_code?: string | null;
+    in_flight_since?: number | string | null;
   };
   const canReconcileFromEvents =
     r.status === "running" ||
@@ -1542,6 +1723,7 @@ export async function getRunByThread(
     dispatchMode: r.dispatch_mode ?? null,
     terminalReason: r.terminal_reason ?? null,
     diagStage: r.diag_stage ?? null,
+    inFlightSince: r.in_flight_since == null ? null : Number(r.in_flight_since),
   };
 }
 
@@ -1706,12 +1888,21 @@ export async function reapAllStaleRuns(): Promise<number> {
   await ensureRunTables();
   const client = getDbExec();
   const now = Date.now();
-  // Background-dispatched runs use the wider window; everything else 15s.
+  // Background-dispatched runs use the wider window; everything else 15s. The
+  // in-flight grace clause is applied identically to this SELECT and the
+  // UPDATE below (both derive `stale.rows`/the terminal-event-append loop
+  // from the SAME predicate) so a row the grace clause protects from the
+  // UPDATE is never mistakenly given a terminal event anyway — see
+  // `inFlightGraceSql` and `IN_FLIGHT_RUN_STALE_GRACE_MS`. This runs at
+  // server startup across possibly-multiple isolates, so a sibling isolate's
+  // still-alive, in-flight run must not be reaped just because THIS isolate
+  // just booted and has no heartbeat history for it.
   const stale = await client.execute({
     sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
-    args: [now],
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
+    args: [now, now],
   });
   for (const row of stale.rows) {
     const id = (row as { id?: unknown }).id;
@@ -1729,13 +1920,15 @@ export async function reapAllStaleRuns(): Promise<number> {
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
     args: [
       completedAt,
       STALE_RUN_ERROR_EVENT.errorCode,
       STALE_RUN_ERROR_EVENT.details,
       STALE_RUN_ERROR_EVENT.errorCode,
       completedAt,
+      completedAt,
     ],
   });
   for (const row of stale.rows) {
@@ -1770,15 +1963,24 @@ export async function cleanupOldRuns(
   // SELECT covers BOTH UPDATE conditions so the terminal-event-append loop
   // below catches every row we're about to flip — a 24h-old row with a
   // somehow-fresh heartbeat would slip past a heartbeat-only SELECT.
+  //
+  // The in-flight grace clause is applied ONLY to the heartbeat-based branch
+  // (mirroring the second UPDATE below), never the absolute-age branch:
+  // nothing can legitimately hold in-flight work anywhere near `olderThanMs`
+  // (default on the order of a day) — `IN_FLIGHT_RUN_STALE_GRACE_MS` bounds
+  // any real grace at ~14.5 minutes — so a row that old is dead regardless of
+  // its in-flight marker. This runs opportunistically after EVERY run
+  // completes in ANY isolate, so a different thread's still-in-flight A2A
+  // call must not be reaped as a side effect of an unrelated run finishing.
   const now = Date.now();
   const stale = await client.execute({
     sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
             AND (
-              ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+              (${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()})
               OR started_at < ?
     )`,
-    args: [now, cutoff],
+    args: [now, now, cutoff],
   });
   for (const row of stale.rows) {
     const id = (row as { id?: unknown }).id;
@@ -1806,7 +2008,13 @@ export async function cleanupOldRuns(
     ],
   });
   // Also expire runs whose heartbeat is stale — producer has died. Uses the
-  // background-aware window so a slow background cold-start isn't reaped early.
+  // background-aware window so a slow background cold-start isn't reaped
+  // early, and the in-flight grace so a demonstrably-alive run holding a tool
+  // call / A2A delegation survives a heartbeat write failure. Must match the
+  // SELECT's heartbeat branch above exactly — the terminal-event-append loop
+  // below fires for every row the SELECT returned, so a mismatch would
+  // silently inject a terminal error event onto a row this UPDATE left
+  // status='running'.
   await client.execute({
     sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1816,13 +2024,15 @@ export async function cleanupOldRuns(
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
     args: [
       completedAt,
       STALE_RUN_ERROR_EVENT.errorCode,
       STALE_RUN_ERROR_EVENT.details,
       STALE_RUN_ERROR_EVENT.errorCode,
       completedAt,
+      completedAt,
     ],
   });
   for (const row of stale.rows) {

package/corpus/core/src/agent/types.ts

@@ -274,6 +274,29 @@ export type AgentChatEvent =
       agent: string;
       status: "start" | "done" | "error";
     }
+  | {
+      /**
+       * Periodic liveness for an in-flight cross-app A2A call. Emitted by the
+       * `call-agent` action once per throttle window ONLY when a real poll
+       * round-trip to the remote agent succeeds and reports a non-terminal
+       * state — never on a timer, so a hung/dead remote emits nothing and the
+       * stuck-detector can still fire. Counts as real progress in
+       * `run-manager`'s `shouldBumpProgressForEvent` (any non-special event
+       * type does), which keeps `last_progress_at` fresh so a slow-but-healthy
+       * sub-agent call doesn't trip the client's stuck banner. A distinct
+       * event type (not an `agent_call` status) so existing `agent_call`
+       * consumers that treat "not start/done" as a failure don't render an
+       * in-flight tick as an error.
+       */
+      type: "agent_call_progress";
+      agent: string;
+      /** Remote A2A task state for this poll, e.g. "working" | "submitted". */
+      state: string;
+      /** Elapsed wall-clock seconds since the cross-app call began. */
+      elapsedSeconds: number;
+      /** Optional short text surfaced from the remote poll, when present. */
+      detail?: string;
+    }
   | { type: "agent_call_text"; agent: string; text: string }
   | {
       type: "agent_task";