@agent-native/core 0.92.11 → 0.93.0

package/corpus/core/src/agent/run-store.ts

@@ -7,6 +7,10 @@ import { getDbExec, intType, isPostgres } from "../db/client.js";
 import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
 import { captureError } from "../server/capture-error.js";
+import {
+  LLM_MISSING_CREDENTIALS_ERROR_CODE,
+  LLM_MISSING_CREDENTIALS_MESSAGE,
+} from "./engine/credential-errors.js";
 import type { AgentChatEvent } from "./types.js";
 
 let _initPromise: Promise<void> | undefined;
@@ -114,6 +118,84 @@ export const UNCLAIMED_BACKGROUND_RUN_GRACE_MS = 25_000;
  */
 export const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS = 5 * 60_000;
 
+/**
+ * Tick interval for the DEDICATED fast redispatch sweep in
+ * agent-chat-plugin.ts (distinct from that file's general-purpose 2-minute
+ * orphan/reap sweep). Only attempts redispatch for rows still inside
+ * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — it never reaps, so it
+ * cannot race the loud-failure fallback onto an earlier trigger.
+ *
+ * This constant exists because the general sweep's 2-minute cadence puts the
+ * FIRST redispatch attempt uncomfortably close to (and on a slow tick, past)
+ * `BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS` (150s, agent-chat-adapter.ts) — the
+ * client following a deferred successor would give up and report a fatal
+ * error for a turn the server was silently about to recover. The whole
+ * budget is a derived chain, each bound following from the one before it:
+ *
+ *   UNCLAIMED_BACKGROUND_RUN_GRACE_MS        (25s)  row must look abandoned
+ * + UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS   (20s)  worst-case tick latency
+ * = ~45s worst-case time-to-first-redispatch-attempt, ~65s to a second
+ *   attempt if the first fails — both comfortably under the client's 150s
+ *   idle timeout, which additionally no longer counts a known-deferred row
+ *   against its idle window at all (see `awaitingRedispatch` surfaced by
+ *   `/runs/active` and consumed by the client follow loop).
+ * < BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS       (150s) client's own backstop
+ * < UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS (300s) hard, unresettable
+ *   ceiling — untouched by this constant — past which the slow sweep's
+ *   existing loud reap (`background_worker_never_started`) still fires.
+ */
+export const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20_000;
+
+/**
+ * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
+ * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
+ * row that is marked in-flight (`in_flight_since`, see `setRunInFlightMarker`)
+ * even though its heartbeat/progress liveness basis
+ * (`livenessBasisSql`/`backgroundAwareStaleCutoffSql`) has gone stale.
+ *
+ * WHY a marker column at all: `inFlightWorkCount` in run-manager.ts (the
+ * no-progress backstop's guard) is in-memory, per-isolate — but all three
+ * reapers above can run in a DIFFERENT isolate than the one holding the
+ * producing run (a client's SQL-subscription poll, a sibling isolate's
+ * opportunistic `cleanupOldRuns` after ITS OWN run completes, or a fresh
+ * boot's `reapAllStaleRuns`). None of them can read another isolate's
+ * in-memory counter, so the counter's 0->1 / 1->0 transitions are mirrored
+ * into this column (`setRunInFlightMarker`, called from
+ * run-manager.ts's `trackInFlightWork`) so it is observable from SQL. This is
+ * exactly the gap that let a demonstrably-alive run holding a long tool call
+ * or A2A `call-agent` delegation get reaped: the heartbeat WRITE can fail
+ * silently (Neon pooler saturation) for the whole `BACKGROUND_RUN_STALE_MS`
+ * window while the run is provably still doing work.
+ *
+ * BOUNDED, not a silent hang — derived from two independent ceilings already
+ * in the codebase, not picked by feel:
+ *   - `DEFAULT_TOOL_TIMEOUT_MS` (12 min, production-agent.ts) is the longest
+ *     any SINGLE tool call or `agent_call` (A2A delegation) may legitimately
+ *     stay in flight — past that its own `AbortSignal.timeout` forces a
+ *     tool_done/error and clears the marker.
+ *   - `BACKGROUND_SOFT_TIMEOUT_CEILING_MS` (13 min, run-manager.ts) is the
+ *     background chunk's OWN soft-timeout ceiling. Unlike the no-progress
+ *     backstop, this timer is NOT gated on in-flight work (see the "secondary"
+ *     hazard documented next to the soft-timeout timer in run-manager.ts) —
+ *     it fires unconditionally and checkpoints/continues the run, so by 13
+ *     minutes the row leaves status='running' via that path regardless of
+ *     what the marker says.
+ * This grace is the LARGER of the two (13 min) plus one `BACKGROUND_RUN_STALE_MS`
+ * (90s) buffer for that checkpoint's own completion write to land under the
+ * same DB pressure that could have caused the heartbeat to lapse in the first
+ * place: 780_000 + 90_000 = 870_000ms (14.5 min). Past that, a "running" row
+ * that still shows in-flight work AND a stale liveness basis is not a slow
+ * producer anymore — every backstop that should have ended it has ALSO failed
+ * to write, and it is reaped loud like any other stale run.
+ *
+ * Never applied when a caller passes an explicit `maxStaleMs` override to
+ * `reapIfStale` — that escape hatch is an exact, caller-chosen window and
+ * stays exact. Never weakens the no-in-flight case: a row with no marker set
+ * evaluates this grace clause to a no-op and is reaped at the original
+ * `BACKGROUND_RUN_STALE_MS` / `RUN_STALE_MS` exactly as before.
+ */
+export const IN_FLIGHT_RUN_STALE_GRACE_MS = 14.5 * 60_000; // 870_000
+
 async function ensureRunTables(): Promise<void> {
   if (!_initPromise) {
     _initPromise = (async () => {
@@ -221,6 +303,14 @@ async function ensureRunTables(): Promise<void> {
           // rehydrates the body from this column via the marker's payloadRef.
           // Cleared on terminal status writes.
           ["dispatch_payload", "TEXT"],
+          // in_flight_since = ms epoch when run-manager's in-memory
+          // `inFlightWorkCount` last transitioned 0->1 (a tool call or nested
+          // `agent_call`/A2A delegation started), NULL once it drops back to 0.
+          // Lets the cross-isolate stale reapers grant a bounded grace to a
+          // demonstrably-alive run even when the SAME-isolate heartbeat write
+          // has failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and
+          // `setRunInFlightMarker`.
+          ["in_flight_since", intType()],
         ] as const) {
           await ensureColumnExists(
             "agent_runs",
@@ -246,6 +336,7 @@ async function ensureRunTables(): Promise<void> {
           "completed_at",
           "heartbeat_at",
           "last_progress_at",
+          "in_flight_since",
         ]);
         await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
         await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -282,6 +373,18 @@ async function ensureRunTables(): Promise<void> {
       } catch {
         // Column already exists — ignore
       }
+      // Backfill in_flight_since — ms epoch when run-manager's in-memory
+      // `inFlightWorkCount` last transitioned 0->1, NULL once back to 0. Lets
+      // the cross-isolate stale reapers grant a bounded grace to a
+      // demonstrably-alive run even when the heartbeat write itself has
+      // failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and `setRunInFlightMarker`.
+      try {
+        await client.execute(
+          `ALTER TABLE agent_runs ADD COLUMN in_flight_since ${intType()}`,
+        );
+      } catch {
+        // Column already exists — ignore
+      }
       // Backfill turn_id / error_code / error_detail.
       //   turn_id    = stable identity for one logical assistant turn that may
       //                span several continuation runs, so the durable record
@@ -335,6 +438,7 @@ async function ensureRunTables(): Promise<void> {
         "completed_at",
         "heartbeat_at",
         "last_progress_at",
+        "in_flight_since",
       ]);
       await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
       await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -525,6 +629,23 @@ function livenessBasisSql(): string {
   return `(CASE WHEN COALESCE(last_progress_at, started_at) > COALESCE(heartbeat_at, started_at) THEN COALESCE(last_progress_at, started_at) ELSE COALESCE(heartbeat_at, started_at) END)`;
 }
 
+/**
+ * Additive grace clause for the default (no explicit `maxStaleMs` override)
+ * heartbeat/liveness-based stale reap conditions — TRUE (row remains eligible
+ * for the surrounding staleness check) unless `in_flight_since` is set AND
+ * still inside `IN_FLIGHT_RUN_STALE_GRACE_MS`. A row with no marker set
+ * (`in_flight_since IS NULL`, the common case and every pre-existing row
+ * before this migration) always evaluates TRUE here, so this can only make
+ * reaping MORE conservative — the no-in-flight `BACKGROUND_RUN_STALE_MS` /
+ * `RUN_STALE_MS` behavior is unchanged. See `IN_FLIGHT_RUN_STALE_GRACE_MS`'s
+ * doc comment for why this is sound and bounded.
+ *
+ * Binds one param: the same `now` value the surrounding cutoff clause binds.
+ */
+function inFlightGraceSql(): string {
+  return `(in_flight_since IS NULL OR in_flight_since <= (CAST(? AS BIGINT) - ${IN_FLIGHT_RUN_STALE_GRACE_MS}))`;
+}
+
 /**
  * Atomically claim a background-dispatched run for processing. The foreground
  * POST inserts the run row with `dispatch_mode = 'background'`; the FIRST
@@ -877,9 +998,9 @@ function terminalStatusForEvent(
   event: AgentChatEvent,
 ): "completed" | "errored" | null {
   if (event.type === "error") return "errored";
+  if (event.type === "missing_api_key") return "errored";
   if (
     event.type === "done" ||
-    event.type === "missing_api_key" ||
     event.type === "loop_limit" ||
     event.type === "auto_continue"
   ) {
@@ -897,32 +1018,78 @@ function terminalReasonForEvent(event: AgentChatEvent): string | null {
   return null;
 }
 
-async function getLatestRunEvent(runId: string): Promise<{
+function isRealFailureTerminalEvent(event: AgentChatEvent): boolean {
+  if (event.type === "missing_api_key") return true;
+  if (event.type !== "error") return false;
+  return event.errorCode !== STALE_RUN_ERROR_EVENT.errorCode;
+}
+
+const RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT = 100;
+
+async function getRunEventForReconciliation(runId: string): Promise<{
   event: AgentChatEvent;
   eventAt: number | null;
 } | null> {
   const client = getDbExec();
   const { rows } = await client.execute({
-    sql: `SELECT seq, event_data, event_at FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,
-    args: [runId],
+    sql: `SELECT seq, event_data, event_at
+          FROM agent_run_events
+          WHERE run_id = ?
+            AND (
+              event_data LIKE '{"type":"done"%'
+              OR event_data LIKE '{"type":"error"%'
+              OR event_data LIKE '{"type":"missing_api_key"%'
+              OR event_data LIKE '{"type":"loop_limit"%'
+              OR event_data LIKE '{"type":"auto_continue"%'
+            )
+          ORDER BY seq DESC
+          LIMIT ?`,
+    args: [runId, RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT],
   });
-  const row = rows[0] as
-    | {
-        event_at?: number | string | null;
-        event_data?: string;
+  let latestTerminal: {
+    event: AgentChatEvent;
+    eventAt: number | null;
+  } | null = null;
+  for (const row of rows as Array<{
+    event_at?: number | string | null;
+    event_data?: string;
+  }>) {
+    const raw = row.event_data;
+    if (!raw) continue;
+    try {
+      const event = JSON.parse(raw) as AgentChatEvent;
+      if (!terminalStatusForEvent(event) || !terminalReasonForEvent(event)) {
+        continue;
       }
-    | undefined;
-  const raw = row?.event_data;
-  if (!raw) return null;
-  try {
-    const eventAt = row.event_at == null ? NaN : Number(row.event_at);
-    return {
-      event: JSON.parse(raw) as AgentChatEvent,
-      eventAt: Number.isFinite(eventAt) && eventAt > 0 ? eventAt : null,
-    };
-  } catch {
-    return null;
+      const rawEventAt = row.event_at == null ? NaN : Number(row.event_at);
+      const parsed = {
+        event,
+        eventAt:
+          Number.isFinite(rawEventAt) && rawEventAt > 0 ? rawEventAt : null,
+      };
+      if (!latestTerminal) latestTerminal = parsed;
+      // A real stream failure must not be laundered into success by a later
+      // done/continuation boundary. Keep the synthetic stale-run error special:
+      // that repair marker may be superseded by a later durable done event.
+      if (isRealFailureTerminalEvent(event)) return parsed;
+    } catch {
+      continue;
+    }
   }
+  return latestTerminal;
+}
+
+function errorCodeForTerminalEvent(event: AgentChatEvent): string | null {
+  if (event.type === "missing_api_key")
+    return LLM_MISSING_CREDENTIALS_ERROR_CODE;
+  if (event.type === "error") return event.errorCode ?? null;
+  return null;
+}
+
+function errorDetailForTerminalEvent(event: AgentChatEvent): string | null {
+  if (event.type === "missing_api_key") return LLM_MISSING_CREDENTIALS_MESSAGE;
+  if (event.type !== "error") return null;
+  return (event.details || event.error || "").slice(0, 2000) || null;
 }
 
 /**
@@ -938,20 +1105,15 @@ export async function reconcileTerminalRunFromEvents(
   runId: string,
 ): Promise<boolean> {
   await ensureRunTables();
-  const latest = await getLatestRunEvent(runId);
+  const latest = await getRunEventForReconciliation(runId);
   if (!latest) return false;
   const status = terminalStatusForEvent(latest.event);
   const terminalReason = terminalReasonForEvent(latest.event);
   if (!status || !terminalReason) return false;
 
   const client = getDbExec();
-  const errorCode =
-    latest.event.type === "error" ? (latest.event.errorCode ?? null) : null;
-  const errorDetail =
-    latest.event.type === "error"
-      ? (latest.event.details || latest.event.error || "").slice(0, 2000) ||
-        null
-      : null;
+  const errorCode = errorCodeForTerminalEvent(latest.event);
+  const errorDetail = errorDetailForTerminalEvent(latest.event);
   const { rowsAffected } = await client.execute({
     sql: `UPDATE agent_runs
           SET status = ?,
@@ -1094,12 +1256,57 @@ export async function updateRunHeartbeat(runId: string): Promise<void> {
 export async function bumpRunProgress(runId: string): Promise<void> {
   await ensureRunTables();
   const client = getDbExec();
+  const now = Date.now();
   await client.execute({
-    sql: `UPDATE agent_runs SET last_progress_at = ? WHERE id = ?`,
-    args: [Date.now(), runId],
+    // Multiple event-persistence paths and serverless isolates can bump the
+    // same run concurrently. A slower, older write must never land after a
+    // newer one and move the user-visible no-progress clock backwards.
+    // CASE keeps this portable across SQLite and Postgres.
+    sql: `UPDATE agent_runs SET last_progress_at = CASE WHEN last_progress_at IS NULL OR last_progress_at < ? THEN ? ELSE last_progress_at END WHERE id = ? AND status = 'running'`,
+    args: [now, now, runId],
   });
 }
 
+/**
+ * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
+ * SQL so a stale reaper running in a DIFFERENT isolate can tell a
+ * demonstrably-alive run (holding a tool call or A2A `agent_call` delegation)
+ * apart from a genuinely dead one — see `IN_FLIGHT_RUN_STALE_GRACE_MS`'s doc
+ * comment for the full reasoning.
+ *
+ * `inFlight: true` only writes when the row is still `NULL` — a defense-in-
+ * depth belt-and-suspenders against a nested 1->2 transition clobbering the
+ * ORIGINAL start time with a later one (the caller's own counter already
+ * dedupes 0->1 transitions; this WHERE just makes the write itself
+ * idempotent/order-independent too). `inFlight: false` always clears
+ * unconditionally — if it races a fresh 0->1 write from a *different* tool
+ * finishing/starting back to back, worst case is losing a few seconds of
+ * grace, never gaining an incorrect one.
+ *
+ * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
+ * failure here never blocks event emission or aborts the run. If this write
+ * itself fails (the same DB pressure that could be starving the heartbeat),
+ * the row simply gets no grace — never worse than today's behavior.
+ */
+export async function setRunInFlightMarker(
+  runId: string,
+  inFlight: boolean,
+): Promise<void> {
+  await ensureRunTables();
+  const client = getDbExec();
+  if (inFlight) {
+    await client.execute({
+      sql: `UPDATE agent_runs SET in_flight_since = ? WHERE id = ? AND status = 'running' AND in_flight_since IS NULL`,
+      args: [Date.now(), runId],
+    });
+  } else {
+    await client.execute({
+      sql: `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ?`,
+      args: [runId],
+    });
+  }
+}
+
 /**
  * If the given run is marked "running" in SQL but its heartbeat is stale
  * (producer likely crashed), flip it to "errored" so watchers stop waiting.
@@ -1115,13 +1322,20 @@ export async function reapIfStale(
   const completedAt = Date.now();
   // Background-dispatched runs get the wider stale window so a slow cold-start
   // isn't reaped; foreground runs keep the tight 15s window. An explicit
-  // override forces a flat window for callers that want one.
+  // override forces a flat window for callers that want one — and, like that
+  // override already bypasses the background-aware widening, it also bypasses
+  // the in-flight grace below: an explicit `maxStaleMs` is an exact,
+  // caller-chosen window and stays exact.
   const staleClause =
     typeof maxStaleMs === "number"
       ? `${livenessBasisSql()} < ?`
-      : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`;
+      : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()}`;
   const staleArgs =
-    typeof maxStaleMs === "number" ? [completedAt - maxStaleMs] : [completedAt];
+    typeof maxStaleMs === "number"
+      ? [completedAt - maxStaleMs]
+      : // First `?` is backgroundAwareStaleCutoffSql's CAST param, second is
+        // inFlightGraceSql's — both bound to the same "now".
+        [completedAt, completedAt];
   const { rowsAffected } = await client.execute({
     sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1455,12 +1669,23 @@ export async function getRunByThread(
   dispatchMode: string | null;
   terminalReason: string | null;
   diagStage: string | null;
+  /**
+   * Raw `in_flight_since` marker (see `setRunInFlightMarker`) — non-null
+   * exactly when run-manager's in-memory `inFlightWorkCount` was last known
+   * (from THIS row's own producer) to be > 0: a tool call or A2A `agent_call`
+   * delegation is open and has not yet resolved. Callers that want the
+   * authoritative "does this run currently hold live work" signal (e.g. the
+   * `hasInFlightWork` wire field on `/runs/active`) should test this for
+   * non-null, not re-derive their own notion of in-flight — see
+   * `getActiveRunForThreadAsync` in run-manager.ts.
+   */
+  inFlightSince: number | null;
 } | null> {
   await ensureRunTables();
   const client = getDbExec();
   const sql = options?.includeTerminal
-    ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
-    : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
+    ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
+    : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
   const { rows } = await client.execute({ sql, args: [threadId] });
   if (rows.length === 0) return null;
   const r = rows[0] as {
@@ -1476,6 +1701,7 @@ export async function getRunByThread(
     terminal_reason?: string | null;
     diag_stage?: string | null;
     error_code?: string | null;
+    in_flight_since?: number | string | null;
   };
   const canReconcileFromEvents =
     r.status === "running" ||
@@ -1497,6 +1723,7 @@ export async function getRunByThread(
     dispatchMode: r.dispatch_mode ?? null,
     terminalReason: r.terminal_reason ?? null,
     diagStage: r.diag_stage ?? null,
+    inFlightSince: r.in_flight_since == null ? null : Number(r.in_flight_since),
   };
 }
 
@@ -1661,12 +1888,21 @@ export async function reapAllStaleRuns(): Promise<number> {
   await ensureRunTables();
   const client = getDbExec();
   const now = Date.now();
-  // Background-dispatched runs use the wider window; everything else 15s.
+  // Background-dispatched runs use the wider window; everything else 15s. The
+  // in-flight grace clause is applied identically to this SELECT and the
+  // UPDATE below (both derive `stale.rows`/the terminal-event-append loop
+  // from the SAME predicate) so a row the grace clause protects from the
+  // UPDATE is never mistakenly given a terminal event anyway — see
+  // `inFlightGraceSql` and `IN_FLIGHT_RUN_STALE_GRACE_MS`. This runs at
+  // server startup across possibly-multiple isolates, so a sibling isolate's
+  // still-alive, in-flight run must not be reaped just because THIS isolate
+  // just booted and has no heartbeat history for it.
   const stale = await client.execute({
     sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
-    args: [now],
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
+    args: [now, now],
   });
   for (const row of stale.rows) {
     const id = (row as { id?: unknown }).id;
@@ -1684,13 +1920,15 @@ export async function reapAllStaleRuns(): Promise<number> {
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
     args: [
       completedAt,
       STALE_RUN_ERROR_EVENT.errorCode,
       STALE_RUN_ERROR_EVENT.details,
       STALE_RUN_ERROR_EVENT.errorCode,
       completedAt,
+      completedAt,
     ],
   });
   for (const row of stale.rows) {
@@ -1725,15 +1963,24 @@ export async function cleanupOldRuns(
   // SELECT covers BOTH UPDATE conditions so the terminal-event-append loop
   // below catches every row we're about to flip — a 24h-old row with a
   // somehow-fresh heartbeat would slip past a heartbeat-only SELECT.
+  //
+  // The in-flight grace clause is applied ONLY to the heartbeat-based branch
+  // (mirroring the second UPDATE below), never the absolute-age branch:
+  // nothing can legitimately hold in-flight work anywhere near `olderThanMs`
+  // (default on the order of a day) — `IN_FLIGHT_RUN_STALE_GRACE_MS` bounds
+  // any real grace at ~14.5 minutes — so a row that old is dead regardless of
+  // its in-flight marker. This runs opportunistically after EVERY run
+  // completes in ANY isolate, so a different thread's still-in-flight A2A
+  // call must not be reaped as a side effect of an unrelated run finishing.
   const now = Date.now();
   const stale = await client.execute({
     sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
             AND (
-              ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+              (${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()})
               OR started_at < ?
     )`,
-    args: [now, cutoff],
+    args: [now, now, cutoff],
   });
   for (const row of stale.rows) {
     const id = (row as { id?: unknown }).id;
@@ -1761,7 +2008,13 @@ export async function cleanupOldRuns(
     ],
   });
   // Also expire runs whose heartbeat is stale — producer has died. Uses the
-  // background-aware window so a slow background cold-start isn't reaped early.
+  // background-aware window so a slow background cold-start isn't reaped
+  // early, and the in-flight grace so a demonstrably-alive run holding a tool
+  // call / A2A delegation survives a heartbeat write failure. Must match the
+  // SELECT's heartbeat branch above exactly — the terminal-event-append loop
+  // below fires for every row the SELECT returned, so a mismatch would
+  // silently inject a terminal error event onto a row this UPDATE left
+  // status='running'.
   await client.execute({
     sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1771,13 +2024,15 @@ export async function cleanupOldRuns(
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
     args: [
       completedAt,
       STALE_RUN_ERROR_EVENT.errorCode,
       STALE_RUN_ERROR_EVENT.details,
       STALE_RUN_ERROR_EVENT.errorCode,
       completedAt,
+      completedAt,
     ],
   });
   for (const row of stale.rows) {

package/corpus/core/src/agent/types.ts

@@ -274,6 +274,29 @@ export type AgentChatEvent =
       agent: string;
       status: "start" | "done" | "error";
     }
+  | {
+      /**
+       * Periodic liveness for an in-flight cross-app A2A call. Emitted by the
+       * `call-agent` action once per throttle window ONLY when a real poll
+       * round-trip to the remote agent succeeds and reports a non-terminal
+       * state — never on a timer, so a hung/dead remote emits nothing and the
+       * stuck-detector can still fire. Counts as real progress in
+       * `run-manager`'s `shouldBumpProgressForEvent` (any non-special event
+       * type does), which keeps `last_progress_at` fresh so a slow-but-healthy
+       * sub-agent call doesn't trip the client's stuck banner. A distinct
+       * event type (not an `agent_call` status) so existing `agent_call`
+       * consumers that treat "not start/done" as a failure don't render an
+       * in-flight tick as an error.
+       */
+      type: "agent_call_progress";
+      agent: string;
+      /** Remote A2A task state for this poll, e.g. "working" | "submitted". */
+      state: string;
+      /** Elapsed wall-clock seconds since the cross-app call began. */
+      elapsedSeconds: number;
+      /** Optional short text surfaced from the remote poll, when present. */
+      detail?: string;
+    }
   | { type: "agent_call_text"; agent: string; text: string }
   | {
       type: "agent_task";

package/corpus/core/src/audit/record.ts

@@ -1,3 +1,4 @@
+import { getIntegrationRequestContext } from "../server/request-context.js";
 /**
  * Audit capture entry point, called from the `defineAction` audit wrapper after
  * an action runs (success or error). Best-effort: any failure here is swallowed
@@ -112,6 +113,7 @@ export async function recordActionAudit(
     const recordInputs = input.config?.recordInputs !== false;
     const inputJson = recordInputs ? redactArgsToJson(input.args) : null;
 
+    const hasExplicitTargetVisibility = target?.visibility !== undefined;
     const event: AuditEvent = {
       id: crypto.randomUUID(),
       createdAt: Date.now(),
@@ -133,6 +135,35 @@ export async function recordActionAudit(
       ownerEmail: target?.ownerEmail ?? actorEmail,
       visibility: target?.visibility ?? "private",
     };
+    const lineage = getIntegrationRequestContext()?.lineage;
+    const integration = getIntegrationRequestContext();
+    if (integration) {
+      if (
+        ctx?.orgId &&
+        event.visibility === "private" &&
+        !hasExplicitTargetVisibility
+      ) {
+        event.visibility = "org";
+      }
+      event.runId = lineage?.runId ?? null;
+      event.taskId = integration.taskId;
+      event.parentTaskId = lineage?.parentTaskId ?? null;
+      event.sourceKind = lineage?.source?.kind ?? null;
+      event.sourcePlatform = lineage?.source?.platform ?? null;
+      event.sourceId = lineage?.source?.id ?? null;
+      event.sourceUrl = lineage?.source?.url ?? null;
+      event.networkProtocol = lineage?.network?.protocol ?? null;
+      event.networkId = lineage?.network?.id ?? null;
+      event.networkPeer = lineage?.network?.peer ?? null;
+      if (!event.networkProtocol && actionName === "provider-api-request") {
+        event.networkProtocol = "provider-api";
+        event.networkId = target?.id ?? "provider-api-request";
+      }
+      if (!event.networkProtocol && actionName === "call-agent") {
+        event.networkProtocol = "a2a";
+        event.networkId = target?.id ?? "call-agent";
+      }
+    }
     // org_id used for scoping defaults to the target's, else the actor's org.
     if (target?.orgId !== undefined) event.orgId = target.orgId;