@agent-native/core 0.92.11 → 0.93.0

package/dist/agent/run-store.d.ts

@@ -89,6 +89,82 @@ export declare const UNCLAIMED_BACKGROUND_RUN_GRACE_MS = 25000;
  * bound how long a real user turn is worth waiting on before failing loud.
  */
 export declare const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS: number;
+/**
+ * Tick interval for the DEDICATED fast redispatch sweep in
+ * agent-chat-plugin.ts (distinct from that file's general-purpose 2-minute
+ * orphan/reap sweep). Only attempts redispatch for rows still inside
+ * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — it never reaps, so it
+ * cannot race the loud-failure fallback onto an earlier trigger.
+ *
+ * This constant exists because the general sweep's 2-minute cadence puts the
+ * FIRST redispatch attempt uncomfortably close to (and on a slow tick, past)
+ * `BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS` (150s, agent-chat-adapter.ts) — the
+ * client following a deferred successor would give up and report a fatal
+ * error for a turn the server was silently about to recover. The whole
+ * budget is a derived chain, each bound following from the one before it:
+ *
+ *   UNCLAIMED_BACKGROUND_RUN_GRACE_MS        (25s)  row must look abandoned
+ * + UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS   (20s)  worst-case tick latency
+ * = ~45s worst-case time-to-first-redispatch-attempt, ~65s to a second
+ *   attempt if the first fails — both comfortably under the client's 150s
+ *   idle timeout, which additionally no longer counts a known-deferred row
+ *   against its idle window at all (see `awaitingRedispatch` surfaced by
+ *   `/runs/active` and consumed by the client follow loop).
+ * < BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS       (150s) client's own backstop
+ * < UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS (300s) hard, unresettable
+ *   ceiling — untouched by this constant — past which the slow sweep's
+ *   existing loud reap (`background_worker_never_started`) still fires.
+ */
+export declare const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20000;
+/**
+ * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
+ * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
+ * row that is marked in-flight (`in_flight_since`, see `setRunInFlightMarker`)
+ * even though its heartbeat/progress liveness basis
+ * (`livenessBasisSql`/`backgroundAwareStaleCutoffSql`) has gone stale.
+ *
+ * WHY a marker column at all: `inFlightWorkCount` in run-manager.ts (the
+ * no-progress backstop's guard) is in-memory, per-isolate — but all three
+ * reapers above can run in a DIFFERENT isolate than the one holding the
+ * producing run (a client's SQL-subscription poll, a sibling isolate's
+ * opportunistic `cleanupOldRuns` after ITS OWN run completes, or a fresh
+ * boot's `reapAllStaleRuns`). None of them can read another isolate's
+ * in-memory counter, so the counter's 0->1 / 1->0 transitions are mirrored
+ * into this column (`setRunInFlightMarker`, called from
+ * run-manager.ts's `trackInFlightWork`) so it is observable from SQL. This is
+ * exactly the gap that let a demonstrably-alive run holding a long tool call
+ * or A2A `call-agent` delegation get reaped: the heartbeat WRITE can fail
+ * silently (Neon pooler saturation) for the whole `BACKGROUND_RUN_STALE_MS`
+ * window while the run is provably still doing work.
+ *
+ * BOUNDED, not a silent hang — derived from two independent ceilings already
+ * in the codebase, not picked by feel:
+ *   - `DEFAULT_TOOL_TIMEOUT_MS` (12 min, production-agent.ts) is the longest
+ *     any SINGLE tool call or `agent_call` (A2A delegation) may legitimately
+ *     stay in flight — past that its own `AbortSignal.timeout` forces a
+ *     tool_done/error and clears the marker.
+ *   - `BACKGROUND_SOFT_TIMEOUT_CEILING_MS` (13 min, run-manager.ts) is the
+ *     background chunk's OWN soft-timeout ceiling. Unlike the no-progress
+ *     backstop, this timer is NOT gated on in-flight work (see the "secondary"
+ *     hazard documented next to the soft-timeout timer in run-manager.ts) —
+ *     it fires unconditionally and checkpoints/continues the run, so by 13
+ *     minutes the row leaves status='running' via that path regardless of
+ *     what the marker says.
+ * This grace is the LARGER of the two (13 min) plus one `BACKGROUND_RUN_STALE_MS`
+ * (90s) buffer for that checkpoint's own completion write to land under the
+ * same DB pressure that could have caused the heartbeat to lapse in the first
+ * place: 780_000 + 90_000 = 870_000ms (14.5 min). Past that, a "running" row
+ * that still shows in-flight work AND a stale liveness basis is not a slow
+ * producer anymore — every backstop that should have ended it has ALSO failed
+ * to write, and it is reaped loud like any other stale run.
+ *
+ * Never applied when a caller passes an explicit `maxStaleMs` override to
+ * `reapIfStale` — that escape hatch is an exact, caller-chosen window and
+ * stays exact. Never weakens the no-in-flight case: a row with no marker set
+ * evaluates this grace clause to a no-op and is reaped at the original
+ * `BACKGROUND_RUN_STALE_MS` / `RUN_STALE_MS` exactly as before.
+ */
+export declare const IN_FLIGHT_RUN_STALE_GRACE_MS: number;
 /**
  * Persist a zombie tool-call completion to the ledger. Called by the detached
  * promise continuation after `Promise.race` abandons it. Best-effort — never
@@ -321,6 +397,28 @@ export declare function updateRunHeartbeat(runId: string): Promise<void>;
  * "process dead." Callers should throttle (run-manager debounces to ~1/s).
  */
 export declare function bumpRunProgress(runId: string): Promise<void>;
+/**
+ * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
+ * SQL so a stale reaper running in a DIFFERENT isolate can tell a
+ * demonstrably-alive run (holding a tool call or A2A `agent_call` delegation)
+ * apart from a genuinely dead one — see `IN_FLIGHT_RUN_STALE_GRACE_MS`'s doc
+ * comment for the full reasoning.
+ *
+ * `inFlight: true` only writes when the row is still `NULL` — a defense-in-
+ * depth belt-and-suspenders against a nested 1->2 transition clobbering the
+ * ORIGINAL start time with a later one (the caller's own counter already
+ * dedupes 0->1 transitions; this WHERE just makes the write itself
+ * idempotent/order-independent too). `inFlight: false` always clears
+ * unconditionally — if it races a fresh 0->1 write from a *different* tool
+ * finishing/starting back to back, worst case is losing a few seconds of
+ * grace, never gaining an incorrect one.
+ *
+ * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
+ * failure here never blocks event emission or aborts the run. If this write
+ * itself fails (the same DB pressure that could be starving the heartbeat),
+ * the row simply gets no grace — never worse than today's behavior.
+ */
+export declare function setRunInFlightMarker(runId: string, inFlight: boolean): Promise<void>;
 /**
  * If the given run is marked "running" in SQL but its heartbeat is stale
  * (producer likely crashed), flip it to "errored" so watchers stop waiting.
@@ -416,6 +514,17 @@ export declare function getRunByThread(threadId: string, options?: {
     dispatchMode: string | null;
     terminalReason: string | null;
     diagStage: string | null;
+    /**
+     * Raw `in_flight_since` marker (see `setRunInFlightMarker`) — non-null
+     * exactly when run-manager's in-memory `inFlightWorkCount` was last known
+     * (from THIS row's own producer) to be > 0: a tool call or A2A `agent_call`
+     * delegation is open and has not yet resolved. Callers that want the
+     * authoritative "does this run currently hold live work" signal (e.g. the
+     * `hasInFlightWork` wire field on `/runs/active`) should test this for
+     * non-null, not re-derive their own notion of in-flight — see
+     * `getActiveRunForThreadAsync` in run-manager.ts.
+     */
+    inFlightSince: number | null;
 } | null>;
 export interface AgentRunSummary {
     id: string;

package/dist/agent/run-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,QAAS,CAAC;AAEnC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,uBAAuB,QAAS,CAAC;AAE9C,eAAO,MAAM,qBAAqB;aAChC,IAAI,EAAE,OAAO;aACb,KAAK,EACH,qHAAqH;aACvH,SAAS,EAAE,WAAW;aACtB,WAAW;aACX,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX;;;;;;;;GAQG;AACH,eAAO,MAAM,oCAAoC;aAC/C,IAAI,EAAE,OAAO;aACb,KAAK,EACH,gHAAgH;aAClH,SAAS,EAAE,iCAAiC;aAC5C,WAAW;aACX,OAAO,EACL,oNAAoN;CAC9M,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,4CAA4C;aACvD,IAAI,EAAE,OAAO;aACb,KAAK,EACH,oHAAoH;aACtH,SAAS,EAAE,0BAA0B;aACrC,WAAW;aACX,OAAO,EACL,yGAAyG;CACnG,CAAC;AAEX;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iCAAiC,QAAS,CAAC;AAExD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,4CAA4C,QAAa,CAAC;AA0PvE;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAcxB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1E;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,YAAY,CAAC,EAAE,YAAY,GAAG,uBAAuB,GAAG,YAAY,CAAC;IACrE;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,OAAO,CAAC,IAAI,CAAC,CAiBf;AA4DD;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAYxE;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACnE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GAAG,IAAI,CAAC,CA2BR;AAED;;;;;;GAMG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB;AAED;;;;GAIG;AACH,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAO1E;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,6BAA6B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAkBvE;AAED,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX;;;+BAG2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAsB,8BAA8B,IAAI,OAAO,CAC7D,yBAAyB,EAAE,CAC5B,CAyBA;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sCAAsC,CACpD,GAAG,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAC1B,GAAG,GAAE,MAAmB,GACvB,OAAO,CAET;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAS5E;AAED;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAsC3D;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,WAAW,EAAE,MAAM,GAAG,SAAS,GAC9B,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,IAAI,CAAC,CAYf;AAsDD;;;;;;;;GAQG;AACH,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAuClB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc;IACzB,gFAAgF;aAChF,YAAY,EAAE,eAAe;IAC7B,sEAAsE;aACtE,UAAU,EAAE,aAAa;IACzB,8EAA8E;aAC9E,UAAU,EAAE,aAAa;IACzB,4EAA4E;aAC5E,aAAa,EAAE,gBAAgB;IAC/B,yDAAyD;aACzD,aAAa,EAAE,gBAAgB;IAC/B,6EAA6E;aAC7E,eAAe,EAAE,mBAAmB;IACpC,+CAA+C;aAC/C,aAAa,EAAE,gBAAgB;IAC/B,oFAAoF;aACpF,eAAe,EAAE,mBAAmB;IACpC,qDAAqD;aACrD,YAAY,EAAE,eAAe;IAC7B,kFAAkF;aAClF,WAAW,EAAE,cAAc;IAC3B,oEAAoE;aACpE,UAAU,EAAE,aAAa;IACzB;;;;;OAKG;aACH,wBAAwB,EAAE,4BAA4B;CAC9C,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,OAAO,cAAc,CAAC,CAAC;AAEhF;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAiCf;AAED,+EAA+E;AAC/E,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYrE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOlE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,OAAO,CAAC,CA4ClB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAuClB;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,OAAO,CAAC,CAUlB;AAED,kFAAkF;AAClF,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CASxE;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAElE;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAchD;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWpD;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GAAG,IAAI,CAAC,CAwBR;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,CA0BjE;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,GAAG,EAAE;IAClD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,GAAG;IACF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;CACxB,CAmBA;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE,GACtC,OAAO,CAAC;IACT,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,GAAG,IAAI,CAAC,CA2CR;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4EAA4E;IAC5E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/B,OAAO,CAAC,eAAe,EAAE,CAAC,CA0E5B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAoC3B;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CA+CxD;AAED;;;;wDAIwD;AACxD,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,IAAI,CAAC,CA4Ff;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,OAAO,CAAC,EAAE;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC,CACH,CA2CA;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,OAAO,CAAC,IAAI,CAAC,CAEf"}
+{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,QAAS,CAAC;AAEnC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,uBAAuB,QAAS,CAAC;AAE9C,eAAO,MAAM,qBAAqB;aAChC,IAAI,EAAE,OAAO;aACb,KAAK,EACH,qHAAqH;aACvH,SAAS,EAAE,WAAW;aACtB,WAAW;aACX,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX;;;;;;;;GAQG;AACH,eAAO,MAAM,oCAAoC;aAC/C,IAAI,EAAE,OAAO;aACb,KAAK,EACH,gHAAgH;aAClH,SAAS,EAAE,iCAAiC;aAC5C,WAAW;aACX,OAAO,EACL,oNAAoN;CAC9M,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,4CAA4C;aACvD,IAAI,EAAE,OAAO;aACb,KAAK,EACH,oHAAoH;aACtH,SAAS,EAAE,0BAA0B;aACrC,WAAW;aACX,OAAO,EACL,yGAAyG;CACnG,CAAC;AAEX;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iCAAiC,QAAS,CAAC;AAExD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,4CAA4C,QAAa,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,sCAAsC,QAAS,CAAC;AAE7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,eAAO,MAAM,4BAA4B,QAAgB,CAAC;AAgR1D;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAcxB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1E;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,YAAY,CAAC,EAAE,YAAY,GAAG,uBAAuB,GAAG,YAAY,CAAC;IACrE;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,OAAO,CAAC,IAAI,CAAC,CAiBf;AA6ED;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAYxE;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACnE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GAAG,IAAI,CAAC,CA2BR;AAED;;;;;;GAMG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB;AAED;;;;GAIG;AACH,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAO1E;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,6BAA6B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAkBvE;AAED,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX;;;+BAG2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAsB,8BAA8B,IAAI,OAAO,CAC7D,yBAAyB,EAAE,CAC5B,CAyBA;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sCAAsC,CACpD,GAAG,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAC1B,GAAG,GAAE,MAAmB,GACvB,OAAO,CAET;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAS5E;AAED;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAsC3D;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,WAAW,EAAE,MAAM,GAAG,SAAS,GAC9B,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,IAAI,CAAC,CAYf;AAoGD;;;;;;;;GAQG;AACH,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAkClB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc;IACzB,gFAAgF;aAChF,YAAY,EAAE,eAAe;IAC7B,sEAAsE;aACtE,UAAU,EAAE,aAAa;IACzB,8EAA8E;aAC9E,UAAU,EAAE,aAAa;IACzB,4EAA4E;aAC5E,aAAa,EAAE,gBAAgB;IAC/B,yDAAyD;aACzD,aAAa,EAAE,gBAAgB;IAC/B,6EAA6E;aAC7E,eAAe,EAAE,mBAAmB;IACpC,+CAA+C;aAC/C,aAAa,EAAE,gBAAgB;IAC/B,oFAAoF;aACpF,eAAe,EAAE,mBAAmB;IACpC,qDAAqD;aACrD,YAAY,EAAE,eAAe;IAC7B,kFAAkF;aAClF,WAAW,EAAE,cAAc;IAC3B,oEAAoE;aACpE,UAAU,EAAE,aAAa;IACzB;;;;;OAKG;aACH,wBAAwB,EAAE,4BAA4B;CAC9C,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,OAAO,cAAc,CAAC,CAAC;AAEhF;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAiCf;AAED,+EAA+E;AAC/E,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYrE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC,CAcf;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,OAAO,CAAC,CAmDlB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAuClB;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,OAAO,CAAC,CAUlB;AAED,kFAAkF;AAClF,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CASxE;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAElE;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAchD;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWpD;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GAAG,IAAI,CAAC,CAwBR;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,CA0BjE;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,GAAG,EAAE;IAClD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,GAAG;IACF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;CACxB,CAmBA;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE,GACtC,OAAO,CAAC;IACT,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;;;;;;;;;OASG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,GAAG,IAAI,CAAC,CA6CR;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4EAA4E;IAC5E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/B,OAAO,CAAC,eAAe,EAAE,CAAC,CA0E5B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAoC3B;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CA0DxD;AAED;;;;wDAIwD;AACxD,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Gf;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,OAAO,CAAC,EAAE;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC,CACH,CA2CA;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,OAAO,CAAC,IAAI,CAAC,CAEf"}

package/dist/agent/run-store.js

@@ -7,6 +7,7 @@ import { getDbExec, intType, isPostgres } from "../db/client.js";
 import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
 import { captureError } from "../server/capture-error.js";
+import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
 let _initPromise;
 /**
  * Max time without a heartbeat before a "running" run is considered dead.
@@ -98,6 +99,82 @@ export const UNCLAIMED_BACKGROUND_RUN_GRACE_MS = 25_000;
  * bound how long a real user turn is worth waiting on before failing loud.
  */
 export const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS = 5 * 60_000;
+/**
+ * Tick interval for the DEDICATED fast redispatch sweep in
+ * agent-chat-plugin.ts (distinct from that file's general-purpose 2-minute
+ * orphan/reap sweep). Only attempts redispatch for rows still inside
+ * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — it never reaps, so it
+ * cannot race the loud-failure fallback onto an earlier trigger.
+ *
+ * This constant exists because the general sweep's 2-minute cadence puts the
+ * FIRST redispatch attempt uncomfortably close to (and on a slow tick, past)
+ * `BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS` (150s, agent-chat-adapter.ts) — the
+ * client following a deferred successor would give up and report a fatal
+ * error for a turn the server was silently about to recover. The whole
+ * budget is a derived chain, each bound following from the one before it:
+ *
+ *   UNCLAIMED_BACKGROUND_RUN_GRACE_MS        (25s)  row must look abandoned
+ * + UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS   (20s)  worst-case tick latency
+ * = ~45s worst-case time-to-first-redispatch-attempt, ~65s to a second
+ *   attempt if the first fails — both comfortably under the client's 150s
+ *   idle timeout, which additionally no longer counts a known-deferred row
+ *   against its idle window at all (see `awaitingRedispatch` surfaced by
+ *   `/runs/active` and consumed by the client follow loop).
+ * < BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS       (150s) client's own backstop
+ * < UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS (300s) hard, unresettable
+ *   ceiling — untouched by this constant — past which the slow sweep's
+ *   existing loud reap (`background_worker_never_started`) still fires.
+ */
+export const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20_000;
+/**
+ * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
+ * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
+ * row that is marked in-flight (`in_flight_since`, see `setRunInFlightMarker`)
+ * even though its heartbeat/progress liveness basis
+ * (`livenessBasisSql`/`backgroundAwareStaleCutoffSql`) has gone stale.
+ *
+ * WHY a marker column at all: `inFlightWorkCount` in run-manager.ts (the
+ * no-progress backstop's guard) is in-memory, per-isolate — but all three
+ * reapers above can run in a DIFFERENT isolate than the one holding the
+ * producing run (a client's SQL-subscription poll, a sibling isolate's
+ * opportunistic `cleanupOldRuns` after ITS OWN run completes, or a fresh
+ * boot's `reapAllStaleRuns`). None of them can read another isolate's
+ * in-memory counter, so the counter's 0->1 / 1->0 transitions are mirrored
+ * into this column (`setRunInFlightMarker`, called from
+ * run-manager.ts's `trackInFlightWork`) so it is observable from SQL. This is
+ * exactly the gap that let a demonstrably-alive run holding a long tool call
+ * or A2A `call-agent` delegation get reaped: the heartbeat WRITE can fail
+ * silently (Neon pooler saturation) for the whole `BACKGROUND_RUN_STALE_MS`
+ * window while the run is provably still doing work.
+ *
+ * BOUNDED, not a silent hang — derived from two independent ceilings already
+ * in the codebase, not picked by feel:
+ *   - `DEFAULT_TOOL_TIMEOUT_MS` (12 min, production-agent.ts) is the longest
+ *     any SINGLE tool call or `agent_call` (A2A delegation) may legitimately
+ *     stay in flight — past that its own `AbortSignal.timeout` forces a
+ *     tool_done/error and clears the marker.
+ *   - `BACKGROUND_SOFT_TIMEOUT_CEILING_MS` (13 min, run-manager.ts) is the
+ *     background chunk's OWN soft-timeout ceiling. Unlike the no-progress
+ *     backstop, this timer is NOT gated on in-flight work (see the "secondary"
+ *     hazard documented next to the soft-timeout timer in run-manager.ts) —
+ *     it fires unconditionally and checkpoints/continues the run, so by 13
+ *     minutes the row leaves status='running' via that path regardless of
+ *     what the marker says.
+ * This grace is the LARGER of the two (13 min) plus one `BACKGROUND_RUN_STALE_MS`
+ * (90s) buffer for that checkpoint's own completion write to land under the
+ * same DB pressure that could have caused the heartbeat to lapse in the first
+ * place: 780_000 + 90_000 = 870_000ms (14.5 min). Past that, a "running" row
+ * that still shows in-flight work AND a stale liveness basis is not a slow
+ * producer anymore — every backstop that should have ended it has ALSO failed
+ * to write, and it is reaped loud like any other stale run.
+ *
+ * Never applied when a caller passes an explicit `maxStaleMs` override to
+ * `reapIfStale` — that escape hatch is an exact, caller-chosen window and
+ * stays exact. Never weakens the no-in-flight case: a row with no marker set
+ * evaluates this grace clause to a no-op and is reaped at the original
+ * `BACKGROUND_RUN_STALE_MS` / `RUN_STALE_MS` exactly as before.
+ */
+export const IN_FLIGHT_RUN_STALE_GRACE_MS = 14.5 * 60_000; // 870_000
 async function ensureRunTables() {
     if (!_initPromise) {
         _initPromise = (async () => {
@@ -203,6 +280,14 @@ async function ensureRunTables() {
                     // rehydrates the body from this column via the marker's payloadRef.
                     // Cleared on terminal status writes.
                     ["dispatch_payload", "TEXT"],
+                    // in_flight_since = ms epoch when run-manager's in-memory
+                    // `inFlightWorkCount` last transitioned 0->1 (a tool call or nested
+                    // `agent_call`/A2A delegation started), NULL once it drops back to 0.
+                    // Lets the cross-isolate stale reapers grant a bounded grace to a
+                    // demonstrably-alive run even when the SAME-isolate heartbeat write
+                    // has failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and
+                    // `setRunInFlightMarker`.
+                    ["in_flight_since", intType()],
                 ]) {
                     await ensureColumnExists("agent_runs", col, `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS ${col} ${colType}`);
                 }
@@ -220,6 +305,7 @@ async function ensureRunTables() {
                     "completed_at",
                     "heartbeat_at",
                     "last_progress_at",
+                    "in_flight_since",
                 ]);
                 await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
                 await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -252,6 +338,17 @@ async function ensureRunTables() {
             catch {
                 // Column already exists — ignore
             }
+            // Backfill in_flight_since — ms epoch when run-manager's in-memory
+            // `inFlightWorkCount` last transitioned 0->1, NULL once back to 0. Lets
+            // the cross-isolate stale reapers grant a bounded grace to a
+            // demonstrably-alive run even when the heartbeat write itself has
+            // failed. See `IN_FLIGHT_RUN_STALE_GRACE_MS` and `setRunInFlightMarker`.
+            try {
+                await client.execute(`ALTER TABLE agent_runs ADD COLUMN in_flight_since ${intType()}`);
+            }
+            catch {
+                // Column already exists — ignore
+            }
             // Backfill turn_id / error_code / error_detail.
             //   turn_id    = stable identity for one logical assistant turn that may
             //                span several continuation runs, so the durable record
@@ -305,6 +402,7 @@ async function ensureRunTables() {
                 "completed_at",
                 "heartbeat_at",
                 "last_progress_at",
+                "in_flight_since",
             ]);
             await widenIntColumnsToBigInt("agent_run_events", ["event_at"]);
             await widenIntColumnsToBigInt("agent_tool_ledger", ["completed_at"]);
@@ -467,6 +565,22 @@ function terminalRunEventExclusionSql(runIdColumn = "id") {
 function livenessBasisSql() {
     return `(CASE WHEN COALESCE(last_progress_at, started_at) > COALESCE(heartbeat_at, started_at) THEN COALESCE(last_progress_at, started_at) ELSE COALESCE(heartbeat_at, started_at) END)`;
 }
+/**
+ * Additive grace clause for the default (no explicit `maxStaleMs` override)
+ * heartbeat/liveness-based stale reap conditions — TRUE (row remains eligible
+ * for the surrounding staleness check) unless `in_flight_since` is set AND
+ * still inside `IN_FLIGHT_RUN_STALE_GRACE_MS`. A row with no marker set
+ * (`in_flight_since IS NULL`, the common case and every pre-existing row
+ * before this migration) always evaluates TRUE here, so this can only make
+ * reaping MORE conservative — the no-in-flight `BACKGROUND_RUN_STALE_MS` /
+ * `RUN_STALE_MS` behavior is unchanged. See `IN_FLIGHT_RUN_STALE_GRACE_MS`'s
+ * doc comment for why this is sound and bounded.
+ *
+ * Binds one param: the same `now` value the surrounding cutoff clause binds.
+ */
+function inFlightGraceSql() {
+    return `(in_flight_since IS NULL OR in_flight_since <= (CAST(? AS BIGINT) - ${IN_FLIGHT_RUN_STALE_GRACE_MS}))`;
+}
 /**
  * Atomically claim a background-dispatched run for processing. The foreground
  * POST inserts the run row with `dispatch_mode = 'background'`; the FIRST
@@ -767,8 +881,9 @@ export async function setRunTerminalReason(runId, terminalReason) {
 function terminalStatusForEvent(event) {
     if (event.type === "error")
         return "errored";
+    if (event.type === "missing_api_key")
+        return "errored";
     if (event.type === "done" ||
-        event.type === "missing_api_key" ||
         event.type === "loop_limit" ||
         event.type === "auto_continue") {
         return "completed";
@@ -788,26 +903,73 @@ function terminalReasonForEvent(event) {
         return "done";
     return null;
 }
-async function getLatestRunEvent(runId) {
+function isRealFailureTerminalEvent(event) {
+    if (event.type === "missing_api_key")
+        return true;
+    if (event.type !== "error")
+        return false;
+    return event.errorCode !== STALE_RUN_ERROR_EVENT.errorCode;
+}
+const RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT = 100;
+async function getRunEventForReconciliation(runId) {
     const client = getDbExec();
     const { rows } = await client.execute({
-        sql: `SELECT seq, event_data, event_at FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,
-        args: [runId],
+        sql: `SELECT seq, event_data, event_at
+          FROM agent_run_events
+          WHERE run_id = ?
+            AND (
+              event_data LIKE '{"type":"done"%'
+              OR event_data LIKE '{"type":"error"%'
+              OR event_data LIKE '{"type":"missing_api_key"%'
+              OR event_data LIKE '{"type":"loop_limit"%'
+              OR event_data LIKE '{"type":"auto_continue"%'
+            )
+          ORDER BY seq DESC
+          LIMIT ?`,
+        args: [runId, RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT],
     });
-    const row = rows[0];
-    const raw = row?.event_data;
-    if (!raw)
-        return null;
-    try {
-        const eventAt = row.event_at == null ? NaN : Number(row.event_at);
-        return {
-            event: JSON.parse(raw),
-            eventAt: Number.isFinite(eventAt) && eventAt > 0 ? eventAt : null,
-        };
+    let latestTerminal = null;
+    for (const row of rows) {
+        const raw = row.event_data;
+        if (!raw)
+            continue;
+        try {
+            const event = JSON.parse(raw);
+            if (!terminalStatusForEvent(event) || !terminalReasonForEvent(event)) {
+                continue;
+            }
+            const rawEventAt = row.event_at == null ? NaN : Number(row.event_at);
+            const parsed = {
+                event,
+                eventAt: Number.isFinite(rawEventAt) && rawEventAt > 0 ? rawEventAt : null,
+            };
+            if (!latestTerminal)
+                latestTerminal = parsed;
+            // A real stream failure must not be laundered into success by a later
+            // done/continuation boundary. Keep the synthetic stale-run error special:
+            // that repair marker may be superseded by a later durable done event.
+            if (isRealFailureTerminalEvent(event))
+                return parsed;
+        }
+        catch {
+            continue;
+        }
     }
-    catch {
+    return latestTerminal;
+}
+function errorCodeForTerminalEvent(event) {
+    if (event.type === "missing_api_key")
+        return LLM_MISSING_CREDENTIALS_ERROR_CODE;
+    if (event.type === "error")
+        return event.errorCode ?? null;
+    return null;
+}
+function errorDetailForTerminalEvent(event) {
+    if (event.type === "missing_api_key")
+        return LLM_MISSING_CREDENTIALS_MESSAGE;
+    if (event.type !== "error")
         return null;
-    }
+    return (event.details || event.error || "").slice(0, 2000) || null;
 }
 /**
  * Repair a run whose terminal event was durably appended but whose final
@@ -820,7 +982,7 @@ async function getLatestRunEvent(runId) {
  */
 export async function reconcileTerminalRunFromEvents(runId) {
     await ensureRunTables();
-    const latest = await getLatestRunEvent(runId);
+    const latest = await getRunEventForReconciliation(runId);
     if (!latest)
         return false;
     const status = terminalStatusForEvent(latest.event);
@@ -828,11 +990,8 @@ export async function reconcileTerminalRunFromEvents(runId) {
     if (!status || !terminalReason)
         return false;
     const client = getDbExec();
-    const errorCode = latest.event.type === "error" ? (latest.event.errorCode ?? null) : null;
-    const errorDetail = latest.event.type === "error"
-        ? (latest.event.details || latest.event.error || "").slice(0, 2000) ||
-            null
-        : null;
+    const errorCode = errorCodeForTerminalEvent(latest.event);
+    const errorDetail = errorDetailForTerminalEvent(latest.event);
     const { rowsAffected } = await client.execute({
         sql: `UPDATE agent_runs
           SET status = ?,
@@ -967,11 +1126,53 @@ export async function updateRunHeartbeat(runId) {
 export async function bumpRunProgress(runId) {
     await ensureRunTables();
     const client = getDbExec();
+    const now = Date.now();
     await client.execute({
-        sql: `UPDATE agent_runs SET last_progress_at = ? WHERE id = ?`,
-        args: [Date.now(), runId],
+        // Multiple event-persistence paths and serverless isolates can bump the
+        // same run concurrently. A slower, older write must never land after a
+        // newer one and move the user-visible no-progress clock backwards.
+        // CASE keeps this portable across SQLite and Postgres.
+        sql: `UPDATE agent_runs SET last_progress_at = CASE WHEN last_progress_at IS NULL OR last_progress_at < ? THEN ? ELSE last_progress_at END WHERE id = ? AND status = 'running'`,
+        args: [now, now, runId],
     });
 }
+/**
+ * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
+ * SQL so a stale reaper running in a DIFFERENT isolate can tell a
+ * demonstrably-alive run (holding a tool call or A2A `agent_call` delegation)
+ * apart from a genuinely dead one — see `IN_FLIGHT_RUN_STALE_GRACE_MS`'s doc
+ * comment for the full reasoning.
+ *
+ * `inFlight: true` only writes when the row is still `NULL` — a defense-in-
+ * depth belt-and-suspenders against a nested 1->2 transition clobbering the
+ * ORIGINAL start time with a later one (the caller's own counter already
+ * dedupes 0->1 transitions; this WHERE just makes the write itself
+ * idempotent/order-independent too). `inFlight: false` always clears
+ * unconditionally — if it races a fresh 0->1 write from a *different* tool
+ * finishing/starting back to back, worst case is losing a few seconds of
+ * grace, never gaining an incorrect one.
+ *
+ * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
+ * failure here never blocks event emission or aborts the run. If this write
+ * itself fails (the same DB pressure that could be starving the heartbeat),
+ * the row simply gets no grace — never worse than today's behavior.
+ */
+export async function setRunInFlightMarker(runId, inFlight) {
+    await ensureRunTables();
+    const client = getDbExec();
+    if (inFlight) {
+        await client.execute({
+            sql: `UPDATE agent_runs SET in_flight_since = ? WHERE id = ? AND status = 'running' AND in_flight_since IS NULL`,
+            args: [Date.now(), runId],
+        });
+    }
+    else {
+        await client.execute({
+            sql: `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ?`,
+            args: [runId],
+        });
+    }
+}
 /**
  * If the given run is marked "running" in SQL but its heartbeat is stale
  * (producer likely crashed), flip it to "errored" so watchers stop waiting.
@@ -985,11 +1186,18 @@ export async function reapIfStale(runId, maxStaleMs) {
     const completedAt = Date.now();
     // Background-dispatched runs get the wider stale window so a slow cold-start
     // isn't reaped; foreground runs keep the tight 15s window. An explicit
-    // override forces a flat window for callers that want one.
+    // override forces a flat window for callers that want one — and, like that
+    // override already bypasses the background-aware widening, it also bypasses
+    // the in-flight grace below: an explicit `maxStaleMs` is an exact,
+    // caller-chosen window and stays exact.
     const staleClause = typeof maxStaleMs === "number"
         ? `${livenessBasisSql()} < ?`
-        : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`;
-    const staleArgs = typeof maxStaleMs === "number" ? [completedAt - maxStaleMs] : [completedAt];
+        : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()}`;
+    const staleArgs = typeof maxStaleMs === "number"
+        ? [completedAt - maxStaleMs]
+        : // First `?` is backgroundAwareStaleCutoffSql's CAST param, second is
+            // inFlightGraceSql's — both bound to the same "now".
+            [completedAt, completedAt];
     const { rowsAffected } = await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1247,8 +1455,8 @@ export async function getRunByThread(threadId, options) {
     await ensureRunTables();
     const client = getDbExec();
     const sql = options?.includeTerminal
-        ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
-        : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
+        ? `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`
+        : `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, dispatch_mode, terminal_reason, diag_stage, error_code, in_flight_since FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;
     const { rows } = await client.execute({ sql, args: [threadId] });
     if (rows.length === 0)
         return null;
@@ -1271,6 +1479,7 @@ export async function getRunByThread(threadId, options) {
         dispatchMode: r.dispatch_mode ?? null,
         terminalReason: r.terminal_reason ?? null,
         diagStage: r.diag_stage ?? null,
+        inFlightSince: r.in_flight_since == null ? null : Number(r.in_flight_since),
     };
 }
 export async function listRunsForThread(threadId, options = {}) {
@@ -1395,12 +1604,21 @@ export async function reapAllStaleRuns() {
     await ensureRunTables();
     const client = getDbExec();
     const now = Date.now();
-    // Background-dispatched runs use the wider window; everything else 15s.
+    // Background-dispatched runs use the wider window; everything else 15s. The
+    // in-flight grace clause is applied identically to this SELECT and the
+    // UPDATE below (both derive `stale.rows`/the terminal-event-append loop
+    // from the SAME predicate) so a row the grace clause protects from the
+    // UPDATE is never mistakenly given a terminal event anyway — see
+    // `inFlightGraceSql` and `IN_FLIGHT_RUN_STALE_GRACE_MS`. This runs at
+    // server startup across possibly-multiple isolates, so a sibling isolate's
+    // still-alive, in-flight run must not be reaped just because THIS isolate
+    // just booted and has no heartbeat history for it.
     const stale = await client.execute({
         sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
-        args: [now],
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
+        args: [now, now],
     });
     for (const row of stale.rows) {
         const id = row.id;
@@ -1418,13 +1636,15 @@ export async function reapAllStaleRuns() {
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
         args: [
             completedAt,
             STALE_RUN_ERROR_EVENT.errorCode,
             STALE_RUN_ERROR_EVENT.details,
             STALE_RUN_ERROR_EVENT.errorCode,
             completedAt,
+            completedAt,
         ],
     });
     for (const row of stale.rows) {
@@ -1450,15 +1670,24 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
     // SELECT covers BOTH UPDATE conditions so the terminal-event-append loop
     // below catches every row we're about to flip — a 24h-old row with a
     // somehow-fresh heartbeat would slip past a heartbeat-only SELECT.
+    //
+    // The in-flight grace clause is applied ONLY to the heartbeat-based branch
+    // (mirroring the second UPDATE below), never the absolute-age branch:
+    // nothing can legitimately hold in-flight work anywhere near `olderThanMs`
+    // (default on the order of a day) — `IN_FLIGHT_RUN_STALE_GRACE_MS` bounds
+    // any real grace at ~14.5 minutes — so a row that old is dead regardless of
+    // its in-flight marker. This runs opportunistically after EVERY run
+    // completes in ANY isolate, so a different thread's still-in-flight A2A
+    // call must not be reaped as a side effect of an unrelated run finishing.
     const now = Date.now();
     const stale = await client.execute({
         sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
             AND (
-              ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+              (${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()})
               OR started_at < ?
     )`,
-        args: [now, cutoff],
+        args: [now, now, cutoff],
     });
     for (const row of stale.rows) {
         const id = row.id;
@@ -1486,7 +1715,13 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
         ],
     });
     // Also expire runs whose heartbeat is stale — producer has died. Uses the
-    // background-aware window so a slow background cold-start isn't reaped early.
+    // background-aware window so a slow background cold-start isn't reaped
+    // early, and the in-flight grace so a demonstrably-alive run holding a tool
+    // call / A2A delegation survives a heartbeat write failure. Must match the
+    // SELECT's heartbeat branch above exactly — the terminal-event-append loop
+    // below fires for every row the SELECT returned, so a mismatch would
+    // silently inject a terminal error event onto a row this UPDATE left
+    // status='running'.
     await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored',
@@ -1496,13 +1731,15 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
               terminal_reason = ?
           WHERE status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}`,
+            AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
+            AND ${inFlightGraceSql()}`,
         args: [
             completedAt,
             STALE_RUN_ERROR_EVENT.errorCode,
             STALE_RUN_ERROR_EVENT.details,
             STALE_RUN_ERROR_EVENT.errorCode,
             completedAt,
+            completedAt,
         ],
     });
     for (const row of stale.rows) {