@agent-native/core 0.122.3 → 0.123.0

package/dist/agent/run-store.js

@@ -3,6 +3,7 @@ import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
 import { captureError } from "../server/capture-error.js";
 import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
+import { isContinuationTerminalReason } from "./types.js";
 let _initPromise;
 /**
  * Max time without a heartbeat before a "running" run is considered dead.
@@ -220,6 +221,26 @@ const STALE_RUN_RECOVERY_NO_PROGRESS_WINDOW_MS = 20_000;
  * `BACKGROUND_RUN_STALE_MS` / `RUN_STALE_MS` exactly as before.
  */
 export const IN_FLIGHT_RUN_STALE_GRACE_MS = 14.5 * 60_000; // 870_000
+/**
+ * Ceiling on how far the liveness basis (`livenessBasisSql`) may lag before
+ * `IN_FLIGHT_RUN_STALE_GRACE_MS` stops applying at all. The grace exists for
+ * ONE scenario: a demonstrably-alive producer whose heartbeat WRITE is failing.
+ * It was never meant to cover a producer that has stopped writing anything —
+ * but as originally written it did, because the marker is set on tool_start and
+ * only cleared on tool_done, so a worker that dies mid-tool leaves the marker
+ * latched and inherits the full 14.5 minutes. Prod: 23 such corpse rows across
+ * five apps sat the entire grace before being reaped (mean age 715-908s), which
+ * made the one mechanism protecting slow tools the reason a dead worker took a
+ * quarter hour to surface.
+ *
+ * Requiring recent liveness for the grace to hold collapses that to this
+ * window. 120s = 80 consecutive missed 1.5s heartbeat writes, and 30s past the
+ * widest normal window (`BACKGROUND_RUN_STALE_MS`) — a write outage that
+ * outlasts it is not "the heartbeat lagged", it is the whole producer being
+ * gone. Beyond it the row falls back to the normal background-aware window and
+ * is reaped like any other stale run.
+ */
+export const IN_FLIGHT_GRACE_MAX_LIVENESS_GAP_MS = 120_000;
 async function ensureRunTables() {
     if (!_initPromise) {
         _initPromise = (async () => {
@@ -242,7 +263,8 @@ async function ensureRunTables() {
           terminal_reason TEXT,
           dispatch_mode TEXT,
           diag_stage TEXT,
-          dispatch_payload TEXT
+          dispatch_payload TEXT,
+          peak_rss_mb ${intType()}
         )
       `;
             const agentRunEventsCreateSql = `
@@ -253,6 +275,22 @@ async function ensureRunTables() {
           event_data TEXT NOT NULL,
           PRIMARY KEY (run_id, seq)
         )
+      `;
+            // Daily terminal-outcome counters, rolled up from `agent_runs` right
+            // before those rows are pruned (see `pruneAndRollUpPrunedRunOutcomes`). Completed
+            // runs are pruned after ~1 day and unsuccessful ones after ~7, so any
+            // window wider than the shorter retention read from `agent_runs` alone
+            // reports N days of failures against 1 day of successes — the distortion
+            // behind the "~25% completion rate" headline. These counters outlive the
+            // rows, so outcome rates stay queryable (see `getRunOutcomeCounters`).
+            const agentRunOutcomeDailyCreateSql = `
+        CREATE TABLE IF NOT EXISTS agent_run_outcome_daily (
+          day TEXT NOT NULL,
+          status TEXT NOT NULL,
+          terminal_reason TEXT NOT NULL DEFAULT '',
+          run_count ${intType()} NOT NULL DEFAULT 0,
+          PRIMARY KEY (day, status, terminal_reason)
+        )
       `;
             // Tool-call result ledger: persists the outcome of write tool calls that
             // completed AFTER their chunk was abandoned (zombie completions). A
@@ -318,6 +356,14 @@ async function ensureRunTables() {
                     ["dispatch_mode", "TEXT"],
                     ["diag_stage", "TEXT"],
                     ["worker_stage", "TEXT"],
+                    // peak_rss_mb: high-water resident memory, bumped on the heartbeat
+                    // write that already happens. An OOM kill is SIGKILL — no handler
+                    // runs, nothing is logged, and the run just stops mid-token — so the
+                    // ONLY way to tell "ran out of memory" from "hit a wall clock" after
+                    // the fact is a memory trace that climbs to a ceiling and stops.
+                    // Assets background workers die ~10s into a 780s budget with this
+                    // signature and Netlify's function logs are not retrievable.
+                    ["peak_rss_mb", intType()],
                     // dispatch_payload holds the JSON request body for a background
                     // dispatch so the self-POST to the Netlify background function can
                     // stay tiny (Netlify caps background-function request bodies at
@@ -339,6 +385,7 @@ async function ensureRunTables() {
                 await ensureTableExists("agent_run_events", agentRunEventsCreateSql);
                 await ensureColumnExists("agent_run_events", "event_at", `ALTER TABLE agent_run_events ADD COLUMN IF NOT EXISTS event_at ${intType()}`);
                 await ensureTableExists("agent_tool_ledger", agentToolLedgerCreateSql);
+                await ensureTableExists("agent_run_outcome_daily", agentRunOutcomeDailyCreateSql);
                 // Widen millisecond-timestamp columns that older deployments created as
                 // 32-bit `INTEGER`. `insertRun()` writes `Date.now()` into `started_at`
                 // on every turn, so an int4 column makes every agent prompt fail on
@@ -436,6 +483,7 @@ async function ensureRunTables() {
                 // Column already exists — ignore
             }
             await client.execute(agentToolLedgerCreateSql);
+            await client.execute(agentRunOutcomeDailyCreateSql);
             // Widen millisecond-timestamp columns that older deployments created as
             // 32-bit `INTEGER`. `insertRun()` writes `Date.now()` into `started_at`
             // on every turn, so an int4 column makes every agent prompt fail on
@@ -621,10 +669,17 @@ function livenessBasisSql() {
  * `RUN_STALE_MS` behavior is unchanged. See `IN_FLIGHT_RUN_STALE_GRACE_MS`'s
  * doc comment for why this is sound and bounded.
  *
- * Binds one param: the same `now` value the surrounding cutoff clause binds.
+ * The grace additionally requires the producer to still be demonstrably
+ * heartbeating: a marker on a row whose liveness basis is itself dead by more
+ * than `IN_FLIGHT_GRACE_MAX_LIVENESS_GAP_MS` is a latched corpse, not a slow
+ * tool, and gets no extension.
+ *
+ * Binds two params, both the same `now` the surrounding cutoff clause binds.
  */
 function inFlightGraceSql() {
-    return `(in_flight_since IS NULL OR in_flight_since <= (CAST(? AS BIGINT) - ${IN_FLIGHT_RUN_STALE_GRACE_MS}))`;
+    return `(in_flight_since IS NULL
+    OR in_flight_since <= (CAST(? AS BIGINT) - ${IN_FLIGHT_RUN_STALE_GRACE_MS})
+    OR ${livenessBasisSql()} < (CAST(? AS BIGINT) - ${IN_FLIGHT_GRACE_MAX_LIVENESS_GAP_MS}))`;
 }
 /**
  * Atomically claim a background-dispatched run for processing. The foreground
@@ -904,9 +959,15 @@ export async function setRunError(runId, errorCode, errorDetail) {
     }
 }
 /**
- * Record why a run reached its terminal status. Unlike error_code/error_detail,
- * this is set for successful checkpoint boundaries too (for example
- * status='completed' + terminal_reason='run_timeout').
+ * Record why a run reached its terminal status, and correct the status when the
+ * reason says the run did not actually finish.
+ *
+ * Every writer sets the status first and the reason second, so this is the one
+ * place that sees both — correcting it here keeps all of them honest instead of
+ * making each caller re-derive "was that completion real?". Only `completed` is
+ * corrected: an `errored`/`aborted` row must never be softened, and a still
+ * `running` row must not be terminated early (`persistRunCheckpointEvent`
+ * records the reason mid-run, before the boundary is final).
  */
 export async function setRunTerminalReason(runId, terminalReason) {
     if (!terminalReason)
@@ -914,24 +975,32 @@ export async function setRunTerminalReason(runId, terminalReason) {
     try {
         await ensureRunTables();
         const client = getDbExec();
+        const reason = terminalReason.slice(0, 200);
         await client.execute({
-            sql: `UPDATE agent_runs SET terminal_reason = ? WHERE id = ?`,
-            args: [terminalReason.slice(0, 200), runId],
+            sql: isContinuationTerminalReason(reason)
+                ? `UPDATE agent_runs SET terminal_reason = ?, status = CASE WHEN status = 'completed' THEN 'truncated' ELSE status END WHERE id = ?`
+                : `UPDATE agent_runs SET terminal_reason = ? WHERE id = ?`,
+            args: [reason, runId],
         });
     }
     catch {
         // Diagnostics are best-effort; never let them break completion.
     }
 }
+/**
+ * INVARIANT: a terminal event yields `completed` if and only if its reason is
+ * `done`. Everything else either failed (`errored`) or stopped short
+ * (`truncated`). Keep this in lockstep with `terminalReasonForEvent` below.
+ */
 function terminalStatusForEvent(event) {
     if (event.type === "error")
         return "errored";
     if (event.type === "missing_api_key")
         return "errored";
-    if (event.type === "done" ||
-        event.type === "loop_limit" ||
-        event.type === "auto_continue") {
+    if (event.type === "done")
         return "completed";
+    if (event.type === "loop_limit" || event.type === "auto_continue") {
+        return "truncated";
     }
     return null;
 }
@@ -1167,10 +1236,31 @@ export async function updateRunHeartbeat(runId) {
     // prod: heartbeat continued ~400s past completed_at), which confuses
     // triage and can keep /runs/active looking "fresh" after failure.
     await client.execute({
-        sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ? AND status = 'running'`,
-        args: [Date.now(), runId],
+        sql: `UPDATE agent_runs
+          SET heartbeat_at = ?,
+              peak_rss_mb = CASE
+                WHEN peak_rss_mb IS NULL OR peak_rss_mb < ? THEN ?
+                ELSE peak_rss_mb
+              END
+          WHERE id = ? AND status = 'running'`,
+        args: [Date.now(), currentRssMb(), currentRssMb(), runId],
     });
 }
+/**
+ * Resident memory in whole MB, or 0 where the runtime does not expose it
+ * (Workers, Deno). Piggybacks the heartbeat rather than adding a timer: a run
+ * killed by the platform never gets to report anything, so the last persisted
+ * high-water mark is the only evidence that survives.
+ */
+function currentRssMb() {
+    try {
+        const rss = process.memoryUsage?.().rss;
+        return typeof rss === "number" ? Math.round(rss / 1024 / 1024) : 0;
+    }
+    catch {
+        return 0;
+    }
+}
 /**
  * Bump `last_progress_at` — call this whenever the agent actually emits an
  * event (token, tool call, message). Distinct from `heartbeat_at` so the
@@ -1426,9 +1516,9 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
         : `${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()}`;
     const staleArgs = typeof maxStaleMs === "number"
         ? [completedAt - maxStaleMs]
-        : // First `?` is backgroundAwareStaleCutoffSql's CAST param, second is
-            // inFlightGraceSql's — both bound to the same "now".
-            [completedAt, completedAt];
+        : // First `?` is backgroundAwareStaleCutoffSql's CAST param, the next two
+            // are inFlightGraceSql's — all bound to the same "now".
+            [completedAt, completedAt, completedAt];
     const updateSql = `UPDATE agent_runs
           SET status = 'errored',
               completed_at = ?,
@@ -1602,6 +1692,47 @@ export async function getRunStatus(runId) {
         return null;
     return String(rows[0].status);
 }
+/**
+ * Abort reasons whose turn genuinely ended, so `done` is the truthful wire
+ * event. `displaced` belongs here because whoever displaced this run already
+ * wrote the row's real terminal state — synthesizing our own would fabricate
+ * a second, competing truth.
+ */
+const TURN_ENDING_ABORT_REASONS = new Set(["user", "displaced"]);
+// Only infrastructure interruptions that the client can safely resume are
+// recoverable. An arbitrary caller-supplied abort reason must never become an
+// auto-continue signal, because that turns an explicit stop into new work.
+const RECOVERABLE_ABORT_REASONS = new Set(["background_worker_died"]);
+/**
+ * Truthful terminal event for an aborted run.
+ *
+ * A synthetic `done` is indistinguishable from a real finish on the wire, so
+ * every recovery abort used to render as "The agent stopped without sending a
+ * final message" with the streamed work apparently thrown away. Recoverable
+ * chunk-boundary reasons ride the `auto_continue` channel the client already
+ * routes into continuation; anything else that isn't a user stop surfaces as a
+ * reason-coded error. Only known infrastructure aborts are recoverable;
+ * every other abort is terminal and non-recoverable.
+ */
+export function terminalEventForAbortReason(reason) {
+    const normalized = (reason ?? "").trim() || "user";
+    if (isContinuationTerminalReason(normalized)) {
+        return {
+            type: "auto_continue",
+            reason: normalized,
+        };
+    }
+    if (TURN_ENDING_ABORT_REASONS.has(normalized) ||
+        normalized.startsWith("user_")) {
+        return { type: "done" };
+    }
+    return {
+        type: "error",
+        error: "The agent run was stopped before it finished.",
+        errorCode: `aborted_${normalized}`,
+        recoverable: RECOVERABLE_ABORT_REASONS.has(normalized),
+    };
+}
 export async function markRunAborted(runId, reason) {
     await ensureRunTables();
     const client = getDbExec();
@@ -1610,7 +1741,7 @@ export async function markRunAborted(runId, reason) {
         args: [reason ?? "user", Date.now(), `aborted:${reason ?? "user"}`, runId],
     });
     if ((rowsAffected ?? 0) > 0) {
-        await safeAppendTerminalRunEvent(runId, { type: "done" }, "mark-aborted");
+        await safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-aborted");
     }
 }
 function turnAbortMarkerRunId(turnId) {
@@ -1648,7 +1779,7 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
         sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ? WHERE thread_id = ? AND turn_id = ? AND status = 'running'`,
         args: [reason, Date.now(), `aborted:${reason}`, threadId, turnId],
     });
-    await Promise.all(runIds.map((runId) => safeAppendTerminalRunEvent(runId, { type: "done" }, "mark-turn-aborted")));
+    await Promise.all(runIds.map((runId) => safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-turn-aborted")));
 }
 export async function isTurnAborted(threadId, turnId) {
     await ensureRunTables();
@@ -1702,12 +1833,39 @@ export async function insertRunEvent(runId, seq, eventData) {
         args: [runId, seq, Date.now(), eventData, runId],
     });
 }
+/**
+ * Reserved seq for a checkpoint terminal event written BEFORE the agent loop
+ * unwinds. Stream events use contiguous 0-based seqs, so a value this high can
+ * never collide with one and always sorts last — which is what lets
+ * `reconcileTerminalRunFromEvents` / `getLastTerminalRunEvent` read the
+ * checkpoint as the run's real terminal state even when the process is killed
+ * mid-unwind, and what keeps `appendTerminalRunEvent` from stamping a
+ * `stale_run` lie over it.
+ */
+export const CHECKPOINT_TERMINAL_EVENT_SEQ = 1_000_000_000;
+/**
+ * Durably record a chunk-boundary terminal event the moment the boundary is
+ * decided, not after the loop unwinds. Wind-down regularly overruns the
+ * remaining serverless budget; without this the auto_continue is never
+ * persisted and the row is reaped as a `stale_run` lie instead of a
+ * sweep-continuable checkpoint.
+ */
+export async function persistRunCheckpointEvent(runId, event, terminalReason) {
+    await insertRunEvent(runId, CHECKPOINT_TERMINAL_EVENT_SEQ, JSON.stringify(event));
+    await setRunTerminalReason(runId, terminalReason);
+}
 export async function getRunEventsSince(runId, fromSeq) {
     await ensureRunTables();
     const client = getDbExec();
+    // The reserved checkpoint band is deliberately invisible here. Streaming it
+    // to a live subscriber would close the SSE stream at the chunk boundary
+    // BEFORE onComplete has made thread_data durable, and the client would
+    // continue the turn from a half-saved message. The checkpoint is replayed on
+    // the terminal/reconnect paths instead (`getLastTerminalRunEvent`,
+    // `reconcileTerminalRunFromEvents`), which run after the row is terminal.
     const { rows } = await client.execute({
-        sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? ORDER BY seq ASC`,
-        args: [runId, fromSeq],
+        sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? AND seq < ? ORDER BY seq ASC`,
+        args: [runId, fromSeq, CHECKPOINT_TERMINAL_EVENT_SEQ],
     });
     return rows.map((r) => {
         const row = r;
@@ -1897,20 +2055,25 @@ export async function listRunsForThread(threadId, options = {}) {
  * Returns an empty array when the thread has no run yet or no parseable events.
  * Best-effort on parse: malformed ledger rows are skipped rather than thrown.
  */
-export async function getCurrentTurnEventsForThread(threadId) {
+export async function getCurrentTurnEventsForThread(threadId, knownTurnId) {
     await ensureRunTables();
     const client = getDbExec();
-    // Find the latest run for this thread (terminal or running) to learn the
-    // logical turn id. The journal is consulted on the resume path, where the
-    // just-interrupted run is typically already terminal.
-    const latest = await client.execute({
-        sql: `SELECT id, turn_id FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`,
-        args: [threadId],
-    });
-    if (latest.rows.length === 0)
-        return [];
-    const latestRow = latest.rows[0];
-    const turnId = latestRow.turn_id ?? latestRow.id;
+    // Callers that already know their turn MUST pass it: `startRun` persists the
+    // run row without awaiting the INSERT, so inferring the turn from the latest
+    // row can read a moment before this turn's row commits and return the
+    // PREVIOUS turn's events — which the read-only journal replay would then
+    // serve as a cache hit for an identical tool+input.
+    let turnId = knownTurnId;
+    if (!turnId) {
+        const latest = await client.execute({
+            sql: `SELECT id, turn_id FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`,
+            args: [threadId],
+        });
+        if (latest.rows.length === 0)
+            return [];
+        const latestRow = latest.rows[0];
+        turnId = latestRow.turn_id ?? latestRow.id;
+    }
     // Gather every run that belongs to this logical turn, oldest chunk first, and
     // read their events in seq order. COALESCE(turn_id, id) folds older rows that
     // predate the turn_id backfill into a turn keyed by their own run id.
@@ -1961,7 +2124,7 @@ export async function reapAllStaleRuns() {
           WHERE status = 'running'
             AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
             AND ${inFlightGraceSql()}`,
-        args: [now, now],
+        args: [now, now, now],
     });
     for (const row of stale.rows) {
         const id = row.id;
@@ -1997,11 +2160,128 @@ export async function reapAllStaleRuns() {
     }
     return reapedCount;
 }
+const RUN_OUTCOME_DAY_MS = 86_400_000;
+/**
+ * Terminal statuses kept on the long retention window. A truncated run is a
+ * failure — it stopped mid-task — so it must be retained with the errors it
+ * belongs with. Filing truncations as `completed` deleted them at 24h while the
+ * genuine failures they should be compared against survived for 7 days, so
+ * every run id a user pasted into a bug report was gone before anyone looked.
+ */
+const UNSUCCESSFUL_STATUS_SQL_LIST = `('errored', 'aborted', 'truncated')`;
+/**
+ * Fold the terminal outcomes of the rows `cleanupOldRuns` is about to delete
+ * into `agent_run_outcome_daily`, so success/failure RATES survive pruning even
+ * though the rows do not. Counters cover exactly the pruned rows and `agent_runs`
+ * covers exactly the unpruned ones, so a rate over any window is
+ * `getRunOutcomeCounters()` plus the live rows — no gap, no double count.
+ *
+ * The DELETE ... RETURNING is the claim: concurrent cleanup calls can both
+ * observe a row, but only the caller that deletes it receives it to roll up.
+ * Grouping the returned rows in TypeScript avoids dialect-specific date SQL.
+ * Counter upserts run in the same transaction as the delete; a failed upsert
+ * rolls back the claim so the source rows remain available for a retry.
+ */
+async function pruneAndRollUpPrunedRunOutcomes(client, cutoff, erroredCutoff) {
+    const prune = async (tx) => {
+        await tx.execute({
+            sql: `DELETE FROM agent_run_events WHERE run_id IN (
+        SELECT id FROM agent_runs
+        WHERE (status = 'completed' AND completed_at < ?)
+           OR (status IN ${UNSUCCESSFUL_STATUS_SQL_LIST} AND completed_at < ?)
+      )`,
+            args: [cutoff, erroredCutoff],
+        });
+        const { rows } = await tx.execute({
+            sql: `DELETE FROM agent_runs
+            WHERE (status = 'completed' AND completed_at < ?)
+               OR (status IN ${UNSUCCESSFUL_STATUS_SQL_LIST} AND completed_at < ?)
+            RETURNING status, completed_at, terminal_reason`,
+            args: [cutoff, erroredCutoff],
+        });
+        const groups = new Map();
+        for (const row of rows) {
+            const outcome = row;
+            const dayIndex = Number(outcome.completed_at) / RUN_OUTCOME_DAY_MS;
+            if (!Number.isFinite(dayIndex) || !outcome.status)
+                continue;
+            const day = new Date(Math.floor(dayIndex) * RUN_OUTCOME_DAY_MS)
+                .toISOString()
+                .slice(0, 10);
+            const terminalReason = outcome.terminal_reason ?? "";
+            const key = `${day}\u0000${outcome.status}\u0000${terminalReason}`;
+            const existing = groups.get(key);
+            if (existing) {
+                existing.count += 1;
+            }
+            else {
+                groups.set(key, {
+                    day,
+                    status: outcome.status,
+                    terminalReason,
+                    count: 1,
+                });
+            }
+        }
+        for (const group of groups.values()) {
+            await tx.execute({
+                sql: `INSERT INTO agent_run_outcome_daily (day, status, terminal_reason, run_count)
+              VALUES (?, ?, ?, ?)
+              ON CONFLICT (day, status, terminal_reason)
+              DO UPDATE SET run_count = agent_run_outcome_daily.run_count + excluded.run_count`,
+                args: [group.day, group.status, group.terminalReason, group.count],
+            });
+        }
+    };
+    try {
+        if (client.transaction) {
+            await client.transaction(prune);
+            return;
+        }
+        await client.execute(isPostgres() ? "BEGIN" : "BEGIN IMMEDIATE");
+        try {
+            await prune(client);
+            await client.execute("COMMIT");
+        }
+        catch (error) {
+            await client.execute("ROLLBACK").catch(() => { });
+            throw error;
+        }
+    }
+    catch {
+        // A transactional failure leaves the rows available for the next sweep.
+    }
+}
+/**
+ * Read the daily terminal-outcome counters rolled up from pruned runs. Pair
+ * with live `agent_runs` rows for a complete picture — see
+ * `pruneAndRollUpPrunedRunOutcomes`.
+ */
+export async function getRunOutcomeCounters(options) {
+    await ensureRunTables();
+    const client = getDbExec();
+    const { rows } = await client.execute({
+        sql: `SELECT day, status, terminal_reason, run_count
+          FROM agent_run_outcome_daily
+          WHERE day >= ?
+          ORDER BY day DESC`,
+        args: [options?.sinceDay ?? ""],
+    });
+    return rows.map((r) => {
+        const row = r;
+        return {
+            day: row.day,
+            status: row.status,
+            terminalReason: row.terminal_reason ?? "",
+            count: Number(row.run_count ?? 0),
+        };
+    });
+}
 /** Delete old runs and expire stale "running" rows that haven't had activity
- *  (e.g. worker crashed before updating status). Completed runs are pruned at
- *  `olderThanMs`; errored/aborted runs are kept until `erroredOlderThanMs` (a
- *  longer window, falling back to `olderThanMs`) so their event log survives
- *  for cut-off pattern analysis via listErroredRuns. */
+ *  (e.g. worker crashed before updating status). Genuinely completed runs are
+ *  pruned at `olderThanMs`; errored/aborted/truncated runs are kept until
+ *  `erroredOlderThanMs` (a longer window, falling back to `olderThanMs`) so
+ *  their event log survives for cut-off pattern analysis via listErroredRuns. */
 export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
     await ensureRunTables();
     const client = getDbExec();
@@ -2029,7 +2309,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
               (${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()} AND ${inFlightGraceSql()})
               OR started_at < ?
     )`,
-        args: [now, now, cutoff],
+        args: [now, now, now, cutoff],
     });
     for (const row of stale.rows) {
         const id = row.id;
@@ -2082,6 +2362,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
             STALE_RUN_ERROR_EVENT.errorCode,
             completedAt,
             completedAt,
+            completedAt,
         ],
     });
     for (const row of stale.rows) {
@@ -2090,28 +2371,19 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
             await safeAppendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT, "cleanup-old-runs");
         }
     }
-    // Delete events for old terminal runs. Completed runs prune at `cutoff`;
-    // errored/aborted runs are retained until the (longer) `erroredCutoff`.
-    await client.execute({
-        sql: `DELETE FROM agent_run_events WHERE run_id IN (
-      SELECT id FROM agent_runs
-      WHERE (status = 'completed' AND completed_at < ?)
-         OR (status IN ('errored', 'aborted') AND completed_at < ?)
-    )`,
-        args: [cutoff, erroredCutoff],
-    });
-    await client.execute({
-        sql: `DELETE FROM agent_runs
-          WHERE (status = 'completed' AND completed_at < ?)
-             OR (status IN ('errored', 'aborted') AND completed_at < ?)`,
-        args: [cutoff, erroredCutoff],
-    });
+    // Claim old terminal runs and roll up only rows this invocation actually
+    // deleted. The transaction prevents concurrent cleanup calls from both
+    // counting the same source rows.
+    await pruneAndRollUpPrunedRunOutcomes(client, cutoff, erroredCutoff);
 }
 /**
- * List recent errored/aborted runs for cut-off pattern analysis. Read-only,
- * bounded, and ordered newest-first. Surfaced via the list-errored-runs action
- * so the team can see why chats are failing (terminal error code, duration,
- * turn linkage) instead of discovering it ad hoc.
+ * List recent unsuccessful runs (errored, aborted, and truncated) for cut-off
+ * pattern analysis. Read-only, bounded, and ordered newest-first. Surfaced via
+ * the list-errored-runs action so the team can see why chats are failing
+ * (terminal error code, duration, turn linkage) instead of discovering it ad
+ * hoc. Truncations belong here: a run that stopped at a budget boundary is a
+ * cut-off, and they outnumbered genuine completions on Plan in prod while being
+ * invisible to this query.
  */
 export async function listErroredRuns(options) {
     await ensureRunTables();
@@ -2121,7 +2393,7 @@ export async function listErroredRuns(options) {
     const { rows } = await client.execute({
         sql: `SELECT id, thread_id, turn_id, status, error_code, error_detail, terminal_reason, started_at, completed_at
           FROM agent_runs
-          WHERE status IN ('errored', 'aborted')
+          WHERE status IN ${UNSUCCESSFUL_STATUS_SQL_LIST}
             AND COALESCE(completed_at, started_at) >= ?
           ORDER BY COALESCE(completed_at, started_at) DESC
           LIMIT ${limit}`,