@agent-native/core 0.219.0 → 0.220.0-nightly-20261023212848

package/dist/agent/run-store.js

@@ -7,7 +7,9 @@ import { fromMigrationEntries } from "../db/versioned-store-migrations.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
 import { automationRecoveryMessagesForLocale } from "../localization/automation-recovery-messages.js";
 import { lockAndAssertServicePrincipalMayStartRun, prepareServicePrincipalRunStart, recordServicePrincipalDenial, } from "../org/service-principal-guard.js";
+import { platformWaitUntil } from "../server/background-work.js";
 import { captureError } from "../server/capture-error.js";
+import { runWithRequestContext } from "../server/request-context.js";
 import { assertNoInlineImageBytes, stripInlineBytes, stripInlineBytesFromJson, } from "../shared/inline-bytes.js";
 import { isRequestedStopAbortReason } from "./abort-reasons.js";
 import { approvalKeyForHistory } from "./approval-key.js";
@@ -17,6 +19,7 @@ import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, }
 import { currentExecutionFence, currentExecutionFenceForScope, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
 import { AGENT_RUN_MIGRATIONS, AGENT_RUN_MIGRATIONS_TABLE, } from "./run-migrations.js";
 import { lockThreadRunSlot } from "./run-slot-lock.js";
+import { trackRunTerminal, } from "./run-terminal-tracking.js";
 import { isConsistentToolCallInputFingerprint, isUnavailableToolCallInput, toolCallInputFingerprint, } from "./tool-call-journal.js";
 import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
 import { readWriteReceipt } from "./write-receipts.js";
@@ -1537,6 +1540,14 @@ export async function setRunTerminalReason(runId, terminalReason) {
         // Diagnostics are best-effort; never let them break completion.
     }
 }
+export async function setRunTerminalReasonIfStatus(runId, expectedStatus, terminalReason) {
+    await ensureRunTables();
+    const { rowsAffected } = await withFencedExecutionWrite((tx) => tx.execute({
+        sql: `UPDATE agent_runs SET terminal_reason = ? WHERE id = ? AND status = ? AND (terminal_reason IS NULL OR terminal_reason = '')`,
+        args: [terminalReason.slice(0, 200), runId, expectedStatus],
+    }), { runId });
+    return (rowsAffected ?? 0) > 0;
+}
 /**
  * INVARIANT: a terminal event yields `completed` if and only if its reason is
  * `done`. Everything else either failed (`errored`) or stopped short
@@ -2018,29 +2029,37 @@ function priorDiagStageLabel(raw) {
 async function reconcileTerminalRunInRecoveryTransaction(tx, runId) {
     const latest = await getRunEventForReconciliation(runId, tx);
     if (!latest)
-        return false;
+        return null;
     const status = terminalStatusForEvent(latest.event);
     const terminalReason = terminalReasonForEvent(latest.event);
     if (!status || !terminalReason)
-        return false;
-    const { rowsAffected } = await tx.execute({
+        return null;
+    const errorCode = errorCodeForTerminalEvent(latest.event);
+    const { rows, rowsAffected } = await tx.execute({
         sql: `UPDATE agent_runs
           SET status = ?,
               completed_at = COALESCE(completed_at, ?, ${livenessBasisSql()}),
               error_code = ?,
               error_detail = ?,
               terminal_reason = ?
-          WHERE id = ? AND status = 'running'`,
+          WHERE id = ? AND status = 'running'
+          ${ENDED_RUN_RETURNING}`,
         args: [
             status,
             latest.eventAt,
-            errorCodeForTerminalEvent(latest.event),
+            errorCode,
             errorDetailForTerminalEvent(latest.event),
             terminalReason,
             runId,
         ],
     });
-    return (rowsAffected ?? 0) === 1;
+    if ((rowsAffected ?? 0) !== 1)
+        return null;
+    return {
+        runId,
+        ending: { status, terminalReason, errorCode: errorCode ?? undefined },
+        row: rows?.[0],
+    };
 }
 async function reapSingleStaleRun(runId, source, maxStaleMs) {
     const completedAt = Date.now();
@@ -2057,7 +2076,8 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
           WHERE id = ?
             AND status = 'running'
             AND ${terminalRunEventExclusionSql()}
-            AND ${staleClause}`;
+            AND ${staleClause}
+          ${ENDED_RUN_RETURNING}`;
     const updateArgs = [
         STALE_RUN_ERROR_EVENT.errorCode,
         STALE_RUN_ERROR_EVENT.details,
@@ -2096,7 +2116,8 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
         if (lease?.isLive && generationMatches) {
             return { reaped: false, outcome: null };
         }
-        if (await reconcileTerminalRunInRecoveryTransaction(tx, runId)) {
+        const repaired = await reconcileTerminalRunInRecoveryTransaction(tx, runId);
+        if (repaired) {
             if (generationMatches && lease) {
                 await tx.execute({
                     sql: `UPDATE agent_execution_leases
@@ -2114,9 +2135,9 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
                     args: [runId],
                 });
             }
-            return { reaped: false, outcome: null };
+            return { reaped: false, outcome: null, repaired };
         }
-        const { rowsAffected } = await tx.execute({
+        const { rows, rowsAffected } = await tx.execute({
             sql: updateSql,
             args: updateArgs,
         });
@@ -2150,9 +2171,32 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
                 }
             }
         }
-        return { reaped, outcome };
+        return {
+            reaped,
+            outcome,
+            reapedRow: rows?.[0],
+        };
     });
     const { reaped, outcome } = reapResult;
+    // Taken from the committed attempt: a rolled-back one ended nothing.
+    const ended = "repaired" in reapResult && reapResult.repaired
+        ? reapResult.repaired
+        : reaped
+            ? {
+                runId,
+                ending: reapedRunEnding({
+                    source,
+                    errorCode: STALE_RUN_ERROR_EVENT.errorCode,
+                    terminalReason: STALE_RUN_TERMINAL_REASON,
+                    recoveryOutcome: staleRunRecoveryLabel(outcome),
+                }),
+                row: "reapedRow" in reapResult ? reapResult.reapedRow : undefined,
+            }
+            : null;
+    // Handed off as soon as it commits: a sweep cut short must not lose the
+    // runs it already ended, which no later sweep can see.
+    if (ended)
+        trackRunsEndedInSql([ended]);
     let forensics = "";
     let priorStageInfo = "";
     if (reaped) {
@@ -2242,8 +2286,8 @@ export async function reapUnclaimedBackgroundRun(runId) {
     }
     const completedAt = Date.now();
     const cutoff = completedAt - UNCLAIMED_BACKGROUND_RUN_GRACE_MS;
-    const reaped = await client.transaction(async (tx) => {
-        const { rowsAffected } = await tx.execute({
+    const { reaped, row } = await client.transaction(async (tx) => {
+        const { rows, rowsAffected } = await tx.execute({
             sql: `UPDATE agent_runs
             SET status = 'errored',
                 completed_at = ?,
@@ -2253,7 +2297,8 @@ export async function reapUnclaimedBackgroundRun(runId) {
             WHERE id = ?
               AND status = 'running'
               AND dispatch_mode = 'background'
-              AND COALESCE(heartbeat_at, started_at) < ?`,
+              AND COALESCE(heartbeat_at, started_at) < ?
+            ${ENDED_RUN_RETURNING}`,
             args: [
                 completedAt,
                 UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
@@ -2267,7 +2312,7 @@ export async function reapUnclaimedBackgroundRun(runId) {
         if (updated) {
             await appendTerminalRunEventInTransaction(tx, runId, UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT);
         }
-        return updated;
+        return { reaped: updated, row: rows?.[0] };
     });
     if (reaped) {
         try {
@@ -2282,6 +2327,11 @@ export async function reapUnclaimedBackgroundRun(runId) {
                 extra: { runId },
             });
         }
+        trackReapedRun(runId, {
+            source: "reap-unclaimed-background",
+            errorCode: UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
+            terminalReason: UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
+        }, row);
     }
     return reaped;
 }
@@ -2351,7 +2401,7 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
         return false;
     }
     const expectedGeneration = scope.worker_generation == null ? null : String(scope.worker_generation);
-    return client.transaction(async (tx) => {
+    const ended = await client.transaction(async (tx) => {
         await lockThreadRunSlot(tx, scope.thread_id);
         const lease = await lockTurnExecutionLease(tx, scope.thread_id, scope.turn_id);
         const selected = await tx.execute({
@@ -2364,7 +2414,7 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
             run.dispatch_mode !== "background-processing" ||
             (run.worker_generation == null ? null : String(run.worker_generation)) !==
                 expectedGeneration) {
-            return false;
+            return null;
         }
         const generationMatches = run.worker_generation == null
             ? lease === null || lease.ownerRunId === runId
@@ -2372,14 +2422,15 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
                 lease.ownerRunId === runId &&
                 String(run.worker_generation) === lease.generation;
         if (!generationMatches)
-            return false;
+            return null;
         const updated = await tx.execute({
             sql: `UPDATE agent_runs
             SET status = 'errored', completed_at = ?, error_code = ?,
                 error_detail = ?, terminal_reason = ?, dispatch_payload = NULL
             WHERE id = ? AND status = 'running'
               AND dispatch_mode = 'background-processing'
-              AND ${terminalRunEventExclusionSql()}`,
+              AND ${terminalRunEventExclusionSql()}
+            ${ENDED_RUN_RETURNING}`,
             args: [
                 Date.now(),
                 failure.errorCode,
@@ -2389,7 +2440,7 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
             ],
         });
         if ((updated.rowsAffected ?? 0) !== 1)
-            return false;
+            return null;
         const lastEvent = await tx.execute({
             sql: `SELECT seq FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,
             args: [runId],
@@ -2409,8 +2460,16 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
         if ((inserted.rowsAffected ?? 0) !== 1) {
             throw new Error(`Failed to persist failure event for run ${runId}`);
         }
-        return true;
+        return { row: updated.rows?.[0] };
     });
+    if (!ended)
+        return false;
+    trackRunEndedInSql(runId, {
+        status: "errored",
+        terminalReason: `error:${failure.errorCode}`.slice(0, 200),
+        errorCode: failure.errorCode,
+    }, ended.row);
+    return true;
 }
 export async function failRunBeforeDispatch(runId, errorCode, errorDetail, terminalReason) {
     await ensureRunTables();
@@ -2428,7 +2487,7 @@ export async function failRunBeforeDispatch(runId, errorCode, errorDetail, termi
         typeof scope.turn_id !== "string") {
         return false;
     }
-    return client.transaction(async (tx) => {
+    const ended = await client.transaction(async (tx) => {
         const lease = await lockTurnExecutionLease(tx, scope.thread_id, scope.turn_id);
         const run = await tx.execute({
             sql: `SELECT status, worker_generation FROM agent_runs WHERE id = ? FOR UPDATE`,
@@ -2436,19 +2495,20 @@ export async function failRunBeforeDispatch(runId, errorCode, errorDetail, termi
         });
         const currentRun = run.rows[0];
         if (currentRun?.status !== "running") {
-            return false;
+            return null;
         }
         if (lease?.isLive &&
             lease.ownerRunId === runId &&
             (currentRun.worker_generation == null ||
                 String(currentRun.worker_generation) === lease.generation)) {
-            return false;
+            return null;
         }
         const updated = await tx.execute({
             sql: `UPDATE agent_runs
             SET status = 'errored', completed_at = ?, error_code = ?,
                 error_detail = ?, terminal_reason = ?, dispatch_payload = NULL
-            WHERE id = ? AND status = 'running'`,
+            WHERE id = ? AND status = 'running'
+            ${ENDED_RUN_RETURNING}`,
             args: [
                 Date.now(),
                 errorCode,
@@ -2457,20 +2517,34 @@ export async function failRunBeforeDispatch(runId, errorCode, errorDetail, termi
                 runId,
             ],
         });
-        return (updated.rowsAffected ?? 0) === 1;
+        return (updated.rowsAffected ?? 0) === 1
+            ? { row: updated.rows?.[0] }
+            : null;
     });
+    if (!ended)
+        return false;
+    // No run loop ever started for this row, so nothing else will report it.
+    trackRunEndedInSql(runId, {
+        status: "errored",
+        terminalReason: terminalReason.slice(0, 200),
+        errorCode,
+    }, ended.row);
+    return true;
 }
 /** The caller owns this claim and has not called startRun yet. */
 export async function releaseBackgroundRunBeforeStart(runId, errorCode, errorDetail) {
     await ensureRunTables();
-    const { rowsAffected } = await getDbExec().execute({
+    const { rowsAffected, rows } = await getDbExec().execute({
         sql: `UPDATE agent_runs
           SET status = 'errored', completed_at = ?, error_code = ?, error_detail = ?, dispatch_payload = NULL
-          WHERE id = ? AND status = 'running' AND dispatch_mode = 'background-processing'`,
+          WHERE id = ? AND status = 'running' AND dispatch_mode = 'background-processing'
+          ${ENDED_RUN_RETURNING}`,
         args: [Date.now(), errorCode, errorDetail, runId],
     });
     if (rowsAffected !== 1)
         throw new Error(`Unstarted background worker ${runId} could not be released`);
+    // No run loop ever started for this row, so nothing else will report it.
+    trackRunEndedInSql(runId, { status: "errored", terminalReason: `error:${errorCode}`, errorCode }, rows?.[0]);
 }
 export async function getRunStatus(runId) {
     await ensureRunTables();
@@ -2556,7 +2630,7 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
             }
         }
         const result = await tx.execute({
-            sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ? WHERE id = ? AND status = 'running'${requireSafeBoundary ? " AND in_flight_since IS NULL" : ""}`,
+            sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ? WHERE id = ? AND status = 'running'${requireSafeBoundary ? " AND in_flight_since IS NULL" : ""} ${ENDED_RUN_RETURNING}`,
             args: [
                 reason ?? "user",
                 Date.now(),
@@ -2568,7 +2642,11 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
         if (rowsAffected > 0) {
             await revokeExecutionLeasesForRuns(tx, [runId]);
             await appendAbortTerminalRunEventInTransaction(tx, runId, abortEvent);
-            return { rowsAffected, appendTerminalEvent: true };
+            return {
+                rowsAffected,
+                appendTerminalEvent: true,
+                endedRow: result.rows?.[0],
+            };
         }
         const terminalState = await tx.execute({
             sql: `SELECT status, terminal_reason, in_flight_since FROM agent_runs WHERE id = ?`,
@@ -2585,6 +2663,9 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
         return { rowsAffected: 0, appendTerminalEvent: false };
     };
     const abortResult = await retryAbortTerminalEventTransaction("mark-aborted", () => client.transaction(commitAbort));
+    if (abortResult.rowsAffected > 0) {
+        trackAbortedRun(runId, reason ?? "user", "endedRow" in abortResult ? abortResult.endedRow : undefined);
+    }
     return abortResult.rowsAffected > 0 || abortResult.appendTerminalEvent;
 }
 async function lockExecutionLeasesForRuns(db, runIds) {
@@ -2659,7 +2740,10 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
     if (!client.transaction) {
         throw new Error("Atomic turn cancellation requires transaction support");
     }
+    let abortedRows = [];
     const commitAbort = async (tx) => {
+        // A retried transaction runs this again; only the committed attempt counts.
+        abortedRows = [];
         const runIds = [];
         await lockThreadRunSlot(tx, threadId);
         await lockAndAssertCurrentExecutionFences(tx);
@@ -2682,9 +2766,10 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
         await revokeExecutionLeasesForRuns(tx, candidateRunIds);
         if (candidateRunIds.length > 0) {
             const aborted = await tx.execute({
-                sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ? WHERE thread_id = ? AND COALESCE(turn_id, id) = ? AND status = 'running' RETURNING id`,
+                sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ? WHERE thread_id = ? AND COALESCE(turn_id, id) = ? AND status = 'running' ${ENDED_RUN_RETURNING}`,
                 args: [reason, now, `aborted:${reason}`, threadId, turnId],
             });
+            abortedRows = aborted.rows;
             runIds.push(...aborted.rows
                 .map((row) => String(row.id ?? ""))
                 .filter(Boolean));
@@ -2789,6 +2874,13 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
         return "aborted";
     };
     const outcome = await retryAbortTerminalEventTransaction("mark-turn-aborted", () => client.transaction(commitAbort));
+    trackRunsEndedInSql(abortedRows
+        .filter((row) => typeof row.id === "string")
+        .map((row) => ({
+        runId: row.id,
+        ending: abortedRunEnding(reason),
+        row,
+    })));
     return outcome;
 }
 export async function getRunTurnRef(runId) {
@@ -3904,6 +3996,143 @@ export async function listErroredRuns(options) {
 export async function ensureTerminalRunEvent(runId, event) {
     return appendTerminalRunEvent(runId, event);
 }
+/**
+ * A duplicate request, or a worker that lost its turn's execution lease, is
+ * aborted before it does any work; the run it stood in for reports the turn.
+ */
+const NOT_A_RUN_ABORT_REASONS = new Set([
+    "duplicate_submission",
+    "execution-lease-busy",
+]);
+function trackAbortedRun(runId, abortReason, returnedRow) {
+    trackRunEndedInSql(runId, abortedRunEnding(abortReason), returnedRow);
+}
+function abortedRunEnding(abortReason) {
+    return {
+        status: "aborted",
+        terminalReason: `aborted:${abortReason}`,
+        abortReason,
+    };
+}
+/**
+ * The ended run and its turn's initiator, read in the write that ended it: a
+ * later read can find the initiator row already pruned.
+ */
+const ENDED_RUN_COLUMNS = `id, thread_id, turn_id, dispatch_mode, started_at, completed_at,
+  (SELECT principal_email FROM agent_turn_initiators i
+    WHERE i.thread_id = agent_runs.thread_id
+      AND i.turn_id = COALESCE(agent_runs.turn_id, agent_runs.id)) AS principal_email,
+  (SELECT auth_user_id FROM agent_turn_initiators i
+    WHERE i.thread_id = agent_runs.thread_id
+      AND i.turn_id = COALESCE(agent_runs.turn_id, agent_runs.id)) AS auth_user_id,
+  (SELECT is_anonymous FROM agent_turn_initiators i
+    WHERE i.thread_id = agent_runs.thread_id
+      AND i.turn_id = COALESCE(agent_runs.turn_id, agent_runs.id)) AS is_anonymous`;
+const ENDED_RUN_RETURNING = `RETURNING ${ENDED_RUN_COLUMNS}`;
+function finiteMs(value) {
+    if (value == null)
+        return undefined;
+    const ms = Number(value);
+    return Number.isFinite(ms) ? ms : undefined;
+}
+/**
+ * Sends `agent_run_terminal` for a run this process ended on another
+ * process's behalf. Call it only once the reap's UPDATE has affected the run:
+ * the conditional `status = 'running'` write is what makes each run's terminal
+ * event fire once across the reaper and the run's own process.
+ *
+ * Returns synchronously; the reap never waits on tracking or sees it fail.
+ */
+function trackReapedRun(runId, reap, returnedRow) {
+    trackRunEndedInSql(runId, reapedRunEnding(reap), returnedRow);
+}
+/** `recovered` and `not_background` stand alone; any other outcome declined. */
+function staleRunRecoveryLabel(recovery) {
+    if (!recovery)
+        return undefined;
+    return recovery.outcome === "recovered" ||
+        recovery.outcome === "not_background"
+        ? recovery.outcome
+        : `declined:${recovery.outcome}`;
+}
+function reapedRunEnding(reap) {
+    return {
+        status: "errored",
+        terminalReason: reap.terminalReason,
+        errorCode: reap.errorCode,
+        reap: { source: reap.source, recoveryOutcome: reap.recoveryOutcome },
+    };
+}
+/**
+ * Sends `agent_run_terminal` for a run a SQL write ended (a reap, stop,
+ * repair, or pre-start release),
+ * which the run's own process then cannot report: its own conditional
+ * terminal write finds no running row. Call it only for runs the ending
+ * UPDATE affected, so whichever write wins is the one that reports.
+ */
+function trackRunEndedInSql(runId, ending, returnedRow) {
+    trackRunsEndedInSql([{ runId, ending, row: returnedRow }]);
+}
+/** One background send for the runs one write ended. */
+function trackRunsEndedInSql(endedRuns) {
+    const runs = endedRuns.filter((run) => !run.ending.abortReason ||
+        !NOT_A_RUN_ABORT_REASONS.has(run.ending.abortReason));
+    if (runs.length === 0)
+        return;
+    const label = () => runs.map((run) => run.runId).join(", ");
+    try {
+        // The ending request is whoever happened to sweep or stop the run: its
+        // synthetic flag, email and client platform must not decide or describe
+        // another user's run, so the send runs without them.
+        const work = Promise.resolve(runWithRequestContext({ isSyntheticTraffic: false }, () => sendSqlEndedRunTerminals(runs))).catch((error) => {
+            console.error("[run-store] agent_run_terminal was not sent for a run ended in SQL:", label(), error instanceof Error ? error.message : error);
+        });
+        platformWaitUntil()?.(work);
+    }
+    catch (error) {
+        console.error("[run-store] agent_run_terminal was not sent for a run ended in SQL:", label(), error instanceof Error ? error.message : error);
+    }
+}
+async function sendSqlEndedRunTerminals(runs) {
+    await Promise.all(runs.map(async (run) => {
+        const row = run.row;
+        if (typeof row?.thread_id !== "string") {
+            console.error("[run-store] agent_run_terminal was not sent: ended run row is unreadable:", run.runId);
+            return;
+        }
+        const startedAt = finiteMs(row.started_at);
+        const completedAt = finiteMs(row.completed_at);
+        await trackRunTerminal({
+            ...run.ending,
+            runId: run.runId,
+            threadId: row.thread_id,
+            // Same turn the initiator row and the run's own process key on.
+            turnId: typeof row.turn_id === "string" && row.turn_id
+                ? row.turn_id
+                : run.runId,
+            dispatchMode: typeof row.dispatch_mode === "string"
+                ? row.dispatch_mode
+                : undefined,
+            durationMs: startedAt !== undefined && completedAt !== undefined
+                ? completedAt - startedAt
+                : undefined,
+        }, 
+        // The turn's initiator, not the request that ended the run; `track()`
+        // drops it when it is a test identity. An anonymous one is not a user.
+        row.is_anonymous === true || row.is_anonymous === "t"
+            ? undefined
+            : {
+                userId: typeof row.principal_email === "string"
+                    ? row.principal_email
+                    : undefined,
+                authUserId: typeof row.auth_user_id === "string"
+                    ? row.auth_user_id
+                    : undefined,
+            });
+    }));
+    const { flushTracking } = await import("../tracking/registry.js");
+    await flushTracking();
+}
 /** Projects a run after its stale status and terminal event committed. */
 async function finalizeStaleRun(runId, source, projectThread = true) {
     if (!projectThread)

package/dist/agent/run-terminal-tracking.d.ts

@@ -0,0 +1,29 @@
+import type { TrackingMeta } from "../tracking/registry.js";
+export interface RunTerminalTrackingFields {
+    runId: string;
+    threadId: string;
+    turnId: string;
+    status: "completed" | "errored" | "aborted" | "truncated";
+    terminalReason: string;
+    errorCode?: string;
+    dispatchMode?: string;
+    abortReason?: string;
+    durationMs?: number;
+    model?: string;
+    engineName?: string;
+    attemptCount?: number;
+    /** For `write_tool_outcome_unknown`: the call held back, and why. */
+    blockedTool?: string;
+    blockedBy?: "rejection" | "interruption";
+    /** Set only by a reap, which ends a run outside the process running it. */
+    reap?: {
+        source: string;
+        recoveryOutcome?: string;
+    };
+}
+/**
+ * The one `agent_run_terminal` sender, shared by the run's own process and the
+ * SQL writes that end a run for it, so the two shapes cannot drift apart. The returned promise never
+ * rejects, so a caller can hand it to `waitUntil` without guarding it.
+ */
+export declare function trackRunTerminal(fields: RunTerminalTrackingFields, source: TrackingMeta | undefined): Promise<void>;

package/dist/agent/run-terminal-tracking.js

@@ -0,0 +1,56 @@
+import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
+/**
+ * The one `agent_run_terminal` sender, shared by the run's own process and the
+ * SQL writes that end a run for it, so the two shapes cannot drift apart. The returned promise never
+ * rejects, so a caller can hand it to `waitUntil` without guarding it.
+ */
+export function trackRunTerminal(fields, source) {
+    try {
+        const properties = {
+            source: "agent_run_manager",
+            run_id: fields.runId,
+            thread_id: fields.threadId,
+            turn_id: fields.turnId,
+            status: fields.status,
+            terminal_reason: fields.terminalReason,
+            error_code: fields.errorCode,
+            ...(fields.errorCode
+                ? runErrorTelemetryProperties(fields.errorCode)
+                : {}),
+            dispatch_mode: fields.dispatchMode,
+            abort_reason: fields.abortReason,
+            duration_ms: fields.durationMs,
+            model: fields.model,
+            engine: fields.engineName,
+            attempt_count: fields.attemptCount,
+            blocked_tool: fields.blockedTool,
+            blocked_by: fields.blockedBy,
+            ...(fields.reap
+                ? {
+                    reaped: true,
+                    reap_source: fields.reap.source,
+                    recovery_outcome: fields.reap.recoveryOutcome,
+                }
+                : {}),
+        };
+        for (const key of Object.keys(properties)) {
+            if (properties[key] === undefined)
+                delete properties[key];
+        }
+        return Promise.all([
+            import("../tracking/registry.js"),
+            import("../observability/tracking-identity.js"),
+        ])
+            .then(([{ track }, { trackingIdentityProperties }]) => {
+            track("agent_run_terminal", { ...properties, ...trackingIdentityProperties() }, source);
+        })
+            .catch((error) => {
+            console.error("[run-terminal-tracking] agent_run_terminal was not sent:", fields.runId, error instanceof Error ? error.message : error);
+        });
+    }
+    catch (error) {
+        // Tracking must never affect the agent run or its persisted status.
+        console.error("[run-terminal-tracking] agent_run_terminal was not sent:", fields.runId, error instanceof Error ? error.message : error);
+        return Promise.resolve();
+    }
+}

package/dist/agent/run-turn-id-header.d.ts

@@ -0,0 +1,9 @@
+import { setResponseHeader } from "h3";
+/**
+ * The turn this request's run is stored under, for the client to report. The
+ * id can come from the request body, and a header value outside visible ASCII
+ * throws once the run has started, so such an id is left to the client's own.
+ * With no id (a replayed run held by another process) no header is set, and
+ * the client reports the replay's turn as unknown rather than guessing it.
+ */
+export declare function setRunTurnIdHeader(event: Parameters<typeof setResponseHeader>[0], turnId: string | undefined): void;

package/dist/agent/run-turn-id-header.js

@@ -0,0 +1,13 @@
+import { setResponseHeader } from "h3";
+/**
+ * The turn this request's run is stored under, for the client to report. The
+ * id can come from the request body, and a header value outside visible ASCII
+ * throws once the run has started, so such an id is left to the client's own.
+ * With no id (a replayed run held by another process) no header is set, and
+ * the client reports the replay's turn as unknown rather than guessing it.
+ */
+export function setRunTurnIdHeader(event, turnId) {
+    if (turnId && /^[\x20-\x7e]+$/.test(turnId)) {
+        setResponseHeader(event, "X-Turn-Id", turnId);
+    }
+}

package/dist/agent/types.d.ts

@@ -240,6 +240,11 @@ export interface AgentChatRichEventEnvelope {
     references?: AgentChatRichEventReference[];
     metadata?: Record<string, unknown>;
 }
+/**
+ * Why a failed tool call failed, where the loop could tell: its arguments
+ * failed validation before it ran, or it failed having written nothing.
+ */
+export type ToolErrorClass = "input_invalid" | "rejected";
 export type AgentChatEvent = {
     type: "text";
     text: string;
@@ -308,6 +313,8 @@ export type AgentChatEvent = {
     /** False marks current completed events; absence is reserved for older journals. */
     approvalPause?: boolean;
     outcomeUnknown?: true;
+    /** Why a failed call failed, where the loop could tell. */
+    errorClass?: ToolErrorClass;
     replayed?: true;
     fileMutation?: AgentFileMutationProof;
     artifacts?: ArtifactReceipt[];
@@ -392,6 +399,10 @@ export type AgentChatEvent = {
     details?: string;
     recoverable?: boolean;
     providerRetryable?: boolean;
+    /** For `write_tool_outcome_unknown`: the call held back. */
+    blockedTool?: string;
+    /** What left the earlier call of `blockedTool` unknown. */
+    blockedBy?: "rejection" | "interruption";
 }
 /**
  * Legacy SSE terminal event. New streams emit