@agent-native/core 0.210.1-nightly-20261020093350 → 0.210.1

package/dist/agent/run-store.js

@@ -14,7 +14,7 @@ import { approvalKeyForHistory } from "./approval-key.js";
 import { admitAutoContinue, admitManualContinue, } from "./auto-continue.js";
 import { AGENT_CHAT_RECOVERY_OF_RUN_FIELD } from "./durable-background.js";
 import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
-import { currentExecutionFence, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
+import { currentExecutionFence, currentExecutionFenceForScope, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
 import { AGENT_RUN_MIGRATIONS, AGENT_RUN_MIGRATIONS_TABLE, } from "./run-migrations.js";
 import { isConsistentToolCallInputFingerprint, isUnavailableToolCallInput, toolCallInputFingerprint, } from "./tool-call-journal.js";
 import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
@@ -2228,32 +2228,51 @@ export async function reapIfStale(runId, maxStaleMs) {
 export async function reapUnclaimedBackgroundRun(runId) {
     await ensureRunTables();
     const client = getDbExec();
+    if (!client.transaction) {
+        throw new Error("Atomic unclaimed-background reaping requires transaction support");
+    }
     const completedAt = Date.now();
     const cutoff = completedAt - UNCLAIMED_BACKGROUND_RUN_GRACE_MS;
-    const { rowsAffected } = await client.execute({
-        sql: `UPDATE agent_runs
-          SET status = 'errored',
-              completed_at = ?,
-              error_code = ?,
-              error_detail = ?,
-              terminal_reason = ?
-          WHERE id = ?
-            AND status = 'running'
-            AND dispatch_mode = 'background'
-            AND COALESCE(heartbeat_at, started_at) < ?`,
-        args: [
-            completedAt,
-            UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
-            UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.details,
-            UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
-            runId,
-            cutoff,
-        ],
+    const reaped = await client.transaction(async (tx) => {
+        const { rowsAffected } = await tx.execute({
+            sql: `UPDATE agent_runs
+            SET status = 'errored',
+                completed_at = ?,
+                error_code = ?,
+                error_detail = ?,
+                terminal_reason = ?
+            WHERE id = ?
+              AND status = 'running'
+              AND dispatch_mode = 'background'
+              AND COALESCE(heartbeat_at, started_at) < ?`,
+            args: [
+                completedAt,
+                UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
+                UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.details,
+                UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
+                runId,
+                cutoff,
+            ],
+        });
+        const updated = (rowsAffected ?? 0) > 0;
+        if (updated) {
+            await appendTerminalRunEventInTransaction(tx, runId, UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT);
+        }
+        return updated;
     });
-    const reaped = (rowsAffected ?? 0) > 0;
     if (reaped) {
-        await recordRunDiagnostic(runId, RUN_DIAG_STAGE.workerThrew, "unclaimed background dispatch reaped (worker never claimed the run)");
-        await safeAppendTerminalRunEvent(runId, UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT, "reap-unclaimed-background");
+        try {
+            await recordRunDiagnostic(runId, RUN_DIAG_STAGE.workerThrew, "unclaimed background dispatch reaped (worker never claimed the run)");
+        }
+        catch (error) {
+            captureError(error, {
+                tags: {
+                    component: "agent-run-store",
+                    operation: "record-unclaimed-background-reap-diagnostic",
+                },
+                extra: { runId },
+            });
+        }
     }
     return reaped;
 }
@@ -2478,7 +2497,7 @@ export function terminalEventForAbortReason(reason) {
     };
 }
 export async function markRunAborted(runId, reason) {
-    await updateRunAborted(runId, reason, false);
+    return updateRunAborted(runId, reason, false);
 }
 /** Abort only when no model or tool work has crossed its dispatch boundary. */
 export async function markRunAbortedAtSafeBoundary(runId, reason) {
@@ -2486,22 +2505,32 @@ export async function markRunAbortedAtSafeBoundary(runId, reason) {
 }
 async function updateRunAborted(runId, reason, requireSafeBoundary) {
     await ensureRunTables();
+    const fence = currentExecutionFenceForScope({ runId });
     const client = getDbExec();
     if (!client.transaction) {
         throw new Error("Atomic run cancellation requires transaction support");
     }
-    const rowsAffected = await client.transaction(async (tx) => {
+    const abortResult = await client.transaction(async (tx) => {
         await lockExecutionLeasesForRuns(tx, [runId]);
+        await lockAndAssertCurrentExecutionFences(tx);
+        const runResult = await tx.execute({
+            sql: `SELECT thread_id, COALESCE(turn_id, id) AS turn_id,
+                   worker_generation
+            FROM agent_runs WHERE id = ? FOR UPDATE`,
+            args: [runId],
+        });
+        const lockedRun = runResult.rows[0];
+        if (fence &&
+            (lockedRun?.thread_id !== fence.threadId ||
+                lockedRun.turn_id !== fence.turnId ||
+                lockedRun.worker_generation == null ||
+                String(lockedRun.worker_generation) !== fence.generation)) {
+            throw new StaleExecutionFenceError(fence);
+        }
         if (requireSafeBoundary) {
-            const run = await tx.execute({
-                sql: `SELECT thread_id, COALESCE(turn_id, id) AS turn_id
-              FROM agent_runs WHERE id = ?`,
-                args: [runId],
-            });
-            const runRef = run.rows[0];
-            if (runRef) {
-                if (typeof runRef.thread_id !== "string" ||
-                    typeof runRef.turn_id !== "string") {
+            if (lockedRun) {
+                if (typeof lockedRun.thread_id !== "string" ||
+                    typeof lockedRun.turn_id !== "string") {
                     throw new Error(`Run ${runId} has an invalid thread or turn ID`);
                 }
                 const dispatchedStep = await tx.execute({
@@ -2509,10 +2538,11 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
                 WHERE thread_id = ? AND turn_id = ? AND status = 'dispatched'
                   AND run_id = ?
                 LIMIT 1`,
-                    args: [runRef.thread_id, runRef.turn_id, runId],
+                    args: [lockedRun.thread_id, lockedRun.turn_id, runId],
                 });
-                if (dispatchedStep.rows.length > 0)
-                    return 0;
+                if (dispatchedStep.rows.length > 0) {
+                    return { rowsAffected: 0, appendTerminalEvent: false };
+                }
             }
         }
         const result = await tx.execute({
@@ -2524,15 +2554,32 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
                 runId,
             ],
         });
-        if ((result.rowsAffected ?? 0) > 0) {
+        const rowsAffected = result.rowsAffected ?? 0;
+        if (rowsAffected > 0) {
             await revokeExecutionLeasesForRuns(tx, [runId]);
+            return { rowsAffected, appendTerminalEvent: true };
         }
-        return result.rowsAffected ?? 0;
+        const terminalState = await tx.execute({
+            sql: `SELECT status, terminal_reason, in_flight_since FROM agent_runs WHERE id = ?`,
+            args: [runId],
+        });
+        const terminalRun = terminalState.rows[0];
+        return {
+            rowsAffected,
+            appendTerminalEvent: terminalRun?.status === "aborted" &&
+                terminalRun.terminal_reason === `aborted:${reason ?? "user"}` &&
+                (!requireSafeBoundary || terminalRun.in_flight_since == null),
+        };
     });
-    if ((rowsAffected ?? 0) > 0) {
-        await safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-aborted");
+    if (abortResult.appendTerminalEvent) {
+        try {
+            await safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-aborted");
+        }
+        catch (error) {
+            throw new AgentRunEventNotPersistedError(runId, null, true, error);
+        }
     }
-    return (rowsAffected ?? 0) > 0;
+    return abortResult.rowsAffected > 0 || abortResult.appendTerminalEvent;
 }
 async function lockExecutionLeasesForRuns(db, runIds) {
     const ids = [...new Set(runIds)].sort();
@@ -2596,6 +2643,10 @@ function turnAbortMarkerRunId(threadId, turnId) {
     return `turn-abort:${encodeURIComponent(threadId)}:${encodeURIComponent(turnId)}`;
 }
 export async function markTurnAborted(threadId, turnId, reason = "user") {
+    const fence = currentExecutionFence();
+    if (fence && (fence.threadId !== threadId || fence.turnId !== turnId)) {
+        throw new StaleExecutionFenceError(fence);
+    }
     await ensureRunTables();
     const now = Date.now();
     const client = getDbExec();
@@ -2608,7 +2659,15 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
             sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
             args: [`agent-native:run-slot:${threadId}`],
         });
-        await lockTurnExecutionLease(tx, threadId, turnId);
+        await lockAndAssertCurrentExecutionFences(tx);
+        const lease = await lockTurnExecutionLease(tx, threadId, turnId);
+        if (fence &&
+            (!lease ||
+                !lease.isLive ||
+                lease.ownerRunId !== fence.runId ||
+                lease.generation !== fence.generation)) {
+            throw new StaleExecutionFenceError(fence);
+        }
         const running = await tx.execute({
             sql: `SELECT id FROM agent_runs WHERE thread_id = ? AND COALESCE(turn_id, id) = ? AND status = 'running' AND dispatch_mode IS DISTINCT FROM 'turn-abort'`,
             args: [threadId, turnId],
@@ -2629,7 +2688,7 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
         }
         if (runIds.length === 0) {
             const existing = await tx.execute({
-                sql: `SELECT id, terminal_reason,
+                sql: `SELECT id, status, terminal_reason,
                      EXISTS (
                        SELECT 1 FROM agent_run_events
                        WHERE agent_run_events.run_id = agent_runs.id
@@ -2650,6 +2709,17 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
                 ],
             });
             const latest = existing.rows[0];
+            const aborted = await tx.execute({
+                sql: `SELECT id FROM agent_runs
+              WHERE thread_id = ? AND COALESCE(turn_id, id) = ?
+                AND status = 'aborted'
+                AND dispatch_mode IS DISTINCT FROM 'turn-abort'
+                AND terminal_reason = ?`,
+                args: [threadId, turnId, `aborted:${reason}`],
+            });
+            runIds.push(...aborted.rows
+                .map((row) => String(row.id ?? ""))
+                .filter(Boolean));
             if (latest &&
                 latest.continuation_pending !== true &&
                 !isContinuationTerminalReason(latest.terminal_reason ?? "")) {
@@ -2681,8 +2751,6 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
         });
         return "aborted";
     });
-    if (outcome === "already_terminal")
-        return outcome;
     await Promise.all(runIds.map((runId) => safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-turn-aborted")));
     return outcome;
 }
@@ -2756,40 +2824,100 @@ function persistedRunEventData(eventData, inputSource) {
     });
 }
 export class AgentRunEventNotPersistedError extends Error {
-    constructor(runId, seq) {
-        super(`Run event was not persisted: ${runId} sequence ${seq}`);
+    terminalRun;
+    constructor(runId, seq, terminalRun = false, cause) {
+        const detail = cause instanceof Error ? `: ${cause.message}` : "";
+        super(seq === null
+            ? `Terminal run event was not persisted: ${runId}${detail}`
+            : `Run event was not persisted: ${runId} sequence ${seq}${detail}`);
         this.name = "AgentRunEventNotPersistedError";
+        this.terminalRun = terminalRun;
+        if (cause !== undefined) {
+            Object.defineProperty(this, "cause", {
+                value: cause,
+                configurable: true,
+            });
+        }
     }
 }
 export async function insertRunEvent(runId, seq, eventData, options) {
     await ensureRunTables();
     const storedEventData = persistedRunEventData(eventData, options?.toolInputSource);
+    const eventAt = options?.eventAt ?? Date.now();
     // The reaper must wait for the marker's commit before exposing a successor.
-    const { rowsAffected } = await withFencedExecutionWrite((tx) => tx.execute({
-        sql: options?.requireInserted
-            ? `WITH running_run AS MATERIALIZED (
+    let result;
+    try {
+        result = await withFencedExecutionWrite(async (tx) => {
+            const { rowsAffected } = await tx.execute({
+                sql: options?.requireInserted
+                    ? `WITH running_run AS MATERIALIZED (
           SELECT id FROM agent_runs WHERE id = ? AND status = 'running' FOR UPDATE
         )
         INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
         SELECT id, ?, ?, ? FROM running_run LIMIT 1
         ON CONFLICT (run_id, seq) DO NOTHING`
-            : `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
+                    : `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
       SELECT ?, ?, ?, ?
       WHERE NOT EXISTS (
         SELECT 1 FROM agent_runs
         WHERE id = ? AND status <> 'running'
       )
       ON CONFLICT (run_id, seq) DO NOTHING`,
-        args: [
-            runId,
-            seq,
-            options?.eventAt ?? Date.now(),
-            storedEventData,
-            ...(options?.requireInserted ? [] : [runId]),
-        ],
-    }), { runId });
-    if (options?.requireInserted && rowsAffected !== 1)
-        throw new AgentRunEventNotPersistedError(runId, seq);
+                args: [
+                    runId,
+                    seq,
+                    eventAt,
+                    storedEventData,
+                    ...(options?.requireInserted ? [] : [runId]),
+                ],
+            });
+            if (!options?.requireInserted || rowsAffected === 1)
+                return {
+                    rowsAffected,
+                    terminalRun: false,
+                    persistedIdentically: rowsAffected === 1,
+                };
+            const existing = await tx.execute({
+                sql: `SELECT event_at, event_data FROM agent_run_events
+              WHERE run_id = ? AND seq = ?`,
+                args: [runId, seq],
+            });
+            const existingRow = existing.rows[0];
+            const persistedIdentically = existingRow !== undefined &&
+                Number(existingRow.event_at) === eventAt &&
+                String(existingRow.event_data) === storedEventData;
+            const run = await tx.execute({
+                sql: "SELECT status FROM agent_runs WHERE id = ?",
+                args: [runId],
+            });
+            const status = run.rows[0]?.status;
+            return {
+                rowsAffected,
+                terminalRun: status !== "running",
+                persistedIdentically,
+            };
+        }, { runId });
+    }
+    catch (error) {
+        if (error instanceof StaleExecutionFenceError) {
+            let status;
+            try {
+                status = await getRunStatus(runId);
+            }
+            catch (statusError) {
+                throw new AggregateError([error, statusError], `Could not confirm run status after a stale execution fence: ${runId}`);
+            }
+            if (status !== null && status !== "running") {
+                throw new AgentRunEventNotPersistedError(runId, seq, true, error);
+            }
+        }
+        throw error;
+    }
+    const { rowsAffected, terminalRun, persistedIdentically } = result;
+    if (options?.requireInserted &&
+        rowsAffected !== 1 &&
+        (!persistedIdentically || terminalRun))
+        throw new AgentRunEventNotPersistedError(runId, seq, terminalRun);
 }
 // Stays well under PostgreSQL's 65,535 bind-parameter limit at 4 per row.
 const RUN_EVENT_BATCH_ROWS = 500;
@@ -2801,9 +2929,13 @@ export async function insertRunEvents(runId, events) {
     if (events.length === 0)
         return;
     await ensureRunTables();
-    await withFencedExecutionWrite(async (tx) => {
+    const writeBatch = async (tx) => {
         for (let start = 0; start < events.length; start += RUN_EVENT_BATCH_ROWS) {
             const chunk = events.slice(start, start + RUN_EVENT_BATCH_ROWS);
+            const storedChunk = chunk.map((event) => ({
+                ...event,
+                eventData: persistedRunEventData(event.eventData),
+            }));
             await tx.execute({
                 sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
         SELECT v.run_id, v.seq, v.event_at, v.event_data
@@ -2816,17 +2948,41 @@ export async function insertRunEvents(runId, events) {
         )
         ON CONFLICT (run_id, seq) DO NOTHING`,
                 args: [
-                    ...chunk.flatMap(({ seq, eventAt, eventData }) => [
+                    ...storedChunk.flatMap(({ seq, eventAt, eventData }) => [
                         runId,
                         seq,
                         eventAt,
-                        persistedRunEventData(eventData),
+                        eventData,
                     ]),
                     runId,
                 ],
             });
+            const persisted = await tx.execute({
+                sql: `SELECT seq, event_at, event_data
+                FROM agent_run_events
+                WHERE run_id = ? AND seq IN (${chunk.map(() => "?").join(", ")})`,
+                args: [runId, ...chunk.map(({ seq }) => seq)],
+            });
+            const persistedBySeq = new Map(persisted.rows.map((row) => [Number(row.seq), row]));
+            for (const event of storedChunk) {
+                const row = persistedBySeq.get(event.seq);
+                if (!row ||
+                    Number(row.event_at) !== event.eventAt ||
+                    String(row.event_data) !== event.eventData) {
+                    const run = await tx.execute({
+                        sql: "SELECT status FROM agent_runs WHERE id = ?",
+                        args: [runId],
+                    });
+                    const status = run.rows[0]
+                        ?.status;
+                    throw new AgentRunEventNotPersistedError(runId, event.seq, status !== "running");
+                }
+            }
         }
-    }, { runId });
+    };
+    await withFencedExecutionWrite((tx) => currentExecutionFence()
+        ? writeBatch(tx)
+        : (tx.transaction?.(writeBatch) ?? writeBatch(tx)), { runId });
 }
 export const CHECKPOINT_TERMINAL_EVENT_SEQ = 1_000_000_000;
 export async function persistRunCheckpointEvent(runId, event, terminalReason) {
@@ -3670,6 +3826,7 @@ async function safeAppendTerminalRunEvent(runId, event, source) {
                 firstError: firstError instanceof Error ? firstError.message : String(firstError),
             },
         });
+        throw retryErr;
     }
 }
 async function appendTerminalRunEvent(runId, event) {
@@ -3696,8 +3853,8 @@ async function appendTerminalRunEvent(runId, event) {
         }
     }
     const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
-    await client.execute({
-        sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,
+    const inserted = await client.execute({
+        sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING RETURNING seq`,
         args: [
             runId,
             nextSeq,
@@ -3705,4 +3862,7 @@ async function appendTerminalRunEvent(runId, event) {
             stripInlineBytesFromJson(JSON.stringify(event), "placeholder"),
         ],
     });
+    if (inserted.rows.length === 0) {
+        throw new Error(`Run ${runId} terminal event conflicted with sequence ${nextSeq}`);
+    }
 }

package/dist/chat-threads/schema-migrations.js

@@ -101,4 +101,12 @@ export const CHAT_THREAD_SCHEMA_MIGRATIONS = [
         ON agent_chat_submission_inbox (state, retry_after, interrupt_active_run DESC, move_to_top_at DESC, queue_order ASC)
     `,
     },
+    {
+        version: 6,
+        name: "chat-thread-inbox-interrupt-replay",
+        sql: `
+      ALTER TABLE agent_chat_submission_inbox
+        ADD COLUMN IF NOT EXISTS interrupt_run_id TEXT
+    `,
+    },
 ];

package/dist/chat-threads/store.d.ts

@@ -259,6 +259,7 @@ export interface QueuedSubmissionReceipt {
     message?: QueuedMessage;
     payloadHash: string;
     submittedRunId?: string;
+    interruptRunId?: string;
     retryAfter?: number;
     createdAt: number;
     updatedAt: number;
@@ -294,6 +295,27 @@ export declare function retryQueuedSubmissionReceipt(threadId: string, messageId
 export declare function blockQueuedSubmissionReceipt(threadId: string, messageId: string, options?: {
     claimId?: string;
 }): Promise<boolean>;
+/** Persist the active run before interrupting it so a later inbox claim can finish the abort. */
+export declare function recordQueuedSubmissionInterruptRun(input: {
+    threadId: string;
+    messageId: string;
+    runId: string;
+    claimId: string;
+}): Promise<boolean>;
+/** Renews a live interruption claim before its recorded run is aborted. */
+export declare function renewQueuedSubmissionInterruptClaim(input: {
+    threadId: string;
+    messageId: string;
+    runId: string;
+    claimId: string;
+}): Promise<boolean>;
+/** Clear a recorded interruption only after its turn abort is durable. */
+export declare function clearQueuedSubmissionInterruptRun(input: {
+    threadId: string;
+    messageId: string;
+    runId: string;
+    claimId: string;
+}): Promise<boolean>;
 export type ThreadQueuedMessageMutation = {
     type: "append";
     message: QueuedMessage;
@@ -321,6 +343,16 @@ export type ThreadQueuedMessageMutation = {
     type: "blockReceipt";
     messageId: string;
     claimId?: string;
+} | {
+    type: "recordInterruptRun" | "clearInterruptRun";
+    messageId: string;
+    runId: string;
+    claimId: string;
+} | {
+    type: "renewClaim";
+    messageId: string;
+    runId: string;
+    claimId: string;
 } | {
     type: "release";
     messageId: string;
@@ -338,6 +370,7 @@ export interface ThreadQueuedMessageMutationResult {
     receiptUnavailable?: boolean;
     receiptConflict?: boolean;
     receiptUpdated?: boolean;
+    claimLost?: boolean;
     released?: boolean;
 }
 /** Applies a queue operation to the latest durable thread state on every CAS retry. */