@agent-native/core 0.210.1-nightly-20261020124228 → 0.210.2

package/dist/agent/run-manager.js

@@ -11,7 +11,7 @@ import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
 import { getAgentEngineEntry } from "./engine/registry.js";
 import { EngineError } from "./engine/types.js";
 import { currentExecutionFence, MissingExecutionFenceError, StaleExecutionFenceError, withExecutionFence, withExecutionLeaseRequested, } from "./execution-fence.js";
-import { insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
+import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
 import { isContinuationTerminalReason } from "./types.js";
 const activeRuns = new Map();
 const threadToRun = new Map();
@@ -37,11 +37,17 @@ function reportAbortFlushFailure(run, error) {
 }
 async function flushRunEventsBeforeAbort(run) {
     try {
+        await run?.waitForPersistence?.();
         await run?.flushEvents?.();
+        return null;
     }
     catch (error) {
         if (run)
             reportAbortFlushFailure(run, error);
+        if (error instanceof AgentRunEventNotPersistedError && error.terminalRun) {
+            return error;
+        }
+        throw error;
     }
 }
 const CLEANUP_DELAY_MS = 5 * 60 * 1000;
@@ -543,7 +549,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         });
     }
     const runControl = {
-        flushEvents: () => persistenceChain,
+        flushEvents: () => flushEventWrites(true),
         get turnSignal() {
             return abort.signal;
         },
@@ -678,7 +684,85 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         });
     };
     let persistenceChain = Promise.resolve();
-    run.flushEvents = () => persistenceChain;
+    const pendingEventWrites = [];
+    let persistenceDrain = null;
+    let persistenceFailure;
+    let persistenceFailureBlocked = false;
+    const flushEventWrites = (retryAfterFailure) => {
+        if (persistenceDrain) {
+            const activeDrain = persistenceDrain;
+            const settledDrain = retryAfterFailure
+                ? activeDrain.catch((error) => {
+                    if (error instanceof AgentRunEventNotPersistedError &&
+                        error.terminalRun) {
+                        throw error;
+                    }
+                })
+                : activeDrain;
+            const flush = settledDrain.then(() => pendingEventWrites.length > 0
+                ? flushEventWrites(retryAfterFailure)
+                : undefined);
+            persistenceChain = flush;
+            return flush;
+        }
+        if (pendingEventWrites.length === 0) {
+            persistenceChain = Promise.resolve();
+            return persistenceChain;
+        }
+        if (persistenceFailureBlocked && !retryAfterFailure) {
+            const blocked = Promise.reject(persistenceFailure);
+            persistenceChain = blocked;
+            return blocked;
+        }
+        if (retryAfterFailure) {
+            persistenceFailure = undefined;
+            persistenceFailureBlocked = false;
+        }
+        const drain = Promise.resolve().then(async () => {
+            while (pendingEventWrites.length > 0) {
+                const pending = pendingEventWrites[0];
+                try {
+                    await pending.write();
+                }
+                catch (error) {
+                    const terminalRun = error instanceof AgentRunEventNotPersistedError &&
+                        error.terminalRun;
+                    persistenceFailure = terminalRun ? undefined : error;
+                    persistenceFailureBlocked = !terminalRun;
+                    for (const queued of pendingEventWrites) {
+                        for (const publication of queued.publications) {
+                            if (publication.settled)
+                                continue;
+                            publication.settled = true;
+                            publication.reject(error);
+                        }
+                    }
+                    if (terminalRun) {
+                        pendingEventWrites.length = 0;
+                        openEventBatch = null;
+                    }
+                    throw error;
+                }
+                pendingEventWrites.shift();
+                for (const publication of pending.publications) {
+                    publishRunEvent(publication.runEvent);
+                    if (publication.settled)
+                        continue;
+                    publication.settled = true;
+                    publication.resolve();
+                }
+            }
+        });
+        const activeDrain = drain.finally(() => {
+            if (persistenceDrain === activeDrain)
+                persistenceDrain = null;
+        });
+        persistenceDrain = activeDrain;
+        persistenceChain = activeDrain;
+        return activeDrain;
+    };
+    run.waitForPersistence = () => insertRunPromise;
+    run.flushEvents = () => flushEventWrites(true);
     // Streamed deltas arrive hundreds per tool call, and a write tool waits for
     // every queued event to persist first; one round trip per event put tens of
     // seconds between the model finishing and the tool starting.
@@ -1189,6 +1273,28 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             bumpProgressIfDue();
         }
     };
+    const queueRunEvent = (pendingWrite, runEvent) => {
+        let resolve;
+        let reject;
+        const publication = new Promise((res, rej) => {
+            resolve = res;
+            reject = rej;
+        });
+        const pendingPublication = {
+            runEvent,
+            resolve,
+            reject,
+            settled: false,
+        };
+        pendingWrite.publications.push(pendingPublication);
+        void flushEventWrites(false).catch((error) => {
+            if (pendingPublication.settled)
+                return;
+            pendingPublication.settled = true;
+            pendingPublication.reject(error);
+        });
+        return publication;
+    };
     const emitRunEvent = (runEvent, options) => {
         const eventData = JSON.stringify(runEvent.event);
         const eventAt = Date.now();
@@ -1204,40 +1310,47 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                         eventType: runEvent.event.type,
                     });
                 }
+                if (error instanceof AgentRunEventNotPersistedError &&
+                    error.terminalRun) {
+                    throw error;
+                }
                 await write();
             }
         };
-        let persistence;
+        let publication;
         if (isBatchableRunEvent(runEvent.event) && !persistEventHook) {
             if (!openEventBatch) {
                 const batch = {
                     events: [],
-                    write: persistenceChain.then(() => {
-                        if (openEventBatch === batch)
-                            openEventBatch = null;
-                        return retryOnce(() => withWorkerFence(() => insertRunEvents(runId, batch.events)));
-                    }),
+                    pendingWrite: {
+                        publications: [],
+                        write: () => {
+                            if (openEventBatch === batch)
+                                openEventBatch = null;
+                            return retryOnce(() => withWorkerFence(() => insertRunEvents(runId, batch.events)));
+                        },
+                    },
                 };
                 openEventBatch = batch;
-                persistenceChain = batch.write;
+                pendingEventWrites.push(batch.pendingWrite);
             }
             openEventBatch.events.push({
                 seq: runEvent.seq,
                 eventAt,
                 eventData,
             });
-            persistence = openEventBatch.write;
+            publication = queueRunEvent(openEventBatch.pendingWrite, runEvent);
         }
         else {
             // Anything emitted after this event must not persist before it.
             openEventBatch = null;
-            persistence = persistenceChain.then(() => retryOnce(() => persistRunEvent(runEvent, eventData, eventAt)));
-            persistenceChain = persistence;
-        }
-        if (!options?.surfacePersistenceError) {
-            persistence.catch(() => { });
+            const pendingWrite = {
+                publications: [],
+                write: () => retryOnce(() => persistRunEvent(runEvent, eventData, eventAt)),
+            };
+            pendingEventWrites.push(pendingWrite);
+            publication = queueRunEvent(pendingWrite, runEvent);
         }
-        const publication = persistence.then(() => publishRunEvent(runEvent));
         if (!options?.surfacePersistenceError)
             publication.catch(() => { });
         checkSqlAbort();
@@ -2207,11 +2320,33 @@ export async function getActiveRunForThreadAsync(threadId) {
             const isUnclaimedBackgroundDispatch = sqlRun.dispatchMode === "background";
             const stillInsideRedispatchBound = shouldRedispatchUnclaimedBackgroundRun({ startedAt: sqlRun.startedAt });
             if (isUnclaimedBackgroundDispatch && !stillInsideRedispatchBound) {
-                const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch(() => false);
+                const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch((error) => {
+                    captureError(error, {
+                        route: "/_agent-native/agent-chat/runs/active",
+                        aiTraceId: sqlRun.id,
+                        tags: {
+                            source: "agent-run-manager",
+                            phase: "unclaimed-background-reap",
+                        },
+                        extra: { runId: sqlRun.id, threadId },
+                    });
+                    return false;
+                });
                 if (recovered)
                     return null;
             }
-            const reaped = await reapIfStale(sqlRun.id).catch(() => false);
+            const reaped = await reapIfStale(sqlRun.id).catch((error) => {
+                captureError(error, {
+                    route: "/_agent-native/agent-chat/runs/active",
+                    aiTraceId: sqlRun.id,
+                    tags: {
+                        source: "agent-run-manager",
+                        phase: "stale-run-reap",
+                    },
+                    extra: { runId: sqlRun.id, threadId },
+                });
+                return false;
+            });
             if (reaped)
                 return null;
             return {
@@ -2299,16 +2434,17 @@ function abortRunInMemory(runId, reason) {
 export function abortRun(runId, reason = "user") {
     const run = activeRuns.get(runId);
     const abortedInMemory = abortRunInMemory(runId, reason);
+    const markAborted = () => markRunAborted(runId, reason);
     void flushRunEventsBeforeAbort(run)
-        .then(() => markRunAborted(runId, reason))
+        .then(markAborted, markAborted)
         .catch((error) => {
         captureError(error, {
             route: "/_agent-native/agent-chat/runs/:id/abort",
             aiTraceId: runId,
-            tags: { source: "agent-run-manager", phase: "abort-run" },
+            tags: { source: "agent-run-manager", phase: "abort-run-status" },
             extra: { runId, threadId: run?.threadId, reason },
         });
-        console.error("[run-manager] durable abort persistence failed:", error instanceof Error ? error.message : error);
+        console.error("[run-manager] durable abort status write failed:", error instanceof Error ? error.message : error);
     });
     return abortedInMemory;
 }
@@ -2318,12 +2454,27 @@ export function abortRun(runId, reason = "user") {
  * must use this path; otherwise the recovery POST can race the old row while
  * it is still marked running.
  */
-export async function abortRunDurably(runId, reason = "user") {
+export async function abortRunDurably(runId, reason = "user", options) {
     const run = activeRuns.get(runId);
     const abortedInMemory = abortRunInMemory(runId, reason);
     try {
-        await flushRunEventsBeforeAbort(run);
-        await markRunAborted(runId, reason);
+        const eventFlushFailure = await flushRunEventsBeforeAbort(run);
+        const abortPersisted = await markRunAborted(runId, reason);
+        if (!abortPersisted || eventFlushFailure) {
+            const persistedStatus = await getRunStatus(runId);
+            if (persistedStatus === null) {
+                // Agent Teams persists Stop intent before run registration.
+                // Workers recheck it before dispatch.
+                if (options?.allowNotStarted && !run && !eventFlushFailure)
+                    return false;
+                throw new Error(`Cannot confirm durable abort for missing run ${runId}`);
+            }
+            if (!abortPersisted && persistedStatus === "running") {
+                throw new Error(`Durable abort did not transition run ${runId}`);
+            }
+        }
+        if (eventFlushFailure)
+            throw eventFlushFailure;
     }
     catch (error) {
         captureError(error, {
@@ -2341,19 +2492,57 @@ export async function abortRunDurably(runId, reason = "user") {
             },
         });
         console.error("[run-manager] durable abort persistence failed:", error instanceof Error ? error.message : error);
+        throw error;
     }
     return abortedInMemory;
 }
-/** Abort after the durable run store confirms no model or tool work is in flight. */
-export async function abortRunAtSafeBoundaryDurably(runId, reason = "user") {
+async function abortRunAtSafeBoundaryDurablyResult(runId, reason = "user") {
     const run = activeRuns.get(runId);
-    await flushRunEventsBeforeAbort(run);
-    const aborted = await markRunAbortedAtSafeBoundary(runId, reason);
-    if (!aborted)
-        return false;
+    const eventFlushFailure = await flushRunEventsBeforeAbort(run);
+    let aborted;
+    try {
+        aborted = await markRunAbortedAtSafeBoundary(runId, reason);
+    }
+    catch (error) {
+        try {
+            const state = await getRunAbortState(runId);
+            if (state.aborted && run) {
+                abortRunInMemory(runId, state.reason ?? reason);
+            }
+        }
+        catch (stateError) {
+            captureError(stateError, {
+                route: "/_agent-native/agent-chat/runs/:id/abort",
+                aiTraceId: runId,
+                tags: {
+                    source: "agent-run-manager",
+                    phase: "safe-boundary-abort-state-read",
+                },
+                extra: {
+                    runId,
+                    reason,
+                    abortError: error instanceof Error ? error.message : String(error),
+                },
+            });
+        }
+        if (eventFlushFailure) {
+            throw new AggregateError([eventFlushFailure, error], "Run abort event flush and marker persistence failed");
+        }
+        throw error;
+    }
+    if (!aborted) {
+        return { aborted: false, eventFlushFailure };
+    }
     if (run)
         abortInMemoryRun(run, reason);
-    return true;
+    return { aborted: true, eventFlushFailure };
+}
+/** Abort after the durable run store confirms no model or tool work is in flight. */
+export async function abortRunAtSafeBoundaryDurably(runId, reason = "user") {
+    const result = await abortRunAtSafeBoundaryDurablyResult(runId, reason);
+    if (result.eventFlushFailure)
+        throw result.eventFlushFailure;
+    return result.aborted;
 }
 export async function abortTurnByRefDurably(threadId, turnId, reason = "user") {
     const fence = currentExecutionFence();
@@ -2369,8 +2558,22 @@ export async function abortTurnByRefDurably(threadId, turnId, reason = "user") {
             runsToFlush.push(run);
         }
     }
-    await Promise.all(runsToFlush.map((run) => flushRunEventsBeforeAbort(run)));
-    return markTurnAborted(threadId, turnId, reason);
+    const flushFailures = await Promise.all(runsToFlush.map((run) => flushRunEventsBeforeAbort(run)));
+    let result;
+    try {
+        result = await markTurnAborted(threadId, turnId, reason);
+    }
+    catch (error) {
+        const eventFlushFailure = flushFailures.find(Boolean);
+        if (eventFlushFailure) {
+            throw new AggregateError([eventFlushFailure, error], "Turn abort event flush and marker persistence failed");
+        }
+        throw error;
+    }
+    const eventFlushFailure = flushFailures.find(Boolean);
+    if (eventFlushFailure)
+        throw eventFlushFailure;
+    return result;
 }
 export async function abortTurnDurably(runId, reason = "user") {
     const memRun = activeRuns.get(runId);
@@ -2393,4 +2596,40 @@ export async function abortTurnDurably(runId, reason = "user") {
         throw error;
     }
 }
+/** Persist a Stop's turn marker even when a terminal event flush must be reported. */
+export async function abortRunAndTurnDurably(runId, reason = "user", options) {
+    let abortedInMemory = false;
+    let terminalFlushFailure;
+    try {
+        if (options?.safeBoundary) {
+            const result = await abortRunAtSafeBoundaryDurablyResult(runId, reason);
+            abortedInMemory = result.aborted;
+            terminalFlushFailure = result.eventFlushFailure ?? undefined;
+        }
+        else {
+            abortedInMemory = await abortRunDurably(runId, reason);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof AgentRunEventNotPersistedError) ||
+            !error.terminalRun) {
+            throw error;
+        }
+        terminalFlushFailure = error;
+    }
+    if (!options?.safeBoundary || abortedInMemory) {
+        try {
+            await abortTurnDurably(runId, reason);
+        }
+        catch (error) {
+            if (terminalFlushFailure) {
+                throw new AggregateError([terminalFlushFailure, error], "Run event flush and turn abort persistence failed");
+            }
+            throw error;
+        }
+    }
+    if (terminalFlushFailure)
+        throw terminalFlushFailure;
+    return abortedInMemory;
+}
 export { getSlotHoldingRunId, tryClaimRunSlot } from "./run-store.js";

package/dist/agent/run-store.d.ts

@@ -353,7 +353,7 @@ export declare function failRunBeforeDispatch(runId: string, errorCode: string,
 export declare function releaseBackgroundRunBeforeStart(runId: string, errorCode: string, errorDetail: string): Promise<void>;
 export declare function getRunStatus(runId: string): Promise<string | null>;
 export declare function terminalEventForAbortReason(reason: string | undefined): AgentChatEvent;
-export declare function markRunAborted(runId: string, reason?: string): Promise<void>;
+export declare function markRunAborted(runId: string, reason?: string): Promise<boolean>;
 /** Abort only when no model or tool work has crossed its dispatch boundary. */
 export declare function markRunAbortedAtSafeBoundary(runId: string, reason?: string): Promise<boolean>;
 /** Serialize a task lifecycle change with worker lease claims and writes. */
@@ -372,7 +372,8 @@ export declare function getRunAbortState(runId: string): Promise<{
     reason?: string;
 }>;
 export declare class AgentRunEventNotPersistedError extends Error {
-    constructor(runId: string, seq: number);
+    readonly terminalRun: boolean;
+    constructor(runId: string, seq: number | null, terminalRun?: boolean, cause?: unknown);
 }
 export declare function insertRunEvent(runId: string, seq: number, eventData: string, options?: {
     toolInputSource?: "execution";

package/dist/agent/run-store.fencing.worker.js

@@ -4,6 +4,7 @@ import { ensureChatThreadTables, getThread, markQueuedSubmissionSubmitted, updat
 import { toPostgresParams, withDbExec } from "../db/client.js";
 import { withMigrationRuntime } from "../db/migration-runtime.js";
 import { withExecutionFence } from "./execution-fence.js";
+import { abortRunAtSafeBoundaryDurably } from "./run-manager.js";
 import { beginAgentExecutionStep, claimTurnExecutionLease, completeAgentExecutionStep, ensureRunTables, insertRun, insertRunEvent, markAgentExecutionStepDispatched, } from "./run-store.js";
 import { applySubmittedUserMessage, buildUserMessage, extractThreadMeta, } from "./thread-data-builder.js";
 function executor(sql) {
@@ -38,6 +39,7 @@ function waitForContinue() {
         process.stdin.resume();
     });
 }
+let databaseBackendPid;
 async function main() {
     const databaseUrl = process.env.AGENT_RUN_POSTGRES_TEST_URL; // config-ok: isolated integration-test database selector, not app runtime config
     if (!databaseUrl)
@@ -112,6 +114,17 @@ async function main() {
                 }));
                 return;
             }
+            if (mode === "durable-abort") {
+                const [{ pid }] = await sql `SELECT pg_backend_pid() AS pid`;
+                databaseBackendPid = Number(pid);
+                const aborted = await abortRunAtSafeBoundaryDurably(runId, stepId ?? "user");
+                process.stdout.write(JSON.stringify({
+                    aborted,
+                    processId: process.pid,
+                    databaseBackendPid,
+                }));
+                return;
+            }
             await insertRun(runId, threadId, turnId);
             const fence = await claimTurnExecutionLease(threadId, turnId, runId);
             if (mode === "claim") {
@@ -170,6 +183,11 @@ main().catch((error) => {
     const code = error && typeof error === "object" && "code" in error
         ? String(error.code)
         : undefined;
-    process.stdout.write(JSON.stringify({ error: String(error), code }));
+    process.stdout.write(JSON.stringify({
+        error: String(error),
+        code,
+        processId: process.pid,
+        databaseBackendPid,
+    }));
     process.exitCode = code === "turn_execution_lease_busy" ? 3 : 1;
 });