@agent-native/core 0.219.0 → 0.220.0-nightly-20261023212848

package/dist/agent/run-manager.js

@@ -5,13 +5,14 @@ import { recordAgentRun } from "../observability/metrics.js";
 import { parseServiceIdentityEmail } from "../org/service-identity.js";
 import { ServicePrincipalRefusedError } from "../org/service-principal-guard.js";
 import { captureError } from "../server/capture-error.js";
+import { EXECUTION_LEASE_LOST_ABORT_REASON } from "./abort-reasons.js";
 import { isLlmCredentialError, LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
 import { classifyTerminalErrorCode, describeErrorWithCauses, isProviderConnectionError, } from "./engine/error-detail.js";
-import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
 import { getAgentEngineEntry } from "./engine/registry.js";
 import { EngineError } from "./engine/types.js";
 import { currentExecutionFence, MissingExecutionFenceError, StaleExecutionFenceError, withExecutionFence, withExecutionLeaseRequested, } from "./execution-fence.js";
-import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
+import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, setRunTerminalReasonIfStatus, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
+import { trackRunTerminal } from "./run-terminal-tracking.js";
 import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
 const activeRuns = new Map();
 const threadToRun = new Map();
@@ -435,39 +436,26 @@ function emitRunTerminalTrackingEvent(args) {
     catch {
         // Metrics must never affect the agent run or its persisted status.
     }
-    const properties = {
-        source: "agent_run_manager",
-        run_id: args.runId,
-        thread_id: args.threadId,
-        turn_id: args.turnId,
+    // An abort is reported by the SQL write that ended the row, which may be
+    // another process's stop; reporting it here too would double-count it.
+    if (args.status === "aborted")
+        return;
+    void trackRunTerminal({
+        runId: args.runId,
+        threadId: args.threadId,
+        turnId: args.turnId,
         status: args.status,
-        terminal_reason: args.terminalReason,
-        error_code: args.errorCode,
-        ...(args.errorCode ? runErrorTelemetryProperties(args.errorCode) : {}),
-        dispatch_mode: args.dispatchMode,
-        abort_reason: args.abortReason,
-        duration_ms: args.durationMs,
+        terminalReason: args.terminalReason,
+        errorCode: args.errorCode,
+        dispatchMode: args.dispatchMode,
+        abortReason: args.abortReason,
+        durationMs: args.durationMs,
         model: args.model,
-        engine: args.engineName,
-        attempt_count: args.attemptCount,
-    };
-    for (const key of Object.keys(properties)) {
-        if (properties[key] === undefined)
-            delete properties[key];
-    }
-    try {
-        void Promise.all([
-            import("../tracking/registry.js"),
-            import("../observability/tracking-identity.js"),
-        ])
-            .then(([{ track }, { trackingIdentityProperties }]) => {
-            track("agent_run_terminal", { ...properties, ...trackingIdentityProperties() }, { userId: args.userId });
-        })
-            .catch(() => { });
-    }
-    catch {
-        // Tracking must never affect the agent run or its persisted status.
-    }
+        engineName: args.engineName,
+        attemptCount: args.attemptCount,
+        blockedTool: args.blockedTool,
+        blockedBy: args.blockedBy,
+    }, { userId: args.userId });
 }
 function abortInMemoryRun(run, reason = "user") {
     run.abortReason = reason;
@@ -1208,6 +1196,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
     let completionInProgress = false;
     let ownTerminalStatusEstablished = false;
     let terminalStatusWrite = null;
+    let terminalStatusWriteOutcomeUnknown = false;
     const checkSqlAbort = () => {
         const now = Date.now();
         if (now - lastAbortCheck < 3000)
@@ -1223,8 +1212,18 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                 const status = await getRunStatus(runId);
                 if (status !== null && status !== "running") {
                     const pendingStatusWrite = terminalStatusWrite;
-                    if (pendingStatusWrite)
-                        await pendingStatusWrite;
+                    if (pendingStatusWrite) {
+                        try {
+                            await pendingStatusWrite;
+                        }
+                        catch (error) {
+                            if (!terminalStatusWriteOutcomeUnknown)
+                                throw error;
+                            return;
+                        }
+                    }
+                    if (terminalStatusWriteOutcomeUnknown)
+                        return;
                 }
                 if (status !== null &&
                     status !== "running" &&
@@ -1272,7 +1271,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                 if (options?.executionLease &&
                     error?.code === "stale_execution_fence" &&
                     !completionInProgress) {
-                    abortInMemoryRun(run, "execution-lease-lost");
+                    abortInMemoryRun(run, EXECUTION_LEASE_LOST_ABORT_REASON);
                 }
                 consecutiveHeartbeatFailures += 1;
                 if (consecutiveHeartbeatFailures >= 3) {
@@ -1607,6 +1606,8 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         let terminalPersistenceError = null;
         let eventPersistenceError = null;
         let runTerminalErrorCode;
+        let runTerminalBlockedTool;
+        let runTerminalBlockedBy;
         let terminalPersistenceEstablished = false;
         let continuationStatusPersistedBeforeEvent = false;
         let suppressContinuationEvent = false;
@@ -1758,12 +1759,21 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             ? updateRunStatusIfRunning(runId, persistedStatus)
             : updateRunStatusIfRunning(runId, persistedStatus, terminalStatusOptions);
         const persistOwnTerminalState = async (persist) => {
-            const write = withWorkerFence(persist);
+            let writeStarted = false;
+            const write = withWorkerFence(() => {
+                writeStarted = true;
+                return persist();
+            }).catch((error) => {
+                if (writeStarted)
+                    terminalStatusWriteOutcomeUnknown = true;
+                throw error;
+            });
             terminalStatusWrite = write;
             try {
                 const established = await write;
-                if (established)
+                if (established && !terminalStatusWriteOutcomeUnknown) {
                     ownTerminalStatusEstablished = true;
+                }
                 return established;
             }
             finally {
@@ -1772,11 +1782,45 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             }
         };
         const persistOwnTerminalStatus = () => persistOwnTerminalState(persistTerminalStatus);
-        const reconcileTerminalRun = () => persistOwnTerminalState(() => threadProjectionPending
-            ? reconcileTerminalRunFromEvents(runId, {
-                threadProjectionPending: true,
-            })
-            : reconcileTerminalRunFromEvents(runId));
+        const confirmAmbiguousTerminalStatus = async () => {
+            const persistedRun = await getRunById(runId);
+            if (persistedRun?.status !== persistedStatus)
+                return false;
+            const expectedTerminalReason = terminalReason.slice(0, 200);
+            if (persistedRun.terminalReason === expectedTerminalReason) {
+                return true;
+            }
+            if (persistedRun.terminalReason)
+                return false;
+            return withWorkerFence(() => setRunTerminalReasonIfStatus(runId, persistedStatus, expectedTerminalReason));
+        };
+        const reconcileTerminalRun = async () => {
+            let reconciled = false;
+            try {
+                reconciled = await persistOwnTerminalState(() => threadProjectionPending
+                    ? reconcileTerminalRunFromEvents(runId, {
+                        threadProjectionPending: true,
+                    })
+                    : reconcileTerminalRunFromEvents(runId));
+            }
+            catch (error) {
+                if (!terminalStatusWriteOutcomeUnknown)
+                    throw error;
+            }
+            if (reconciled) {
+                terminalStatusWriteOutcomeUnknown = false;
+                ownTerminalStatusEstablished = true;
+                return true;
+            }
+            if (!terminalStatusWriteOutcomeUnknown) {
+                return reconciled;
+            }
+            const confirmed = await confirmAmbiguousTerminalStatus();
+            terminalStatusWriteOutcomeUnknown = false;
+            if (confirmed)
+                ownTerminalStatusEstablished = true;
+            return confirmed;
+        };
         const reportTerminalStatusPersistenceError = (error) => {
             captureError(error, {
                 route: "/_agent-native/agent-chat",
@@ -1888,7 +1932,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             }
             else if (!executionLeaseClaimFailed &&
                 (run.abortReason === "superseded" ||
-                    run.abortReason === "execution_lease_lost")) {
+                    run.abortReason === EXECUTION_LEASE_LOST_ABORT_REASON)) {
                 terminalPersistenceEstablished = false;
             }
             else {
@@ -1955,6 +1999,8 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                 else if (ev.type === "error") {
                     errorCode = ev.errorCode;
                     errorDetail = ev.error ?? ev.details;
+                    runTerminalBlockedTool = ev.blockedTool;
+                    runTerminalBlockedBy = ev.blockedBy;
                     break;
                 }
             }
@@ -2002,6 +2048,8 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                 status: persistedStatus,
                 terminalReason,
                 errorCode: runTerminalErrorCode,
+                blockedTool: runTerminalBlockedTool,
+                blockedBy: runTerminalBlockedBy,
                 dispatchMode: options?.dispatchMode,
                 abortReason: run.abortReason,
                 durationMs: Date.now() - run.startedAt,

package/dist/agent/run-store.d.ts

@@ -276,6 +276,7 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
 }>;
 export declare function setRunError(runId: string, errorCode: string | undefined, errorDetail: string | undefined): Promise<void>;
 export declare function setRunTerminalReason(runId: string, terminalReason: string | undefined): Promise<void>;
+export declare function setRunTerminalReasonIfStatus(runId: string, expectedStatus: "completed" | "truncated" | "errored" | "aborted", terminalReason: string): Promise<boolean>;
 /**
  * Repair a run whose terminal event was durably appended but whose final
  * `agent_runs.status` write lost a race with reconnect/reaper code.