@agent-native/core 0.219.0 → 0.220.0-nightly-20261023183804

package/dist/agent/execution-fence.js

@@ -1,5 +1,5 @@
 import { AsyncLocalStorage } from "node:async_hooks";
-import { getDbExec } from "../db/client.js";
+import { getDbExec, getScopedTransactionDbExec, } from "../db/client.js";
 export class ExecutionFenceError extends Error {
     code;
     constructor(code, message) {
@@ -106,6 +106,10 @@ export async function withFencedExecutionWrite(write, scope) {
     if (!scopedFence) {
         throw new StaleExecutionFenceError(fence);
     }
+    if (getScopedTransactionDbExec() === client) {
+        await lockAndAssertCurrentExecutionFences(client);
+        return write(client);
+    }
     if (!client.transaction) {
         throw new Error("Fenced execution writes require transaction support");
     }

package/dist/agent/production-agent.d.ts

@@ -384,7 +384,7 @@ export interface ProductionAgentOptions {
     runSoftTimeoutMs?: number;
     runNoProgressTimeoutMs?: number;
     durableBackgroundRuns?: boolean;
-    onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string) => void | Promise<void>;
+    onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string, turnId: string) => void | Promise<void>;
     /**
      * Called when the run's engine, model, and effort resolve, and again before
      * each model call, because a call can run on a rate-limit fallback model or

package/dist/agent/production-agent.js

@@ -90,6 +90,7 @@ import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js
 import { applyUncheckedDefaultModelReplacement } from "./provider-model-selection.js";
 import { startRun, subscribeToRun, replayCompletedTurn, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, getSlotHoldingRunId, tryClaimRunSlot, isHostedRuntime, resolveRunSoftTimeoutMs, resolveRunToolTimeoutCeilingMs, endsAfterToolResultWithoutAssistantFinal, endsDuringActionPreparation, } from "./run-manager.js";
 import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
+import { setRunTurnIdHeader } from "./run-turn-id-header.js";
 import { buildCurrentTimeUserContext } from "./runtime-context.js";
 import { claimSetupResume, setupResumeRefusedRunId, } from "./setup-resume-claim.js";
 import { getStrictToolInputValidator } from "./strict-tool-input.js";
@@ -4448,6 +4449,20 @@ async function prepareToolCall(entry, input, signal, timeoutMs) {
 function intentIsReadOnly(intent) {
     return intent?.valid === true && intent.readOnly === true;
 }
+function isBoundedTaskWait(toolName, input) {
+    if (toolName !== "agent-teams" || !input || typeof input !== "object") {
+        return false;
+    }
+    const args = input;
+    // Only these operations honor the task reader's bounded wait contract.
+    return ((args.action === "status" || args.action === "read-result") &&
+        typeof args.taskId === "string" &&
+        args.taskId.trim().length > 0 &&
+        typeof args.waitMs === "number" &&
+        Number.isInteger(args.waitMs) &&
+        args.waitMs > 0 &&
+        args.waitMs <= 20_000);
+}
 function normalizeOptionalToolPlaceholders(schema, input) {
     if (!schema?.properties || !input || typeof input !== "object") {
         return { input, changed: false };
@@ -5977,7 +5992,12 @@ export async function runAgentLoop(opts) {
             const actionRetrySafe = actionEntry && intent?.valid
                 ? actionCallIsRetrySafe(actionEntry, preparedCallArguments(intent))
                 : false;
-            const repeatGuardStopFromThisCall = noteRepeatedToolCall(toolCall.name, toolCall.input, toolCall.id);
+            const repeatGuardStopFromThisCall = actionIsReadOnly &&
+                actionEntry?.dedupe === false &&
+                intent?.valid === true &&
+                isBoundedTaskWait(toolCall.name, preparedCallArguments(intent))
+                ? null
+                : noteRepeatedToolCall(toolCall.name, toolCall.input, toolCall.id);
             const rollbackRepeatedToolCall = () => {
                 const repeatKey = toolCallCacheKey(toolCall.name, toolCall.input);
                 const repeatCall = inFlightRepeatToolCalls.get(toolCall.id);
@@ -9398,6 +9418,7 @@ export function createProductionAgentHandler(options) {
                 setResponseHeader(event, "Cache-Control", "no-cache");
                 setResponseHeader(event, "Connection", "keep-alive");
                 setResponseHeader(event, "X-Run-Id", result.submittedRunId);
+                setRunTurnIdHeader(event, getRun(result.submittedRunId)?.turnId);
                 setResponseHeader(event, "X-Dispatch-Mode", "replay");
                 return stream;
             }
@@ -9486,6 +9507,7 @@ export function createProductionAgentHandler(options) {
                     setResponseHeader(event, "Cache-Control", "no-cache");
                     setResponseHeader(event, "Connection", "keep-alive");
                     setResponseHeader(event, "X-Run-Id", submittedRunId);
+                    setRunTurnIdHeader(event, getRun(submittedRunId)?.turnId);
                     setResponseHeader(event, "X-Dispatch-Mode", "replay");
                     return stream;
                 }
@@ -10780,6 +10802,11 @@ export function createProductionAgentHandler(options) {
                     !queuedMessageId &&
                     !queuedMessageClaimId &&
                     !isBackgroundWorker);
+                const queuedSubmissionStore = shouldPreserveQueuedOrder
+                    ? await import("../chat-threads/store.js")
+                    : undefined;
+                // The inbox read runs under the slot lock, but its schema must be ready first.
+                await queuedSubmissionStore?.ensureChatThreadTables();
                 slot = await tryClaimRunSlot(threadId, runId, undefined, {
                     turnId: effectiveTurnId,
                     ...(turnInitiator ? { turnInitiator } : {}),
@@ -10795,12 +10822,9 @@ export function createProductionAgentHandler(options) {
                         ? { dispatchPayload: durableDispatchPayload }
                         : {}),
                     ...(continueOf ? { continueOf } : {}),
-                    ...(shouldPreserveQueuedOrder
+                    ...(queuedSubmissionStore
                         ? {
-                            hasPendingQueuedSubmission: async (tx) => {
-                                const { hasPendingQueuedSubmissionReceipt } = await import("../chat-threads/store.js");
-                                return hasPendingQueuedSubmissionReceipt(tx, runThreadId);
-                            },
+                            hasPendingQueuedSubmission: (tx) => queuedSubmissionStore.hasPendingQueuedSubmissionReceipt(tx, runThreadId),
                         }
                         : {}),
                 });
@@ -10833,6 +10857,9 @@ export function createProductionAgentHandler(options) {
                 setResponseHeader(event, "Cache-Control", "no-cache");
                 setResponseHeader(event, "Connection", "keep-alive");
                 setResponseHeader(event, "X-Run-Id", slot.completedRunId);
+                // The client leaves a replay's turn out unless named here, and this
+                // replay was found by its turn, so the stored turn is known.
+                setRunTurnIdHeader(event, effectiveTurnId);
                 setResponseHeader(event, "X-Dispatch-Mode", "replay");
                 return stream;
             }
@@ -11074,6 +11101,7 @@ export function createProductionAgentHandler(options) {
                     setResponseHeader(event, "Cache-Control", "no-cache");
                     setResponseHeader(event, "Connection", "keep-alive");
                     setResponseHeader(event, "X-Run-Id", duplicateRunId);
+                    setRunTurnIdHeader(event, getRun(duplicateRunId)?.turnId);
                     setResponseHeader(event, "X-Dispatch-Mode", "replay");
                     return stream;
                 }
@@ -11174,6 +11202,7 @@ export function createProductionAgentHandler(options) {
                     setResponseHeader(event, "Cache-Control", "no-cache");
                     setResponseHeader(event, "Connection", "keep-alive");
                     setResponseHeader(event, "X-Run-Id", runId);
+                    setRunTurnIdHeader(event, effectiveTurnId);
                     setResponseHeader(event, "X-Dispatch-Mode", "background");
                     return stream;
                 }
@@ -11503,7 +11532,7 @@ export function createProductionAgentHandler(options) {
                 void retainRequestDbPoolScope(recordRunDiagnostic(runId, RUN_DIAG_STAGE.setupTimings, setupDetail).catch(() => { })).catch(() => { });
             }
             if (options.onRunStart) {
-                await options.onRunStart(send, threadId ?? runId, runId);
+                await options.onRunStart(send, threadId ?? runId, runId, effectiveTurnId);
             }
             const unknownOutcomes = noUnknownOutcomes();
             if (customAgentRefs.length > 0) {
@@ -11938,6 +11967,7 @@ export function createProductionAgentHandler(options) {
                             loopOpts: agentLoopOpts,
                             runId,
                             threadId: threadId ?? null,
+                            turnId: effectiveTurnId,
                             userId: ownerEmail,
                             config: obsConfig,
                             spanName: turnUsageLabel && turnUsageLabel !== "chat"
@@ -12047,10 +12077,12 @@ export function createProductionAgentHandler(options) {
                     : "foreground",
             runRowAlreadyInserted: foregroundRunRowInserted,
             executionLease: true,
+            // An anonymous run is not a user; `run_started` does not name one either.
+            userId: turnInitiator?.anonymous
+                ? undefined
+                : (ownerEmail ?? undefined),
             // Resolved AFTER stored-model/experiment overrides — the same value
             // actually sent to the engine, not the raw client-requested model.
-            // No userId here: `ownerEmail` is the only identity known at this
-            // scope and is PII (email), which the terminal event must not carry.
             model: effectiveModel,
             engineName: engine.name,
             attemptCount: backgroundContinuationCount,
@@ -12068,6 +12100,7 @@ export function createProductionAgentHandler(options) {
         setResponseHeader(event, "Cache-Control", "no-cache");
         setResponseHeader(event, "Connection", "keep-alive");
         setResponseHeader(event, "X-Run-Id", runId);
+        setRunTurnIdHeader(event, effectiveTurnId);
         setResponseHeader(event, "X-Dispatch-Mode", foregroundSelfChainEligible ? "foreground-self-chain" : "foreground");
         return stream;
     });

package/dist/agent/run-manager.js

@@ -7,11 +7,11 @@ import { ServicePrincipalRefusedError } from "../org/service-principal-guard.js"
 import { captureError } from "../server/capture-error.js";
 import { isLlmCredentialError, LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
 import { classifyTerminalErrorCode, describeErrorWithCauses, isProviderConnectionError, } from "./engine/error-detail.js";
-import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
 import { getAgentEngineEntry } from "./engine/registry.js";
 import { EngineError } from "./engine/types.js";
 import { currentExecutionFence, MissingExecutionFenceError, StaleExecutionFenceError, withExecutionFence, withExecutionLeaseRequested, } from "./execution-fence.js";
-import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
+import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, setRunTerminalReasonIfStatus, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
+import { trackRunTerminal } from "./run-terminal-tracking.js";
 import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
 const activeRuns = new Map();
 const threadToRun = new Map();
@@ -435,39 +435,24 @@ function emitRunTerminalTrackingEvent(args) {
     catch {
         // Metrics must never affect the agent run or its persisted status.
     }
-    const properties = {
-        source: "agent_run_manager",
-        run_id: args.runId,
-        thread_id: args.threadId,
-        turn_id: args.turnId,
+    // An abort is reported by the SQL write that ended the row, which may be
+    // another process's stop; reporting it here too would double-count it.
+    if (args.status === "aborted")
+        return;
+    void trackRunTerminal({
+        runId: args.runId,
+        threadId: args.threadId,
+        turnId: args.turnId,
         status: args.status,
-        terminal_reason: args.terminalReason,
-        error_code: args.errorCode,
-        ...(args.errorCode ? runErrorTelemetryProperties(args.errorCode) : {}),
-        dispatch_mode: args.dispatchMode,
-        abort_reason: args.abortReason,
-        duration_ms: args.durationMs,
+        terminalReason: args.terminalReason,
+        errorCode: args.errorCode,
+        dispatchMode: args.dispatchMode,
+        abortReason: args.abortReason,
+        durationMs: args.durationMs,
         model: args.model,
-        engine: args.engineName,
-        attempt_count: args.attemptCount,
-    };
-    for (const key of Object.keys(properties)) {
-        if (properties[key] === undefined)
-            delete properties[key];
-    }
-    try {
-        void Promise.all([
-            import("../tracking/registry.js"),
-            import("../observability/tracking-identity.js"),
-        ])
-            .then(([{ track }, { trackingIdentityProperties }]) => {
-            track("agent_run_terminal", { ...properties, ...trackingIdentityProperties() }, { userId: args.userId });
-        })
-            .catch(() => { });
-    }
-    catch {
-        // Tracking must never affect the agent run or its persisted status.
-    }
+        engineName: args.engineName,
+        attemptCount: args.attemptCount,
+    }, { userId: args.userId });
 }
 function abortInMemoryRun(run, reason = "user") {
     run.abortReason = reason;
@@ -1208,6 +1193,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
     let completionInProgress = false;
     let ownTerminalStatusEstablished = false;
     let terminalStatusWrite = null;
+    let terminalStatusWriteOutcomeUnknown = false;
     const checkSqlAbort = () => {
         const now = Date.now();
         if (now - lastAbortCheck < 3000)
@@ -1223,8 +1209,18 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                 const status = await getRunStatus(runId);
                 if (status !== null && status !== "running") {
                     const pendingStatusWrite = terminalStatusWrite;
-                    if (pendingStatusWrite)
-                        await pendingStatusWrite;
+                    if (pendingStatusWrite) {
+                        try {
+                            await pendingStatusWrite;
+                        }
+                        catch (error) {
+                            if (!terminalStatusWriteOutcomeUnknown)
+                                throw error;
+                            return;
+                        }
+                    }
+                    if (terminalStatusWriteOutcomeUnknown)
+                        return;
                 }
                 if (status !== null &&
                     status !== "running" &&
@@ -1758,12 +1754,21 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             ? updateRunStatusIfRunning(runId, persistedStatus)
             : updateRunStatusIfRunning(runId, persistedStatus, terminalStatusOptions);
         const persistOwnTerminalState = async (persist) => {
-            const write = withWorkerFence(persist);
+            let writeStarted = false;
+            const write = withWorkerFence(() => {
+                writeStarted = true;
+                return persist();
+            }).catch((error) => {
+                if (writeStarted)
+                    terminalStatusWriteOutcomeUnknown = true;
+                throw error;
+            });
             terminalStatusWrite = write;
             try {
                 const established = await write;
-                if (established)
+                if (established && !terminalStatusWriteOutcomeUnknown) {
                     ownTerminalStatusEstablished = true;
+                }
                 return established;
             }
             finally {
@@ -1772,11 +1777,45 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             }
         };
         const persistOwnTerminalStatus = () => persistOwnTerminalState(persistTerminalStatus);
-        const reconcileTerminalRun = () => persistOwnTerminalState(() => threadProjectionPending
-            ? reconcileTerminalRunFromEvents(runId, {
-                threadProjectionPending: true,
-            })
-            : reconcileTerminalRunFromEvents(runId));
+        const confirmAmbiguousTerminalStatus = async () => {
+            const persistedRun = await getRunById(runId);
+            if (persistedRun?.status !== persistedStatus)
+                return false;
+            const expectedTerminalReason = terminalReason.slice(0, 200);
+            if (persistedRun.terminalReason === expectedTerminalReason) {
+                return true;
+            }
+            if (persistedRun.terminalReason)
+                return false;
+            return withWorkerFence(() => setRunTerminalReasonIfStatus(runId, persistedStatus, expectedTerminalReason));
+        };
+        const reconcileTerminalRun = async () => {
+            let reconciled = false;
+            try {
+                reconciled = await persistOwnTerminalState(() => threadProjectionPending
+                    ? reconcileTerminalRunFromEvents(runId, {
+                        threadProjectionPending: true,
+                    })
+                    : reconcileTerminalRunFromEvents(runId));
+            }
+            catch (error) {
+                if (!terminalStatusWriteOutcomeUnknown)
+                    throw error;
+            }
+            if (reconciled) {
+                terminalStatusWriteOutcomeUnknown = false;
+                ownTerminalStatusEstablished = true;
+                return true;
+            }
+            if (!terminalStatusWriteOutcomeUnknown) {
+                return reconciled;
+            }
+            const confirmed = await confirmAmbiguousTerminalStatus();
+            terminalStatusWriteOutcomeUnknown = false;
+            if (confirmed)
+                ownTerminalStatusEstablished = true;
+            return confirmed;
+        };
         const reportTerminalStatusPersistenceError = (error) => {
             captureError(error, {
                 route: "/_agent-native/agent-chat",

package/dist/agent/run-store.d.ts

@@ -276,6 +276,7 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
 }>;
 export declare function setRunError(runId: string, errorCode: string | undefined, errorDetail: string | undefined): Promise<void>;
 export declare function setRunTerminalReason(runId: string, terminalReason: string | undefined): Promise<void>;
+export declare function setRunTerminalReasonIfStatus(runId: string, expectedStatus: "completed" | "truncated" | "errored" | "aborted", terminalReason: string): Promise<boolean>;
 /**
  * Repair a run whose terminal event was durably appended but whose final
  * `agent_runs.status` write lost a race with reconnect/reaper code.