@agent-native/core 0.149.2 → 0.149.3

package/dist/agent/run-manager.js

@@ -499,10 +499,24 @@ function abortInMemoryRun(run, reason = "user") {
         threadToRun.delete(run.threadId);
     }
     run.abort.abort(reason);
-    const terminalEvent = terminalEventForAbortReason(reason);
+    // Keep the abort terminal in the replay buffer as well as sending it to
+    // current subscribers. A reconnect can arrive after the live subscriber
+    // was removed but while this run is still warm in memory; closing that
+    // replay with no terminal frame makes the client manufacture a false
+    // frozen response and retry against a run that is already done.
+    const existingTerminalEvent = [...run.events]
+        .reverse()
+        .find((event) => isTerminalRunEvent(event.event));
+    const terminalRunEvent = existingTerminalEvent ?? {
+        seq: run.events.length,
+        event: terminalEventForAbortReason(reason),
+    };
+    if (!existingTerminalEvent) {
+        run.events.push(terminalRunEvent);
+    }
     for (const subscriber of run.subscribers) {
         try {
-            subscriber({ seq: run.events.length, event: terminalEvent });
+            subscriber(terminalRunEvent);
         }
         catch {
             // ignore — subscriber is being removed below
@@ -596,18 +610,116 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
     // Throttle the durable progress timestamp to at most once per second so
     // a chatty token-by-token stream doesn't translate into one DB write per
     // chunk. The stuck-detector threshold is on the order of tens of seconds,
-    // so 1s resolution is plenty.
+    // so 1s resolution is plenty. Keep one write in flight and coalesce a
+    // trailing request: overlapping progress updates can exhaust a pooler and
+    // starve the heartbeat that is supposed to prove this run is alive.
+    const PROGRESS_BUMP_INTERVAL_MS = 1_000;
     let lastProgressBumpAt = 0;
+    let progressWriteInFlight = null;
+    let progressWriteTimer = null;
+    let progressWritePending = false;
+    let progressWriteAttempts = 0;
+    let progressWriteFailures = 0;
+    let consecutiveProgressWriteFailures = 0;
     const preparingActivityBytes = new Map();
     const preparingActivityTools = new Map();
     const preparingActivityRestartHighWater = new Map();
     let eventPersistenceErrorCaptured = false;
-    const bumpProgressIfDue = () => {
-        const now = Date.now();
-        if (now - lastProgressBumpAt < 1000)
+    const recordProgressWriteFailure = (error, kind) => {
+        progressWriteFailures += 1;
+        consecutiveProgressWriteFailures += 1;
+        // Keep every failure counted and retried, but sample the error capture so
+        // a database outage does not turn one long streamed input into hundreds of
+        // identical reports. The run id and event count make the sampled report
+        // useful for correlating the durable clock with the event ledger.
+        if (progressWriteFailures === 1 || progressWriteFailures % 10 === 0) {
+            const capturedError = error instanceof Error ? error : new Error(String(error));
+            console.error(`[run-manager] durable progress write ${kind} failed`, runId, {
+                attempts: progressWriteAttempts,
+                failures: progressWriteFailures,
+                consecutiveFailures: consecutiveProgressWriteFailures,
+                eventCount: run.events.length,
+            }, capturedError);
+            captureError(capturedError, {
+                route: "/_agent-native/agent-chat",
+                tags: {
+                    source: "agent-run-manager",
+                    phase: "progress",
+                    kind,
+                    consecutiveFailures: String(consecutiveProgressWriteFailures),
+                },
+                aiTraceId: runId,
+                extra: {
+                    runId,
+                    threadId,
+                    attempts: progressWriteAttempts,
+                    failures: progressWriteFailures,
+                    eventCount: run.events.length,
+                },
+            });
+        }
+    };
+    const writeProgress = async () => {
+        let updated = await bumpRunProgress(runId);
+        if (updated === false) {
+            // The initial run insert is intentionally asynchronous so the fast SSE
+            // path does not wait on SQL. If this first UPDATE wins the race, retry
+            // after that insert commits instead of permanently losing the first
+            // durable progress signal.
+            await insertRunPromise;
+            updated = await bumpRunProgress(runId);
+        }
+        if (updated === false) {
+            if (run.status !== "running")
+                return;
+            recordProgressWriteFailure(new Error("Durable progress update affected no running run row"), "no-row");
+            progressWritePending = true;
             return;
-        lastProgressBumpAt = now;
-        bumpRunProgress(runId).catch(() => { });
+        }
+        consecutiveProgressWriteFailures = 0;
+    };
+    function startProgressWrite() {
+        if (!progressWritePending ||
+            progressWriteInFlight ||
+            run.status !== "running") {
+            return;
+        }
+        progressWritePending = false;
+        lastProgressBumpAt = Date.now();
+        progressWriteAttempts += 1;
+        const write = writeProgress()
+            .catch((error) => {
+            recordProgressWriteFailure(error, "error");
+            // Retry even if no later event arrives. A transient failure must not
+            // freeze the durable clock until the next user-visible event.
+            progressWritePending = true;
+        })
+            .finally(() => {
+            progressWriteInFlight = null;
+            scheduleProgressWrite();
+        });
+        progressWriteInFlight = write;
+    }
+    function scheduleProgressWrite() {
+        if (!progressWritePending ||
+            progressWriteInFlight ||
+            progressWriteTimer ||
+            run.status !== "running") {
+            return;
+        }
+        const delayMs = Math.max(0, lastProgressBumpAt + PROGRESS_BUMP_INTERVAL_MS - Date.now());
+        if (delayMs === 0) {
+            startProgressWrite();
+            return;
+        }
+        progressWriteTimer = setTimeout(() => {
+            progressWriteTimer = null;
+            startProgressWrite();
+        }, delayMs);
+    }
+    const bumpProgressIfDue = () => {
+        progressWritePending = true;
+        scheduleProgressWrite();
     };
     const shouldBumpProgressForEvent = (event) => {
         if (event.type === "stream_keepalive")
@@ -683,6 +795,25 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
     // emits nothing for minutes and carries its own 12-min timeout.
     let lastRealProgressAt = Date.now();
     let inFlightWorkCount = 0;
+    let inFlightMarkerSince = null;
+    let inFlightMarkerWrite = Promise.resolve();
+    const mirrorInFlightMarker = (inFlight) => {
+        const markerSince = inFlight ? Date.now() : inFlightMarkerSince;
+        if (inFlight) {
+            inFlightMarkerSince = markerSince;
+        }
+        else {
+            inFlightMarkerSince = null;
+        }
+        // Preserve transition order: a delayed clear must not erase the marker
+        // written for the next tool that starts before the clear reaches SQL.
+        inFlightMarkerWrite = inFlightMarkerWrite
+            .catch(() => undefined)
+            .then(() => setRunInFlightMarker(runId, inFlight, markerSince ?? undefined))
+            .catch((error) => {
+            console.error(`[run-manager] failed to mirror in-flight marker (${inFlight ? "set" : "clear"})`, runId, error instanceof Error ? error.message : error);
+        });
+    };
     const trackInFlightWork = (event) => {
         const wasIdle = inFlightWorkCount === 0;
         if (event.type === "tool_start") {
@@ -708,15 +839,15 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         // grant this demonstrably-alive run a bounded grace even when THIS
         // isolate's own heartbeat write is failing (e.g. Neon pooler saturation)
         // — `inFlightWorkCount` itself is in-memory and invisible to those other
-        // isolates. Fire-and-forget: never block event emission on this write,
-        // and a write failure here is no worse than today's behavior (the row
-        // just gets no grace). See `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS`
-        // in run-store.ts for the full reasoning and the bounded-grace derivation.
+        // isolates. Fire-and-forget: never block event emission on this write, but
+        // keep failures observable and preserve transition order. See
+        // `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS` in run-store.ts
+        // for the full reasoning and bounded-grace derivation.
         if (wasIdle && inFlightWorkCount > 0) {
-            setRunInFlightMarker(runId, true).catch(() => { });
+            mirrorInFlightMarker(true);
         }
         else if (!wasIdle && inFlightWorkCount === 0) {
-            setRunInFlightMarker(runId, false).catch(() => { });
+            mirrorInFlightMarker(false);
         }
     };
     // Make a chunk boundary durable at the instant it is decided. The terminal
@@ -1171,6 +1302,17 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         clearInterval(heartbeatTimer);
         if (softTimeoutTimer)
             clearTimeout(softTimeoutTimer);
+        if (progressWriteTimer)
+            clearTimeout(progressWriteTimer);
+        progressWriteTimer = null;
+        progressWritePending = false;
+        // A tool can throw before its matching event is emitted, or the process
+        // can reach terminal cleanup with an in-flight call still open. Do not
+        // leave the SQL grace marker attached to a terminal run.
+        if (inFlightWorkCount > 0 || inFlightMarkerSince !== null) {
+            inFlightWorkCount = 0;
+            mirrorInFlightMarker(false);
+        }
         // 5. Persist final status to SQL. Use the conditional write so a zombie
         //    run (reaped or displaced while executing) cannot clobber the newer
         //    status written by the reaper or a replacement run.

package/dist/agent/run-store.d.ts

@@ -457,7 +457,7 @@ export declare function updateRunHeartbeat(runId: string): Promise<void>;
  * stuck-detector can tell "process alive but nothing happening" from
  * "process dead." Callers should throttle (run-manager debounces to ~1/s).
  */
-export declare function bumpRunProgress(runId: string): Promise<void>;
+export declare function bumpRunProgress(runId: string): Promise<boolean>;
 /**
  * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
  * SQL so a stale reaper running in a DIFFERENT isolate can tell a
@@ -469,17 +469,18 @@ export declare function bumpRunProgress(runId: string): Promise<void>;
  * depth belt-and-suspenders against a nested 1->2 transition clobbering the
  * ORIGINAL start time with a later one (the caller's own counter already
  * dedupes 0->1 transitions; this WHERE just makes the write itself
- * idempotent/order-independent too). `inFlight: false` always clears
- * unconditionally — if it races a fresh 0->1 write from a *different* tool
- * finishing/starting back to back, worst case is losing a few seconds of
- * grace, never gaining an incorrect one.
+ * idempotent/order-independent too). When `markerSince` is supplied for a
+ * clear, the UPDATE is conditional on the marker still having that value. The
+ * run manager serializes marker writes and supplies this token so a delayed
+ * clear from one tool cannot erase a newer tool's marker.
  *
- * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
- * failure here never blocks event emission or aborts the run. If this write
- * itself fails (the same DB pressure that could be starving the heartbeat),
- * the row simply gets no grace — never worse than today's behavior.
+ * Best-effort: callers fire-and-forget so a write failure here never blocks
+ * event emission or aborts the run. The run manager logs marker-write failures
+ * while preserving transition order. If this write itself fails (the same DB
+ * pressure that could be starving the heartbeat), the row simply gets no grace
+ * - never worse than today's behavior.
  */
-export declare function setRunInFlightMarker(runId: string, inFlight: boolean): Promise<void>;
+export declare function setRunInFlightMarker(runId: string, inFlight: boolean, markerSince?: number): Promise<void>;
 /**
  * If the given run is marked "running" in SQL but its heartbeat is stale
  * (producer likely crashed), flip it to "errored" so watchers stop waiting.

package/dist/agent/run-store.js

@@ -1351,7 +1351,7 @@ export async function bumpRunProgress(runId) {
     await ensureRunTables();
     const client = getDbExec();
     const now = Date.now();
-    await client.execute({
+    const { rowsAffected } = await client.execute({
         // Multiple event-persistence paths and serverless isolates can bump the
         // same run concurrently. A slower, older write must never land after a
         // newer one and move the user-visible no-progress clock backwards.
@@ -1359,6 +1359,7 @@ export async function bumpRunProgress(runId) {
         sql: `UPDATE agent_runs SET last_progress_at = CASE WHEN last_progress_at IS NULL OR last_progress_at < ? THEN ? ELSE last_progress_at END WHERE id = ? AND status = 'running'`,
         args: [now, now, runId],
     });
+    return (rowsAffected ?? 0) > 0;
 }
 /**
  * Mirror run-manager's in-memory `inFlightWorkCount` 0<->N transitions into
@@ -1371,29 +1372,32 @@ export async function bumpRunProgress(runId) {
  * depth belt-and-suspenders against a nested 1->2 transition clobbering the
  * ORIGINAL start time with a later one (the caller's own counter already
  * dedupes 0->1 transitions; this WHERE just makes the write itself
- * idempotent/order-independent too). `inFlight: false` always clears
- * unconditionally — if it races a fresh 0->1 write from a *different* tool
- * finishing/starting back to back, worst case is losing a few seconds of
- * grace, never gaining an incorrect one.
+ * idempotent/order-independent too). When `markerSince` is supplied for a
+ * clear, the UPDATE is conditional on the marker still having that value. The
+ * run manager serializes marker writes and supplies this token so a delayed
+ * clear from one tool cannot erase a newer tool's marker.
  *
- * Best-effort: callers fire-and-forget (`.catch(() => {})`) so a write
- * failure here never blocks event emission or aborts the run. If this write
- * itself fails (the same DB pressure that could be starving the heartbeat),
- * the row simply gets no grace — never worse than today's behavior.
+ * Best-effort: callers fire-and-forget so a write failure here never blocks
+ * event emission or aborts the run. The run manager logs marker-write failures
+ * while preserving transition order. If this write itself fails (the same DB
+ * pressure that could be starving the heartbeat), the row simply gets no grace
+ * - never worse than today's behavior.
  */
-export async function setRunInFlightMarker(runId, inFlight) {
+export async function setRunInFlightMarker(runId, inFlight, markerSince) {
     await ensureRunTables();
     const client = getDbExec();
     if (inFlight) {
         await client.execute({
             sql: `UPDATE agent_runs SET in_flight_since = ? WHERE id = ? AND status = 'running' AND in_flight_since IS NULL`,
-            args: [Date.now(), runId],
+            args: [markerSince ?? Date.now(), runId],
         });
     }
     else {
         await client.execute({
-            sql: `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ?`,
-            args: [runId],
+            sql: markerSince == null
+                ? `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ?`
+                : `UPDATE agent_runs SET in_flight_since = NULL WHERE id = ? AND in_flight_since = ?`,
+            args: markerSince == null ? [runId] : [runId, markerSince],
         });
     }
 }

package/dist/client/AssistantChat.d.ts

@@ -34,7 +34,7 @@ export declare function createUserMessageRunConfig(references?: Reference[], req
     model?: string;
     engine?: string;
     effort?: ReasoningEffort;
-}): {
+}, turnId?: string): {
     runConfig?: {
         custom: {
             references?: Reference[];
@@ -45,6 +45,7 @@ export declare function createUserMessageRunConfig(references?: Reference[], req
             model?: string;
             engine?: string;
             effort?: ReasoningEffort;
+            turnId?: string;
         };
     };
     metadata?: {
@@ -98,13 +99,14 @@ export declare function resolveAssistantChatSubmitIntent({ isRunning, requestedI
     isRunning: boolean;
     requestedIntent?: ComposerSubmitIntent;
 }): ComposerSubmitIntent;
-export declare function resolveAssistantChatRunningState({ forceStopped, isRuntimeRunning, isReconnecting, optimisticRunning, isAutoResuming, hasActiveServerRun, }: {
+export declare function resolveAssistantChatRunningState({ forceStopped, isRuntimeRunning, isReconnecting, optimisticRunning, isAutoResuming, hasActiveServerRun, hasTerminalRunError, }: {
     forceStopped: boolean;
     isRuntimeRunning: boolean;
     isReconnecting: boolean;
     optimisticRunning: boolean;
     isAutoResuming: boolean;
     hasActiveServerRun?: boolean;
+    hasTerminalRunError?: boolean;
 }): {
     isRunning: boolean;
     showRunningInUI: boolean;
@@ -137,6 +139,8 @@ type QueuedMessage = {
     trackInRunsTray?: boolean;
     hideUserMessage?: boolean;
     approvedToolCalls?: string[];
+    /** Preserve the logical turn when a hidden reconnect recovery is re-issued. */
+    turnId?: string;
     /**
      * Model/engine/effort snapshotted at enqueue time, for the same reason
      * `requestMode` is: the picker is global and live, so a queue that flushes