@agent-native/core 0.153.4 → 0.153.6

package/dist/agent/run-manager.js

@@ -937,12 +937,15 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
     // false-stale-reap zombie scenario where the reaper flipped the row while
     // this isolate was briefly unable to heartbeat (DB latency / GC pause).
     let lastAbortCheck = Date.now() - 3000;
-    // A read failure here used to be indistinguishable from "not aborted" —
-    // exactly the coerced-to-false pattern that lets a real Stop go unseen for
-    // the rest of the run. Count consecutive failures like the heartbeat-write
-    // handler above; past the same threshold, fail closed (self-abort with a
-    // reason outside TURN_ENDING_ABORT_REASONS/RECOVERABLE_ABORT_REASONS, so it
-    // surfaces as a typed error) instead of silently retrying forever.
+    // A read failure here is not "not aborted" — but it is not grounds to kill
+    // the run either. This check is only a cross-isolate backstop: the outage
+    // that hides a Stop from us hides it from every other reader too, so
+    // self-aborting delivers nobody's Stop any sooner while destroying in-flight
+    // work in the far more common case where nobody pressed Stop at all. Fail
+    // open and stay bounded by the soft timeout, no-progress backstop, and
+    // iteration/token limits — none of which touch the DB. Report the outage to
+    // Sentry (once at the threshold, then ~once a minute) so a long unreadable
+    // window is visible and its duration is queryable instead of silent.
     let consecutiveAbortCheckFailures = 0;
     const checkSqlAbort = () => {
         const now = Date.now();
@@ -970,7 +973,9 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         })
             .catch((error) => {
             consecutiveAbortCheckFailures += 1;
-            if (consecutiveAbortCheckFailures >= 3) {
+            // 3 ticks ≈ 9s of unreadability, then every 20 ticks ≈ 60s.
+            if (consecutiveAbortCheckFailures === 3 ||
+                consecutiveAbortCheckFailures % 20 === 0) {
                 captureError(error, {
                     route: "/_agent-native/agent-chat",
                     tags: {
@@ -979,11 +984,12 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
                         consecutiveFailures: String(consecutiveAbortCheckFailures),
                     },
                     aiTraceId: runId,
-                    extra: { runId, threadId },
+                    extra: {
+                        runId,
+                        threadId,
+                        unreadableForMs: consecutiveAbortCheckFailures * 3000,
+                    },
                 });
-                if (!abort.signal.aborted) {
-                    abortInMemoryRun(run, "abort_check_unavailable");
-                }
             }
         });
     };

package/dist/agent/run-store.d.ts

@@ -44,6 +44,22 @@ export declare const STALE_RUN_ERROR_EVENT: {
     readonly recoverable: true;
     readonly details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.";
 };
+/**
+ * `agent_runs.terminal_reason` recorded by the stale reapers.
+ *
+ * Every other error path records `error:<code>` (`terminalReasonForEvent`), and
+ * `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
+ * replays the terminal event a reaper itself appended. The reapers used to write
+ * the bare code, which split one outcome across two permanent
+ * `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
+ * `error:stale_run`) purely by whether anything happened to read the row after
+ * the reap — and made the reconciliation convergence guard, which compares
+ * `terminal_reason` for equality, rewrite the row on every read forever.
+ *
+ * Rows reaped before this change keep the bare `stale_run`, so anything matching
+ * on the value must accept both forms for one retention window.
+ */
+export declare const STALE_RUN_TERMINAL_REASON: string;
 /**
  * Terminal error for a background-dispatched run whose worker NEVER claimed it
  * (the foreground fired the self-dispatch, Netlify acked it async with a 202,

package/dist/agent/run-store.js

@@ -50,6 +50,22 @@ export const STALE_RUN_ERROR_EVENT = {
     recoverable: true,
     details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
 };
+/**
+ * `agent_runs.terminal_reason` recorded by the stale reapers.
+ *
+ * Every other error path records `error:<code>` (`terminalReasonForEvent`), and
+ * `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
+ * replays the terminal event a reaper itself appended. The reapers used to write
+ * the bare code, which split one outcome across two permanent
+ * `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
+ * `error:stale_run`) purely by whether anything happened to read the row after
+ * the reap — and made the reconciliation convergence guard, which compares
+ * `terminal_reason` for equality, rewrite the row on every read forever.
+ *
+ * Rows reaped before this change keep the bare `stale_run`, so anything matching
+ * on the value must accept both forms for one retention window.
+ */
+export const STALE_RUN_TERMINAL_REASON = `error:${STALE_RUN_ERROR_EVENT.errorCode}`;
 /**
  * Terminal error for a background-dispatched run whose worker NEVER claimed it
  * (the foreground fired the self-dispatch, Netlify acked it async with a 202,
@@ -1306,6 +1322,22 @@ export async function recordRunDiagnostic(runId, stage, detail) {
         // Diagnostics are best-effort; never let them break the run or the route.
     }
 }
+/**
+ * Wall-clock budget for ONE heartbeat write.
+ *
+ * The default `DbExec` budget is `dbOpTimeoutMs()` (8s on serverless) across 3
+ * connection attempts, so a single heartbeat can occupy up to 24s — against a
+ * 15s `RUN_STALE_MS`. The run-manager single-flights the write, so that one
+ * attempt suppresses every 1.5s tick behind it: a live foreground run in a quiet
+ * phase becomes reapable while its own heartbeat is still in flight. The same
+ * attempt also pins a pooler connection for those 24s, feeding the saturation
+ * that starves heartbeats in the first place.
+ *
+ * A heartbeat that cannot land promptly has no value — the 1.5s tick IS the
+ * retry — so it gets ONE attempt inside a third of the stale window, leaving
+ * room for two more to land before the reap cutoff.
+ */
+const HEARTBEAT_WRITE_TIMEOUT_MS = Math.floor(RUN_STALE_MS / 3);
 /** Update the run's liveness heartbeat. Called periodically by run-manager. */
 export async function updateRunHeartbeat(runId) {
     await ensureRunTables();
@@ -1324,6 +1356,8 @@ export async function updateRunHeartbeat(runId) {
               END
           WHERE id = ? AND status = 'running'`,
         args: [Date.now(), currentRssMb(), currentRssMb(), runId],
+        timeoutMs: HEARTBEAT_WRITE_TIMEOUT_MS,
+        maxAttempts: 1,
     });
 }
 /**
@@ -1612,9 +1646,15 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
         : // First `?` is backgroundAwareStaleCutoffSql's CAST param, the next two
             // are inFlightGraceSql's — all bound to the same "now".
             [completedAt, completedAt, completedAt];
+    // `completed_at` is when the run DIED, not when a reaper noticed. Stamping
+    // reap time made every stale row report its own detection latency as run
+    // duration (prod: 30-59 minute "runs" against a 15s window), which is the
+    // measurement used to judge whether reaping is timely at all. The liveness
+    // basis is the last proof the producer was alive, and it is what
+    // `reconcileTerminalRunFromEvents` already records for the same failure.
     const updateSql = `UPDATE agent_runs
           SET status = 'errored',
-              completed_at = ?,
+              completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
               error_code = ?,
               error_detail = ?,
               terminal_reason = ?
@@ -1623,10 +1663,9 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
             AND ${terminalRunEventExclusionSql()}
             AND ${staleClause}`;
     const updateArgs = [
-        completedAt,
         STALE_RUN_ERROR_EVENT.errorCode,
         STALE_RUN_ERROR_EVENT.details,
-        STALE_RUN_ERROR_EVENT.errorCode,
+        STALE_RUN_TERMINAL_REASON,
         runId,
         ...staleArgs,
     ];
@@ -2433,11 +2472,16 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
             await reconcileTerminalRunFromEvents(id);
         }
     }
+    // Both UPDATEs below backdate `completed_at` to the liveness basis for the
+    // reason given on `reapSingleStaleRun`'s UPDATE: this pass can fire a DAY
+    // after the producer died, and stamping now would file that whole gap as run
+    // duration. On the absolute-age branch the basis degrades to `started_at`,
+    // which is still the truest thing known about a row that never heartbeat.
     const completedAt = Date.now();
     await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored',
-              completed_at = ?,
+              completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
               error_code = ?,
               error_detail = ?,
               terminal_reason = ?
@@ -2445,10 +2489,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
             AND ${terminalRunEventExclusionSql()}
             AND started_at < ?`,
         args: [
-            completedAt,
             STALE_RUN_ERROR_EVENT.errorCode,
             STALE_RUN_ERROR_EVENT.details,
-            STALE_RUN_ERROR_EVENT.errorCode,
+            STALE_RUN_TERMINAL_REASON,
             cutoff,
         ],
     });
@@ -2463,7 +2506,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
     await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored',
-              completed_at = ?,
+              completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
               error_code = ?,
               error_detail = ?,
               terminal_reason = ?
@@ -2472,10 +2515,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
             AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
             AND ${inFlightGraceSql()}`,
         args: [
-            completedAt,
             STALE_RUN_ERROR_EVENT.errorCode,
             STALE_RUN_ERROR_EVENT.details,
-            STALE_RUN_ERROR_EVENT.errorCode,
+            STALE_RUN_TERMINAL_REASON,
             completedAt,
             completedAt,
             completedAt,

package/dist/agent/thread-data-builder.d.ts

@@ -17,6 +17,7 @@ interface ContentPart {
     completedSideEffect?: boolean;
     mcpApp?: AgentMcpAppPayload;
     chatUI?: ActionChatUIConfig;
+    activity?: boolean;
     approval?: {
         approvalKey: string;
         dismissed?: boolean;

package/dist/agent/thread-data-builder.js

@@ -243,18 +243,31 @@ export function buildAssistantMessage(events, runId, options = {}) {
         metadata,
     };
 }
+/**
+ * The rebuild half of the live client's `clearAssistantDraftContent`
+ * (client/sse-event-processor.ts). The two bodies are asserted identical by
+ * `keeps clearAssistantDraftContent identical to the live client copy` in
+ * thread-data-builder.spec.ts — a rebuild that clears more than the live stream
+ * did makes narration vanish on reload, which is worse than clearing nothing.
+ */
 function clearAssistantDraftContent(content) {
     for (let index = content.length - 1; index >= 0; index--) {
         const part = content[index];
         if (!part)
             continue;
+        if (part.type === "tool-call" &&
+            part.activity !== true &&
+            part.result !== undefined) {
+            return;
+        }
         if (part.type === "text" || part.type === "reasoning") {
             content.splice(index, 1);
             continue;
         }
         if (part.type === "tool-call" && part.result === undefined) {
-            // Keep materialized in-flight tool cards across retry clears so persisted
-            // thread rebuilds match the live SSE processor and avoid hide→show flicker.
+            // Only drop ephemeral placeholders. Materialized in-flight tool cards
+            // (real args from tool_start) stay mounted so a retry/auto-continue clear
+            // does not hide→show the same call when the next chunk re-emits it.
             const isEphemeral = part.activity === true ||
                 part.argsText === "" ||
                 Object.keys(part.args ?? {}).length === 0;

package/dist/client/RouteTransitionIndicator.js

@@ -37,5 +37,5 @@ export function RouteTransitionIndicator() {
     }, [destination]);
     if (!destination || visibleDestination !== destination)
         return null;
-    return (_jsx("div", { "aria-label": "Loading the next page", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
+    return (_jsx("div", { "aria-label": "Loading page...", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
 }