@agent-native/core 0.84.67 → 0.85.0

package/dist/client/agent-chat-adapter.js

@@ -78,6 +78,38 @@ const LARGE_INPUT_TOOL_NAMES = new Set([
 ]);
 const MAX_HISTORY_LARGE_TOOL_ARGS_CHARS = 200_000;
 const STARTUP_RESPONSE_TIMEOUT_MS = 45_000;
+// ── Background follow mode ──────────────────────────────────────────────────
+// For background-dispatched runs (dispatchMode starts with "background") the
+// SERVER is the sole recovery brain: it chains continuation chunks itself
+// (fresh runId, same turnId), pre-inserts the successor run row BEFORE the old
+// chunk completes (so /runs/active shows an active run continuously across
+// chunk boundaries), and a server sweep reaps lost handoffs into loud terminal
+// errors. The client therefore never POSTs synthetic continuations for these
+// runs — it demotes itself to a READER of server state: poll /runs/active,
+// (re)attach to whichever run of this turn is live, and fold its events into
+// the same assistant message. Read-only, so multiple tabs following the same
+// run are safe (no localStorage/Web-Locks claim needed).
+const BACKGROUND_FOLLOW_POLL_INTERVAL_MS = 1_000;
+// How long the follow loop tolerates seeing NO active run for this turn before
+// treating the turn as ended. The server pre-inserts the successor row before
+// the old chunk completes, so a healthy chain never shows an idle gap; 45s
+// also comfortably covers the server's 25s unclaimed-handoff grace + sweep
+// cadence so a reaped handoff resurfaces as a terminal errored run (which the
+// follow loop replays) before this window expires.
+const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 45_000;
+/**
+ * User-facing copy for the server's background terminal reasons
+ * (`agent_runs.terminal_reason`, surfaced by /runs/active). These runs died in
+ * server-side handoff machinery where no richer error event may exist, so the
+ * client owns the message. Keys are matched after stripping an optional
+ * `error:` prefix (`terminalReasonForEvent` records `error:<code>`).
+ */
+const BACKGROUND_TERMINAL_REASON_MESSAGES = {
+    background_worker_never_started: "The agent run was handed off to a background worker that never started. It was recovered so you can try again.",
+    background_continuation_dispatch_failed: "The agent's background worker could not hand off the next step of this run. Retry to continue from the preserved context.",
+    dispatch_payload_missing: "The agent's background run lost its saved request data and could not continue. Retry to start a fresh run.",
+    turn_continuation_budget_exhausted: "This request needed more automatic continuations than allowed and was stopped. Try breaking it into smaller steps.",
+};
 function normalizeMentions(text) {
     return text.replace(/@\[([^\]|]+)\|[^\]]+\]/g, "@$1");
 }
@@ -939,6 +971,75 @@ function missingCredentialFailure(message) {
         };
     }
 }
+function runtimeDebugUrlForApiUrl(apiUrl) {
+    if (typeof window === "undefined")
+        return null;
+    try {
+        const url = new URL(apiUrl, window.location.href);
+        const marker = "/_agent-native/";
+        const markerIndex = url.pathname.indexOf(marker);
+        if (markerIndex < 0)
+            return null;
+        url.pathname = `${url.pathname.slice(0, markerIndex)}${marker}debug/runtime`;
+        url.search = "";
+        return url.toString();
+    }
+    catch {
+        return null;
+    }
+}
+function stringValue(value) {
+    return typeof value === "string" ? value : "";
+}
+function formatRuntimeDebugDetails(payload) {
+    if (!payload || typeof payload !== "object")
+        return "";
+    const root = payload;
+    const runtime = root.runtime && typeof root.runtime === "object" ? root.runtime : {};
+    const database = runtime.database && typeof runtime.database === "object"
+        ? runtime.database
+        : {};
+    const schema = root.schema && typeof root.schema === "object" ? root.schema : {};
+    const lines = [
+        stringValue(runtime.app) ? `runtime_app: ${runtime.app}` : "",
+        stringValue(runtime.environment)
+            ? `runtime_environment: ${runtime.environment}`
+            : "",
+        stringValue(runtime.deployContext)
+            ? `deploy_context: ${runtime.deployContext}`
+            : "",
+        stringValue(runtime.deployId) ? `deploy_id: ${runtime.deployId}` : "",
+        stringValue(runtime.commitRef) ? `commit_ref: ${runtime.commitRef}` : "",
+        stringValue(runtime.branch) ? `branch: ${runtime.branch}` : "",
+        stringValue(runtime.siteName) ? `site_name: ${runtime.siteName}` : "",
+        typeof database.configured === "boolean"
+            ? `db_configured: ${database.configured}`
+            : "",
+        stringValue(database.source) ? `db_source: ${database.source}` : "",
+        stringValue(database.dialect) ? `db_dialect: ${database.dialect}` : "",
+        stringValue(database.protocol) ? `db_protocol: ${database.protocol}` : "",
+        stringValue(database.host) ? `db_host: ${database.host}` : "",
+        stringValue(database.database) ? `db_database: ${database.database}` : "",
+        stringValue(database.urlHash) ? `db_url_hash: ${database.urlHash}` : "",
+        database.neon?.endpointId
+            ? `db_neon_endpoint: ${database.neon.endpointId}`
+            : "",
+        typeof database.neon?.pooled === "boolean"
+            ? `db_neon_pooled: ${database.neon.pooled}`
+            : "",
+        typeof schema.ok === "boolean" ? `schema_ok: ${schema.ok}` : "",
+        Array.isArray(schema.missingTables) && schema.missingTables.length
+            ? `schema_missing_tables: ${schema.missingTables.join(", ")}`
+            : "",
+        Array.isArray(schema.missingColumns) && schema.missingColumns.length
+            ? `schema_missing_columns: ${schema.missingColumns
+                .map((entry) => `${entry.table}.${entry.column}`)
+                .join(", ")}`
+            : "",
+        stringValue(schema.error) ? `schema_error: ${schema.error}` : "",
+    ].filter(Boolean);
+    return lines.join("\n");
+}
 export function createAgentChatAdapter(options) {
     const apiUrl = options?.apiUrl ?? agentNativePath("/_agent-native/agent-chat");
     const tabId = options?.tabId;
@@ -950,6 +1051,16 @@ export function createAgentChatAdapter(options) {
     const browserTabId = options?.browserTabId;
     const scopeRef = options?.scopeRef;
     const surface = options?.surface ?? "app";
+    let runtimeDebugDetails = "";
+    const runtimeDebugUrl = runtimeDebugUrlForApiUrl(apiUrl);
+    if (runtimeDebugUrl && typeof fetch === "function") {
+        void fetch(runtimeDebugUrl, { credentials: "include" })
+            .then((res) => (res.ok ? res.json() : null))
+            .then((payload) => {
+            runtimeDebugDetails = formatRuntimeDebugDetails(payload);
+        })
+            .catch(() => { });
+    }
     return {
         async *run({ messages, abortSignal, runConfig }) {
             // Extract latest user message and build history from prior messages
@@ -1114,6 +1225,7 @@ export function createAgentChatAdapter(options) {
                     attemptedRunIds.length > 0
                         ? `attempted_runs: ${attemptedRunIds.join(", ")}`
                         : "",
+                    runtimeDebugDetails,
                 ]
                     .filter(Boolean)
                     .join("\n");
@@ -1220,6 +1332,11 @@ export function createAgentChatAdapter(options) {
                 if (mode)
                     currentRunDispatchMode = mode;
             };
+            // Mode switch for recovery ownership: background-dispatched runs are
+            // recovered by the SERVER (chained continuations, sweeps); the client
+            // only follows. Foreground (null/"foreground"/unknown) keeps the full
+            // client-side continuation machinery unchanged.
+            const isBackgroundDispatch = () => currentRunDispatchMode?.startsWith("background") === true;
             const rememberRunSeq = (seq) => {
                 lastSeq = seq;
                 if (runId) {
@@ -1518,6 +1635,279 @@ export function createAgentChatAdapter(options) {
                     lastSeq = interruptedLastSeq;
                     return false;
                 };
+                // ── Background follow mode (see module comment on the constants) ──
+                // Emits a terminal chat error using the same runError/content/yield
+                // shape as the exhausted-recovery paths, so background failures render
+                // identically to foreground ones.
+                const emitBackgroundTerminalError = function* (args) {
+                    const runError = {
+                        message: args.message,
+                        details: [args.details, connectionRecoveryDetails()]
+                            .filter(Boolean)
+                            .join("\n\n"),
+                        errorCode: args.errorCode,
+                        recoverable: true,
+                        ...(runId ? { runId } : {}),
+                    };
+                    if (typeof window !== "undefined") {
+                        window.dispatchEvent(new CustomEvent("agent-chat:run-error", {
+                            detail: { ...runError, tabId },
+                        }));
+                    }
+                    settleInterruptedToolCalls(content, undefined, {
+                        includeActivity: true,
+                    });
+                    content.push({
+                        type: "text",
+                        text: formatChatErrorText(args.message, undefined, args.errorCode),
+                    });
+                    yield {
+                        content: [...content],
+                        status: { type: "incomplete", reason: "error" },
+                        metadata: { custom: { ...(runId ? { runId } : {}), runError } },
+                    };
+                    clearActiveRun();
+                };
+                // Final outcome for a background turn the follow loop can no longer
+                // follow: either the run went terminal server-side (surface its
+                // error, mapping terminal_reason to clear copy) or it vanished
+                // (finalize with received content when it completed, loud error
+                // otherwise — never a silent stop).
+                const emitBackgroundTerminalOutcome = function* (lastKnown) {
+                    const status = typeof lastKnown?.status === "string" ? lastKnown.status : "";
+                    const rawTerminalReason = typeof lastKnown?.terminalReason === "string"
+                        ? lastKnown.terminalReason
+                        : "";
+                    // terminal_reason is either a bare reason ("dispatch_payload_missing")
+                    // or "error:<errorCode>" when derived from a terminal error event.
+                    const terminalReason = rawTerminalReason.replace(/^error:/, "");
+                    if (status === "completed") {
+                        // The turn finished server-side; the events we managed to fold are
+                        // the durable transcript. Finalize with them.
+                        settleInterruptedToolCalls(content, undefined, {
+                            includeActivity: true,
+                        });
+                        yield {
+                            content: [...content],
+                            status: { type: "complete", reason: "stop" },
+                            metadata: { custom: { ...(runId ? { runId } : {}) } },
+                        };
+                        clearActiveRun();
+                        return;
+                    }
+                    const mappedMessage = terminalReason
+                        ? BACKGROUND_TERMINAL_REASON_MESSAGES[terminalReason]
+                        : undefined;
+                    if (mappedMessage) {
+                        yield* emitBackgroundTerminalError({
+                            message: mappedMessage,
+                            errorCode: terminalReason,
+                            details: `terminal_reason: ${rawTerminalReason}`,
+                        });
+                        return;
+                    }
+                    if (lastRecoverableRunError) {
+                        // A replayed terminal error event already carried the real
+                        // message (recoverable errors replay as auto-continue signals
+                        // whose errorInfo we captured while following).
+                        yield* emitBackgroundTerminalError({
+                            message: lastRecoverableRunError.message,
+                            errorCode: lastRecoverableRunError.errorCode ?? "connection_error",
+                            details: lastRecoverableRunError.details,
+                        });
+                        return;
+                    }
+                    yield* emitBackgroundTerminalError({
+                        message: "The agent's background run stopped before finishing and no continuation appeared. You can retry from the preserved chat context.",
+                        errorCode: terminalReason || "background_run_lost",
+                        details: rawTerminalReason
+                            ? `terminal_reason: ${rawTerminalReason}`
+                            : undefined,
+                    });
+                };
+                // One read-only attach to the current run's event stream. Unlike
+                // reconnectCurrentRun this does NOT retry internally and lets the
+                // follow loop see every auto-continue signal (with errorInfo intact),
+                // because the loop — not this reader — decides what a detach means.
+                // Return values: "completed" = terminal event consumed (turn over);
+                // "aborted" = user abort; "detached" = we ATTACHED to a live stream
+                // that ended/stalled without a terminal event (resets the follow
+                // loop's idle window); "gone" = could not attach at all (404, HTTP
+                // error, network failure — the idle window keeps accumulating so a
+                // persistently unattachable run still terminates loudly).
+                const followAttachOnce = async function* () {
+                    if (!runId)
+                        return "gone";
+                    let attached = false;
+                    try {
+                        const eventsRes = await fetch(`${apiUrl}/runs/${encodeURIComponent(runId)}/events?after=${lastSeq + 1}`, { signal: abortSignal });
+                        if (!eventsRes.ok || !eventsRes.body) {
+                            return "gone";
+                        }
+                        attached = true;
+                        updateCurrentRunDispatchMode(eventsRes.headers.get("X-Dispatch-Mode"));
+                        for await (const result of readSSEStream(eventsRes.body, content, toolCallCounter, tabId, (seq) => {
+                            rememberRunSeq(seq);
+                            if (threadId)
+                                updateActiveRunSeq(seq);
+                        }, runId, currentSSEOptions())) {
+                            yield withRequestModeMetadata(result);
+                        }
+                        // readSSEStream returned normally: a terminal done/error was
+                        // consumed and rendered — the turn is over.
+                        clearActiveRun();
+                        return "completed";
+                    }
+                    catch (attachErr) {
+                        if (attachErr instanceof Error && attachErr.name === "AbortError") {
+                            clearActiveRun();
+                            return "aborted";
+                        }
+                        if (attachErr instanceof AgentAutoContinueSignal) {
+                            lastAutoContinueReason = attachErr.reason;
+                            if (attachErr.activityTrail.length > 0) {
+                                lastActivityTrail = [...attachErr.activityTrail];
+                            }
+                            if (attachErr.errorInfo) {
+                                lastRecoverableRunError = attachErr.errorInfo;
+                            }
+                        }
+                        // A fetch that never yielded a response body counts as "gone" for
+                        // the idle window; a broken mid-stream read counts as attached.
+                        return attached ? "detached" : "gone";
+                    }
+                };
+                // The background follow loop. Entered instead of ANY synthetic
+                // continuation POST when the current run is background-dispatched.
+                // Keeps following as long as an active run for this turn exists — the
+                // server keeps chaining until its own per-turn budget or a real
+                // terminal error, so a successor that stalls again is simply
+                // re-followed, not counted against any client budget. Only a
+                // continuous BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS with no active run
+                // ends the turn from the client side.
+                const followBackgroundTurn = async function* (initialSignal) {
+                    lastAutoContinueReason = initialSignal.reason;
+                    if (initialSignal.activityTrail.length > 0) {
+                        lastActivityTrail = [...initialSignal.activityTrail];
+                    }
+                    if (initialSignal.errorInfo) {
+                        lastRecoverableRunError = initialSignal.errorInfo;
+                    }
+                    // Show "Resuming…" instead of a frozen Thinking label while waiting
+                    // for the server-chained successor chunk.
+                    const dispatchResumingUiEvent = () => {
+                        if (typeof window !== "undefined") {
+                            window.dispatchEvent(new CustomEvent("agent-chat:auto-continue", {
+                                detail: { tabId },
+                            }));
+                        }
+                    };
+                    dispatchResumingUiEvent();
+                    let idleSince = null;
+                    let lastSeenActive = null;
+                    // Terminal runs already replayed once. A recoverable terminal error
+                    // event replays as an auto-continue signal (isAutoRecoverableError),
+                    // which used to re-drive a POST in foreground mode. In follow mode
+                    // the server owns recovery, so a SECOND visit to the same terminal
+                    // run means the turn is over — surface the outcome instead of
+                    // re-replaying it every poll for the whole reconnect window.
+                    const replayedTerminalRunIds = new Set();
+                    while (true) {
+                        if (abortSignal.aborted) {
+                            clearActiveRun();
+                            return;
+                        }
+                        let active = null;
+                        try {
+                            const activeRes = await fetch(`${apiUrl}/runs/active?threadId=${encodeURIComponent(threadId)}`, { signal: abortSignal });
+                            if (activeRes.ok) {
+                                active = await activeRes.json().catch(() => null);
+                            }
+                        }
+                        catch (pollErr) {
+                            if (pollErr instanceof Error && pollErr.name === "AbortError") {
+                                clearActiveRun();
+                                return;
+                            }
+                            // Transient poll failure — counts as "no active run" this tick.
+                        }
+                        const activeRunId = active?.active === true && active.runId
+                            ? String(active.runId)
+                            : null;
+                        const activeTurnId = typeof active?.turnId === "string" ? active.turnId : "";
+                        const activeStatus = typeof active?.status === "string" ? active.status : "";
+                        // Only follow runs belonging to THIS turn (server-chained
+                        // successors reuse the turnId) or runs we already attached to.
+                        const isOurRun = activeRunId !== null &&
+                            (attemptedRunIds.includes(activeRunId) ||
+                                !activeTurnId ||
+                                activeTurnId === turnId);
+                        if (activeRunId && isOurRun) {
+                            lastSeenActive = active;
+                            updateCurrentRunDispatchMode(active?.dispatchMode);
+                            const isTerminal = activeStatus === "completed" || activeStatus === "errored";
+                            if (isTerminal && replayedTerminalRunIds.has(activeRunId)) {
+                                yield* emitBackgroundTerminalOutcome(active);
+                                return;
+                            }
+                            const previousRunId = runId;
+                            runId = activeRunId;
+                            if (!attemptedRunIds.includes(activeRunId)) {
+                                attemptedRunIds.push(activeRunId);
+                            }
+                            reconnectCursorForRun(activeRunId, previousRunId);
+                            if (threadId) {
+                                setActiveRun({ threadId, runId: activeRunId, lastSeq });
+                            }
+                            const attach = yield* followAttachOnce();
+                            if (attach === "completed" || attach === "aborted") {
+                                return;
+                            }
+                            if (isTerminal) {
+                                replayedTerminalRunIds.add(activeRunId);
+                            }
+                            if (attach === "detached") {
+                                // We really attached to a live stream (chunk boundary or
+                                // transport blip) — the run demonstrably exists; keep
+                                // following with a fresh idle window.
+                                idleSince = null;
+                            }
+                            else {
+                                // "gone": /runs/active reported the run but its event stream
+                                // 404s (reaped row / cross-isolate lag). Let the idle window
+                                // ACCUMULATE instead of resetting, so a persistently
+                                // inconsistent state still terminates loudly instead of
+                                // hot-looping for the whole reconnect window.
+                                if (idleSince === null)
+                                    idleSince = Date.now();
+                                if (Date.now() - idleSince >=
+                                    BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS) {
+                                    yield* emitBackgroundTerminalOutcome(lastSeenActive);
+                                    return;
+                                }
+                            }
+                            dispatchResumingUiEvent();
+                        }
+                        else {
+                            if (idleSince === null)
+                                idleSince = Date.now();
+                            if (Date.now() - idleSince >= BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS) {
+                                captureChatClientError(new Error("Background run went idle with no successor run appearing"), "background-follow-idle", {
+                                    lastSeenStatus: typeof lastSeenActive?.status === "string"
+                                        ? lastSeenActive.status
+                                        : null,
+                                });
+                                yield* emitBackgroundTerminalOutcome(lastSeenActive);
+                                return;
+                            }
+                        }
+                        await delay(BACKGROUND_FOLLOW_POLL_INTERVAL_MS, abortSignal);
+                        if (abortSignal.aborted) {
+                            clearActiveRun();
+                            return;
+                        }
+                    }
+                };
                 const visibleContentForContinuation = () => {
                     return contentAfterContinuationPrefix(content, visibleContinuationPrefix);
                 };
@@ -2027,6 +2417,17 @@ export function createAgentChatAdapter(options) {
                             return;
                         }
                         if (err instanceof AgentAutoContinueSignal) {
+                            // Background-dispatched runs: the server chains continuations
+                            // itself (successor row pre-inserted before the old chunk
+                            // completes). Never POST a synthetic continuation and never
+                            // abort the live server-side run — switch to read-only
+                            // following of server state instead. This is the fix for the
+                            // client/server recovery race: client watchdog signals here are
+                            // just "reattach", not "recover".
+                            if (isBackgroundDispatch() && threadId) {
+                                yield* followBackgroundTurn(err);
+                                return;
+                            }
                             if (err.reason === "no_progress") {
                                 await abortCurrentRun();
                             }
@@ -2181,6 +2582,15 @@ export function createAgentChatAdapter(options) {
                         const activeReconnected = yield* reconnectActiveRunForThread();
                         if (activeReconnected)
                             return;
+                        // Background-dispatched run whose transport failed and could not
+                        // be reconnected above: follow server state instead of the
+                        // synthetic-continuation POST below. (An initial POST that failed
+                        // before any response headers arrived has no dispatch mode yet
+                        // and keeps the foreground startup-retry path.)
+                        if (isBackgroundDispatch() && threadId) {
+                            yield* followBackgroundTurn(new AgentAutoContinueSignal({ reason: "stream_ended" }));
+                            return;
+                        }
                         if (err instanceof AgentStartupTimeoutError) {
                             if (startupRecoveryAttempts < MAX_STARTUP_RECOVERY_ATTEMPTS) {
                                 await retryDelay(startupRecoveryAttempts++, abortSignal);