@agent-native/core 0.70.3 → 0.72.0

package/dist/agent/production-agent.js

@@ -12,6 +12,8 @@ import { isDemoModeEnabled } from "../demo/config.js";
 import { redactDemoData, redactDemoString } from "../demo/redact.js";
 import { redactSensitiveFields, sanitizeToolErrorText, sanitizeToolErrorValue, } from "./tool-error-redaction.js";
 import { startRun, subscribeToRun, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, tryClaimRunSlot, } from "./run-manager.js";
+import { AGENT_CHAT_PROCESS_RUN_PATH, AGENT_CHAT_BACKGROUND_RUN_FIELD, isAgentChatDurableBackgroundEnabled, } from "./durable-background.js";
+import { fireInternalDispatch } from "../server/self-dispatch.js";
 import { readBody } from "../server/h3-helpers.js";
 import { isReadOnlyShellCommand } from "../coding-tools/index.js";
 import { getRequestRunContext, ensureRequestRunContext, getRequestOrgId, getRequestUserEmail, } from "../server/request-context.js";
@@ -22,7 +24,7 @@ import { createToolSearchEntry, TOOL_SEARCH_ACTION_NAME, } from "./tool-search.j
 import { getDefaultMaxIterations, normalizeMaxIterations, readAgentLoopSettings, } from "./loop-settings.js";
 import { isReasoningEffort, normalizeReasoningEffortForModel, } from "../shared/reasoning-effort.js";
 import { isAgentActionStopError } from "../action.js";
-import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, } from "./run-store.js";
+import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, insertRun, updateRunHeartbeat, updateRunStatusIfRunning, claimBackgroundRun, } from "./run-store.js";
 import { classifyToolCallJournal, findCompletedJournalEntry, } from "./tool-call-journal.js";
 import { preUploadAttachments } from "../file-upload/pre-upload-attachments.js";
 import { extensionIdFromPathname } from "../extensions/path.js";
@@ -2158,6 +2160,29 @@ export async function runAgentLoop(opts) {
                         approvalKey,
                         ...(toolCall.id ? { toolCallId: toolCall.id } : {}),
                     });
+                    // Audit the blocked attempt: the action did NOT run, but "the agent
+                    // tried to do X and was gated" is itself worth recording. Best-effort,
+                    // but AWAITED (not fire-and-forget) so the row isn't lost to a
+                    // serverless freeze / request teardown when the turn pauses.
+                    try {
+                        const { recordActionAudit } = await import("../audit/record.js");
+                        await recordActionAudit({
+                            config: undefined,
+                            args: toolCall.input,
+                            ctx: {
+                                actionName: toolCall.name,
+                                caller: "tool",
+                                userEmail: getRequestUserEmail(),
+                                orgId: getRequestOrgId() ?? null,
+                                ...(opts.threadId ? { threadId: opts.threadId } : {}),
+                                ...(opts.turnId ? { turnId: opts.turnId } : {}),
+                            },
+                            status: "denied",
+                        });
+                    }
+                    catch {
+                        // Best-effort — auditing must never break the approval pause.
+                    }
                     const result = `Awaiting human approval to run "${toolCall.name}". This action did ` +
                         `NOT execute — a human must approve this specific call before it ` +
                         `can run. The turn is paused; do not retry.`;
@@ -2378,6 +2403,11 @@ export async function runAgentLoop(opts) {
                     caller: "tool",
                     attachments: opts.attachments,
                     signal,
+                    // Audit attribution: the action name + the agent thread/turn that
+                    // triggered this call, so a mutation can be traced to its run.
+                    actionName: toolCall.name,
+                    ...(opts.threadId ? { threadId: opts.threadId } : {}),
+                    ...(opts.turnId ? { turnId: opts.turnId } : {}),
                 }));
                 // When the run is aborted (soft-timeout / user cancel) while this tool
                 // call is in flight, Promise.race below will throw "Run aborted" and the
@@ -2675,6 +2705,27 @@ function endsAtInternalContinuationBoundary(run) {
     }
     return last.type === "error" && isRecoverableContinuationError(last);
 }
+/**
+ * Hard cap on server-driven background→background continuation chunks for a
+ * single logical turn. A `backgroundFunction` run gets a ~13-min soft timeout,
+ * so reaching this boundary at all is the rare exception (most turns finish in
+ * one chunk). The cap bounds a pathological turn that would otherwise chain
+ * background invocations forever, mirroring `MAX_AGENT_TEAM_CONTINUATIONS`.
+ */
+export const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
+/**
+ * Whether the background worker should self-fire the next server-driven
+ * continuation chunk. True only when this is a background worker run that ended
+ * at a recoverable soft-timeout boundary (not aborted/stopped) and the chain is
+ * still under its budget. Extracted so the decision is unit testable without
+ * booting the whole handler.
+ */
+export function shouldChainBackgroundContinuation(opts) {
+    return (opts.isBackgroundWorker &&
+        opts.run.status !== "aborted" &&
+        endsAtInternalContinuationBoundary(opts.run) &&
+        opts.continuationCount < MAX_BACKGROUND_RUN_CONTINUATIONS);
+}
 function progressStepFromAgentChatEvent(event) {
     switch (event.type) {
         case "activity":
@@ -2729,14 +2780,46 @@ export function createProductionAgentHandler(options) {
             return { error: "Method not allowed" };
         }
         let body;
-        try {
-            body = await readBody(event);
+        // The durable-background `_process-run` route already consumed and verified
+        // the request body (h3 v2's web Request body stream is single-use, so a
+        // second readBody would fail). It stashes the verified+augmented body here
+        // so this re-entered handler reads it instead of the spent stream.
+        const preInjectedBody = event?.context
+            ?.__agentChatBackgroundBody;
+        if (preInjectedBody && typeof preInjectedBody === "object") {
+            body = preInjectedBody;
         }
-        catch {
-            setResponseStatus(event, 400);
-            return { error: "Invalid request body" };
+        else {
+            try {
+                body = await readBody(event);
+            }
+            catch {
+                setResponseStatus(event, 400);
+                return { error: "Invalid request body" };
+            }
         }
         const { message, history = [], structuredHistory, references = [], threadId, attachments, displayMessage, internalContinuation, turnId: requestTurnId, model: requestModel, engine: requestEngine, effort: requestEffort, browserTabId, scope, trackInRunsTray, } = body;
+        // Durable-background marker. Present ONLY when this handler was re-entered
+        // as the Netlify background worker via the `_process-run` self-dispatch
+        // (the route HMAC-verifies the dispatch before invoking us). When set, we
+        // run the loop inline with the background soft-timeout, reusing the
+        // pre-claimed runId/turnId — we must NOT re-claim the slot or re-dispatch.
+        const backgroundRunMarker = body[AGENT_CHAT_BACKGROUND_RUN_FIELD] &&
+            typeof body[AGENT_CHAT_BACKGROUND_RUN_FIELD] === "object" &&
+            typeof body[AGENT_CHAT_BACKGROUND_RUN_FIELD].runId === "string"
+            ? body[AGENT_CHAT_BACKGROUND_RUN_FIELD]
+            : null;
+        const isBackgroundWorker = backgroundRunMarker !== null;
+        // How many server-driven background continuations have already chained into
+        // this logical turn (0 on the first chunk). Used to bound the chain.
+        const backgroundContinuationCount = isBackgroundWorker &&
+            typeof backgroundRunMarker?.continuationCount === "number" &&
+            Number.isFinite(backgroundRunMarker.continuationCount)
+            ? Math.max(0, Math.floor(backgroundRunMarker.continuationCount))
+            : 0;
+        // The foreground POST decides whether to dispatch into a background
+        // function. The background worker itself never re-dispatches.
+        const dispatchToBackground = !isBackgroundWorker && isAgentChatDurableBackgroundEnabled();
         const requestBrowserTabId = normalizeBrowserTabId(browserTabId);
         const requestChatScope = normalizeChatScope(scope);
         const requestRunCtx = ensureRequestRunContext();
@@ -3154,7 +3237,11 @@ export function createProductionAgentHandler(options) {
         // different serverless isolates both see the correct state — a plain
         // read-then-act check races on multi-isolate deployments because both
         // reads see no running row before either insert commits.
-        if (threadId) {
+        //
+        // The background worker SKIPS this: the foreground POST already claimed the
+        // slot and inserted the run row before dispatching, so re-claiming here
+        // would falsely 409 against the row the foreground holds.
+        if (threadId && !isBackgroundWorker) {
             const slot = await tryClaimRunSlot(threadId);
             if (!slot.claimed) {
                 setResponseStatus(event, 409);
@@ -3164,17 +3251,62 @@ export function createProductionAgentHandler(options) {
                 };
             }
         }
-        // Start agent loop in background via run-manager
-        const runId = generateRunId();
+        // Start agent loop in background via run-manager. The background worker
+        // reuses the runId carried in the marker (signed into the dispatch token):
+        //  - First background chunk (count 0): the foreground generated + INSERTED
+        //    this runId, so the event stream the client is already subscribed to is
+        //    the one we write to.
+        //  - Chained continuation chunk (count > 0): the prior chunk minted a FRESH
+        //    runId for this one (a reused runId would restart `startRun`'s in-memory
+        //    seq log at 0 and collide with the prior chunk's persisted seqs, which
+        //    insertRunEvent's ON CONFLICT would drop — making the continuation
+        //    invisible). A fresh runId on the SAME thread + SAME turnId folds onto
+        //    one assistant message and is surfaced by the existing
+        //    `/runs/active?threadId` reconnect path. The continuation worker inserts
+        //    its own background row below (the foreground only inserted chunk-0's).
+        const isChainedBackgroundContinuation = isBackgroundWorker && backgroundContinuationCount > 0;
+        const runId = backgroundRunMarker?.runId ?? generateRunId();
         const effectiveThreadId = threadId ?? runId;
-        const effectiveTurnId = typeof requestTurnId === "string" && requestTurnId.trim()
-            ? requestTurnId.trim()
-            : runId;
+        const effectiveTurnId = typeof backgroundRunMarker?.turnId === "string" &&
+            backgroundRunMarker.turnId.trim()
+            ? backgroundRunMarker.turnId.trim()
+            : typeof requestTurnId === "string" && requestTurnId.trim()
+                ? requestTurnId.trim()
+                : runId;
         const messageToPersist = typeof requestDisplayMessage === "string" &&
             requestDisplayMessage.trim().length > 0
             ? requestDisplayMessage
             : requestMessage;
-        if (options.onRunPrepared && !internalContinuation) {
+        // Server-driven background continuation: when the background worker re-fired
+        // itself at a soft-timeout boundary (a chained continuation chunk), rebuild
+        // the conversation from the persisted thread_data so the next chunk resumes
+        // from committed progress instead of restarting from the original user
+        // message (which would re-do work — the re-hydration thrash the design doc
+        // calls out). Mirrors the agent-teams continuation, which seeds from
+        // thread_data + appends a continuation nudge. Falls back to the body-derived
+        // `messages` if thread_data is empty/unreadable — a continuation that
+        // restarts is worse than one that resumes, but both are correct.
+        if (isChainedBackgroundContinuation && effectiveThreadId) {
+            try {
+                const { getThread } = await import("../chat-threads/store.js");
+                const { threadDataToEngineMessages } = await import("./thread-data-builder.js");
+                const priorThreadData = (await getThread(effectiveThreadId))
+                    ?.threadData;
+                const resumed = threadDataToEngineMessages(priorThreadData);
+                if (resumed.length > 0) {
+                    appendAgentLoopContinuation(resumed, "run_timeout");
+                    messages.length = 0;
+                    messages.push(...resumed);
+                }
+            }
+            catch {
+                // Keep the body-derived messages — never drop the run.
+            }
+        }
+        // Persist the user's turn exactly once. The foreground POST does this
+        // before dispatching; the background worker must NOT repeat it (it re-enters
+        // with the same body, which would double-persist the user message).
+        if (options.onRunPrepared && !internalContinuation && !isBackgroundWorker) {
             await options.onRunPrepared({
                 runId,
                 threadId,
@@ -3182,6 +3314,73 @@ export function createProductionAgentHandler(options) {
                 attachments: requestAttachments,
             });
         }
+        // ─── Durable-background dispatch decision ──────────────────────────────
+        // Flag active (hosted + A2A_SECRET + AGENT_CHAT_DURABLE_BACKGROUND) and we
+        // are the FOREGROUND POST: insert the run row (marked background), fire an
+        // HMAC-signed self-dispatch into the Netlify background function (15-min
+        // budget), and return the SSE subscription immediately. The client streams
+        // the same events via the cross-isolate SQL-poll path with no client
+        // change. With the flag OFF this whole branch is skipped and the inline
+        // `startRun` path below runs exactly as before (byte-for-byte).
+        if (dispatchToBackground) {
+            try {
+                // Insert the run row up front so /runs/active sees it immediately and
+                // the slot stays held while the background function cold-starts. Mark
+                // it background-dispatched so the stale reaper uses the wider window.
+                await insertRun(runId, effectiveThreadId, effectiveTurnId, {
+                    dispatchMode: "background",
+                });
+            }
+            catch (err) {
+                // A duplicate-PK collision means the row already exists (ret­ried POST);
+                // any other failure means we can't safely hand off — fall back to the
+                // inline path rather than dropping the turn.
+                console.error("[agent-chat] background insertRun failed; falling back to inline:", err instanceof Error ? err.message : err);
+            }
+            let dispatched = false;
+            try {
+                await fireInternalDispatch({
+                    event,
+                    path: AGENT_CHAT_PROCESS_RUN_PATH,
+                    taskId: runId,
+                    body: {
+                        ...body,
+                        // Carry the pre-claimed identity so the worker reuses this run.
+                        [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
+                            runId,
+                            turnId: effectiveTurnId,
+                        },
+                    },
+                });
+                dispatched = true;
+            }
+            catch (err) {
+                console.error("[agent-chat] background dispatch failed; falling back to inline:", err instanceof Error ? err.message : err);
+            }
+            if (dispatched) {
+                const stream = subscribeToRun(runId, 0);
+                if (stream) {
+                    setResponseHeader(event, "Content-Type", "text/event-stream");
+                    setResponseHeader(event, "Cache-Control", "no-cache");
+                    setResponseHeader(event, "Connection", "keep-alive");
+                    setResponseHeader(event, "X-Run-Id", runId);
+                    return stream;
+                }
+                // Subscription failed even though dispatch landed — surface an error
+                // rather than silently running inline (the background worker is already
+                // processing this runId, so an inline second run would double-execute).
+                setResponseStatus(event, 500);
+                return { error: "Failed to subscribe to background run" };
+            }
+            // Dispatch failed before any worker could claim the run. Fail loud: flip
+            // the row terminal so the held slot is released (and any reconnect sees a
+            // terminal status instead of spinning), then 500 so the client can retry.
+            // We do NOT silently fall through to inline here — that risks a later,
+            // delayed background delivery double-executing the same runId.
+            await updateRunStatusIfRunning(runId, "errored").catch(() => { });
+            setResponseStatus(event, 500);
+            return { error: "Failed to dispatch background run" };
+        }
         const trackedProgressOwner = trackInRunsTray === true && ownerEmail ? ownerEmail : null;
         const trackedProgressRunId = trackedProgressOwner
             ? backgroundChatProgressRunId(effectiveTurnId)
@@ -3246,7 +3445,18 @@ export function createProductionAgentHandler(options) {
                 metadata: trackedProgressMetadata ?? undefined,
             }).catch(() => { });
         }
-        const handleRunComplete = options.onRunComplete || trackedProgressRunId
+        // The background worker must AWAIT the run to completion before returning,
+        // or Netlify freezes/kills the function the instant the handler returns and
+        // the detached run dies mid-turn (mirrors the agent-teams processor, which
+        // wraps startRun in `await new Promise(resolve => startRun(..., onComplete:
+        // () => resolve()))`). We resolve this when the run's onComplete fires.
+        let resolveBackgroundRunDone = null;
+        const backgroundRunDone = isBackgroundWorker
+            ? new Promise((resolve) => {
+                resolveBackgroundRunDone = resolve;
+            })
+            : null;
+        const baseHandleRunComplete = options.onRunComplete || trackedProgressRunId
             ? async (run) => {
                 try {
                     await options.onRunComplete?.(run, threadId);
@@ -3258,6 +3468,89 @@ export function createProductionAgentHandler(options) {
                 await completeTrackedProgressRun(run);
             }
             : undefined;
+        // Wrap so the background worker is unblocked even when there is no app
+        // onRunComplete / tracked-progress callback configured.
+        const handleRunComplete = isBackgroundWorker || baseHandleRunComplete
+            ? async (run) => {
+                try {
+                    // Persist the (partial) assistant turn to thread_data FIRST — the
+                    // server-driven continuation below rebuilds from it, so it must be
+                    // committed before we re-fire.
+                    await baseHandleRunComplete?.(run);
+                    // Server-driven background→background continuation. If this chunk
+                    // hit its soft-timeout still unfinished (ended at an auto_continue
+                    // / loop_limit / recoverable boundary), chain the next chunk by
+                    // re-firing the `_process-run` self-dispatch with mode "continue"
+                    // (carried as internalContinuation + an incremented count),
+                    // instead of relying on the client to re-POST. Mirrors the
+                    // agent-teams `fireInternalDispatch({ body: { mode: "continue" }})`
+                    // chain. Bounded by MAX_BACKGROUND_RUN_CONTINUATIONS. Aborted /
+                    // user-stopped runs do NOT chain.
+                    if (shouldChainBackgroundContinuation({
+                        isBackgroundWorker,
+                        run,
+                        continuationCount: backgroundContinuationCount,
+                    })) {
+                        // Mint the next chunk's runId here and sign the dispatch token
+                        // over it, so the `_process-run` route's HMAC check and the
+                        // worker's run identity agree. Fresh runId (not this chunk's) so
+                        // its seq log starts clean; same turnId folds the assistant
+                        // message across chunks.
+                        const nextRunId = generateRunId();
+                        try {
+                            await fireInternalDispatch({
+                                event,
+                                path: AGENT_CHAT_PROCESS_RUN_PATH,
+                                taskId: nextRunId,
+                                body: {
+                                    ...body,
+                                    internalContinuation: true,
+                                    [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
+                                        runId: nextRunId,
+                                        turnId: effectiveTurnId,
+                                        continuationCount: backgroundContinuationCount + 1,
+                                    },
+                                },
+                            });
+                        }
+                        catch (chainErr) {
+                            // Chain dispatch failed — fail loud so the held row goes
+                            // terminal instead of spinning. The reaper would also catch
+                            // it, but this is immediate and truthful.
+                            console.error("[agent-chat] background continuation dispatch failed:", chainErr instanceof Error ? chainErr.message : chainErr);
+                            await updateRunStatusIfRunning(runId, "errored").catch(() => { });
+                        }
+                    }
+                }
+                finally {
+                    resolveBackgroundRunDone?.();
+                }
+            }
+            : undefined;
+        // Background worker: claim the pre-inserted run idempotently before
+        // executing. A duplicate Netlify delivery loses the claim and no-ops here,
+        // so the run can never be double-executed. Bump the heartbeat immediately
+        // on entry so a slow cold-start doesn't leave the row looking stale to the
+        // reaper before startRun's 1.5s heartbeat timer takes over.
+        if (isBackgroundWorker) {
+            // A chained continuation chunk's runId was minted by the prior chunk and
+            // never inserted, so insert its background row now (idempotently — a
+            // duplicate Netlify delivery that already inserted it just PK-collides and
+            // the claim below dedups). The first chunk's row was inserted by the
+            // foreground, so skip the insert there.
+            if (isChainedBackgroundContinuation) {
+                await insertRun(runId, effectiveThreadId, effectiveTurnId, {
+                    dispatchMode: "background",
+                }).catch(() => { });
+            }
+            const won = await claimBackgroundRun(runId);
+            if (!won) {
+                // Already claimed by an earlier delivery — return a benign ack so
+                // Netlify doesn't retry a successful handoff.
+                return { ok: true, skipped: "already-claimed" };
+            }
+            await updateRunHeartbeat(runId).catch(() => { });
+        }
         startRun(runId, effectiveThreadId, async (rawSend, signal) => {
             const send = (event) => {
                 rawSend(event);
@@ -3591,11 +3884,24 @@ export function createProductionAgentHandler(options) {
         }, handleRunComplete, {
             softTimeoutMs: options.runSoftTimeoutMs,
             useHostedSoftTimeoutDefault: true,
+            // Inside the Netlify background function there is no ~60s wall, so lift
+            // the soft-timeout clamp to ~13min for THIS run only. Foreground runs
+            // never set this, so their 40s clamp is unchanged.
+            backgroundFunction: isBackgroundWorker,
             // Fold continuation runs of one logical turn onto a single durable
             // assistant message. Falls back to the runId (turn == run) when the
             // client doesn't supply a turnId.
             turnId: effectiveTurnId,
         });
+        // Background worker: await the run to completion so Netlify keeps the
+        // background function alive for the whole turn (the client is streaming the
+        // same events via the foreground POST's cross-isolate SQL subscription).
+        // The onComplete wrapper above resolves `backgroundRunDone`.
+        if (isBackgroundWorker) {
+            if (backgroundRunDone)
+                await backgroundRunDone;
+            return { ok: true, runId };
+        }
         // Subscribe to the run and stream events to the client
         const stream = subscribeToRun(runId, 0);
         if (!stream) {