@agent-native/core 0.84.67 → 0.85.1

package/dist/agent/production-agent.js

@@ -32,7 +32,7 @@ import { getDefaultMaxIterations, MAX_AGENT_MAX_ITERATIONS, MIN_AGENT_MAX_ITERAT
 import { maybeCompactThread, buildObservationalContext, hasObservationalMemory, serializeObservationalMemoryBlock, } from "./observational-memory/index.js";
 import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js";
 import { startRun, subscribeToRun, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, tryClaimRunSlot, } from "./run-manager.js";
-import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, insertRun, updateRunHeartbeat, updateRunStatusIfRunning, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, } from "./run-store.js";
+import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, insertRun, updateRunHeartbeat, updateRunStatusIfRunning, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, } from "./run-store.js";
 import { classifyToolCallJournal, findCompletedJournalEntry, } from "./tool-call-journal.js";
 import { redactSensitiveFields, sanitizeToolErrorText, sanitizeToolErrorValue, } from "./tool-error-redaction.js";
 import { createToolSearchEntry, TOOL_SEARCH_ACTION_NAME, } from "./tool-search.js";
@@ -3460,6 +3460,15 @@ export function shouldChainBackgroundContinuation(opts) {
         endsAtContinuationBoundary(opts.run) &&
         opts.continuationCount < MAX_BACKGROUND_RUN_CONTINUATIONS);
 }
+export async function markBackgroundContinuationChunkTerminal(opts) {
+    const updateStatus = opts.deps?.updateRunStatusIfRunning ?? updateRunStatusIfRunning;
+    const setTerminalReason = opts.deps?.setRunTerminalReason ?? setRunTerminalReason;
+    const updated = await updateStatus(opts.runId, "completed");
+    if (updated) {
+        await setTerminalReason(opts.runId, opts.continuationReason);
+    }
+    return updated;
+}
 export async function claimBackgroundWorkerRunEarly(opts) {
     const record = opts.deps?.recordRunDiagnostic ?? recordRunDiagnostic;
     const insert = opts.deps?.insertRun ?? insertRun;
@@ -4246,8 +4255,13 @@ export function createProductionAgentHandler(options) {
                 // Insert the run row up front so /runs/active sees it immediately and
                 // the slot stays held while the background function cold-starts. Mark
                 // it background-dispatched so the stale reaper uses the wider window.
+                // The full request body is persisted ON the row (dispatch_payload) so
+                // the self-POST below can carry only the tiny marker — Netlify caps
+                // background-function request bodies at 256KB, and a large chat
+                // history (inline attachments especially) silently exceeded that.
                 await insertRun(runId, effectiveThreadId, effectiveTurnId, {
                     dispatchMode: "background",
+                    dispatchPayload: JSON.stringify(body),
                 });
                 backgroundRowInserted = true;
             }
@@ -4275,15 +4289,29 @@ export function createProductionAgentHandler(options) {
                     // the Authorization Bearer HMAC is preserved either way.
                     path: backgroundDispatchPath,
                     taskId: runId,
-                    body: {
-                        ...body,
-                        // Carry the pre-claimed identity so the worker reuses this run.
-                        [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
-                            runId,
-                            turnId: effectiveTurnId,
-                            backgroundFunctionRuntimeExpected: expectsNetlifyBackgroundFunction,
+                    // When the row (and its persisted payload) landed, send only the
+                    // marker — the worker rehydrates the body from dispatch_payload
+                    // (`payloadRef`), keeping the self-POST far under Netlify's 256KB
+                    // background-function body cap. If the insert failed we fall back to
+                    // carrying the full body inline, exactly as before.
+                    body: backgroundRowInserted
+                        ? {
+                            [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
+                                runId,
+                                turnId: effectiveTurnId,
+                                backgroundFunctionRuntimeExpected: expectsNetlifyBackgroundFunction,
+                                payloadRef: true,
+                            },
+                        }
+                        : {
+                            ...body,
+                            // Carry the pre-claimed identity so the worker reuses this run.
+                            [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
+                                runId,
+                                turnId: effectiveTurnId,
+                                backgroundFunctionRuntimeExpected: expectsNetlifyBackgroundFunction,
+                            },
                         },
-                    },
                 });
                 dispatched = true;
             }
@@ -4503,6 +4531,23 @@ export function createProductionAgentHandler(options) {
                     // foreground fallback, which is not a worker and rides the
                     // connected client's auto_continue instead.)
                     if (willChainBackgroundContinuation(run)) {
+                        // DURABLE PER-TURN LEDGER: bound the total number of runs one
+                        // logical turn may consume, counted in SQL. The in-marker
+                        // `continuationCount` resets whenever a fresh POST starts a new
+                        // chain for the same turn (client recovery, duplicate delivery),
+                        // so it cannot bound cross-chain loops — the SQL count survives
+                        // every recovery path and is what actually kills a pathological
+                        // turn (the "dozens of runs on one prompt" incident class).
+                        const turnRunCount = await countRunsForTurn(effectiveThreadId, effectiveTurnId).catch(() => null);
+                        if (turnRunCount !== null &&
+                            turnRunCount > MAX_BACKGROUND_RUN_CONTINUATIONS + 5) {
+                            console.error(`[agent-chat] turn ${effectiveTurnId} consumed ${turnRunCount} runs — refusing to chain further`, runId);
+                            const statusUpdated = await updateRunStatusIfRunning(runId, "errored").catch(() => false);
+                            if (statusUpdated) {
+                                await setRunTerminalReason(runId, "turn_continuation_budget_exhausted").catch(() => { });
+                            }
+                            return;
+                        }
                         // Mint the next chunk's runId here and sign the dispatch token
                         // over it, so the `_process-run` route's HMAC check and the
                         // worker's run identity agree. Fresh runId (not this chunk's) so
@@ -4513,37 +4558,126 @@ export function createProductionAgentHandler(options) {
                         const continuationReason = backgroundContinuationReasonForRun(run);
                         const continuationDispatchPath = resolveAgentChatProcessRunDispatchPath();
                         const continuationExpectsNetlifyBackgroundFunction = dispatchPathTargetsNetlifyBackgroundFunction(continuationDispatchPath);
+                        const continuationMarker = {
+                            runId: nextRunId,
+                            turnId: effectiveTurnId,
+                            continuationCount: backgroundContinuationCount + 1,
+                            continuationReason,
+                            ...(actionPreparationTool ? { actionPreparationTool } : {}),
+                            backgroundFunctionRuntimeExpected: continuationExpectsNetlifyBackgroundFunction,
+                        };
+                        // Strip this chunk's own marker before persisting/forwarding —
+                        // the next chunk gets the fresh marker above.
+                        const continuationBody = {
+                            ...body,
+                            internalContinuation: true,
+                        };
+                        delete continuationBody[AGENT_CHAT_BACKGROUND_RUN_FIELD];
                         try {
                             await recordRunDiagnostic(run.runId, RUN_DIAG_STAGE.workerSetupStep, `chain_dispatch_start nextRunId=${nextRunId} reason=${continuationReason} path=${continuationDispatchPath}`).catch(() => { });
-                            await fireInternalDispatch({
-                                event,
-                                // Continuation chunks use the same path resolution as the
-                                // initial dispatch: on hosted Netlify the background
-                                // function's DEFAULT url (no custom config.path; async via
-                                // background:true; never shadowed because /.netlify/* is
-                                // excluded from the /* catch-all) so each chunk keeps the
-                                // 15-min budget; off-Netlify the in-process framework route.
-                                path: continuationDispatchPath,
-                                taskId: nextRunId,
-                                body: {
-                                    ...body,
+                            // ── TRANSACTIONAL HANDOFF ──────────────────────────────────
+                            // 1. Insert the successor row (with its rehydration payload)
+                            //    BEFORE firing the dispatch, so:
+                            //    - /runs/active shows an active run continuously across
+                            //      the chunk boundary (no idle gap for the client to
+                            //      misread as "the turn ended"), and
+                            //    - a lost dispatch leaves a row the unclaimed-run sweep
+                            //      reaps into a LOUD error instead of a silent hang.
+                            // 2. Await the dispatch response fully (`awaitResponse`) —
+                            //    this worker's Lambda is about to finish, and the old
+                            //    250ms settle race let a still-in-flight handoff fetch be
+                            //    killed by the post-return freeze WITHOUT rejecting: the
+                            //    turn just stopped, silently. A Netlify background
+                            //    function 202s on enqueue (normally well under a second),
+                            //    and this chunk has minutes of budget headroom left, so
+                            //    awaiting is cheap. Retried with backoff for transient
+                            //    network blips.
+                            let nextRowInserted = false;
+                            try {
+                                await insertRun(nextRunId, effectiveThreadId, effectiveTurnId, {
+                                    dispatchMode: "background",
+                                    dispatchPayload: JSON.stringify(continuationBody),
+                                });
+                                nextRowInserted = true;
+                            }
+                            catch (insertErr) {
+                                await recordRunDiagnostic(run.runId, RUN_DIAG_STAGE.workerSetupStep, `chain_successor_insert_failed nextRunId=${nextRunId} ${insertErr instanceof Error
+                                    ? insertErr.message
+                                    : String(insertErr)}`).catch(() => { });
+                                console.error("[agent-chat] continuation insertRun failed; dispatching with inline body:", insertErr instanceof Error
+                                    ? insertErr.message
+                                    : insertErr);
+                            }
+                            const dispatchBody = nextRowInserted
+                                ? {
                                     internalContinuation: true,
                                     [AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
-                                        runId: nextRunId,
-                                        turnId: effectiveTurnId,
-                                        continuationCount: backgroundContinuationCount + 1,
-                                        continuationReason,
-                                        ...(actionPreparationTool
-                                            ? { actionPreparationTool }
-                                            : {}),
-                                        backgroundFunctionRuntimeExpected: continuationExpectsNetlifyBackgroundFunction,
+                                        ...continuationMarker,
+                                        payloadRef: true,
                                     },
-                                },
-                                settleMs: continuationExpectsNetlifyBackgroundFunction
-                                    ? BACKGROUND_CLAIM_GRACE_MS
-                                    : undefined,
-                            });
+                                }
+                                : {
+                                    ...continuationBody,
+                                    [AGENT_CHAT_BACKGROUND_RUN_FIELD]: continuationMarker,
+                                };
+                            let dispatched = false;
+                            let lastDispatchErr;
+                            for (let attempt = 0; attempt < 3 && !dispatched; attempt++) {
+                                try {
+                                    if (attempt > 0) {
+                                        await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** attempt));
+                                        // Keep the pre-inserted successor row visibly alive
+                                        // while we retry: the awaited attempts + backoff can
+                                        // outlast UNCLAIMED_BACKGROUND_RUN_GRACE_MS (25s), and
+                                        // without a fresh heartbeat the unclaimed-run reaper /
+                                        // sweep could error a handoff we are still delivering.
+                                        if (nextRowInserted) {
+                                            await updateRunHeartbeat(nextRunId).catch(() => { });
+                                        }
+                                    }
+                                    await fireInternalDispatch({
+                                        event,
+                                        // Continuation chunks use the same path resolution as
+                                        // the initial dispatch: on hosted Netlify the background
+                                        // function's DEFAULT url (no custom config.path; async
+                                        // via background:true; never shadowed because
+                                        // /.netlify/* is excluded from the /* catch-all) so each
+                                        // chunk keeps the 15-min budget; off-Netlify the
+                                        // in-process framework route.
+                                        path: continuationDispatchPath,
+                                        taskId: nextRunId,
+                                        body: dispatchBody,
+                                        awaitResponse: true,
+                                        responseTimeoutMs: 15_000,
+                                    });
+                                    dispatched = true;
+                                }
+                                catch (dispatchErr) {
+                                    lastDispatchErr = dispatchErr;
+                                    console.error(`[agent-chat] background continuation dispatch attempt ${attempt + 1} failed:`, dispatchErr instanceof Error
+                                        ? dispatchErr.message
+                                        : dispatchErr);
+                                }
+                            }
+                            if (!dispatched) {
+                                // The pre-inserted successor row would otherwise sit
+                                // unclaimed until the sweep reaps it — error it now so the
+                                // failure is immediate and truthful.
+                                if (nextRowInserted) {
+                                    const nextStatusUpdated = await updateRunStatusIfRunning(nextRunId, "errored").catch(() => false);
+                                    if (nextStatusUpdated) {
+                                        await setRunTerminalReason(nextRunId, "background_continuation_dispatch_failed").catch(() => { });
+                                    }
+                                }
+                                throw lastDispatchErr instanceof Error
+                                    ? lastDispatchErr
+                                    : new Error(String(lastDispatchErr));
+                            }
                             await recordRunDiagnostic(run.runId, RUN_DIAG_STAGE.workerSetupStep, `chain_dispatch_sent nextRunId=${nextRunId} reason=${continuationReason}`).catch(() => { });
+                            await markBackgroundContinuationChunkTerminal({
+                                runId: run.runId,
+                                continuationReason,
+                            }).catch(() => { });
                         }
                         catch (chainErr) {
                             // Chain dispatch failed — fail loud so the held row goes