@agent-native/core 0.168.13 → 0.169.1

package/dist/agent/run-manager.js

@@ -1,9 +1,10 @@
 import { getAppConfig } from "../app-config/index.js";
+import { BACKGROUND_AUTOMATION_SOFT_TIMEOUT_HEADROOM_MS, BACKGROUND_SOFT_TIMEOUT_CEILING_MS, RUN_NO_PROGRESS_HARD_TIMEOUT_MS, } from "../app-config/run-lifecycle-invariants.js";
 import { captureError } from "../server/capture-error.js";
 import { isLlmCredentialError, LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
 import { classifyTerminalErrorCode, describeErrorWithCauses, isProviderConnectionError, } from "./engine/error-detail.js";
 import { EngineError } from "./engine/types.js";
-import { insertRun, insertRunEvent, updateRunStatusIfRunning, markRunAborted, getRunAbortState, getRunStatus, getRunEventsSince, getRunById, getRunByThread, getRunTurnRef, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, terminalEventForAbortReason, } from "./run-store.js";
+import { insertRun, insertRunEvent, updateRunStatusIfRunning, markRunAborted, getRunAbortState, getRunStatus, getRunEventsSince, getRunById, getRunByThread, getRunTurnRef, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, terminalEventForAbortReason, } from "./run-store.js";
 import { isContinuationTerminalReason } from "./types.js";
 const activeRuns = new Map();
 const threadToRun = new Map();
@@ -43,68 +44,7 @@ export const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;
  * function (no ~60s wall, 15-min budget) and therefore can safely outlast 40s.
  */
 export const HOSTED_SOFT_TIMEOUT_CEILING_MS = 40_000;
-/**
- * Hard ceiling for the soft timeout when a run executes inside a Netlify
- * background function (any deployed function whose name ends in `-background`).
- * Background functions return 202 immediately and run detached for up to 15
- * minutes, so the ~60s synchronous function wall that 40s defends against does
- * NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard
- * kill to abort, persist the partial turn, write the terminal event, and (for
- * the rare >13-min turn) self-fire another background continuation.
- *
- * This ceiling is used ONLY when a caller explicitly opts in with
- * `backgroundFunction: true`. It does not change the foreground/interactive
- * ceiling and does not fire unless the durable-background path dispatched the
- * run into a background function. Per the design doc Guardrail, the 40s
- * interactive clamp stays correct for every non-background run.
- */
-export const BACKGROUND_SOFT_TIMEOUT_CEILING_MS = 13 * 60_000; // 780_000
-/**
- * Default soft-timeout budget for a background-function run when the caller
- * does not pass an explicit `softTimeoutMs`. Same value as the ceiling — we
- * want a background turn to use nearly its whole 15-min budget before handing
- * off to a chained background continuation.
- */
-export const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS = BACKGROUND_SOFT_TIMEOUT_CEILING_MS;
-/**
- * AUTHORITATIVE no-progress backstop for a run, enforced by the run manager
- * itself (timer-driven, independent of any layer below).
- *
- * The finer-grained watchdogs inside the agent loop (model-stream and
- * action-preparation no-progress, both 90s) only guard the model event stream
- * — a stall in any segment OUTSIDE that guarded loop (engine-call
- * establishment, worker setup between continuation chunks, a wedged transport
- * that emits keepalives while the loop never runs) previously hung forever
- * with the client watching keepalives. This backstop covers every segment by
- * construction: if no REAL progress event (see `shouldBumpProgressForEvent`;
- * keepalives and zero-byte prep activity don't count) lands for this long —
- * and no unit of work is in flight (see `inFlightWorkDelta`: tool calls,
- * cross-app calls, and the model stream all legitimately emit nothing for
- * minutes and each carry a bound of their own) — the run manager emits
- * `auto_continue { reason: "no_progress" }` and aborts the chunk, exactly
- * like the soft timeout, so the normal continuation machinery recovers it.
- *
- * Being numerically larger than the in-loop watchdogs is NOT what keeps this
- * from killing a healthy run, and treating it that way is what made it do so:
- * this clock and the loop's `lastModelStreamProgressAt` measure DIFFERENT
- * events. An extended-thinking phase bumps the inner clock on every engine
- * frame while forwarding nothing, so the inner watchdog correctly stayed quiet
- * and this one saw pure silence — runs whose worst gap crossed 150s died while
- * still streaming, some by a single second. Ordering between two clocks only
- * means something when they watch the same events; suspending on in-flight
- * work is what actually makes the two agree.
- *
- * This is now only the CEILING, not the value: `resolveRunNoProgressTimeoutMs`
- * clamps the foreground backstop to a fraction of the chunk's soft timeout
- * (~30s at a 40s chunk), which is BELOW the 90s in-loop watchdogs rather than
- * above them. That ordering is deliberate — the in-loop watchdogs could never
- * fire inside a hosted foreground chunk anyway, since the serverless wall
- * (~57-59s) arrives first. Proven durable-background chunks keep the full
- * `DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS` so large outputs can use the
- * background budget. Only armed when a soft-timeout regime is active (hosted
- * runs); local dev stays unbounded.
- */
-export const RUN_NO_PROGRESS_HARD_TIMEOUT_MS = 150_000;
+// 780_000
 /**
  * Default no-progress window for a run executing inside a proven durable
  * background function. A background worker that is heartbeating but has no
@@ -157,17 +97,38 @@ export function resolveRunNoProgressTimeoutMs(params) {
     const explicit = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0
         ? value
         : undefined;
+    // Per-call override wins, then configuration, then the shipped default —
+    // the same ladder `resolveRunSoftTimeoutMs` implements.
+    const configured = getAppConfig().agent;
+    // Largest window that can still fire inside the chunk it is guarding. A
+    // backstop at or above the chunk budget is not a loose backstop, it is an
+    // absent one.
+    const budgetCeilingMs = Math.floor(softTimeoutMs * FOREGROUND_NO_PROGRESS_SOFT_TIMEOUT_FRACTION);
     if (backgroundFunction === true) {
+        // The background override exists to RAISE this window, so it is honoured
+        // as given — a caller asking for a longer one is making an explicit choice
+        // and stays bounded by its own hard abort.
         const override = explicit(params.backgroundOverrideMs) ?? explicit(params.overrideMs);
-        if (override !== undefined)
-            return override;
-        return softTimeoutMs > 0 ? DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS : 0;
+        if (!(softTimeoutMs > 0))
+            return override ?? 0;
+        // Honoured as given, because this override exists to RAISE the window —
+        // but still bounded by the chunk it guards. A backstop at or above the
+        // chunk budget is not a longer backstop, it is an absent one, so a caller
+        // asking for one was disabling recovery without meaning to.
+        if (override !== undefined) {
+            return override === 0 ? 0 : Math.min(override, budgetCeilingMs);
+        }
+        // Clamped: returned flat, a deployment that lowered the GLOBAL
+        // `runSoftTimeoutMs` shrank the chunk without shrinking the backstop, and
+        // the backstop silently stopped being reachable. Nothing changes at the
+        // shipped values — min(150s, 0.75 x 13min) is still 150s.
+        return Math.min(configured.backgroundNoProgressTimeoutMs, budgetCeilingMs);
     }
     const override = explicit(params.overrideMs);
     // Local dev keeps runs unbounded unless a caller explicitly asks otherwise.
     if (!(softTimeoutMs > 0))
         return override ?? 0;
-    const ceiling = Math.min(RUN_NO_PROGRESS_HARD_TIMEOUT_MS, Math.floor(softTimeoutMs * FOREGROUND_NO_PROGRESS_SOFT_TIMEOUT_FRACTION));
+    const ceiling = Math.min(RUN_NO_PROGRESS_HARD_TIMEOUT_MS, budgetCeilingMs);
     if (override === undefined)
         return ceiling;
     return override === 0 ? 0 : Math.min(override, ceiling);
@@ -442,7 +403,7 @@ export function resolveRunSoftTimeoutMs(overrideMs, options) {
     // A background-function run uses the full background budget by default; the
     // foreground default (40s) is unchanged.
     if (background) {
-        return hosted ? DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS : 0;
+        return hosted ? BACKGROUND_SOFT_TIMEOUT_CEILING_MS : 0;
     }
     return options?.useHostedDefault && hosted
         ? DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS
@@ -456,6 +417,37 @@ export function resolveErroredRunRetentionMs() {
     return (getAppConfig().agent.erroredRunRetentionMs ??
         DEFAULT_ERRORED_RUN_RETENTION_MS);
 }
+/**
+ * Hard abort for one in-process background automation run.
+ *
+ * This is the host's real function budget for scheduled work, which is exactly
+ * the kind of number that differs between deployments — so it is configuration,
+ * not a module constant nobody outside this package can see.
+ */
+export function resolveBackgroundRunHardTimeoutMs() {
+    return getAppConfig().agent.backgroundRunHardTimeoutMs;
+}
+/**
+ * Chunk budget for a background automation, derived from the runner's OWN hard
+ * abort rather than from the durable-chat background ceiling.
+ *
+ * The shipped build took the 13-minute chat ceiling for a path whose process is
+ * killed at 10 minutes, which made the recoverable soft-timeout boundary dead
+ * code and left the terminal no-progress backstop as the only boundary an
+ * automation could ever reach. Deriving from the hard abort keeps
+ * `soft timeout < hard abort` true by construction; the invariant check asserts
+ * the headroom still fits.
+ */
+export function resolveBackgroundAutomationSoftTimeoutMs(overrideMs) {
+    const budget = Math.max(1_000, resolveBackgroundRunHardTimeoutMs() -
+        BACKGROUND_AUTOMATION_SOFT_TIMEOUT_HEADROOM_MS);
+    const resolved = resolveRunSoftTimeoutMs(overrideMs, {
+        useHostedDefault: true,
+        backgroundFunction: true,
+    });
+    // `0` means "no soft-timeout regime" (local dev) and is never clamped up.
+    return resolved > 0 ? Math.min(resolved, budget) : 0;
+}
 function isTerminalRunEvent(event) {
     return (event.type === "done" ||
         event.type === "error" ||
@@ -519,6 +511,49 @@ function terminalReasonForRun(finalStatus, terminalEvent, abortReason, completio
     return "done";
 }
 const MAX_RUN_ERROR_DETAIL_LENGTH = 500;
+/**
+ * One counter per chunk boundary, dimensioned by reason and by whether the
+ * turn continued past it.
+ *
+ * Boundaries are normal; boundaries that TERMINATE a run are not, and before
+ * this the two were indistinguishable from outside — which is how a 37%
+ * automation failure rate stayed invisible. The ratio between `recovered:true`
+ * and `recovered:false` is the number that belongs on a dashboard.
+ *
+ * Same swallow-everything mechanism as `emitRunTerminalTrackingEvent`: a
+ * missing or broken tracking provider can never affect the run.
+ */
+function emitRunBoundaryTrackingEvent(args) {
+    const properties = {
+        source: "agent_run_manager",
+        run_id: args.runId,
+        thread_id: args.threadId,
+        reason: args.reason,
+        recovered: args.recovered,
+        boundary_index: args.boundaryIndex,
+        dispatch_mode: args.dispatchMode,
+        model: args.model,
+        engine: args.engineName,
+    };
+    for (const key of Object.keys(properties)) {
+        if (properties[key] === undefined)
+            delete properties[key];
+    }
+    try {
+        void Promise.all([
+            import("../tracking/registry.js"),
+            import("../observability/tracking-identity.js"),
+        ])
+            .then(([{ track }, { trackingIdentityProperties }]) => {
+            track("agent_run_boundary", { ...properties, ...trackingIdentityProperties() }, { userId: args.userId });
+        })
+            .catch(() => { });
+        // coercion-ok: a boundary counter must never affect the run it counts.
+    }
+    catch {
+        // Tracking must never affect the agent run or its persisted status.
+    }
+}
 /**
  * Emit one analytics event per terminal run — the seam that makes cutoffs
  * (`run_budget_exhausted`, `loop_limit`, aborts, `truncated` continuation
@@ -547,7 +582,7 @@ function emitRunTerminalTrackingEvent(args) {
                 ? `${args.errorDetail.slice(0, MAX_RUN_ERROR_DETAIL_LENGTH)}…`
                 : args.errorDetail
             : undefined,
-        dispatch_mode: args.dispatchMode ?? "foreground",
+        dispatch_mode: args.dispatchMode,
         abort_reason: args.abortReason,
         duration_ms: args.durationMs,
         model: args.model,
@@ -618,6 +653,66 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         abortRun(existingRunId);
     }
     const abort = new AbortController();
+    // Chunk-scoped controller, only for a runFn that recovers boundaries in this
+    // invocation. `abort` stays the turn: a Stop, the cross-isolate abort check,
+    // and a caller's hard timeout all still end the run through it, and it always
+    // ends whichever chunk is executing under it.
+    const recoverChunkBoundaries = options?.recoverChunkBoundaries === true;
+    let chunkAbort = recoverChunkBoundaries
+        ? new AbortController()
+        : null;
+    let chunkBoundaryReason = null;
+    let recoveredChunkBoundaries = 0;
+    /** A boundary that has been reached but not yet proven recovered. */
+    let pendingBoundary = null;
+    /**
+     * Resolve the outstanding boundary once its fate is known: `true` when the
+     * caller actually opened another round, `false` when the run ended first.
+     */
+    const settleBoundary = (recovered) => {
+        const boundary = pendingBoundary;
+        if (!boundary)
+            return;
+        pendingBoundary = null;
+        recordRunBoundaryDiagnostic(boundary.reason, boundary.diagnostic, recovered ? "recovered" : "terminal");
+        emitRunBoundaryTrackingEvent({
+            runId,
+            threadId,
+            reason: boundary.reason,
+            recovered,
+            boundaryIndex: boundary.index,
+            dispatchMode: options?.dispatchMode,
+            model: options?.model,
+            engineName: options?.engineName,
+            userId: options?.userId,
+        });
+    };
+    if (chunkAbort) {
+        abort.signal.addEventListener("abort", () => {
+            chunkAbort?.abort(abort.signal.reason);
+        });
+    }
+    const runControl = {
+        get turnSignal() {
+            return abort.signal;
+        },
+        get chunkSignal() {
+            return chunkAbort?.signal ?? abort.signal;
+        },
+        chunkBoundaryReason: () => chunkBoundaryReason,
+        beginChunk: () => {
+            if (abort.signal.aborted || !chunkAbort)
+                return abort.signal;
+            settleBoundary(true);
+            chunkBoundaryReason = null;
+            chunkAbort = new AbortController();
+            // The boundary is behind us; the silence clock restarts with the chunk,
+            // or the backstop fires again on the elapsed time of the chunk it just
+            // ended and every recovery round dies instantly.
+            lastRealProgressAt = Date.now();
+            return chunkAbort.signal;
+        },
+    };
     let softTimedOut = false;
     let resolveFinalized = () => { };
     let rejectFinalized = () => { };
@@ -952,29 +1047,114 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             checkpointAbortInFlight = false;
         }
     };
-    const checkNoProgressBackstop = () => {
-        if (noProgressTimeoutMs <= 0)
-            return;
+    /**
+     * Localise the stall. `RUN_DIAG_STAGE`/`recordRunDiagnostic` existed for
+     * exactly this and were wired only into the `_process-run` HTTP path, which
+     * is why a 37%-failure-rate backstop could not name the segment it killed.
+     */
+    const recordRunBoundaryDiagnostic = (reason, diagnostic, disposition) => {
+        void recordRunDiagnostic(runId, RUN_DIAG_STAGE.runBoundaryReached, JSON.stringify({
+            reason,
+            disposition,
+            silentForMs: diagnostic.silentForMs,
+            lastEventType: diagnostic.lastEventType,
+            inFlightWorkCount,
+            eventCount: run.events.length,
+            elapsedMs: Date.now() - run.startedAt,
+        })).catch(() => { });
+    };
+    /**
+     * Reach a server-owned chunk boundary.
+     *
+     * Two outcomes, and the difference is the whole point: a runFn that recovers
+     * boundaries in this invocation gets its CHUNK aborted and keeps the turn;
+     * every other caller gets the turn ended so a fresh invocation can continue
+     * it. The recoverable case deliberately does NOT emit `auto_continue` or
+     * write a checkpoint terminal event — both describe a turn that stopped here,
+     * and this one has not: the checkpoint row is written at a reserved seq that
+     * outranks the real `done` this run is still going to emit, so persisting it
+     * would relabel a recovered run as truncated.
+     */
+    const reachRunBoundary = (reason, diagnostic = {}) => {
         if (run.status !== "running" || abort.signal.aborted)
             return;
-        if (inFlightWorkCount > 0)
-            return;
-        if (Date.now() - lastRealProgressAt < noProgressTimeoutMs)
+        if (chunkAbort) {
+            if (chunkAbort.signal.aborted)
+                return;
+            recoveredChunkBoundaries += 1;
+            console.warn(`[run-manager] chunk boundary (${reason}) — recovering in-invocation`, runId, diagnostic);
+            // NOT counted as recovered yet. `recovered` is the whole point of this
+            // counter — it answers "is the recovery working?" — so it has to mean a
+            // round actually started, not that one was invited to. The caller can
+            // still exhaust its budget or fail to build continuation context, and
+            // counting the invitation would over-report recovery, which is the
+            // direction that hides the failure.
+            pendingBoundary = { reason, diagnostic, index: recoveredChunkBoundaries };
+            chunkBoundaryReason = reason;
+            chunkAbort.abort(reason);
             return;
-        console.error(`[run-manager] no real progress for ${noProgressTimeoutMs}ms with no tool ` +
-            `or model stream in flight — ` +
-            `checkpointing run for continuation`, runId);
+        }
         // Mirror the soft-timeout semantics exactly: the chunk completes (not
         // aborts) at an auto_continue boundary, so the continuation machinery —
         // server-chained for background workers, client-driven for foreground —
         // recovers the turn.
         softTimedOut = true;
-        const event = {
-            type: "auto_continue",
-            reason: "no_progress",
-        };
+        recordRunBoundaryDiagnostic(reason, diagnostic, "terminal");
+        emitRunBoundaryTrackingEvent({
+            runId,
+            threadId,
+            reason,
+            recovered: false,
+            boundaryIndex: recoveredChunkBoundaries + 1,
+            dispatchMode: options?.dispatchMode,
+            model: options?.model,
+            engineName: options?.engineName,
+            userId: options?.userId,
+        });
+        const event = { type: "auto_continue", reason };
         send(event);
-        void checkpointRunBoundary(event, "no_progress");
+        void checkpointRunBoundary(event, reason);
+    };
+    const checkNoProgressBackstop = () => {
+        if (noProgressTimeoutMs <= 0)
+            return;
+        if (run.status !== "running" || abort.signal.aborted)
+            return;
+        if (inFlightWorkCount > 0)
+            return;
+        const silentForMs = Date.now() - lastRealProgressAt;
+        if (silentForMs < noProgressTimeoutMs)
+            return;
+        const lastEventType = run.events.at(-1)?.event.type;
+        if (!chunkAbort) {
+            // This backstop ends the TURN here; whether a successor invocation picks
+            // it up is decided later and elsewhere, so from this vantage it is a run
+            // that died on silence. It reached production for two releases as one
+            // console line nobody read. A boundary recovered in THIS invocation stays
+            // a log line — that distinction is the point.
+            console.error(`[run-manager] no real progress for ${noProgressTimeoutMs}ms with no tool ` +
+                `or model stream in flight — ` +
+                `checkpointing run for continuation`, runId);
+            captureError(new Error(`Agent run checkpointed after ${silentForMs}ms of silence (no_progress)`), {
+                route: "/_agent-native/agent-chat",
+                aiTraceId: runId,
+                tags: {
+                    source: "agent-run-manager",
+                    phase: "no-progress-backstop",
+                    terminalReason: "no_progress",
+                    lastEventType,
+                },
+                extra: {
+                    runId,
+                    threadId,
+                    silentForMs,
+                    noProgressTimeoutMs,
+                    lastEventType,
+                    eventCount: run.events.length,
+                },
+            });
+        }
+        reachRunBoundary("no_progress", { silentForMs, lastEventType });
     };
     // Periodic SQL abort check interval (for cross-isolate abort on Workers).
     // Also self-aborts when our row is no longer status='running' — catches the
@@ -1096,17 +1276,17 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         overrideMs: options?.noProgressTimeoutMs,
         backgroundOverrideMs: options?.backgroundNoProgressTimeoutMs,
     });
-    const softTimeoutTimer = softTimeoutMs > 0
+    // Not armed for a runFn that recovers boundaries in this invocation. That
+    // runFn already races the SAME wall with its own per-round timer, budgeted
+    // against cumulative elapsed time — so this timer fires at the moment the
+    // wrapper has nothing left to continue with, producing a boundary that is
+    // recoverable in name only and burning the tail of the budget on nothing.
+    // One wall, one clock; the caller's hard abort still backstops it.
+    const softTimeoutTimer = softTimeoutMs > 0 && !recoverChunkBoundaries
         ? setTimeout(() => {
-            if (run.status !== "running" || abort.signal.aborted)
-                return;
-            softTimedOut = true;
-            const event = {
-                type: "auto_continue",
-                reason: "run_timeout",
-            };
-            send(event);
-            void checkpointRunBoundary(event, "run_timeout");
+            reachRunBoundary("run_timeout", {
+                lastEventType: run.events.at(-1)?.event.type,
+            });
         }, softTimeoutMs)
         : null;
     let pendingTerminalEvent = null;
@@ -1213,8 +1393,13 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         emitRunEvent(runEvent);
     };
     // Run in background — intentionally detached from any HTTP connection
-    const runPromise = runFn(send, abort.signal)
+    const runPromise = runFn(send, runControl.chunkSignal, runControl)
         .then(() => {
+        // Settled inside the existing handlers rather than a `.finally()`: that
+        // would add a microtask tick to a chain whose ordering callers depend on.
+        // The runFn is done and never opened another round, so an outstanding
+        // boundary ended the run rather than being recovered from.
+        settleBoundary(false);
         if (abort.signal.aborted) {
             run.status = softTimedOut ? "completed" : "aborted";
             return;
@@ -1222,6 +1407,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         run.status = "completed";
     })
         .catch((err) => {
+        settleBoundary(false);
         // Don't surface abort errors — the run was intentionally stopped
         if (abort.signal.aborted) {
             run.status = softTimedOut ? "completed" : "aborted";

package/dist/agent/run-store.d.ts

@@ -148,6 +148,29 @@ export declare const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS: number;
 export declare const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20000;
 /** Test seam — the probe cache is module state, so suites must clear it. */
 export declare function __resetNoRunningRunsProbeForTests(): void;
+/**
+ * Ceiling on run ROWS for one logical turn — the number the continuation-chain
+ * guard and stale-run recovery must agree on.
+ *
+ * This was a literal `25` here plus `MAX_BACKGROUND_RUN_CONTINUATIONS + 5` in
+ * production-agent.ts, kept in step by a comment asking the next editor to
+ * remember, because importing back from this file would have been circular.
+ * It no longer needs to be: the base value is configuration, and `app-config`
+ * imports no agent code, so both sites can read the same resolver.
+ */
+export declare function resolveTurnRunLedgerBudget(): number;
+/**
+ * True when a turn holding `turnRunCount` run rows must not be given another.
+ *
+ * A predicate rather than a number the callers compare themselves, because both
+ * call sites had `turnRunCount > budget` and both were off by one: the current
+ * run's row is already inserted when they check, and the successor's row is
+ * inserted after — so at equality they permitted a row past the documented
+ * ceiling. Two sites, one comparison, no way for them to disagree about the
+ * boundary again. That is the third time in this area that one number had two
+ * spellings.
+ */
+export declare function turnRunLedgerExhausted(turnRunCount: number): boolean;
 /**
  * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
  * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
@@ -449,6 +472,14 @@ export declare const RUN_DIAG_STAGE: {
      * the per-turn budget is exhausted). See `attemptStaleRunRecovery`.
      */
     readonly staleRunRecoveryAttempted: "stale_run_recovery_attempted";
+    /**
+     * The run manager reached a server-owned chunk boundary (`no_progress` or
+     * `run_timeout`). Detail carries the reason, whether it was recovered in the
+     * same invocation or terminated the turn, how long the run had been silent,
+     * and the last event type seen — the segment that went quiet, which is what
+     * no boundary previously recorded anywhere.
+     */
+    readonly runBoundaryReached: "run_boundary_reached";
 };
 export type RunDiagStage = (typeof RUN_DIAG_STAGE)[keyof typeof RUN_DIAG_STAGE];
 /**

package/dist/agent/run-store.js

@@ -1,3 +1,9 @@
+/**
+ * SQL persistence for agent runs and events.
+ * Enables cross-isolate access on Cloudflare Workers and
+ * reliable reconnection after page refreshes.
+ */
+import { MAX_BACKGROUND_RUN_CONTINUATIONS, TURN_RUN_LEDGER_SLACK, } from "../app-config/run-lifecycle-invariants.js";
 import { getDbExec, intType, isPostgres } from "../db/client.js";
 import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
@@ -190,15 +196,32 @@ async function hasRunningRuns() {
     return false;
 }
 /**
- * FIX 3 (durable-background incident) per-turn run-count ceiling for
- * stale-run recovery — mirrors `chainServerDrivenContinuation`'s own ledger
- * guard in production-agent.ts (`MAX_BACKGROUND_RUN_CONTINUATIONS + 5` = 25).
- * Duplicated as a literal rather than imported: production-agent.ts already
- * imports run-manager.ts, which imports this file, so a runtime import back
- * from here would be circular. Keep this numerically in sync if that
- * constant ever changes.
+ * Ceiling on run ROWS for one logical turn — the number the continuation-chain
+ * guard and stale-run recovery must agree on.
+ *
+ * This was a literal `25` here plus `MAX_BACKGROUND_RUN_CONTINUATIONS + 5` in
+ * production-agent.ts, kept in step by a comment asking the next editor to
+ * remember, because importing back from this file would have been circular.
+ * It no longer needs to be: the base value is configuration, and `app-config`
+ * imports no agent code, so both sites can read the same resolver.
+ */
+export function resolveTurnRunLedgerBudget() {
+    return MAX_BACKGROUND_RUN_CONTINUATIONS + TURN_RUN_LEDGER_SLACK;
+}
+/**
+ * True when a turn holding `turnRunCount` run rows must not be given another.
+ *
+ * A predicate rather than a number the callers compare themselves, because both
+ * call sites had `turnRunCount > budget` and both were off by one: the current
+ * run's row is already inserted when they check, and the successor's row is
+ * inserted after — so at equality they permitted a row past the documented
+ * ceiling. Two sites, one comparison, no way for them to disagree about the
+ * boundary again. That is the third time in this area that one number had two
+ * spellings.
  */
-const STALE_RUN_RECOVERY_MAX_TURN_RUNS = 25;
+export function turnRunLedgerExhausted(turnRunCount) {
+    return turnRunCount >= resolveTurnRunLedgerBudget();
+}
 /**
  * Circuit breaker for a DETERMINISTIC dead-on-arrival loop: some request
  * shapes make the worker hang almost immediately every single time (e.g. an
@@ -206,7 +229,7 @@ const STALE_RUN_RECOVERY_MAX_TURN_RUNS = 25;
  * hitting a transient blip. Because `attemptStaleRunRecovery` replays the
  * SAME captured `dispatch_payload` on every successor (never a fresh
  * request), such a turn was retrying an unwinnable request up to
- * `STALE_RUN_RECOVERY_MAX_TURN_RUNS` (25) times — ~25 * 53s ≈ 22 minutes,
+ * `resolveTurnRunLedgerBudget()` (25) times — ~25 * 53s ≈ 22 minutes,
  * each cycle re-billing the full input context — before finally giving up.
  * Confirmed live in prod (assets: one turn cycled 24x, each attempt an
  * identical ~32K-token request that made a token of real progress around
@@ -1271,6 +1294,14 @@ export const RUN_DIAG_STAGE = {
      * the per-turn budget is exhausted). See `attemptStaleRunRecovery`.
      */
     staleRunRecoveryAttempted: "stale_run_recovery_attempted",
+    /**
+     * The run manager reached a server-owned chunk boundary (`no_progress` or
+     * `run_timeout`). Detail carries the reason, whether it was recovered in the
+     * same invocation or terminated the turn, how long the run had been silent,
+     * and the last event type seen — the segment that went quiet, which is what
+     * no boundary previously recorded anywhere.
+     */
+    runBoundaryReached: "run_boundary_reached",
 };
 /**
  * Record the last reached pipeline stage (+ optional short detail) for a run.
@@ -1491,7 +1522,7 @@ function staleRecoveryDispatchPayload(payload) {
  *     caller's own atomic "did I win the reap" gate, this guarantees AT MOST
  *     ONE recovery successor per reaped run even under concurrent reapers.
  *   - the per-turn run ledger (`countRunsForTurn`'s underlying query) has
- *     room (`STALE_RUN_RECOVERY_MAX_TURN_RUNS`) — mirrors
+ *     room (`resolveTurnRunLedgerBudget`) — mirrors
  *     `chainServerDrivenContinuation`'s own budget guard so a pathological
  *     turn can't loop forever through reaper-driven recovery either.
  */
@@ -1535,8 +1566,7 @@ async function attemptStaleRunRecovery(db, runId) {
         args: [threadId, turnId],
     });
     const turnRunCount = Number(countRows?.[0]?.run_count);
-    if (Number.isFinite(turnRunCount) &&
-        turnRunCount > STALE_RUN_RECOVERY_MAX_TURN_RUNS) {
+    if (Number.isFinite(turnRunCount) && turnRunLedgerExhausted(turnRunCount)) {
         return { outcome: "budget_exhausted" };
     }
     // See `STALE_RUN_RECOVERY_CONSECUTIVE_NO_PROGRESS_LIMIT`: a run whose last

package/dist/app-config/agent.d.ts

@@ -15,4 +15,6 @@ export declare const agentConfig: z.ZodObject<{
     runSoftTimeoutMs: z.ZodOptional<z.ZodNumber>;
     completedRunRetentionMs: z.ZodOptional<z.ZodNumber>;
     erroredRunRetentionMs: z.ZodOptional<z.ZodNumber>;
+    backgroundNoProgressTimeoutMs: z.ZodDefault<z.ZodNumber>;
+    backgroundRunHardTimeoutMs: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;

package/dist/app-config/agent.js

@@ -66,4 +66,37 @@ export const agentConfig = z.object({
         env: ["AGENT_ERRORED_RUN_RETENTION_MS"],
         doc: "How long an errored agent run row is kept, in milliseconds.",
     }),
+    // ── Run-lifecycle bounds ────────────────────────────────────────────────
+    //
+    // These are the numbers that can TERMINATE a run, or that encode an
+    // assumption about the host it runs on. They carry today's shipped values as
+    // declared defaults, so a deployment that configures nothing sees no
+    // behaviour change; `agent/run-lifecycle.ts` is the only place that reads
+    // them, one resolver per field, and `assertRunLifecycleInvariants` checks the
+    // ordering between them every time configuration resolves.
+    //
+    // Derived values (the foreground backstop fraction, tool-timeout headroom)
+    // stay internal on purpose: they are relationships, not host facts, and
+    // making them settable is how the ordering below stops being checkable.
+    //
+    // Each default here is the value that shipped as a module constant, and the
+    // constant still exists under its historical name where its reasoning is
+    // written down. `agent-run-lifecycle-config.spec.ts` pins the two together so
+    // editing one alone is a failing test, not a silent divergence.
+    backgroundNoProgressTimeoutMs: z
+        .number()
+        .nonnegative()
+        .default(150_000)
+        .meta({
+        env: ["AGENT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS"],
+        doc: "No-progress backstop for a background-function run, in milliseconds. 0 disables it.",
+    }),
+    backgroundRunHardTimeoutMs: z
+        .number()
+        .positive()
+        .default(10 * 60_000)
+        .meta({
+        env: ["AGENT_BACKGROUND_RUN_HARD_TIMEOUT_MS"],
+        doc: "Hard abort for one in-process background automation run, in milliseconds. This is the host's real function budget for scheduled work.",
+    }),
 });