@agent-native/core 0.168.13 → 0.169.0

package/dist/agent/run-store.d.ts

@@ -148,6 +148,29 @@ export declare const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS: number;
 export declare const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20000;
 /** Test seam — the probe cache is module state, so suites must clear it. */
 export declare function __resetNoRunningRunsProbeForTests(): void;
+/**
+ * Ceiling on run ROWS for one logical turn — the number the continuation-chain
+ * guard and stale-run recovery must agree on.
+ *
+ * This was a literal `25` here plus `MAX_BACKGROUND_RUN_CONTINUATIONS + 5` in
+ * production-agent.ts, kept in step by a comment asking the next editor to
+ * remember, because importing back from this file would have been circular.
+ * It no longer needs to be: the base value is configuration, and `app-config`
+ * imports no agent code, so both sites can read the same resolver.
+ */
+export declare function resolveTurnRunLedgerBudget(): number;
+/**
+ * True when a turn holding `turnRunCount` run rows must not be given another.
+ *
+ * A predicate rather than a number the callers compare themselves, because both
+ * call sites had `turnRunCount > budget` and both were off by one: the current
+ * run's row is already inserted when they check, and the successor's row is
+ * inserted after — so at equality they permitted a row past the documented
+ * ceiling. Two sites, one comparison, no way for them to disagree about the
+ * boundary again. That is the third time in this area that one number had two
+ * spellings.
+ */
+export declare function turnRunLedgerExhausted(turnRunCount: number): boolean;
 /**
  * Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
  * `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
@@ -449,6 +472,14 @@ export declare const RUN_DIAG_STAGE: {
      * the per-turn budget is exhausted). See `attemptStaleRunRecovery`.
      */
     readonly staleRunRecoveryAttempted: "stale_run_recovery_attempted";
+    /**
+     * The run manager reached a server-owned chunk boundary (`no_progress` or
+     * `run_timeout`). Detail carries the reason, whether it was recovered in the
+     * same invocation or terminated the turn, how long the run had been silent,
+     * and the last event type seen — the segment that went quiet, which is what
+     * no boundary previously recorded anywhere.
+     */
+    readonly runBoundaryReached: "run_boundary_reached";
 };
 export type RunDiagStage = (typeof RUN_DIAG_STAGE)[keyof typeof RUN_DIAG_STAGE];
 /**

package/dist/agent/run-store.js

@@ -1,3 +1,9 @@
+/**
+ * SQL persistence for agent runs and events.
+ * Enables cross-isolate access on Cloudflare Workers and
+ * reliable reconnection after page refreshes.
+ */
+import { MAX_BACKGROUND_RUN_CONTINUATIONS, TURN_RUN_LEDGER_SLACK, } from "../app-config/run-lifecycle-invariants.js";
 import { getDbExec, intType, isPostgres } from "../db/client.js";
 import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
@@ -190,15 +196,32 @@ async function hasRunningRuns() {
     return false;
 }
 /**
- * FIX 3 (durable-background incident) per-turn run-count ceiling for
- * stale-run recovery — mirrors `chainServerDrivenContinuation`'s own ledger
- * guard in production-agent.ts (`MAX_BACKGROUND_RUN_CONTINUATIONS + 5` = 25).
- * Duplicated as a literal rather than imported: production-agent.ts already
- * imports run-manager.ts, which imports this file, so a runtime import back
- * from here would be circular. Keep this numerically in sync if that
- * constant ever changes.
+ * Ceiling on run ROWS for one logical turn — the number the continuation-chain
+ * guard and stale-run recovery must agree on.
+ *
+ * This was a literal `25` here plus `MAX_BACKGROUND_RUN_CONTINUATIONS + 5` in
+ * production-agent.ts, kept in step by a comment asking the next editor to
+ * remember, because importing back from this file would have been circular.
+ * It no longer needs to be: the base value is configuration, and `app-config`
+ * imports no agent code, so both sites can read the same resolver.
+ */
+export function resolveTurnRunLedgerBudget() {
+    return MAX_BACKGROUND_RUN_CONTINUATIONS + TURN_RUN_LEDGER_SLACK;
+}
+/**
+ * True when a turn holding `turnRunCount` run rows must not be given another.
+ *
+ * A predicate rather than a number the callers compare themselves, because both
+ * call sites had `turnRunCount > budget` and both were off by one: the current
+ * run's row is already inserted when they check, and the successor's row is
+ * inserted after — so at equality they permitted a row past the documented
+ * ceiling. Two sites, one comparison, no way for them to disagree about the
+ * boundary again. That is the third time in this area that one number had two
+ * spellings.
  */
-const STALE_RUN_RECOVERY_MAX_TURN_RUNS = 25;
+export function turnRunLedgerExhausted(turnRunCount) {
+    return turnRunCount >= resolveTurnRunLedgerBudget();
+}
 /**
  * Circuit breaker for a DETERMINISTIC dead-on-arrival loop: some request
  * shapes make the worker hang almost immediately every single time (e.g. an
@@ -206,7 +229,7 @@ const STALE_RUN_RECOVERY_MAX_TURN_RUNS = 25;
  * hitting a transient blip. Because `attemptStaleRunRecovery` replays the
  * SAME captured `dispatch_payload` on every successor (never a fresh
  * request), such a turn was retrying an unwinnable request up to
- * `STALE_RUN_RECOVERY_MAX_TURN_RUNS` (25) times — ~25 * 53s ≈ 22 minutes,
+ * `resolveTurnRunLedgerBudget()` (25) times — ~25 * 53s ≈ 22 minutes,
  * each cycle re-billing the full input context — before finally giving up.
  * Confirmed live in prod (assets: one turn cycled 24x, each attempt an
  * identical ~32K-token request that made a token of real progress around
@@ -1271,6 +1294,14 @@ export const RUN_DIAG_STAGE = {
      * the per-turn budget is exhausted). See `attemptStaleRunRecovery`.
      */
     staleRunRecoveryAttempted: "stale_run_recovery_attempted",
+    /**
+     * The run manager reached a server-owned chunk boundary (`no_progress` or
+     * `run_timeout`). Detail carries the reason, whether it was recovered in the
+     * same invocation or terminated the turn, how long the run had been silent,
+     * and the last event type seen — the segment that went quiet, which is what
+     * no boundary previously recorded anywhere.
+     */
+    runBoundaryReached: "run_boundary_reached",
 };
 /**
  * Record the last reached pipeline stage (+ optional short detail) for a run.
@@ -1491,7 +1522,7 @@ function staleRecoveryDispatchPayload(payload) {
  *     caller's own atomic "did I win the reap" gate, this guarantees AT MOST
  *     ONE recovery successor per reaped run even under concurrent reapers.
  *   - the per-turn run ledger (`countRunsForTurn`'s underlying query) has
- *     room (`STALE_RUN_RECOVERY_MAX_TURN_RUNS`) — mirrors
+ *     room (`resolveTurnRunLedgerBudget`) — mirrors
  *     `chainServerDrivenContinuation`'s own budget guard so a pathological
  *     turn can't loop forever through reaper-driven recovery either.
  */
@@ -1535,8 +1566,7 @@ async function attemptStaleRunRecovery(db, runId) {
         args: [threadId, turnId],
     });
     const turnRunCount = Number(countRows?.[0]?.run_count);
-    if (Number.isFinite(turnRunCount) &&
-        turnRunCount > STALE_RUN_RECOVERY_MAX_TURN_RUNS) {
+    if (Number.isFinite(turnRunCount) && turnRunLedgerExhausted(turnRunCount)) {
         return { outcome: "budget_exhausted" };
     }
     // See `STALE_RUN_RECOVERY_CONSECUTIVE_NO_PROGRESS_LIMIT`: a run whose last

package/dist/app-config/agent.d.ts

@@ -15,4 +15,6 @@ export declare const agentConfig: z.ZodObject<{
     runSoftTimeoutMs: z.ZodOptional<z.ZodNumber>;
     completedRunRetentionMs: z.ZodOptional<z.ZodNumber>;
     erroredRunRetentionMs: z.ZodOptional<z.ZodNumber>;
+    backgroundNoProgressTimeoutMs: z.ZodDefault<z.ZodNumber>;
+    backgroundRunHardTimeoutMs: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;

package/dist/app-config/agent.js

@@ -66,4 +66,37 @@ export const agentConfig = z.object({
         env: ["AGENT_ERRORED_RUN_RETENTION_MS"],
         doc: "How long an errored agent run row is kept, in milliseconds.",
     }),
+    // ── Run-lifecycle bounds ────────────────────────────────────────────────
+    //
+    // These are the numbers that can TERMINATE a run, or that encode an
+    // assumption about the host it runs on. They carry today's shipped values as
+    // declared defaults, so a deployment that configures nothing sees no
+    // behaviour change; `agent/run-lifecycle.ts` is the only place that reads
+    // them, one resolver per field, and `assertRunLifecycleInvariants` checks the
+    // ordering between them every time configuration resolves.
+    //
+    // Derived values (the foreground backstop fraction, tool-timeout headroom)
+    // stay internal on purpose: they are relationships, not host facts, and
+    // making them settable is how the ordering below stops being checkable.
+    //
+    // Each default here is the value that shipped as a module constant, and the
+    // constant still exists under its historical name where its reasoning is
+    // written down. `agent-run-lifecycle-config.spec.ts` pins the two together so
+    // editing one alone is a failing test, not a silent divergence.
+    backgroundNoProgressTimeoutMs: z
+        .number()
+        .nonnegative()
+        .default(150_000)
+        .meta({
+        env: ["AGENT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS"],
+        doc: "No-progress backstop for a background-function run, in milliseconds. 0 disables it.",
+    }),
+    backgroundRunHardTimeoutMs: z
+        .number()
+        .positive()
+        .default(10 * 60_000)
+        .meta({
+        env: ["AGENT_BACKGROUND_RUN_HARD_TIMEOUT_MS"],
+        doc: "Hard abort for one in-process background automation run, in milliseconds. This is the host's real function budget for scheduled work.",
+    }),
 });

package/dist/app-config/run-lifecycle-invariants.d.ts

@@ -0,0 +1,248 @@
+import type { AppConfig } from "./schema.js";
+/**
+ * Wall-clock reserved between a background automation's round budget and its
+ * own hard abort.
+ *
+ * It covers wind-down only — emit the terminal event, persist the turn, let
+ * `finalized` settle — because the recoverable boundary on this path belongs to
+ * the agent-loop wrapper's own per-round timer, not to a second timer in the
+ * run manager.
+ *
+ * This is the constant that makes `automation soft timeout < automation hard
+ * abort` true by construction rather than by hoping two independently chosen
+ * numbers happen to be ordered — they were not: the shipped build gave the
+ * automation path a 13-minute soft timeout under a 10-minute hard abort, so
+ * the recoverable boundary was dead code and the only boundary an automation
+ * could reach was the terminal one.
+ */
+export declare const BACKGROUND_AUTOMATION_SOFT_TIMEOUT_HEADROOM_MS = 20000;
+/**
+ * The host's hard kill for a background function (Netlify: 15 minutes).
+ *
+ * Not configuration — a deployment does not choose it, the platform does. It is
+ * here because `backgroundSoftTimeoutCeilingMs` IS the clamp that
+ * `resolveRunSoftTimeoutMs` reduces every background soft timeout to, so once
+ * that ceiling became configurable nothing was left bounding it: a deployment
+ * could set 60 minutes and push its own chunk boundary past the wall the
+ * ceiling exists to stay inside, turning every long background turn back into
+ * the silent platform kill it was introduced to prevent. Configurable must not
+ * mean unclamped.
+ */
+export declare const BACKGROUND_FUNCTION_WALL_MS: number;
+/**
+ * Wall-clock a background chunk must leave itself to abort, persist the partial
+ * turn, write the terminal event, and chain a successor before the host kills
+ * the invocation. The shipped 13-minute ceiling under a 15-minute wall is
+ * exactly this margin.
+ */
+export declare const BACKGROUND_FUNCTION_WALL_HEADROOM_MS: number;
+/**
+ * Slack between the CHAIN bound (`agent.maxBackgroundRunContinuations`) and the
+ * per-turn LEDGER bound below.
+ *
+ * They count different things. The chain bound counts handoffs a chunk decided
+ * to make; the ledger counts every run ROW the turn produced, which also
+ * includes sweep redispatches and stale-run recoveries no chunk ever decided.
+ * Without slack the ledger would refuse a turn before the chain bound it is
+ * meant to sit above, so a turn recovered once would die holding unused chain
+ * budget.
+ */
+/**
+ * Shipped run-lifecycle bounds.
+ *
+ * They live beside the relationships that constrain them so a change to one is
+ * checked against the others in the same file. `run-manager.ts` and
+ * `production-agent.ts` re-export them under their historical names; this
+ * module imports no agent code, so nothing here can become circular.
+ */
+/**
+ * Hard ceiling for the soft timeout when a run executes inside a Netlify
+ * background function (any deployed function whose name ends in `-background`).
+ * Background functions return 202 immediately and run detached for up to 15
+ * minutes, so the ~60s synchronous function wall that 40s defends against does
+ * NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard
+ * kill to abort, persist the partial turn, write the terminal event, and (for
+ * the rare >13-min turn) self-fire another background continuation.
+ *
+ * This ceiling is used ONLY when a caller explicitly opts in with
+ * `backgroundFunction: true`. It does not change the foreground/interactive
+ * ceiling and does not fire unless the durable-background path dispatched the
+ * run into a background function. Per the design doc Guardrail, the 40s
+ * interactive clamp stays correct for every non-background run.
+ */
+export declare const BACKGROUND_SOFT_TIMEOUT_CEILING_MS: number;
+/**
+ * AUTHORITATIVE no-progress backstop for a run, enforced by the run manager
+ * itself (timer-driven, independent of any layer below).
+ *
+ * The finer-grained watchdogs inside the agent loop (model-stream and
+ * action-preparation no-progress, both 90s) only guard the model event stream
+ * — a stall in any segment OUTSIDE that guarded loop (engine-call
+ * establishment, worker setup between continuation chunks, a wedged transport
+ * that emits keepalives while the loop never runs) previously hung forever
+ * with the client watching keepalives. This backstop covers every segment by
+ * construction: if no REAL progress event (see `shouldBumpProgressForEvent`;
+ * keepalives and zero-byte prep activity don't count) lands for this long —
+ * and no unit of work is in flight (see `inFlightWorkDelta`: tool calls,
+ * cross-app calls, and the model stream all legitimately emit nothing for
+ * minutes and each carry a bound of their own) — the run manager emits
+ * `auto_continue { reason: "no_progress" }` and aborts the chunk, exactly
+ * like the soft timeout, so the normal continuation machinery recovers it.
+ *
+ * Being numerically larger than the in-loop watchdogs is NOT what keeps this
+ * from killing a healthy run, and treating it that way is what made it do so:
+ * this clock and the loop's `lastModelStreamProgressAt` measure DIFFERENT
+ * events. An extended-thinking phase bumps the inner clock on every engine
+ * frame while forwarding nothing, so the inner watchdog correctly stayed quiet
+ * and this one saw pure silence — runs whose worst gap crossed 150s died while
+ * still streaming, some by a single second. Ordering between two clocks only
+ * means something when they watch the same events; suspending on in-flight
+ * work is what actually makes the two agree.
+ *
+ * This is now only the CEILING, not the value: `resolveRunNoProgressTimeoutMs`
+ * clamps the foreground backstop to a fraction of the chunk's soft timeout
+ * (~30s at a 40s chunk), which is BELOW the 90s in-loop watchdogs rather than
+ * above them. That ordering is deliberate — the in-loop watchdogs could never
+ * fire inside a hosted foreground chunk anyway, since the serverless wall
+ * (~57-59s) arrives first. Proven durable-background chunks keep the full
+ * `DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS` so large outputs can use the
+ * background budget. Only armed when a soft-timeout regime is active (hosted
+ * runs); local dev stays unbounded.
+ */
+export declare const RUN_NO_PROGRESS_HARD_TIMEOUT_MS = 150000;
+/**
+ * Default in-loop watchdog for silence while an action's arguments stream in.
+ * Read through `resolveActionPreparationNoProgressTimeoutMs`, never directly:
+ * a host diagnosing a timeout has to be able to see and change this number.
+ */
+export declare const ACTION_PREPARATION_NO_PROGRESS_TIMEOUT_MS = 90000;
+/**
+ * Default in-loop watchdog for silence between engine stream frames. Read
+ * through `resolveModelStreamNoProgressTimeoutMs`, never directly.
+ */
+export declare const MODEL_STREAM_NO_PROGRESS_TIMEOUT_MS = 90000;
+/**
+ * Consecutive chunks allowed to end on the SAME terminal error code having
+ * produced nothing before the chain stops.
+ *
+ * Two, because two independent recovery layers multiply here and neither can
+ * see the other: the engine already retried this identical request 3x with
+ * backoff before the error was ever emitted, and a recoverable error is also a
+ * continuation boundary, so every chunk that fails costs 4 gateway attempts
+ * and dispatches a fresh one. A production turn spent 27 background runs and
+ * 15 minutes on one message this way. The first repeat is the retry this path
+ * exists for; a second identical failure that moved nothing is evidence the
+ * retrying itself is what is broken, not the request.
+ */
+export declare const MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS = 2;
+/**
+ * Wall-clock ceiling on a single logical turn. The run-count ledger alone is
+ * not a time bound: in durable mode each of the ~25 permitted chunks may burn
+ * ~780s, so the ledger's real worst case is over five hours (production has an
+ * observed 2h34m turn). Nobody is waiting that long, and every minute past
+ * this point is spend on a request the user has abandoned.
+ */
+export declare const MAX_TURN_WALL_CLOCK_MS: number;
+/**
+ * Cap on continuation iterations inside a single
+ * `runAgentLoopDirectWithSoftTimeout` invocation. The host's hard function
+ * timeout usually bounds this naturally — but a defensive cap prevents an
+ * instant-error spiral from looping forever inside hosting environments with a
+ * generous budget.
+ *
+ * 6 leaves room for: 1 normal completion + a few resume rounds for design
+ * generation (prompt + 3 variants ≈ 4 LLM calls), with a small safety margin.
+ */
+export declare const MAX_RUN_LOOP_CONTINUATIONS = 6;
+/**
+ * A delegated turn that is proven to be running inside a durable background
+ * function has the same 15-minute host budget as main chat, but this wrapper
+ * historically kept the foreground-sized six-continuation cap. A healthy
+ * child A2A call can consume several minutes and the receiving model may then
+ * need more than six recovery/model-stream boundaries to finish its own tool
+ * work. Keep a hard cap, but give the proven background path the same bounded
+ * continuation allowance as the durable main-chat runner. The cumulative
+ * soft-timeout below still prevents these rounds from exceeding the one real
+ * background-function wall-clock budget.
+ */
+export declare const MAX_BACKGROUND_RUN_LOOP_CONTINUATIONS = 20;
+export declare const TURN_RUN_LEDGER_SLACK = 5;
+/**
+ * Hard cap on server-driven background→background continuation chunks for a
+ * single logical turn. A `backgroundFunction` run gets a ~13-min soft timeout,
+ * so reaching this boundary at all is the rare exception (most turns finish in
+ * one chunk). The cap bounds a pathological turn that would otherwise chain
+ * background invocations forever, mirroring `MAX_AGENT_TEAM_CONTINUATIONS`.
+ */
+export declare const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
+/**
+ * Per-TURN follow budgets the browser applies while reading a background turn.
+ *
+ * They live here, not in `client/agent-chat-adapter.ts`, because they are one
+ * half of an ordering relationship whose other half is server configuration —
+ * and a relationship checked in only one of its two homes is the failure this
+ * module exists to prevent. This file has no runtime imports (the `AppConfig`
+ * import is type-only and erased), so the browser bundle pays nothing to read
+ * them from here.
+ *
+ * CLIENT-ABOVE-SERVER: these MUST stay above the server's own ceilings. The
+ * client fires on a clock and cannot tell looping from working; the server can,
+ * so the server must always terminate a turn first and write a truthful
+ * terminal reason. They shipped at 10 min / 6 runs while ONE legal background
+ * chunk may run 13 minutes — so the client killed healthy turns the server was
+ * still streaming, measured in production as aborts at 11-25 minutes with
+ * progress recorded right up to the abort. That was the top non-auth cause of
+ * "the chat just stopped".
+ *
+ * Do NOT tighten these to catch a stuck turn. A turn that is not progressing is
+ * already caught twice by mechanisms that read progress rather than a clock:
+ * `BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS` and the repeated-terminal-reason
+ * detector.
+ */
+export declare const MAX_FOLLOWED_BACKGROUND_RUNS = 30;
+export declare const MAX_BACKGROUND_FOLLOW_WALL_TIME_MS: number;
+/**
+ * Ordering relationships between the run-lifecycle bounds.
+ *
+ * Every one of these was already argued for in a source comment somewhere and
+ * enforced by nothing, which is how the framework shipped a violated pair. The
+ * check runs on resolved configuration — including the all-defaults case — so
+ * a relationship broken by a new default fails the same way a relationship
+ * broken by a deployment does.
+ *
+ * DECLARED EXCEPTION, deliberately not asserted:
+ * `backgroundSoftTimeoutCeilingMs` (13 min) sits ABOVE
+ * `backgroundRunHardTimeoutMs` (10 min). Those two bound different paths — the
+ * ceiling belongs to a durable background CHAT chunk, whose wall is the host's
+ * 15-minute background-function budget, while the hard abort belongs to the
+ * in-process automation runner. The automation path does not inherit the
+ * ceiling: it derives its budget from its own hard abort minus
+ * `BACKGROUND_AUTOMATION_SOFT_TIMEOUT_HEADROOM_MS` (see
+ * `resolveBackgroundAutomationSoftTimeoutMs`), which is what invariant 4 below
+ * checks is possible at all.
+ */
+interface Invariant {
+    name: string;
+    smaller: {
+        key: string;
+        value: number;
+    };
+    larger: {
+        key: string;
+        value: number;
+    };
+    relation: "<" | "<=";
+    why: string;
+}
+export declare class RunLifecycleInvariantError extends Error {
+    constructor(violations: readonly Invariant[]);
+}
+/**
+ * Throws when the resolved run-lifecycle bounds cannot all do their job.
+ *
+ * Called from configuration resolution, so it fails at startup naming both
+ * constants and the relationship rather than at 3am when a run dies inside the
+ * window a mis-ordered pair opened.
+ */
+export declare function assertRunLifecycleInvariants(agent: AppConfig["agent"]): void;
+export {};