@agent-native/core 0.153.5 → 0.153.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,7 +22,10 @@ export default defineAction({
22
22
  }),
23
23
  http: { method: "GET" },
24
24
  readOnly: true,
25
- grounding: true,
25
+ // Not `grounding`: this reads the stored program plus its last-run summary,
26
+ // and even `includeRows` replays rows cached by an earlier run rather than
27
+ // querying the source. The flag means evidence retrieved at call time, which
28
+ // is why `get-analysis` is excluded too.
26
29
  run: async (args) => {
27
30
  const ctx = getCredentialContext();
28
31
  if (!ctx) throw new Error("No authenticated context for get-data-program.");
@@ -91,27 +91,46 @@ function isToolName(name: string, expected: string): boolean {
91
91
  let groundingActionNames: ReadonlySet<string> | null = null;
92
92
 
93
93
  /**
94
- * Publish the action names whose definitions declare `grounding: true`.
94
+ * Read the action names whose definitions declare `grounding: true`.
95
95
  *
96
- * The set is pushed in rather than read from `.generated/actions-registry`
97
- * because that registry imports every action, and `save-analysis` imports this
98
- * module — importing it back here would close an evaluation cycle. The
99
- * agent-chat plugin derives the set once at module load; nothing else may.
96
+ * Takes the registry as an argument rather than importing
97
+ * `.generated/actions-registry` because that registry imports every action, and
98
+ * `save-analysis` imports this module — importing it back here would close an
99
+ * evaluation cycle. The agent-chat plugin passes it in once at module load.
100
100
  */
101
+ export function deriveGroundingActionNames(
102
+ registry: Record<string, unknown>,
103
+ ): string[] {
104
+ return Object.entries(registry)
105
+ .filter(([, module]) => {
106
+ // Action modules reach the registry either as the module namespace or as
107
+ // an already-unwrapped definition, the same two shapes
108
+ // `loadActionsFromStaticRegistry` normalizes.
109
+ const candidate = module as
110
+ | { grounding?: boolean; default?: { grounding?: boolean } }
111
+ | undefined;
112
+ return (
113
+ candidate?.grounding === true || candidate?.default?.grounding === true
114
+ );
115
+ })
116
+ .map(([name]) => name);
117
+ }
118
+
119
+ /** Publish the derived set for the response guard to consult. */
101
120
  export function registerGroundingActions(names: Iterable<string>): void {
102
- const normalized = new Set(
121
+ groundingActionNames = new Set(
103
122
  [...names].map((name) => normalizeActionToolName(name)),
104
123
  );
105
124
  // An empty set is never a real deployment: it means the installed core build
106
- // predates `grounding` and dropped it from every definition. Left unchecked,
107
- // the guard would replace every correct grounded answer with "connect data
108
- // sources" — the exact silent failure this flag replaced.
109
- if (normalized.size === 0) {
110
- throw new Error(
111
- "no action declares grounding: true; the installed @agent-native/core cannot carry the flag",
125
+ // predates `grounding` and dropped it from every definition. Registration
126
+ // runs at plugin module scope, so throwing here would take the whole server
127
+ // down for a response-guard heuristic; the failure is raised at first use
128
+ // instead, where it costs one turn.
129
+ if (groundingActionNames.size === 0) {
130
+ console.error(
131
+ "[analytics] no action declares grounding: true; the installed @agent-native/core cannot carry the flag, so the response guard is unusable",
112
132
  );
113
133
  }
114
- groundingActionNames = normalized;
115
134
  }
116
135
 
117
136
  function isGroundingActionName(name: string): boolean {
@@ -120,6 +139,11 @@ function isGroundingActionName(name: string): boolean {
120
139
  "grounding actions were never registered: the analytics response guard cannot tell a grounded turn from an ungrounded one",
121
140
  );
122
141
  }
142
+ if (groundingActionNames.size === 0) {
143
+ throw new Error(
144
+ "no action declares grounding: true; the installed @agent-native/core cannot carry the flag",
145
+ );
146
+ }
123
147
  return groundingActionNames.has(normalizeActionToolName(name));
124
148
  }
125
149
 
@@ -12,6 +12,7 @@ import { ANALYTICS_CONNECTOR_CATALOG } from "../lib/analytics-connector-catalog"
12
12
  import { credentialProviderConfigs } from "../lib/credential-keys";
13
13
  import { isProductionServerlessRuntime } from "../lib/production-serverless-runtime.js";
14
14
  import {
15
+ deriveGroundingActionNames,
15
16
  draftClaimsAnalyticsMetrics,
16
17
  failedDataQueryAttemptMessage,
17
18
  hasDashboardConstructionAttempt,
@@ -35,21 +36,7 @@ import {
35
36
  // Which actions return real data-source evidence is a fact each action states
36
37
  // on itself (`grounding: true`). Reading it off the definitions here is what
37
38
  // keeps the response guard from drifting behind a newly shipped source action.
38
- registerGroundingActions(
39
- Object.entries(actionsRegistry)
40
- .filter(([, module]) => {
41
- // Action modules reach the registry either as the module namespace or as
42
- // an already-unwrapped definition, the same two shapes
43
- // `loadActionsFromStaticRegistry` normalizes.
44
- const candidate = module as
45
- | { grounding?: boolean; default?: { grounding?: boolean } }
46
- | undefined;
47
- return (
48
- candidate?.grounding === true || candidate?.default?.grounding === true
49
- );
50
- })
51
- .map(([name]) => name),
52
- );
39
+ registerGroundingActions(deriveGroundingActionNames(actionsRegistry));
53
40
 
54
41
  const ANALYTICS_BACKGROUND_RUN_SOFT_TIMEOUT_MS = 13 * 60_000;
55
42
  // A background job may legitimately spend minutes inside a provider/tool call,
@@ -1,5 +1,22 @@
1
1
  import { getCurrentTurnEventsForThread } from "../run-store.js";
2
2
  import { classifyToolCallJournal, } from "../tool-call-journal.js";
3
+ const LEDGER_READ_RETRY_MS = 250;
4
+ /**
5
+ * `unreadable` ends the turn with a stop the user sees, so a single pool
6
+ * timeout or connection blip must not produce one. One retry, because the
7
+ * failures worth surviving are transient and the caller is holding the whole
8
+ * turn open while we wait.
9
+ */
10
+ async function readCurrentTurnEventsWithRetry(threadId, turnId) {
11
+ try {
12
+ return await getCurrentTurnEventsForThread(threadId, turnId);
13
+ }
14
+ catch (err) {
15
+ console.warn(`[tool-call-journal] per-turn ledger read failed for thread ${threadId}, retrying once: ${err instanceof Error ? err.message : String(err)}`);
16
+ await new Promise((resolve) => setTimeout(resolve, LEDGER_READ_RETRY_MS));
17
+ return await getCurrentTurnEventsForThread(threadId, turnId);
18
+ }
19
+ }
3
20
  /**
4
21
  * Tool-call journal hard-block (resume safety). Snapshot the per-turn journal
5
22
  * ONCE here, before any tool runs in this chunk, so it reflects only PRIOR
@@ -33,7 +50,7 @@ export async function loadPriorTurnToolCallJournal(threadId, turnId) {
33
50
  }
34
51
  let priorEvents;
35
52
  try {
36
- priorEvents = await getCurrentTurnEventsForThread(threadId, turnId);
53
+ priorEvents = await readCurrentTurnEventsWithRetry(threadId, turnId);
37
54
  }
38
55
  catch (err) {
39
56
  const error = err instanceof Error ? err.message : String(err);
@@ -618,12 +618,12 @@ export declare const MAX_IDENTICAL_TOOL_CALLS = 8;
618
618
  /**
619
619
  * A stop that ends the turn from inside the tool loop.
620
620
  *
621
- * `message` is the last thing the user reads, so it must name the remedy and
622
- * must NOT quote a raw provider/tool error: both `isAutoRecoverableError`
623
- * (client) and `isRecoverableContinuationError` (below) sniff this string for
624
- * "timeout"/"connection"/"network", and a stop whose text happens to contain
625
- * one is auto-continued into the very spiral it was raised to end. The raw
626
- * error belongs in `details`, which nothing retries on.
621
+ * `message` is the last thing the user reads, so it names the remedy in plain
622
+ * words and does not quote a raw provider/tool error; the raw error goes in
623
+ * `details`. Both retry sniffers now honour the `recoverable: false` these
624
+ * stops are sent with, so a stop that happens to say "timeout" is no longer
625
+ * auto-continued — but the prose still has one reader who cannot ignore it,
626
+ * and a pasted stack trace is not a remedy.
627
627
  */
628
628
  export interface TerminalActionStop {
629
629
  message: string;
@@ -720,9 +720,15 @@ export declare function findApprovedStructuredToolCall(history: AgentChatStructu
720
720
  * which is worse than one more retry.
721
721
  */
722
722
  export declare function permanentPreconditionRemedy(message: string): string | null;
723
+ /**
724
+ * Deliberately carries no running call count. This decline is recorded as a
725
+ * tool error, and the breakers key on that text: a message that reports its own
726
+ * tally ("already made 13 call(s)") mints a fresh key on every decline, so the
727
+ * decline that exists to stop a spiral becomes the one thing nothing can count.
728
+ * The fixed threshold below says the same thing without varying.
729
+ */
723
730
  export declare function repeatedSourceSweepGuardMessage(opts: {
724
731
  toolName: string;
725
- priorCalls: number;
726
732
  threshold?: number;
727
733
  scope?: "tool" | "aggregate";
728
734
  }): string;
@@ -2440,10 +2440,12 @@ async function waitForInterruptedToolLedgerEntry(opts) {
2440
2440
  * sentence before it, and two failures that differ ONLY in the arguments they
2441
2441
  * echo are the same failure.
2442
2442
  *
2443
- * Numbers go the same way, for the same reason. A guard that reports its own
2444
- * running tally ("this turn already made 13 call(s)") hands the breaker a new
2445
- * key on every single decline, so the decline that exists to stop a spiral is
2446
- * itself uncountable.
2443
+ * Digits stay. Collapsing them merges failures that are genuinely different:
2444
+ * "Record 41 not found" and "Record 42 not found" are two missing records, so a
2445
+ * seven-item sweep where every item legitimately 404s would trip the
2446
+ * across-arguments breaker at item six. A guard message that varies its own
2447
+ * running tally is fixed where that message is written, not by blinding every
2448
+ * breaker to numbers.
2447
2449
  */
2448
2450
  function normalizeToolErrorForBreaker(error) {
2449
2451
  return (error
@@ -2451,7 +2453,6 @@ function normalizeToolErrorForBreaker(error) {
2451
2453
  .replace(/Received:\s*[\s\S]*?\.\s(?=Expected:|The tool was not executed)/g, "")
2452
2454
  // Bare JSON payloads some providers inline instead of a Received: span.
2453
2455
  .replace(/\{[\s\S]{0,2000}?\}/g, "{}")
2454
- .replace(/\d+/g, "#")
2455
2456
  .replace(/\s+/g, " ")
2456
2457
  .trim());
2457
2458
  }
@@ -2494,16 +2495,27 @@ const PERMANENT_PRECONDITION_PATTERNS = [
2494
2495
  /\b(?:api[ -]?keys?|access tokens?|credentials?|secrets?)\b[^.]{0,60}\bnot (?:configured|set|connected|available)\b/i,
2495
2496
  /\bsave [A-Z][A-Z0-9_]{3,} in (?:the )?settings\b/i,
2496
2497
  // "Connect Builder.io before indexing a design system from Figma or code."
2497
- /\bconnect\b[^;]{1,40}?\b(?:before|first|in settings)\b/i,
2498
- // "…is only available in local Plan runtime." Excludes "only available after
2499
- // …", which names a condition that clears on its own.
2500
- /\bonly available (?:in|on|from|to)\b/i,
2498
+ // Case-sensitive, sentence-initial, and followed by a capitalized service
2499
+ // name, because that is the only shape that separates the imperative remedy
2500
+ // from the retryable network failure: "failed to connect to the warehouse
2501
+ // before the deadline" and "Connect timed out, retry first" both satisfy
2502
+ // every looser reading of the same words.
2503
+ /(?:^|[.:!?]\s+)Connect [A-Z][\w.-]*[^;]{0,40}?\b(?:before|first|in settings)\b/,
2501
2504
  // "Plan mode blocked `update-extension`. Switch to Act mode …" — cleared by
2502
2505
  // the user approving the plan, never by the model trying again.
2503
2506
  /\bplan mode blocked\b/i,
2507
+ // The templates' most-copied throw, and always `if (!getRequestUserEmail())`:
2508
+ // one synchronous AsyncLocalStorage read of a value fixed when the request
2509
+ // context was established. It cannot appear and disappear between tool calls
2510
+ // of the same chunk, so retrying inside this turn lands on it again.
2504
2511
  /\bno authenticated user\b/i,
2505
2512
  // "SSRF blocked: refusing to fetch private/internal address (…)"
2506
2513
  /\bssrf blocked\b/i,
2514
+ // Deliberately absent: "only available in …". Retention windows ("only
2515
+ // available from the last 90 days") share its wording and are fixed by
2516
+ // narrowing the range, so it stopped turns that were one argument away from
2517
+ // succeeding. The count-based breaker still ends a genuine runtime gate
2518
+ // after six.
2507
2519
  ];
2508
2520
  const SOURCE_SWEEP_TOOL_NAME = /\b(?:api|calls?|deals?|docs?|events?|issues?|messages?|metrics?|provider|query|records?|request|search|source|tickets?|transcripts?)\b/i;
2509
2521
  const SOURCE_SWEEP_PROVIDER_TOKEN = /\b(?:amplitude|apollo|bigquery|commonroom|data-source|ga4|github|gong|grafana|hubspot|jira|mixpanel|notion|posthog|postgres|postgresql|pylon|sentry|slack|stripe)\b/i;
@@ -2610,14 +2622,20 @@ function restrictAgentTeamsAfterSourceSweep(tools) {
2610
2622
  };
2611
2623
  });
2612
2624
  }
2625
+ /**
2626
+ * Deliberately carries no running call count. This decline is recorded as a
2627
+ * tool error, and the breakers key on that text: a message that reports its own
2628
+ * tally ("already made 13 call(s)") mints a fresh key on every decline, so the
2629
+ * decline that exists to stop a spiral becomes the one thing nothing can count.
2630
+ * The fixed threshold below says the same thing without varying.
2631
+ */
2613
2632
  export function repeatedSourceSweepGuardMessage(opts) {
2614
2633
  const threshold = opts.threshold ?? SOURCE_SWEEP_TOOL_CALL_THRESHOLD;
2615
2634
  const target = opts.scope === "aggregate"
2616
2635
  ? "read-only source/search tools"
2617
2636
  : "the same read-only source/search tool";
2618
- return (`Skipped ${opts.toolName}: this turn already made ${opts.priorCalls} ` +
2619
- `call(s) to ${target}, which exceeds the ` +
2620
- `${threshold}-call convergence budget. Stop calling ${opts.toolName} ` +
2637
+ return (`Skipped ${opts.toolName}: this turn already exhausted its ` +
2638
+ `${threshold}-call convergence budget for ${target}. Stop calling ${opts.toolName} ` +
2621
2639
  `one item at a time and change strategy before answering. If a broader ` +
2622
2640
  `read-only bulk/source mechanism is available, use it now: provider API ` +
2623
2641
  `catalog/docs/request tools with pagination or staging, code execution ` +
@@ -2647,7 +2665,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
2647
2665
  priorCalls: priorSourceSweepCalls,
2648
2666
  message: repeatedSourceSweepGuardMessage({
2649
2667
  toolName: opts.toolName,
2650
- priorCalls: priorSourceSweepCalls,
2651
2668
  threshold,
2652
2669
  scope: "aggregate",
2653
2670
  }),
@@ -2660,7 +2677,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
2660
2677
  priorCalls,
2661
2678
  message: repeatedSourceSweepGuardMessage({
2662
2679
  toolName: opts.toolName,
2663
- priorCalls,
2664
2680
  threshold,
2665
2681
  scope: "tool",
2666
2682
  }),
@@ -4041,7 +4057,8 @@ export async function runAgentLoop(opts) {
4041
4057
  const permanentRemedy = permanentPreconditionRemedy(sanitizedResult);
4042
4058
  if (permanentRemedy) {
4043
4059
  requestedActionStop ??= {
4044
- message: `I stopped because ${toolCall.name} cannot run until this is fixed: ${permanentRemedy} Retrying would not have changed it, and anything completed before this is saved.`,
4060
+ message: `I stopped because ${toolCall.name} needs a setup step outside this turn — a credential, a role, a connected account, or an approval — before it can run. ` +
4061
+ "Retrying would not have changed it, and anything completed before this is saved.",
4045
4062
  errorCode: "permanent_precondition",
4046
4063
  details: sanitizedResult,
4047
4064
  };
@@ -5056,6 +5073,13 @@ function isRecoverableContinuationError(event) {
5056
5073
  const message = event.error.toLowerCase();
5057
5074
  if (code === "builder_gateway_error")
5058
5075
  return false;
5076
+ // An explicit flag outranks the message sniff below, which exists for events
5077
+ // that carry no flag at all. Every guard stop sets `recoverable: false`, and
5078
+ // sniffing its prose is how a stop that merely NAMES a connection or a
5079
+ // timeout gets continued into the spiral it was raised to end. The client
5080
+ // applies the same precedence in `isAutoRecoverableError`.
5081
+ if (event.recoverable === false)
5082
+ return false;
5059
5083
  return (event.recoverable === true ||
5060
5084
  code === "builder_gateway_timeout" ||
5061
5085
  code === "builder_gateway_network_error" ||
@@ -937,12 +937,15 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
937
937
  // false-stale-reap zombie scenario where the reaper flipped the row while
938
938
  // this isolate was briefly unable to heartbeat (DB latency / GC pause).
939
939
  let lastAbortCheck = Date.now() - 3000;
940
- // A read failure here used to be indistinguishable from "not aborted" —
941
- // exactly the coerced-to-false pattern that lets a real Stop go unseen for
942
- // the rest of the run. Count consecutive failures like the heartbeat-write
943
- // handler above; past the same threshold, fail closed (self-abort with a
944
- // reason outside TURN_ENDING_ABORT_REASONS/RECOVERABLE_ABORT_REASONS, so it
945
- // surfaces as a typed error) instead of silently retrying forever.
940
+ // A read failure here is not "not aborted" — but it is not grounds to kill
941
+ // the run either. This check is only a cross-isolate backstop: the outage
942
+ // that hides a Stop from us hides it from every other reader too, so
943
+ // self-aborting delivers nobody's Stop any sooner while destroying in-flight
944
+ // work in the far more common case where nobody pressed Stop at all. Fail
945
+ // open and stay bounded by the soft timeout, no-progress backstop, and
946
+ // iteration/token limits — none of which touch the DB. Report the outage to
947
+ // Sentry (once at the threshold, then ~once a minute) so a long unreadable
948
+ // window is visible and its duration is queryable instead of silent.
946
949
  let consecutiveAbortCheckFailures = 0;
947
950
  const checkSqlAbort = () => {
948
951
  const now = Date.now();
@@ -970,7 +973,9 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
970
973
  })
971
974
  .catch((error) => {
972
975
  consecutiveAbortCheckFailures += 1;
973
- if (consecutiveAbortCheckFailures >= 3) {
976
+ // 3 ticks ≈ 9s of unreadability, then every 20 ticks ≈ 60s.
977
+ if (consecutiveAbortCheckFailures === 3 ||
978
+ consecutiveAbortCheckFailures % 20 === 0) {
974
979
  captureError(error, {
975
980
  route: "/_agent-native/agent-chat",
976
981
  tags: {
@@ -979,11 +984,12 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
979
984
  consecutiveFailures: String(consecutiveAbortCheckFailures),
980
985
  },
981
986
  aiTraceId: runId,
982
- extra: { runId, threadId },
987
+ extra: {
988
+ runId,
989
+ threadId,
990
+ unreadableForMs: consecutiveAbortCheckFailures * 3000,
991
+ },
983
992
  });
984
- if (!abort.signal.aborted) {
985
- abortInMemoryRun(run, "abort_check_unavailable");
986
- }
987
993
  }
988
994
  });
989
995
  };
@@ -44,6 +44,22 @@ export declare const STALE_RUN_ERROR_EVENT: {
44
44
  readonly recoverable: true;
45
45
  readonly details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.";
46
46
  };
47
+ /**
48
+ * `agent_runs.terminal_reason` recorded by the stale reapers.
49
+ *
50
+ * Every other error path records `error:<code>` (`terminalReasonForEvent`), and
51
+ * `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
52
+ * replays the terminal event a reaper itself appended. The reapers used to write
53
+ * the bare code, which split one outcome across two permanent
54
+ * `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
55
+ * `error:stale_run`) purely by whether anything happened to read the row after
56
+ * the reap — and made the reconciliation convergence guard, which compares
57
+ * `terminal_reason` for equality, rewrite the row on every read forever.
58
+ *
59
+ * Rows reaped before this change keep the bare `stale_run`, so anything matching
60
+ * on the value must accept both forms for one retention window.
61
+ */
62
+ export declare const STALE_RUN_TERMINAL_REASON: string;
47
63
  /**
48
64
  * Terminal error for a background-dispatched run whose worker NEVER claimed it
49
65
  * (the foreground fired the self-dispatch, Netlify acked it async with a 202,
@@ -665,13 +681,26 @@ export declare function listRunsForThread(threadId: string, options?: {
665
681
  * Best-effort on parse: malformed ledger rows are skipped rather than thrown.
666
682
  */
667
683
  export declare function getCurrentTurnEventsForThread(threadId: string, knownTurnId?: string): Promise<AgentChatEvent[]>;
684
+ export interface StaleRunReapResult {
685
+ /** Rows this pass reaped. */
686
+ reaped: number;
687
+ /**
688
+ * Rows whose reap threw. Separate from `reaped` because a pass where every
689
+ * row failed must not read as "nothing was stale" — those were the same
690
+ * number before, and the caller's own `null`-vs-`0` distinction is worthless
691
+ * while the per-row failures underneath it are coerced away.
692
+ */
693
+ failed: number;
694
+ /** More stale rows remain than the batch cap; the next tick continues. */
695
+ truncated: boolean;
696
+ }
668
697
  /**
669
698
  * Expire any "running" rows whose heartbeat is stale — producer died.
670
699
  * Safe to call at server startup on multi-isolate deployments: only rows
671
700
  * without a fresh heartbeat get reaped, so runs owned by OTHER live
672
701
  * isolates (which keep heartbeating) are left alone.
673
702
  */
674
- export declare function reapAllStaleRuns(): Promise<number>;
703
+ export declare function reapAllStaleRuns(): Promise<StaleRunReapResult>;
675
704
  /**
676
705
  * Read the daily terminal-outcome counters rolled up from pruned runs. Pair
677
706
  * with live `agent_runs` rows for a complete picture — see
@@ -50,6 +50,22 @@ export const STALE_RUN_ERROR_EVENT = {
50
50
  recoverable: true,
51
51
  details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
52
52
  };
53
+ /**
54
+ * `agent_runs.terminal_reason` recorded by the stale reapers.
55
+ *
56
+ * Every other error path records `error:<code>` (`terminalReasonForEvent`), and
57
+ * `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
58
+ * replays the terminal event a reaper itself appended. The reapers used to write
59
+ * the bare code, which split one outcome across two permanent
60
+ * `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
61
+ * `error:stale_run`) purely by whether anything happened to read the row after
62
+ * the reap — and made the reconciliation convergence guard, which compares
63
+ * `terminal_reason` for equality, rewrite the row on every read forever.
64
+ *
65
+ * Rows reaped before this change keep the bare `stale_run`, so anything matching
66
+ * on the value must accept both forms for one retention window.
67
+ */
68
+ export const STALE_RUN_TERMINAL_REASON = `error:${STALE_RUN_ERROR_EVENT.errorCode}`;
53
69
  /**
54
70
  * Terminal error for a background-dispatched run whose worker NEVER claimed it
55
71
  * (the foreground fired the self-dispatch, Netlify acked it async with a 202,
@@ -1306,6 +1322,22 @@ export async function recordRunDiagnostic(runId, stage, detail) {
1306
1322
  // Diagnostics are best-effort; never let them break the run or the route.
1307
1323
  }
1308
1324
  }
1325
+ /**
1326
+ * Wall-clock budget for ONE heartbeat write.
1327
+ *
1328
+ * The default `DbExec` budget is `dbOpTimeoutMs()` (8s on serverless) across 3
1329
+ * connection attempts, so a single heartbeat can occupy up to 24s — against a
1330
+ * 15s `RUN_STALE_MS`. The run-manager single-flights the write, so that one
1331
+ * attempt suppresses every 1.5s tick behind it: a live foreground run in a quiet
1332
+ * phase becomes reapable while its own heartbeat is still in flight. The same
1333
+ * attempt also pins a pooler connection for those 24s, feeding the saturation
1334
+ * that starves heartbeats in the first place.
1335
+ *
1336
+ * A heartbeat that cannot land promptly has no value — the 1.5s tick IS the
1337
+ * retry — so it gets ONE attempt inside a third of the stale window, leaving
1338
+ * room for two more to land before the reap cutoff.
1339
+ */
1340
+ const HEARTBEAT_WRITE_TIMEOUT_MS = Math.floor(RUN_STALE_MS / 3);
1309
1341
  /** Update the run's liveness heartbeat. Called periodically by run-manager. */
1310
1342
  export async function updateRunHeartbeat(runId) {
1311
1343
  await ensureRunTables();
@@ -1324,6 +1356,8 @@ export async function updateRunHeartbeat(runId) {
1324
1356
  END
1325
1357
  WHERE id = ? AND status = 'running'`,
1326
1358
  args: [Date.now(), currentRssMb(), currentRssMb(), runId],
1359
+ timeoutMs: HEARTBEAT_WRITE_TIMEOUT_MS,
1360
+ maxAttempts: 1,
1327
1361
  });
1328
1362
  }
1329
1363
  /**
@@ -1612,9 +1646,15 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
1612
1646
  : // First `?` is backgroundAwareStaleCutoffSql's CAST param, the next two
1613
1647
  // are inFlightGraceSql's — all bound to the same "now".
1614
1648
  [completedAt, completedAt, completedAt];
1649
+ // `completed_at` is when the run DIED, not when a reaper noticed. Stamping
1650
+ // reap time made every stale row report its own detection latency as run
1651
+ // duration (prod: 30-59 minute "runs" against a 15s window), which is the
1652
+ // measurement used to judge whether reaping is timely at all. The liveness
1653
+ // basis is the last proof the producer was alive, and it is what
1654
+ // `reconcileTerminalRunFromEvents` already records for the same failure.
1615
1655
  const updateSql = `UPDATE agent_runs
1616
1656
  SET status = 'errored',
1617
- completed_at = ?,
1657
+ completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
1618
1658
  error_code = ?,
1619
1659
  error_detail = ?,
1620
1660
  terminal_reason = ?
@@ -1623,10 +1663,9 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
1623
1663
  AND ${terminalRunEventExclusionSql()}
1624
1664
  AND ${staleClause}`;
1625
1665
  const updateArgs = [
1626
- completedAt,
1627
1666
  STALE_RUN_ERROR_EVENT.errorCode,
1628
1667
  STALE_RUN_ERROR_EVENT.details,
1629
- STALE_RUN_ERROR_EVENT.errorCode,
1668
+ STALE_RUN_TERMINAL_REASON,
1630
1669
  runId,
1631
1670
  ...staleArgs,
1632
1671
  ];
@@ -2214,6 +2253,18 @@ export async function getCurrentTurnEventsForThread(threadId, knownTurnId) {
2214
2253
  }
2215
2254
  return events;
2216
2255
  }
2256
+ /**
2257
+ * Rows one pass will reap. Each costs ~5-10 serial round trips (reconcile,
2258
+ * transactional reap plus recovery, terminal-event append), and this sweep now
2259
+ * runs ahead of `processRecurringJobs` on the shared durable tick — so an
2260
+ * unbounded first pass against a backlog spends the platform wall before jobs
2261
+ * get any. Production carried 1,216 stale rows while nothing periodic reaped
2262
+ * them, so the first tick after that fix lands on exactly such a backlog.
2263
+ * A truncated pass is drained by the next tick (60s durable, 20s in-process),
2264
+ * never dropped — `truncated` is what stops a partial sweep being reported as
2265
+ * a clean one.
2266
+ */
2267
+ const REAP_ALL_STALE_BATCH_LIMIT = 200;
2217
2268
  /**
2218
2269
  * Expire any "running" rows whose heartbeat is stale — producer died.
2219
2270
  * Safe to call at server startup on multi-isolate deployments: only rows
@@ -2221,9 +2272,14 @@ export async function getCurrentTurnEventsForThread(threadId, knownTurnId) {
2221
2272
  * isolates (which keep heartbeating) are left alone.
2222
2273
  */
2223
2274
  export async function reapAllStaleRuns() {
2275
+ const nothingStale = {
2276
+ reaped: 0,
2277
+ failed: 0,
2278
+ truncated: false,
2279
+ };
2224
2280
  await ensureRunTables();
2225
2281
  if (!(await hasRunningRuns()))
2226
- return 0;
2282
+ return nothingStale;
2227
2283
  const client = getDbExec();
2228
2284
  const now = Date.now();
2229
2285
  // Background-dispatched runs use the wider window; everything else 15s. The
@@ -2235,13 +2291,23 @@ export async function reapAllStaleRuns() {
2235
2291
  // server startup across possibly-multiple isolates, so a sibling isolate's
2236
2292
  // still-alive, in-flight run must not be reaped just because THIS isolate
2237
2293
  // just booted and has no heartbeat history for it.
2238
- const stale = await client.execute({
2294
+ // One row over the cap is how a truncated pass is detected without a second
2295
+ // COUNT query against the same predicate.
2296
+ const scanned = await client.execute({
2239
2297
  sql: `SELECT id FROM agent_runs
2240
2298
  WHERE status = 'running'
2241
2299
  AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
2242
- AND ${inFlightGraceSql()}`,
2300
+ AND ${inFlightGraceSql()}
2301
+ ORDER BY started_at ASC
2302
+ LIMIT ${REAP_ALL_STALE_BATCH_LIMIT + 1}`,
2243
2303
  args: [now, now, now],
2244
2304
  });
2305
+ const truncated = scanned.rows.length > REAP_ALL_STALE_BATCH_LIMIT;
2306
+ const stale = {
2307
+ rows: truncated
2308
+ ? scanned.rows.slice(0, REAP_ALL_STALE_BATCH_LIMIT)
2309
+ : scanned.rows,
2310
+ };
2245
2311
  for (const row of stale.rows) {
2246
2312
  const id = row.id;
2247
2313
  if (typeof id === "string") {
@@ -2260,13 +2326,19 @@ export async function reapAllStaleRuns() {
2260
2326
  // already found stale; on an idle app `hasRunningRuns` returns before any of
2261
2327
  // this, so the 20s fast sweep (agent-chat-plugin.ts) costs one probe.
2262
2328
  let reapedCount = 0;
2329
+ let failedCount = 0;
2263
2330
  for (const row of stale.rows) {
2264
2331
  const id = row.id;
2265
2332
  if (typeof id !== "string")
2266
2333
  continue;
2267
- const reaped = await reapSingleStaleRun(id).catch(() => false);
2268
- if (reaped)
2269
- reapedCount += 1;
2334
+ try {
2335
+ if (await reapSingleStaleRun(id))
2336
+ reapedCount += 1;
2337
+ }
2338
+ catch (error) {
2339
+ failedCount += 1;
2340
+ console.error(`[run-store] stale reap failed for run ${id}:`, error);
2341
+ }
2270
2342
  }
2271
2343
  for (const row of stale.rows) {
2272
2344
  const id = row.id;
@@ -2274,7 +2346,7 @@ export async function reapAllStaleRuns() {
2274
2346
  await safeAppendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT, "reap-all-stale");
2275
2347
  }
2276
2348
  }
2277
- return reapedCount;
2349
+ return { reaped: reapedCount, failed: failedCount, truncated };
2278
2350
  }
2279
2351
  const RUN_OUTCOME_DAY_MS = 86_400_000;
2280
2352
  /**
@@ -2433,11 +2505,16 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
2433
2505
  await reconcileTerminalRunFromEvents(id);
2434
2506
  }
2435
2507
  }
2508
+ // Both UPDATEs below backdate `completed_at` to the liveness basis for the
2509
+ // reason given on `reapSingleStaleRun`'s UPDATE: this pass can fire a DAY
2510
+ // after the producer died, and stamping now would file that whole gap as run
2511
+ // duration. On the absolute-age branch the basis degrades to `started_at`,
2512
+ // which is still the truest thing known about a row that never heartbeat.
2436
2513
  const completedAt = Date.now();
2437
2514
  await client.execute({
2438
2515
  sql: `UPDATE agent_runs
2439
2516
  SET status = 'errored',
2440
- completed_at = ?,
2517
+ completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
2441
2518
  error_code = ?,
2442
2519
  error_detail = ?,
2443
2520
  terminal_reason = ?
@@ -2445,10 +2522,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
2445
2522
  AND ${terminalRunEventExclusionSql()}
2446
2523
  AND started_at < ?`,
2447
2524
  args: [
2448
- completedAt,
2449
2525
  STALE_RUN_ERROR_EVENT.errorCode,
2450
2526
  STALE_RUN_ERROR_EVENT.details,
2451
- STALE_RUN_ERROR_EVENT.errorCode,
2527
+ STALE_RUN_TERMINAL_REASON,
2452
2528
  cutoff,
2453
2529
  ],
2454
2530
  });
@@ -2463,7 +2539,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
2463
2539
  await client.execute({
2464
2540
  sql: `UPDATE agent_runs
2465
2541
  SET status = 'errored',
2466
- completed_at = ?,
2542
+ completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
2467
2543
  error_code = ?,
2468
2544
  error_detail = ?,
2469
2545
  terminal_reason = ?
@@ -2472,10 +2548,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
2472
2548
  AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
2473
2549
  AND ${inFlightGraceSql()}`,
2474
2550
  args: [
2475
- completedAt,
2476
2551
  STALE_RUN_ERROR_EVENT.errorCode,
2477
2552
  STALE_RUN_ERROR_EVENT.details,
2478
- STALE_RUN_ERROR_EVENT.errorCode,
2553
+ STALE_RUN_TERMINAL_REASON,
2479
2554
  completedAt,
2480
2555
  completedAt,
2481
2556
  completedAt,
@@ -17,6 +17,7 @@ interface ContentPart {
17
17
  completedSideEffect?: boolean;
18
18
  mcpApp?: AgentMcpAppPayload;
19
19
  chatUI?: ActionChatUIConfig;
20
+ activity?: boolean;
20
21
  approval?: {
21
22
  approvalKey: string;
22
23
  dismissed?: boolean;
@@ -243,18 +243,31 @@ export function buildAssistantMessage(events, runId, options = {}) {
243
243
  metadata,
244
244
  };
245
245
  }
246
+ /**
247
+ * The rebuild half of the live client's `clearAssistantDraftContent`
248
+ * (client/sse-event-processor.ts). The two bodies are asserted identical by
249
+ * `keeps clearAssistantDraftContent identical to the live client copy` in
250
+ * thread-data-builder.spec.ts — a rebuild that clears more than the live stream
251
+ * did makes narration vanish on reload, which is worse than clearing nothing.
252
+ */
246
253
  function clearAssistantDraftContent(content) {
247
254
  for (let index = content.length - 1; index >= 0; index--) {
248
255
  const part = content[index];
249
256
  if (!part)
250
257
  continue;
258
+ if (part.type === "tool-call" &&
259
+ part.activity !== true &&
260
+ part.result !== undefined) {
261
+ return;
262
+ }
251
263
  if (part.type === "text" || part.type === "reasoning") {
252
264
  content.splice(index, 1);
253
265
  continue;
254
266
  }
255
267
  if (part.type === "tool-call" && part.result === undefined) {
256
- // Keep materialized in-flight tool cards across retry clears so persisted
257
- // thread rebuilds match the live SSE processor and avoid hide→show flicker.
268
+ // Only drop ephemeral placeholders. Materialized in-flight tool cards
269
+ // (real args from tool_start) stay mounted so a retry/auto-continue clear
270
+ // does not hide→show the same call when the next chunk re-emits it.
258
271
  const isEphemeral = part.activity === true ||
259
272
  part.argsText === "" ||
260
273
  Object.keys(part.args ?? {}).length === 0;
@@ -37,5 +37,5 @@ export function RouteTransitionIndicator() {
37
37
  }, [destination]);
38
38
  if (!destination || visibleDestination !== destination)
39
39
  return null;
40
- return (_jsx("div", { "aria-label": "Loading the next page", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
40
+ return (_jsx("div", { "aria-label": "Loading page...", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
41
41
  }
@@ -24,20 +24,29 @@ const AUTO_CONTINUE_COMPLETION_GUARD = "Before doing more work, inspect the prio
24
24
  const MAX_RECONNECT_ATTEMPTS = 5;
25
25
  const MAX_STARTUP_RECOVERY_ATTEMPTS = 8;
26
26
  const MAX_QUEUED_CONFLICT_RETRIES = 120;
27
- // How many consecutive continuations that did not advance the turn (see
28
- // `describeContinuationAdvance`) we tolerate before giving up. This replaced
29
- // five separate budgets — stale-run, stalled, empty, repeated-narration and
30
- // repeated-action-preparation — each of which existed because the rung above
27
+ // The consecutive non-advancing continuation (see `describeContinuationAdvance`)
28
+ // that ends the turn: the third one stops it, so two are tolerated. This
29
+ // replaced five separate budgets — stale-run, stalled, empty, repeated-narration
30
+ // and repeated-action-preparation — each of which existed because the rung above
31
31
  // it had a hole the next one patched. One question answers all five: a round
32
32
  // that produced nothing, or produced exactly what the round before it
33
33
  // produced, did not advance.
34
34
  const MAX_NON_ADVANCING_CONTINUATIONS = 3;
35
- // Ceiling across the whole turn. Every attempt re-POSTs the full history plus
36
- // attachments, so a high ceiling only burns tokens and wall time. Non-advancing
37
- // turns are already stopped far earlier by the budget above; this one only ever
38
- // binds turns that keep making real progress, so it stays just above the
39
- // longest recovery we deliberately support.
35
+ // Ceiling across the whole turn for TRANSIENT continuations — rounds that
36
+ // followed a failure (timeout, dropped stream, stale run). Every attempt
37
+ // re-POSTs the full history plus attachments, so a high ceiling only burns
38
+ // tokens and wall time on a connection that keeps breaking.
40
39
  const MAX_TOTAL_TRANSIENT_CONTINUATIONS = 12;
40
+ // Ceiling across the whole turn for WORK-boundary continuations. A `loop_limit`
41
+ // round is not a failure — the server spent a full iteration budget on real
42
+ // tool work and handed the turn back — so counting it against the transient
43
+ // ceiling killed progressing turns at round 13 that had never failed once.
44
+ // Sized against the server's own limits: at its default budget of 400
45
+ // iterations per run this is ~10,000 tool calls, two orders past the deepest
46
+ // legitimate production turn (117 tool calls) that sized that budget. It still
47
+ // has to exist, because the server's per-turn token backstop rides the request
48
+ // body between server-chained chunks and resets on every client re-POST.
49
+ const MAX_LOOP_LIMIT_CONTINUATIONS = 25;
41
50
  const RETRY_BASE_DELAY_MS = 500;
42
51
  const RETRY_MAX_DELAY_MS = 8_000;
43
52
  const MAX_HISTORY_ATTACHMENT_CHARS = 60_000;
@@ -742,7 +751,11 @@ function estimateHistoryMessageCost(message) {
742
751
  const argsText = tool.argsText ?? stableJson(tool.args ?? {});
743
752
  cost += Math.min(argsText.length, argsCap);
744
753
  if (tool.result !== undefined) {
745
- cost += Math.min(String(tool.result).length, MAX_HISTORY_TOOL_RESULT_CHARS);
754
+ cost += Math.min(
755
+ // Price the string the request actually carries. `String(result)` is
756
+ // 15 chars ("[object Object]") for every object result, which is most
757
+ // of them.
758
+ toolResultContent(tool.result, tool.toolName).length, MAX_HISTORY_TOOL_RESULT_CHARS);
746
759
  }
747
760
  }
748
761
  return Math.max(1, cost);
@@ -1634,6 +1647,9 @@ export function createAgentChatAdapter(options) {
1634
1647
  let startupRecoveryAttempts = 0;
1635
1648
  let queuedConflictRetries = 0;
1636
1649
  let totalTransientContinuationAttempts = 0;
1650
+ // Work-boundary continuations (`loop_limit`), counted apart from the
1651
+ // transient ones — see MAX_LOOP_LIMIT_CONTINUATIONS.
1652
+ let loopLimitContinuations = 0;
1637
1653
  // Consecutive continuations whose advance signature matched the previous
1638
1654
  // round's (or was empty). Reset by any real advance.
1639
1655
  let nonAdvancingContinuations = 0;
@@ -1647,12 +1663,13 @@ export function createAgentChatAdapter(options) {
1647
1663
  // continuation history; the advance check needs the per-round delta for
1648
1664
  // every reason code, `loop_limit` included.
1649
1665
  let advanceCheckPrefix = [];
1650
- // Set when the turn is stopped by MAX_TOTAL_TRANSIENT_CONTINUATIONS —
1651
- // the whole-turn continuation ceiling, not a dropped connection. This
1652
- // can trip on a turn that was making real progress the entire time, so
1653
- // it needs its own honest message instead of falling through to the
1654
- // generic "connection kept failing" copy below.
1655
- let recoveryGaveUpOnTransientBudget = false;
1666
+ // Set when the turn is stopped by a whole-turn continuation ceiling
1667
+ // (MAX_TOTAL_TRANSIENT_CONTINUATIONS or MAX_LOOP_LIMIT_CONTINUATIONS)
1668
+ // rather than by a dropped connection. Either can trip on a turn that was
1669
+ // making real progress the entire time, so it needs its own honest
1670
+ // message instead of falling through to the generic "connection kept
1671
+ // failing" copy below.
1672
+ let recoveryGaveUpOnContinuationBudget = false;
1656
1673
  const continuationHistoryFragments = [];
1657
1674
  const structuredContinuationFragments = [];
1658
1675
  let visibleContinuationPrefix = [];
@@ -1708,6 +1725,7 @@ export function createAgentChatAdapter(options) {
1708
1725
  ? `activity_trail: ${formatActivityTrail(lastActivityTrail)}`
1709
1726
  : "",
1710
1727
  `total_transient_continuations: ${totalTransientContinuationAttempts}`,
1728
+ `loop_limit_continuations: ${loopLimitContinuations}`,
1711
1729
  `background_follow_no_progress_detaches: ${backgroundFollowNoProgressDetaches}`,
1712
1730
  `background_follow_consecutive_no_progress_detaches: ${backgroundFollowConsecutiveNoProgressDetaches}`,
1713
1731
  backgroundFollowLastDetachReason
@@ -1738,7 +1756,7 @@ export function createAgentChatAdapter(options) {
1738
1756
  : " the same action";
1739
1757
  return `The agent got stuck preparing${tool} input and never started the tool, so I stopped the automatic retries. Try a smaller first step or a more compact version of the request.`;
1740
1758
  }
1741
- if (recoveryGaveUpOnTransientBudget) {
1759
+ if (recoveryGaveUpOnContinuationBudget) {
1742
1760
  return "This turn reached the limit on how many times it can be automatically continued, so I stopped it here. It may have still been making progress — retry as a single, narrower request so it can finish within fewer continuations.";
1743
1761
  }
1744
1762
  if (content.length === 0 &&
@@ -2815,15 +2833,6 @@ export function createAgentChatAdapter(options) {
2815
2833
  const visibleContent = visibleContentForContinuation();
2816
2834
  let currentPartialHistory = contentToContinuationHistory(visibleContent);
2817
2835
  const madeContentProgress = hasContinuationProgress(visibleContent);
2818
- // An action was streamed but has not returned yet (a tool_start with
2819
- // no tool_done). Only a run_timeout in that window means the SERVER
2820
- // is still executing it and a reconnect can still deliver the result;
2821
- // every other reason means the stream is dead and nothing will
2822
- // complete the tool.
2823
- const hasInFlightTool = hasInFlightToolCall(visibleContent);
2824
- const completedTool = lastCompletedTimeoutCandidateTool(content);
2825
- const currentPreparingToolName = lastUnresolvedToolActivity(visibleContent) ??
2826
- lastPreparingActionTool(signal.activityTrail);
2827
2836
  // THE boundary. One question — did this continuation advance the
2828
2837
  // turn? — decides every reason code, `loop_limit` included. It used
2829
2838
  // to skip this block entirely AND reset two of the budgets, so a
@@ -2832,6 +2841,20 @@ export function createAgentChatAdapter(options) {
2832
2841
  // no answer). The durable per-turn ledger lives server-side inside
2833
2842
  // one run and never sees a client re-POST, so nothing else bounds it.
2834
2843
  const advanceDelta = contentAfterContinuationPrefix(content, advanceCheckPrefix);
2844
+ // Everything that asks "what did THIS round do" reads the delta.
2845
+ // `visibleContent` accumulates across a `loop_limit` chain — its
2846
+ // prefix deliberately does not advance there — so one unresolved
2847
+ // "Preparing X" card from an earlier round would otherwise stall
2848
+ // every later round on the same name and kill a turn that was
2849
+ // streaming new prose the whole time.
2850
+ // An action streamed but not returned yet (a tool_start with no
2851
+ // tool_done) means the SERVER is still executing it, so a run_timeout
2852
+ // in that window can still deliver the result on reconnect; every
2853
+ // other reason means the stream is dead and nothing will complete it.
2854
+ const hasInFlightTool = hasInFlightToolCall(advanceDelta);
2855
+ const completedTool = lastCompletedTimeoutCandidateTool(content);
2856
+ const currentPreparingToolName = lastUnresolvedToolActivity(advanceDelta) ??
2857
+ lastPreparingActionTool(signal.activityTrail);
2835
2858
  const advance = describeContinuationAdvance(advanceDelta, currentPreparingToolName);
2836
2859
  const serverStillRunningTool = signal.reason === "run_timeout" && hasInFlightTool;
2837
2860
  const advanced = serverStillRunningTool ||
@@ -2844,7 +2867,12 @@ export function createAgentChatAdapter(options) {
2844
2867
  nonAdvancingContinuations = advanced
2845
2868
  ? 0
2846
2869
  : nonAdvancingContinuations + 1;
2847
- totalTransientContinuationAttempts += 1;
2870
+ if (isTransient) {
2871
+ totalTransientContinuationAttempts += 1;
2872
+ }
2873
+ else {
2874
+ loopLimitContinuations += 1;
2875
+ }
2848
2876
  advanceCheckPrefix = snapshotContent(content);
2849
2877
  // If a tool already completed, do not turn a missing closing
2850
2878
  // sentence into a scary connection failure. Give the model one
@@ -2868,8 +2896,9 @@ export function createAgentChatAdapter(options) {
2868
2896
  return { ok: false, resetVisibleContent: false };
2869
2897
  }
2870
2898
  if (totalTransientContinuationAttempts >
2871
- MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
2872
- recoveryGaveUpOnTransientBudget = true;
2899
+ MAX_TOTAL_TRANSIENT_CONTINUATIONS ||
2900
+ loopLimitContinuations > MAX_LOOP_LIMIT_CONTINUATIONS) {
2901
+ recoveryGaveUpOnContinuationBudget = true;
2873
2902
  return { ok: false, resetVisibleContent: false };
2874
2903
  }
2875
2904
  if (isTransient) {
@@ -534,6 +534,13 @@ function isAutoRecoverableError(ev, errMsg) {
534
534
  }
535
535
  if (ev.recoverable === true)
536
536
  return true;
537
+ // An explicit flag outranks the message sniff below, which exists only for
538
+ // events that carry no flag at all. The repeat guards stop a turn with
539
+ // `recoverable: false` and a message that names the looping tool, so a stop
540
+ // on `list-workspace-connections` matched the "connection" sniff and
541
+ // auto-continued the very loop it was emitted to break.
542
+ if (ev.recoverable === false)
543
+ return false;
537
544
  if (msg.includes("daily gateway request cap"))
538
545
  return false;
539
546
  // "gateway error" intentionally absent — that's the no-detail Builder
@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
62
62
  error: string;
63
63
  states?: undefined;
64
64
  } | {
65
+ error?: undefined;
65
66
  states: {
66
67
  clientId: number;
67
68
  state: string;
68
69
  }[];
69
- error?: undefined;
70
70
  }>>;
71
71
  /**
72
72
  * GET /_agent-native/collab/:docId/users
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
77
77
  error: string;
78
78
  users?: undefined;
79
79
  } | {
80
+ error?: undefined;
80
81
  users: {
81
82
  clientId: number;
82
83
  lastSeen: number;
83
84
  }[];
84
- error?: undefined;
85
85
  }>>;
@@ -26,8 +26,8 @@ export declare const getCollabState: import("h3").EventHandlerWithFetch<import("
26
26
  * Body: { update: string (base64), requestSource?: string }
27
27
  */
28
28
  export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
29
- ok?: undefined;
30
29
  error: string;
30
+ ok?: undefined;
31
31
  } | {
32
32
  error?: undefined;
33
33
  ok: boolean;
package/dist/db/client.js CHANGED
@@ -1091,7 +1091,7 @@ const _sharedDbPoolReplacementHooks = new Map();
1091
1091
  * database access down with it.
1092
1092
  */
1093
1093
  export function sharedDbPool(driver, url, create) {
1094
- const key = `${driver}${url}`;
1094
+ const key = `${driver}\u0000${url}`;
1095
1095
  const existing = _sharedDbPools.get(key);
1096
1096
  if (existing)
1097
1097
  return existing;
@@ -1105,7 +1105,7 @@ export function sharedDbPool(driver, url, create) {
1105
1105
  * handing out the discarded pool.
1106
1106
  */
1107
1107
  export function replaceSharedDbPool(driver, url, previous, next) {
1108
- const key = `${driver}${url}`;
1108
+ const key = `${driver}\u0000${url}`;
1109
1109
  if (_sharedDbPools.get(key) !== previous)
1110
1110
  return;
1111
1111
  _sharedDbPools.set(key, next);
@@ -28,6 +28,6 @@ export declare function createNotificationsHandler(): import("h3").EventHandlerW
28
28
  } | {
29
29
  count?: undefined;
30
30
  updated?: undefined;
31
- error?: undefined;
32
31
  ok: boolean;
32
+ error?: undefined;
33
33
  }>>;
@@ -4890,18 +4890,44 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
4890
4890
  error: "Recurring-job sweep reached the synchronous server instead of the durable background worker.",
4891
4891
  };
4892
4892
  }
4893
+ // Stale reaping runs FIRST and site-wide, before the open-ended job
4894
+ // sweep can spend the platform wall. It is the durable driver the
4895
+ // in-process fast sweep below cannot be on serverless: that timer is
4896
+ // off wherever `shouldDisableInProcessSweeps` is on — i.e. every
4897
+ // production Lambda — and nothing replaced it, so a claimed run
4898
+ // whose producer died was only reaped when an unrelated request path
4899
+ // happened to look (prod: 1,216 runs, 12% of all runs, sitting
4900
+ // "running" for up to 59 minutes against a 15s window).
4901
+ //
4902
+ // One reap per site-tick instead of one per warm container is also
4903
+ // the point: the per-container timers are what issued 237k queries
4904
+ // in two minutes and earned that kill switch. `reapAllStaleRuns`
4905
+ // short-circuits on `hasRunningRuns()` (negative-cached), so an idle
4906
+ // app pays one probe.
4907
+ //
4908
+ // Reported, never swallowed, and never fatal to the job sweep: a
4909
+ // reap failure must not take recurring jobs down with it, and
4910
+ // `null` must stay distinguishable from "reaped nothing". The
4911
+ // result carries `failed` and `truncated` for the same reason one
4912
+ // level down — a pass where every row threw, and a pass that hit
4913
+ // the batch cap, both used to be reportable as a clean sweep.
4914
+ const { reapAllStaleRuns } = await import("../agent/run-store.js");
4915
+ const staleRunsReaped = await reapAllStaleRuns().catch((error) => {
4916
+ console.error("[agent-chat] durable stale-run reap failed:", error);
4917
+ return null;
4918
+ });
4893
4919
  try {
4894
4920
  // Jobs may request MCP tools, and `getActions` is synchronous —
4895
4921
  // hydrate before the sweep so a serverless container that never
4896
4922
  // eagerly initialized still resolves them.
4897
4923
  await ensureMcpInitialized();
4898
4924
  await processRecurringJobs(schedulerDeps);
4899
- return { ok: true };
4925
+ return { ok: true, staleRunsReaped };
4900
4926
  }
4901
4927
  catch (error) {
4902
4928
  console.error("[recurring-jobs] Sweep route failed:", error);
4903
4929
  setResponseStatus(event, 500);
4904
- return { error: "Recurring-job sweep failed" };
4930
+ return { error: "Recurring-job sweep failed", staleRunsReaped };
4905
4931
  }
4906
4932
  }));
4907
4933
  if (disableRecurringJobsRuntime) {
@@ -5139,7 +5165,16 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
5139
5165
  // against a 45s window). `reapAllStaleRuns` is per-row,
5140
5166
  // idempotent, re-checks staleness at UPDATE time and honours
5141
5167
  // the in-flight grace, so it is safe on this cadence.
5142
- await reapAllStaleRuns().catch(() => { });
5168
+ //
5169
+ // This timer is the driver only where it actually runs — a
5170
+ // long-lived Node host. Wherever `sweepsDisabled` turns it off
5171
+ // (every production serverless function) the signed
5172
+ // RECURRING_JOBS_SWEEP_PATH route above owns the same reap,
5173
+ // driven by the platform scheduler. The two never both fire on
5174
+ // one host, which is why neither needs to coordinate.
5175
+ await reapAllStaleRuns().catch((error) => {
5176
+ console.error("[agent-chat] in-process stale-run reap failed:", error);
5177
+ });
5143
5178
  let rows;
5144
5179
  try {
5145
5180
  rows = await listUnclaimedBackgroundRunRows();
@@ -20,6 +20,6 @@ export declare function createTranscribeVoiceHandler(): import("h3").EventHandle
20
20
  error: string;
21
21
  text?: undefined;
22
22
  } | {
23
- text: string;
24
23
  error?: undefined;
24
+ text: string;
25
25
  }>>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agent-native/core",
3
- "version": "0.153.5",
3
+ "version": "0.153.8",
4
4
  "description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
5
5
  "homepage": "https://github.com/BuilderIO/agent-native#readme",
6
6
  "bugs": {
@@ -421,8 +421,8 @@
421
421
  "y-protocols": "^1.0.7",
422
422
  "yjs": "^13.6.31",
423
423
  "zod": "^4.3.6",
424
- "@agent-native/toolkit": "^0.14.1",
425
- "@agent-native/recap-cli": "0.5.4"
424
+ "@agent-native/recap-cli": "0.5.4",
425
+ "@agent-native/toolkit": "^0.14.1"
426
426
  },
427
427
  "devDependencies": {
428
428
  "@ai-sdk/anthropic": "^3.0.71",