@agent-native/core 0.153.5 → 0.153.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/analytics/actions/get-data-program.ts +4 -1
- package/corpus/templates/analytics/server/lib/real-data-actions.ts +37 -13
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +2 -15
- package/dist/agent/engine/tool-call-journal-seed.js +18 -1
- package/dist/agent/production-agent.d.ts +13 -7
- package/dist/agent/production-agent.js +39 -15
- package/dist/agent/run-manager.js +17 -11
- package/dist/agent/run-store.d.ts +30 -1
- package/dist/agent/run-store.js +91 -16
- package/dist/agent/thread-data-builder.d.ts +1 -0
- package/dist/agent/thread-data-builder.js +15 -2
- package/dist/client/RouteTransitionIndicator.js +1 -1
- package/dist/client/agent-chat-adapter.js +58 -29
- package/dist/client/sse-event-processor.js +7 -0
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/routes.d.ts +1 -1
- package/dist/db/client.js +2 -2
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/server/agent-chat-plugin.js +38 -3
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/package.json +3 -3
|
@@ -22,7 +22,10 @@ export default defineAction({
|
|
|
22
22
|
}),
|
|
23
23
|
http: { method: "GET" },
|
|
24
24
|
readOnly: true,
|
|
25
|
-
grounding
|
|
25
|
+
// Not `grounding`: this reads the stored program plus its last-run summary,
|
|
26
|
+
// and even `includeRows` replays rows cached by an earlier run rather than
|
|
27
|
+
// querying the source. The flag means evidence retrieved at call time, which
|
|
28
|
+
// is why `get-analysis` is excluded too.
|
|
26
29
|
run: async (args) => {
|
|
27
30
|
const ctx = getCredentialContext();
|
|
28
31
|
if (!ctx) throw new Error("No authenticated context for get-data-program.");
|
|
@@ -91,27 +91,46 @@ function isToolName(name: string, expected: string): boolean {
|
|
|
91
91
|
let groundingActionNames: ReadonlySet<string> | null = null;
|
|
92
92
|
|
|
93
93
|
/**
|
|
94
|
-
*
|
|
94
|
+
* Read the action names whose definitions declare `grounding: true`.
|
|
95
95
|
*
|
|
96
|
-
*
|
|
97
|
-
* because that registry imports every action, and
|
|
98
|
-
* module — importing it back here would close an
|
|
99
|
-
* agent-chat plugin
|
|
96
|
+
* Takes the registry as an argument rather than importing
|
|
97
|
+
* `.generated/actions-registry` because that registry imports every action, and
|
|
98
|
+
* `save-analysis` imports this module — importing it back here would close an
|
|
99
|
+
* evaluation cycle. The agent-chat plugin passes it in once at module load.
|
|
100
100
|
*/
|
|
101
|
+
export function deriveGroundingActionNames(
|
|
102
|
+
registry: Record<string, unknown>,
|
|
103
|
+
): string[] {
|
|
104
|
+
return Object.entries(registry)
|
|
105
|
+
.filter(([, module]) => {
|
|
106
|
+
// Action modules reach the registry either as the module namespace or as
|
|
107
|
+
// an already-unwrapped definition, the same two shapes
|
|
108
|
+
// `loadActionsFromStaticRegistry` normalizes.
|
|
109
|
+
const candidate = module as
|
|
110
|
+
| { grounding?: boolean; default?: { grounding?: boolean } }
|
|
111
|
+
| undefined;
|
|
112
|
+
return (
|
|
113
|
+
candidate?.grounding === true || candidate?.default?.grounding === true
|
|
114
|
+
);
|
|
115
|
+
})
|
|
116
|
+
.map(([name]) => name);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Publish the derived set for the response guard to consult. */
|
|
101
120
|
export function registerGroundingActions(names: Iterable<string>): void {
|
|
102
|
-
|
|
121
|
+
groundingActionNames = new Set(
|
|
103
122
|
[...names].map((name) => normalizeActionToolName(name)),
|
|
104
123
|
);
|
|
105
124
|
// An empty set is never a real deployment: it means the installed core build
|
|
106
|
-
// predates `grounding` and dropped it from every definition.
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
125
|
+
// predates `grounding` and dropped it from every definition. Registration
|
|
126
|
+
// runs at plugin module scope, so throwing here would take the whole server
|
|
127
|
+
// down for a response-guard heuristic; the failure is raised at first use
|
|
128
|
+
// instead, where it costs one turn.
|
|
129
|
+
if (groundingActionNames.size === 0) {
|
|
130
|
+
console.error(
|
|
131
|
+
"[analytics] no action declares grounding: true; the installed @agent-native/core cannot carry the flag, so the response guard is unusable",
|
|
112
132
|
);
|
|
113
133
|
}
|
|
114
|
-
groundingActionNames = normalized;
|
|
115
134
|
}
|
|
116
135
|
|
|
117
136
|
function isGroundingActionName(name: string): boolean {
|
|
@@ -120,6 +139,11 @@ function isGroundingActionName(name: string): boolean {
|
|
|
120
139
|
"grounding actions were never registered: the analytics response guard cannot tell a grounded turn from an ungrounded one",
|
|
121
140
|
);
|
|
122
141
|
}
|
|
142
|
+
if (groundingActionNames.size === 0) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
"no action declares grounding: true; the installed @agent-native/core cannot carry the flag",
|
|
145
|
+
);
|
|
146
|
+
}
|
|
123
147
|
return groundingActionNames.has(normalizeActionToolName(name));
|
|
124
148
|
}
|
|
125
149
|
|
|
@@ -12,6 +12,7 @@ import { ANALYTICS_CONNECTOR_CATALOG } from "../lib/analytics-connector-catalog"
|
|
|
12
12
|
import { credentialProviderConfigs } from "../lib/credential-keys";
|
|
13
13
|
import { isProductionServerlessRuntime } from "../lib/production-serverless-runtime.js";
|
|
14
14
|
import {
|
|
15
|
+
deriveGroundingActionNames,
|
|
15
16
|
draftClaimsAnalyticsMetrics,
|
|
16
17
|
failedDataQueryAttemptMessage,
|
|
17
18
|
hasDashboardConstructionAttempt,
|
|
@@ -35,21 +36,7 @@ import {
|
|
|
35
36
|
// Which actions return real data-source evidence is a fact each action states
|
|
36
37
|
// on itself (`grounding: true`). Reading it off the definitions here is what
|
|
37
38
|
// keeps the response guard from drifting behind a newly shipped source action.
|
|
38
|
-
registerGroundingActions(
|
|
39
|
-
Object.entries(actionsRegistry)
|
|
40
|
-
.filter(([, module]) => {
|
|
41
|
-
// Action modules reach the registry either as the module namespace or as
|
|
42
|
-
// an already-unwrapped definition, the same two shapes
|
|
43
|
-
// `loadActionsFromStaticRegistry` normalizes.
|
|
44
|
-
const candidate = module as
|
|
45
|
-
| { grounding?: boolean; default?: { grounding?: boolean } }
|
|
46
|
-
| undefined;
|
|
47
|
-
return (
|
|
48
|
-
candidate?.grounding === true || candidate?.default?.grounding === true
|
|
49
|
-
);
|
|
50
|
-
})
|
|
51
|
-
.map(([name]) => name),
|
|
52
|
-
);
|
|
39
|
+
registerGroundingActions(deriveGroundingActionNames(actionsRegistry));
|
|
53
40
|
|
|
54
41
|
const ANALYTICS_BACKGROUND_RUN_SOFT_TIMEOUT_MS = 13 * 60_000;
|
|
55
42
|
// A background job may legitimately spend minutes inside a provider/tool call,
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
import { getCurrentTurnEventsForThread } from "../run-store.js";
|
|
2
2
|
import { classifyToolCallJournal, } from "../tool-call-journal.js";
|
|
3
|
+
const LEDGER_READ_RETRY_MS = 250;
|
|
4
|
+
/**
|
|
5
|
+
* `unreadable` ends the turn with a stop the user sees, so a single pool
|
|
6
|
+
* timeout or connection blip must not produce one. One retry, because the
|
|
7
|
+
* failures worth surviving are transient and the caller is holding the whole
|
|
8
|
+
* turn open while we wait.
|
|
9
|
+
*/
|
|
10
|
+
async function readCurrentTurnEventsWithRetry(threadId, turnId) {
|
|
11
|
+
try {
|
|
12
|
+
return await getCurrentTurnEventsForThread(threadId, turnId);
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
console.warn(`[tool-call-journal] per-turn ledger read failed for thread ${threadId}, retrying once: ${err instanceof Error ? err.message : String(err)}`);
|
|
16
|
+
await new Promise((resolve) => setTimeout(resolve, LEDGER_READ_RETRY_MS));
|
|
17
|
+
return await getCurrentTurnEventsForThread(threadId, turnId);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
3
20
|
/**
|
|
4
21
|
* Tool-call journal hard-block (resume safety). Snapshot the per-turn journal
|
|
5
22
|
* ONCE here, before any tool runs in this chunk, so it reflects only PRIOR
|
|
@@ -33,7 +50,7 @@ export async function loadPriorTurnToolCallJournal(threadId, turnId) {
|
|
|
33
50
|
}
|
|
34
51
|
let priorEvents;
|
|
35
52
|
try {
|
|
36
|
-
priorEvents = await
|
|
53
|
+
priorEvents = await readCurrentTurnEventsWithRetry(threadId, turnId);
|
|
37
54
|
}
|
|
38
55
|
catch (err) {
|
|
39
56
|
const error = err instanceof Error ? err.message : String(err);
|
|
@@ -618,12 +618,12 @@ export declare const MAX_IDENTICAL_TOOL_CALLS = 8;
|
|
|
618
618
|
/**
|
|
619
619
|
* A stop that ends the turn from inside the tool loop.
|
|
620
620
|
*
|
|
621
|
-
* `message` is the last thing the user reads, so it
|
|
622
|
-
*
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
621
|
+
* `message` is the last thing the user reads, so it names the remedy in plain
|
|
622
|
+
* words and does not quote a raw provider/tool error; the raw error goes in
|
|
623
|
+
* `details`. Both retry sniffers now honour the `recoverable: false` these
|
|
624
|
+
* stops are sent with, so a stop that happens to say "timeout" is no longer
|
|
625
|
+
* auto-continued — but the prose still has one reader who cannot ignore it,
|
|
626
|
+
* and a pasted stack trace is not a remedy.
|
|
627
627
|
*/
|
|
628
628
|
export interface TerminalActionStop {
|
|
629
629
|
message: string;
|
|
@@ -720,9 +720,15 @@ export declare function findApprovedStructuredToolCall(history: AgentChatStructu
|
|
|
720
720
|
* which is worse than one more retry.
|
|
721
721
|
*/
|
|
722
722
|
export declare function permanentPreconditionRemedy(message: string): string | null;
|
|
723
|
+
/**
|
|
724
|
+
* Deliberately carries no running call count. This decline is recorded as a
|
|
725
|
+
* tool error, and the breakers key on that text: a message that reports its own
|
|
726
|
+
* tally ("already made 13 call(s)") mints a fresh key on every decline, so the
|
|
727
|
+
* decline that exists to stop a spiral becomes the one thing nothing can count.
|
|
728
|
+
* The fixed threshold below says the same thing without varying.
|
|
729
|
+
*/
|
|
723
730
|
export declare function repeatedSourceSweepGuardMessage(opts: {
|
|
724
731
|
toolName: string;
|
|
725
|
-
priorCalls: number;
|
|
726
732
|
threshold?: number;
|
|
727
733
|
scope?: "tool" | "aggregate";
|
|
728
734
|
}): string;
|
|
@@ -2440,10 +2440,12 @@ async function waitForInterruptedToolLedgerEntry(opts) {
|
|
|
2440
2440
|
* sentence before it, and two failures that differ ONLY in the arguments they
|
|
2441
2441
|
* echo are the same failure.
|
|
2442
2442
|
*
|
|
2443
|
-
*
|
|
2444
|
-
*
|
|
2445
|
-
*
|
|
2446
|
-
*
|
|
2443
|
+
* Digits stay. Collapsing them merges failures that are genuinely different:
|
|
2444
|
+
* "Record 41 not found" and "Record 42 not found" are two missing records, so a
|
|
2445
|
+
* seven-item sweep where every item legitimately 404s would trip the
|
|
2446
|
+
* across-arguments breaker at item six. A guard message that varies its own
|
|
2447
|
+
* running tally is fixed where that message is written, not by blinding every
|
|
2448
|
+
* breaker to numbers.
|
|
2447
2449
|
*/
|
|
2448
2450
|
function normalizeToolErrorForBreaker(error) {
|
|
2449
2451
|
return (error
|
|
@@ -2451,7 +2453,6 @@ function normalizeToolErrorForBreaker(error) {
|
|
|
2451
2453
|
.replace(/Received:\s*[\s\S]*?\.\s(?=Expected:|The tool was not executed)/g, "")
|
|
2452
2454
|
// Bare JSON payloads some providers inline instead of a Received: span.
|
|
2453
2455
|
.replace(/\{[\s\S]{0,2000}?\}/g, "{}")
|
|
2454
|
-
.replace(/\d+/g, "#")
|
|
2455
2456
|
.replace(/\s+/g, " ")
|
|
2456
2457
|
.trim());
|
|
2457
2458
|
}
|
|
@@ -2494,16 +2495,27 @@ const PERMANENT_PRECONDITION_PATTERNS = [
|
|
|
2494
2495
|
/\b(?:api[ -]?keys?|access tokens?|credentials?|secrets?)\b[^.]{0,60}\bnot (?:configured|set|connected|available)\b/i,
|
|
2495
2496
|
/\bsave [A-Z][A-Z0-9_]{3,} in (?:the )?settings\b/i,
|
|
2496
2497
|
// "Connect Builder.io before indexing a design system from Figma or code."
|
|
2497
|
-
|
|
2498
|
-
//
|
|
2499
|
-
//
|
|
2500
|
-
|
|
2498
|
+
// Case-sensitive, sentence-initial, and followed by a capitalized service
|
|
2499
|
+
// name, because that is the only shape that separates the imperative remedy
|
|
2500
|
+
// from the retryable network failure: "failed to connect to the warehouse
|
|
2501
|
+
// before the deadline" and "Connect timed out, retry first" both satisfy
|
|
2502
|
+
// every looser reading of the same words.
|
|
2503
|
+
/(?:^|[.:!?]\s+)Connect [A-Z][\w.-]*[^;]{0,40}?\b(?:before|first|in settings)\b/,
|
|
2501
2504
|
// "Plan mode blocked `update-extension`. Switch to Act mode …" — cleared by
|
|
2502
2505
|
// the user approving the plan, never by the model trying again.
|
|
2503
2506
|
/\bplan mode blocked\b/i,
|
|
2507
|
+
// The templates' most-copied throw, and always `if (!getRequestUserEmail())`:
|
|
2508
|
+
// one synchronous AsyncLocalStorage read of a value fixed when the request
|
|
2509
|
+
// context was established. It cannot appear and disappear between tool calls
|
|
2510
|
+
// of the same chunk, so retrying inside this turn lands on it again.
|
|
2504
2511
|
/\bno authenticated user\b/i,
|
|
2505
2512
|
// "SSRF blocked: refusing to fetch private/internal address (…)"
|
|
2506
2513
|
/\bssrf blocked\b/i,
|
|
2514
|
+
// Deliberately absent: "only available in …". Retention windows ("only
|
|
2515
|
+
// available from the last 90 days") share its wording and are fixed by
|
|
2516
|
+
// narrowing the range, so it stopped turns that were one argument away from
|
|
2517
|
+
// succeeding. The count-based breaker still ends a genuine runtime gate
|
|
2518
|
+
// after six.
|
|
2507
2519
|
];
|
|
2508
2520
|
const SOURCE_SWEEP_TOOL_NAME = /\b(?:api|calls?|deals?|docs?|events?|issues?|messages?|metrics?|provider|query|records?|request|search|source|tickets?|transcripts?)\b/i;
|
|
2509
2521
|
const SOURCE_SWEEP_PROVIDER_TOKEN = /\b(?:amplitude|apollo|bigquery|commonroom|data-source|ga4|github|gong|grafana|hubspot|jira|mixpanel|notion|posthog|postgres|postgresql|pylon|sentry|slack|stripe)\b/i;
|
|
@@ -2610,14 +2622,20 @@ function restrictAgentTeamsAfterSourceSweep(tools) {
|
|
|
2610
2622
|
};
|
|
2611
2623
|
});
|
|
2612
2624
|
}
|
|
2625
|
+
/**
|
|
2626
|
+
* Deliberately carries no running call count. This decline is recorded as a
|
|
2627
|
+
* tool error, and the breakers key on that text: a message that reports its own
|
|
2628
|
+
* tally ("already made 13 call(s)") mints a fresh key on every decline, so the
|
|
2629
|
+
* decline that exists to stop a spiral becomes the one thing nothing can count.
|
|
2630
|
+
* The fixed threshold below says the same thing without varying.
|
|
2631
|
+
*/
|
|
2613
2632
|
export function repeatedSourceSweepGuardMessage(opts) {
|
|
2614
2633
|
const threshold = opts.threshold ?? SOURCE_SWEEP_TOOL_CALL_THRESHOLD;
|
|
2615
2634
|
const target = opts.scope === "aggregate"
|
|
2616
2635
|
? "read-only source/search tools"
|
|
2617
2636
|
: "the same read-only source/search tool";
|
|
2618
|
-
return (`Skipped ${opts.toolName}: this turn already
|
|
2619
|
-
|
|
2620
|
-
`${threshold}-call convergence budget. Stop calling ${opts.toolName} ` +
|
|
2637
|
+
return (`Skipped ${opts.toolName}: this turn already exhausted its ` +
|
|
2638
|
+
`${threshold}-call convergence budget for ${target}. Stop calling ${opts.toolName} ` +
|
|
2621
2639
|
`one item at a time and change strategy before answering. If a broader ` +
|
|
2622
2640
|
`read-only bulk/source mechanism is available, use it now: provider API ` +
|
|
2623
2641
|
`catalog/docs/request tools with pagination or staging, code execution ` +
|
|
@@ -2647,7 +2665,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
|
|
|
2647
2665
|
priorCalls: priorSourceSweepCalls,
|
|
2648
2666
|
message: repeatedSourceSweepGuardMessage({
|
|
2649
2667
|
toolName: opts.toolName,
|
|
2650
|
-
priorCalls: priorSourceSweepCalls,
|
|
2651
2668
|
threshold,
|
|
2652
2669
|
scope: "aggregate",
|
|
2653
2670
|
}),
|
|
@@ -2660,7 +2677,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
|
|
|
2660
2677
|
priorCalls,
|
|
2661
2678
|
message: repeatedSourceSweepGuardMessage({
|
|
2662
2679
|
toolName: opts.toolName,
|
|
2663
|
-
priorCalls,
|
|
2664
2680
|
threshold,
|
|
2665
2681
|
scope: "tool",
|
|
2666
2682
|
}),
|
|
@@ -4041,7 +4057,8 @@ export async function runAgentLoop(opts) {
|
|
|
4041
4057
|
const permanentRemedy = permanentPreconditionRemedy(sanitizedResult);
|
|
4042
4058
|
if (permanentRemedy) {
|
|
4043
4059
|
requestedActionStop ??= {
|
|
4044
|
-
message: `I stopped because ${toolCall.name}
|
|
4060
|
+
message: `I stopped because ${toolCall.name} needs a setup step outside this turn — a credential, a role, a connected account, or an approval — before it can run. ` +
|
|
4061
|
+
"Retrying would not have changed it, and anything completed before this is saved.",
|
|
4045
4062
|
errorCode: "permanent_precondition",
|
|
4046
4063
|
details: sanitizedResult,
|
|
4047
4064
|
};
|
|
@@ -5056,6 +5073,13 @@ function isRecoverableContinuationError(event) {
|
|
|
5056
5073
|
const message = event.error.toLowerCase();
|
|
5057
5074
|
if (code === "builder_gateway_error")
|
|
5058
5075
|
return false;
|
|
5076
|
+
// An explicit flag outranks the message sniff below, which exists for events
|
|
5077
|
+
// that carry no flag at all. Every guard stop sets `recoverable: false`, and
|
|
5078
|
+
// sniffing its prose is how a stop that merely NAMES a connection or a
|
|
5079
|
+
// timeout gets continued into the spiral it was raised to end. The client
|
|
5080
|
+
// applies the same precedence in `isAutoRecoverableError`.
|
|
5081
|
+
if (event.recoverable === false)
|
|
5082
|
+
return false;
|
|
5059
5083
|
return (event.recoverable === true ||
|
|
5060
5084
|
code === "builder_gateway_timeout" ||
|
|
5061
5085
|
code === "builder_gateway_network_error" ||
|
|
@@ -937,12 +937,15 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
937
937
|
// false-stale-reap zombie scenario where the reaper flipped the row while
|
|
938
938
|
// this isolate was briefly unable to heartbeat (DB latency / GC pause).
|
|
939
939
|
let lastAbortCheck = Date.now() - 3000;
|
|
940
|
-
// A read failure here
|
|
941
|
-
//
|
|
942
|
-
//
|
|
943
|
-
//
|
|
944
|
-
//
|
|
945
|
-
//
|
|
940
|
+
// A read failure here is not "not aborted" — but it is not grounds to kill
|
|
941
|
+
// the run either. This check is only a cross-isolate backstop: the outage
|
|
942
|
+
// that hides a Stop from us hides it from every other reader too, so
|
|
943
|
+
// self-aborting delivers nobody's Stop any sooner while destroying in-flight
|
|
944
|
+
// work in the far more common case where nobody pressed Stop at all. Fail
|
|
945
|
+
// open and stay bounded by the soft timeout, no-progress backstop, and
|
|
946
|
+
// iteration/token limits — none of which touch the DB. Report the outage to
|
|
947
|
+
// Sentry (once at the threshold, then ~once a minute) so a long unreadable
|
|
948
|
+
// window is visible and its duration is queryable instead of silent.
|
|
946
949
|
let consecutiveAbortCheckFailures = 0;
|
|
947
950
|
const checkSqlAbort = () => {
|
|
948
951
|
const now = Date.now();
|
|
@@ -970,7 +973,9 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
970
973
|
})
|
|
971
974
|
.catch((error) => {
|
|
972
975
|
consecutiveAbortCheckFailures += 1;
|
|
973
|
-
|
|
976
|
+
// 3 ticks ≈ 9s of unreadability, then every 20 ticks ≈ 60s.
|
|
977
|
+
if (consecutiveAbortCheckFailures === 3 ||
|
|
978
|
+
consecutiveAbortCheckFailures % 20 === 0) {
|
|
974
979
|
captureError(error, {
|
|
975
980
|
route: "/_agent-native/agent-chat",
|
|
976
981
|
tags: {
|
|
@@ -979,11 +984,12 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
979
984
|
consecutiveFailures: String(consecutiveAbortCheckFailures),
|
|
980
985
|
},
|
|
981
986
|
aiTraceId: runId,
|
|
982
|
-
extra: {
|
|
987
|
+
extra: {
|
|
988
|
+
runId,
|
|
989
|
+
threadId,
|
|
990
|
+
unreadableForMs: consecutiveAbortCheckFailures * 3000,
|
|
991
|
+
},
|
|
983
992
|
});
|
|
984
|
-
if (!abort.signal.aborted) {
|
|
985
|
-
abortInMemoryRun(run, "abort_check_unavailable");
|
|
986
|
-
}
|
|
987
993
|
}
|
|
988
994
|
});
|
|
989
995
|
};
|
|
@@ -44,6 +44,22 @@ export declare const STALE_RUN_ERROR_EVENT: {
|
|
|
44
44
|
readonly recoverable: true;
|
|
45
45
|
readonly details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.";
|
|
46
46
|
};
|
|
47
|
+
/**
|
|
48
|
+
* `agent_runs.terminal_reason` recorded by the stale reapers.
|
|
49
|
+
*
|
|
50
|
+
* Every other error path records `error:<code>` (`terminalReasonForEvent`), and
|
|
51
|
+
* `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
|
|
52
|
+
* replays the terminal event a reaper itself appended. The reapers used to write
|
|
53
|
+
* the bare code, which split one outcome across two permanent
|
|
54
|
+
* `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
|
|
55
|
+
* `error:stale_run`) purely by whether anything happened to read the row after
|
|
56
|
+
* the reap — and made the reconciliation convergence guard, which compares
|
|
57
|
+
* `terminal_reason` for equality, rewrite the row on every read forever.
|
|
58
|
+
*
|
|
59
|
+
* Rows reaped before this change keep the bare `stale_run`, so anything matching
|
|
60
|
+
* on the value must accept both forms for one retention window.
|
|
61
|
+
*/
|
|
62
|
+
export declare const STALE_RUN_TERMINAL_REASON: string;
|
|
47
63
|
/**
|
|
48
64
|
* Terminal error for a background-dispatched run whose worker NEVER claimed it
|
|
49
65
|
* (the foreground fired the self-dispatch, Netlify acked it async with a 202,
|
|
@@ -665,13 +681,26 @@ export declare function listRunsForThread(threadId: string, options?: {
|
|
|
665
681
|
* Best-effort on parse: malformed ledger rows are skipped rather than thrown.
|
|
666
682
|
*/
|
|
667
683
|
export declare function getCurrentTurnEventsForThread(threadId: string, knownTurnId?: string): Promise<AgentChatEvent[]>;
|
|
684
|
+
export interface StaleRunReapResult {
|
|
685
|
+
/** Rows this pass reaped. */
|
|
686
|
+
reaped: number;
|
|
687
|
+
/**
|
|
688
|
+
* Rows whose reap threw. Separate from `reaped` because a pass where every
|
|
689
|
+
* row failed must not read as "nothing was stale" — those were the same
|
|
690
|
+
* number before, and the caller's own `null`-vs-`0` distinction is worthless
|
|
691
|
+
* while the per-row failures underneath it are coerced away.
|
|
692
|
+
*/
|
|
693
|
+
failed: number;
|
|
694
|
+
/** More stale rows remain than the batch cap; the next tick continues. */
|
|
695
|
+
truncated: boolean;
|
|
696
|
+
}
|
|
668
697
|
/**
|
|
669
698
|
* Expire any "running" rows whose heartbeat is stale — producer died.
|
|
670
699
|
* Safe to call at server startup on multi-isolate deployments: only rows
|
|
671
700
|
* without a fresh heartbeat get reaped, so runs owned by OTHER live
|
|
672
701
|
* isolates (which keep heartbeating) are left alone.
|
|
673
702
|
*/
|
|
674
|
-
export declare function reapAllStaleRuns(): Promise<
|
|
703
|
+
export declare function reapAllStaleRuns(): Promise<StaleRunReapResult>;
|
|
675
704
|
/**
|
|
676
705
|
* Read the daily terminal-outcome counters rolled up from pruned runs. Pair
|
|
677
706
|
* with live `agent_runs` rows for a complete picture — see
|
package/dist/agent/run-store.js
CHANGED
|
@@ -50,6 +50,22 @@ export const STALE_RUN_ERROR_EVENT = {
|
|
|
50
50
|
recoverable: true,
|
|
51
51
|
details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
|
|
52
52
|
};
|
|
53
|
+
/**
|
|
54
|
+
* `agent_runs.terminal_reason` recorded by the stale reapers.
|
|
55
|
+
*
|
|
56
|
+
* Every other error path records `error:<code>` (`terminalReasonForEvent`), and
|
|
57
|
+
* `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
|
|
58
|
+
* replays the terminal event a reaper itself appended. The reapers used to write
|
|
59
|
+
* the bare code, which split one outcome across two permanent
|
|
60
|
+
* `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
|
|
61
|
+
* `error:stale_run`) purely by whether anything happened to read the row after
|
|
62
|
+
* the reap — and made the reconciliation convergence guard, which compares
|
|
63
|
+
* `terminal_reason` for equality, rewrite the row on every read forever.
|
|
64
|
+
*
|
|
65
|
+
* Rows reaped before this change keep the bare `stale_run`, so anything matching
|
|
66
|
+
* on the value must accept both forms for one retention window.
|
|
67
|
+
*/
|
|
68
|
+
export const STALE_RUN_TERMINAL_REASON = `error:${STALE_RUN_ERROR_EVENT.errorCode}`;
|
|
53
69
|
/**
|
|
54
70
|
* Terminal error for a background-dispatched run whose worker NEVER claimed it
|
|
55
71
|
* (the foreground fired the self-dispatch, Netlify acked it async with a 202,
|
|
@@ -1306,6 +1322,22 @@ export async function recordRunDiagnostic(runId, stage, detail) {
|
|
|
1306
1322
|
// Diagnostics are best-effort; never let them break the run or the route.
|
|
1307
1323
|
}
|
|
1308
1324
|
}
|
|
1325
|
+
/**
|
|
1326
|
+
* Wall-clock budget for ONE heartbeat write.
|
|
1327
|
+
*
|
|
1328
|
+
* The default `DbExec` budget is `dbOpTimeoutMs()` (8s on serverless) across 3
|
|
1329
|
+
* connection attempts, so a single heartbeat can occupy up to 24s — against a
|
|
1330
|
+
* 15s `RUN_STALE_MS`. The run-manager single-flights the write, so that one
|
|
1331
|
+
* attempt suppresses every 1.5s tick behind it: a live foreground run in a quiet
|
|
1332
|
+
* phase becomes reapable while its own heartbeat is still in flight. The same
|
|
1333
|
+
* attempt also pins a pooler connection for those 24s, feeding the saturation
|
|
1334
|
+
* that starves heartbeats in the first place.
|
|
1335
|
+
*
|
|
1336
|
+
* A heartbeat that cannot land promptly has no value — the 1.5s tick IS the
|
|
1337
|
+
* retry — so it gets ONE attempt inside a third of the stale window, leaving
|
|
1338
|
+
* room for two more to land before the reap cutoff.
|
|
1339
|
+
*/
|
|
1340
|
+
const HEARTBEAT_WRITE_TIMEOUT_MS = Math.floor(RUN_STALE_MS / 3);
|
|
1309
1341
|
/** Update the run's liveness heartbeat. Called periodically by run-manager. */
|
|
1310
1342
|
export async function updateRunHeartbeat(runId) {
|
|
1311
1343
|
await ensureRunTables();
|
|
@@ -1324,6 +1356,8 @@ export async function updateRunHeartbeat(runId) {
|
|
|
1324
1356
|
END
|
|
1325
1357
|
WHERE id = ? AND status = 'running'`,
|
|
1326
1358
|
args: [Date.now(), currentRssMb(), currentRssMb(), runId],
|
|
1359
|
+
timeoutMs: HEARTBEAT_WRITE_TIMEOUT_MS,
|
|
1360
|
+
maxAttempts: 1,
|
|
1327
1361
|
});
|
|
1328
1362
|
}
|
|
1329
1363
|
/**
|
|
@@ -1612,9 +1646,15 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
|
|
|
1612
1646
|
: // First `?` is backgroundAwareStaleCutoffSql's CAST param, the next two
|
|
1613
1647
|
// are inFlightGraceSql's — all bound to the same "now".
|
|
1614
1648
|
[completedAt, completedAt, completedAt];
|
|
1649
|
+
// `completed_at` is when the run DIED, not when a reaper noticed. Stamping
|
|
1650
|
+
// reap time made every stale row report its own detection latency as run
|
|
1651
|
+
// duration (prod: 30-59 minute "runs" against a 15s window), which is the
|
|
1652
|
+
// measurement used to judge whether reaping is timely at all. The liveness
|
|
1653
|
+
// basis is the last proof the producer was alive, and it is what
|
|
1654
|
+
// `reconcileTerminalRunFromEvents` already records for the same failure.
|
|
1615
1655
|
const updateSql = `UPDATE agent_runs
|
|
1616
1656
|
SET status = 'errored',
|
|
1617
|
-
completed_at =
|
|
1657
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
1618
1658
|
error_code = ?,
|
|
1619
1659
|
error_detail = ?,
|
|
1620
1660
|
terminal_reason = ?
|
|
@@ -1623,10 +1663,9 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
|
|
|
1623
1663
|
AND ${terminalRunEventExclusionSql()}
|
|
1624
1664
|
AND ${staleClause}`;
|
|
1625
1665
|
const updateArgs = [
|
|
1626
|
-
completedAt,
|
|
1627
1666
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
1628
1667
|
STALE_RUN_ERROR_EVENT.details,
|
|
1629
|
-
|
|
1668
|
+
STALE_RUN_TERMINAL_REASON,
|
|
1630
1669
|
runId,
|
|
1631
1670
|
...staleArgs,
|
|
1632
1671
|
];
|
|
@@ -2214,6 +2253,18 @@ export async function getCurrentTurnEventsForThread(threadId, knownTurnId) {
|
|
|
2214
2253
|
}
|
|
2215
2254
|
return events;
|
|
2216
2255
|
}
|
|
2256
|
+
/**
|
|
2257
|
+
* Rows one pass will reap. Each costs ~5-10 serial round trips (reconcile,
|
|
2258
|
+
* transactional reap plus recovery, terminal-event append), and this sweep now
|
|
2259
|
+
* runs ahead of `processRecurringJobs` on the shared durable tick — so an
|
|
2260
|
+
* unbounded first pass against a backlog spends the platform wall before jobs
|
|
2261
|
+
* get any. Production carried 1,216 stale rows while nothing periodic reaped
|
|
2262
|
+
* them, so the first tick after that fix lands on exactly such a backlog.
|
|
2263
|
+
* A truncated pass is drained by the next tick (60s durable, 20s in-process),
|
|
2264
|
+
* never dropped — `truncated` is what stops a partial sweep being reported as
|
|
2265
|
+
* a clean one.
|
|
2266
|
+
*/
|
|
2267
|
+
const REAP_ALL_STALE_BATCH_LIMIT = 200;
|
|
2217
2268
|
/**
|
|
2218
2269
|
* Expire any "running" rows whose heartbeat is stale — producer died.
|
|
2219
2270
|
* Safe to call at server startup on multi-isolate deployments: only rows
|
|
@@ -2221,9 +2272,14 @@ export async function getCurrentTurnEventsForThread(threadId, knownTurnId) {
|
|
|
2221
2272
|
* isolates (which keep heartbeating) are left alone.
|
|
2222
2273
|
*/
|
|
2223
2274
|
export async function reapAllStaleRuns() {
|
|
2275
|
+
const nothingStale = {
|
|
2276
|
+
reaped: 0,
|
|
2277
|
+
failed: 0,
|
|
2278
|
+
truncated: false,
|
|
2279
|
+
};
|
|
2224
2280
|
await ensureRunTables();
|
|
2225
2281
|
if (!(await hasRunningRuns()))
|
|
2226
|
-
return
|
|
2282
|
+
return nothingStale;
|
|
2227
2283
|
const client = getDbExec();
|
|
2228
2284
|
const now = Date.now();
|
|
2229
2285
|
// Background-dispatched runs use the wider window; everything else 15s. The
|
|
@@ -2235,13 +2291,23 @@ export async function reapAllStaleRuns() {
|
|
|
2235
2291
|
// server startup across possibly-multiple isolates, so a sibling isolate's
|
|
2236
2292
|
// still-alive, in-flight run must not be reaped just because THIS isolate
|
|
2237
2293
|
// just booted and has no heartbeat history for it.
|
|
2238
|
-
|
|
2294
|
+
// One row over the cap is how a truncated pass is detected without a second
|
|
2295
|
+
// COUNT query against the same predicate.
|
|
2296
|
+
const scanned = await client.execute({
|
|
2239
2297
|
sql: `SELECT id FROM agent_runs
|
|
2240
2298
|
WHERE status = 'running'
|
|
2241
2299
|
AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
|
|
2242
|
-
AND ${inFlightGraceSql()}
|
|
2300
|
+
AND ${inFlightGraceSql()}
|
|
2301
|
+
ORDER BY started_at ASC
|
|
2302
|
+
LIMIT ${REAP_ALL_STALE_BATCH_LIMIT + 1}`,
|
|
2243
2303
|
args: [now, now, now],
|
|
2244
2304
|
});
|
|
2305
|
+
const truncated = scanned.rows.length > REAP_ALL_STALE_BATCH_LIMIT;
|
|
2306
|
+
const stale = {
|
|
2307
|
+
rows: truncated
|
|
2308
|
+
? scanned.rows.slice(0, REAP_ALL_STALE_BATCH_LIMIT)
|
|
2309
|
+
: scanned.rows,
|
|
2310
|
+
};
|
|
2245
2311
|
for (const row of stale.rows) {
|
|
2246
2312
|
const id = row.id;
|
|
2247
2313
|
if (typeof id === "string") {
|
|
@@ -2260,13 +2326,19 @@ export async function reapAllStaleRuns() {
|
|
|
2260
2326
|
// already found stale; on an idle app `hasRunningRuns` returns before any of
|
|
2261
2327
|
// this, so the 20s fast sweep (agent-chat-plugin.ts) costs one probe.
|
|
2262
2328
|
let reapedCount = 0;
|
|
2329
|
+
let failedCount = 0;
|
|
2263
2330
|
for (const row of stale.rows) {
|
|
2264
2331
|
const id = row.id;
|
|
2265
2332
|
if (typeof id !== "string")
|
|
2266
2333
|
continue;
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2334
|
+
try {
|
|
2335
|
+
if (await reapSingleStaleRun(id))
|
|
2336
|
+
reapedCount += 1;
|
|
2337
|
+
}
|
|
2338
|
+
catch (error) {
|
|
2339
|
+
failedCount += 1;
|
|
2340
|
+
console.error(`[run-store] stale reap failed for run ${id}:`, error);
|
|
2341
|
+
}
|
|
2270
2342
|
}
|
|
2271
2343
|
for (const row of stale.rows) {
|
|
2272
2344
|
const id = row.id;
|
|
@@ -2274,7 +2346,7 @@ export async function reapAllStaleRuns() {
|
|
|
2274
2346
|
await safeAppendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT, "reap-all-stale");
|
|
2275
2347
|
}
|
|
2276
2348
|
}
|
|
2277
|
-
return reapedCount;
|
|
2349
|
+
return { reaped: reapedCount, failed: failedCount, truncated };
|
|
2278
2350
|
}
|
|
2279
2351
|
const RUN_OUTCOME_DAY_MS = 86_400_000;
|
|
2280
2352
|
/**
|
|
@@ -2433,11 +2505,16 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2433
2505
|
await reconcileTerminalRunFromEvents(id);
|
|
2434
2506
|
}
|
|
2435
2507
|
}
|
|
2508
|
+
// Both UPDATEs below backdate `completed_at` to the liveness basis for the
|
|
2509
|
+
// reason given on `reapSingleStaleRun`'s UPDATE: this pass can fire a DAY
|
|
2510
|
+
// after the producer died, and stamping now would file that whole gap as run
|
|
2511
|
+
// duration. On the absolute-age branch the basis degrades to `started_at`,
|
|
2512
|
+
// which is still the truest thing known about a row that never heartbeat.
|
|
2436
2513
|
const completedAt = Date.now();
|
|
2437
2514
|
await client.execute({
|
|
2438
2515
|
sql: `UPDATE agent_runs
|
|
2439
2516
|
SET status = 'errored',
|
|
2440
|
-
completed_at =
|
|
2517
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
2441
2518
|
error_code = ?,
|
|
2442
2519
|
error_detail = ?,
|
|
2443
2520
|
terminal_reason = ?
|
|
@@ -2445,10 +2522,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2445
2522
|
AND ${terminalRunEventExclusionSql()}
|
|
2446
2523
|
AND started_at < ?`,
|
|
2447
2524
|
args: [
|
|
2448
|
-
completedAt,
|
|
2449
2525
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
2450
2526
|
STALE_RUN_ERROR_EVENT.details,
|
|
2451
|
-
|
|
2527
|
+
STALE_RUN_TERMINAL_REASON,
|
|
2452
2528
|
cutoff,
|
|
2453
2529
|
],
|
|
2454
2530
|
});
|
|
@@ -2463,7 +2539,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2463
2539
|
await client.execute({
|
|
2464
2540
|
sql: `UPDATE agent_runs
|
|
2465
2541
|
SET status = 'errored',
|
|
2466
|
-
completed_at =
|
|
2542
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
2467
2543
|
error_code = ?,
|
|
2468
2544
|
error_detail = ?,
|
|
2469
2545
|
terminal_reason = ?
|
|
@@ -2472,10 +2548,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2472
2548
|
AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
|
|
2473
2549
|
AND ${inFlightGraceSql()}`,
|
|
2474
2550
|
args: [
|
|
2475
|
-
completedAt,
|
|
2476
2551
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
2477
2552
|
STALE_RUN_ERROR_EVENT.details,
|
|
2478
|
-
|
|
2553
|
+
STALE_RUN_TERMINAL_REASON,
|
|
2479
2554
|
completedAt,
|
|
2480
2555
|
completedAt,
|
|
2481
2556
|
completedAt,
|
|
@@ -243,18 +243,31 @@ export function buildAssistantMessage(events, runId, options = {}) {
|
|
|
243
243
|
metadata,
|
|
244
244
|
};
|
|
245
245
|
}
|
|
246
|
+
/**
|
|
247
|
+
* The rebuild half of the live client's `clearAssistantDraftContent`
|
|
248
|
+
* (client/sse-event-processor.ts). The two bodies are asserted identical by
|
|
249
|
+
* `keeps clearAssistantDraftContent identical to the live client copy` in
|
|
250
|
+
* thread-data-builder.spec.ts — a rebuild that clears more than the live stream
|
|
251
|
+
* did makes narration vanish on reload, which is worse than clearing nothing.
|
|
252
|
+
*/
|
|
246
253
|
function clearAssistantDraftContent(content) {
|
|
247
254
|
for (let index = content.length - 1; index >= 0; index--) {
|
|
248
255
|
const part = content[index];
|
|
249
256
|
if (!part)
|
|
250
257
|
continue;
|
|
258
|
+
if (part.type === "tool-call" &&
|
|
259
|
+
part.activity !== true &&
|
|
260
|
+
part.result !== undefined) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
251
263
|
if (part.type === "text" || part.type === "reasoning") {
|
|
252
264
|
content.splice(index, 1);
|
|
253
265
|
continue;
|
|
254
266
|
}
|
|
255
267
|
if (part.type === "tool-call" && part.result === undefined) {
|
|
256
|
-
//
|
|
257
|
-
//
|
|
268
|
+
// Only drop ephemeral placeholders. Materialized in-flight tool cards
|
|
269
|
+
// (real args from tool_start) stay mounted so a retry/auto-continue clear
|
|
270
|
+
// does not hide→show the same call when the next chunk re-emits it.
|
|
258
271
|
const isEphemeral = part.activity === true ||
|
|
259
272
|
part.argsText === "" ||
|
|
260
273
|
Object.keys(part.args ?? {}).length === 0;
|
|
@@ -37,5 +37,5 @@ export function RouteTransitionIndicator() {
|
|
|
37
37
|
}, [destination]);
|
|
38
38
|
if (!destination || visibleDestination !== destination)
|
|
39
39
|
return null;
|
|
40
|
-
return (_jsx("div", { "aria-label": "Loading
|
|
40
|
+
return (_jsx("div", { "aria-label": "Loading page...", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
|
|
41
41
|
}
|
|
@@ -24,20 +24,29 @@ const AUTO_CONTINUE_COMPLETION_GUARD = "Before doing more work, inspect the prio
|
|
|
24
24
|
const MAX_RECONNECT_ATTEMPTS = 5;
|
|
25
25
|
const MAX_STARTUP_RECOVERY_ATTEMPTS = 8;
|
|
26
26
|
const MAX_QUEUED_CONFLICT_RETRIES = 120;
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
// five separate budgets — stale-run, stalled, empty, repeated-narration
|
|
30
|
-
// repeated-action-preparation — each of which existed because the rung above
|
|
27
|
+
// The consecutive non-advancing continuation (see `describeContinuationAdvance`)
|
|
28
|
+
// that ends the turn: the third one stops it, so two are tolerated. This
|
|
29
|
+
// replaced five separate budgets — stale-run, stalled, empty, repeated-narration
|
|
30
|
+
// and repeated-action-preparation — each of which existed because the rung above
|
|
31
31
|
// it had a hole the next one patched. One question answers all five: a round
|
|
32
32
|
// that produced nothing, or produced exactly what the round before it
|
|
33
33
|
// produced, did not advance.
|
|
34
34
|
const MAX_NON_ADVANCING_CONTINUATIONS = 3;
|
|
35
|
-
// Ceiling across the whole turn
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
// longest recovery we deliberately support.
|
|
35
|
+
// Ceiling across the whole turn for TRANSIENT continuations — rounds that
|
|
36
|
+
// followed a failure (timeout, dropped stream, stale run). Every attempt
|
|
37
|
+
// re-POSTs the full history plus attachments, so a high ceiling only burns
|
|
38
|
+
// tokens and wall time on a connection that keeps breaking.
|
|
40
39
|
const MAX_TOTAL_TRANSIENT_CONTINUATIONS = 12;
|
|
40
|
+
// Ceiling across the whole turn for WORK-boundary continuations. A `loop_limit`
|
|
41
|
+
// round is not a failure — the server spent a full iteration budget on real
|
|
42
|
+
// tool work and handed the turn back — so counting it against the transient
|
|
43
|
+
// ceiling killed progressing turns at round 13 that had never failed once.
|
|
44
|
+
// Sized against the server's own limits: at its default budget of 400
|
|
45
|
+
// iterations per run this is ~10,000 tool calls, two orders past the deepest
|
|
46
|
+
// legitimate production turn (117 tool calls) that sized that budget. It still
|
|
47
|
+
// has to exist, because the server's per-turn token backstop rides the request
|
|
48
|
+
// body between server-chained chunks and resets on every client re-POST.
|
|
49
|
+
const MAX_LOOP_LIMIT_CONTINUATIONS = 25;
|
|
41
50
|
const RETRY_BASE_DELAY_MS = 500;
|
|
42
51
|
const RETRY_MAX_DELAY_MS = 8_000;
|
|
43
52
|
const MAX_HISTORY_ATTACHMENT_CHARS = 60_000;
|
|
@@ -742,7 +751,11 @@ function estimateHistoryMessageCost(message) {
|
|
|
742
751
|
const argsText = tool.argsText ?? stableJson(tool.args ?? {});
|
|
743
752
|
cost += Math.min(argsText.length, argsCap);
|
|
744
753
|
if (tool.result !== undefined) {
|
|
745
|
-
cost += Math.min(
|
|
754
|
+
cost += Math.min(
|
|
755
|
+
// Price the string the request actually carries. `String(result)` is
|
|
756
|
+
// 15 chars ("[object Object]") for every object result, which is most
|
|
757
|
+
// of them.
|
|
758
|
+
toolResultContent(tool.result, tool.toolName).length, MAX_HISTORY_TOOL_RESULT_CHARS);
|
|
746
759
|
}
|
|
747
760
|
}
|
|
748
761
|
return Math.max(1, cost);
|
|
@@ -1634,6 +1647,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1634
1647
|
let startupRecoveryAttempts = 0;
|
|
1635
1648
|
let queuedConflictRetries = 0;
|
|
1636
1649
|
let totalTransientContinuationAttempts = 0;
|
|
1650
|
+
// Work-boundary continuations (`loop_limit`), counted apart from the
|
|
1651
|
+
// transient ones — see MAX_LOOP_LIMIT_CONTINUATIONS.
|
|
1652
|
+
let loopLimitContinuations = 0;
|
|
1637
1653
|
// Consecutive continuations whose advance signature matched the previous
|
|
1638
1654
|
// round's (or was empty). Reset by any real advance.
|
|
1639
1655
|
let nonAdvancingContinuations = 0;
|
|
@@ -1647,12 +1663,13 @@ export function createAgentChatAdapter(options) {
|
|
|
1647
1663
|
// continuation history; the advance check needs the per-round delta for
|
|
1648
1664
|
// every reason code, `loop_limit` included.
|
|
1649
1665
|
let advanceCheckPrefix = [];
|
|
1650
|
-
// Set when the turn is stopped by
|
|
1651
|
-
//
|
|
1652
|
-
//
|
|
1653
|
-
//
|
|
1654
|
-
// generic "connection kept
|
|
1655
|
-
|
|
1666
|
+
// Set when the turn is stopped by a whole-turn continuation ceiling
|
|
1667
|
+
// (MAX_TOTAL_TRANSIENT_CONTINUATIONS or MAX_LOOP_LIMIT_CONTINUATIONS)
|
|
1668
|
+
// rather than by a dropped connection. Either can trip on a turn that was
|
|
1669
|
+
// making real progress the entire time, so it needs its own honest
|
|
1670
|
+
// message instead of falling through to the generic "connection kept
|
|
1671
|
+
// failing" copy below.
|
|
1672
|
+
let recoveryGaveUpOnContinuationBudget = false;
|
|
1656
1673
|
const continuationHistoryFragments = [];
|
|
1657
1674
|
const structuredContinuationFragments = [];
|
|
1658
1675
|
let visibleContinuationPrefix = [];
|
|
@@ -1708,6 +1725,7 @@ export function createAgentChatAdapter(options) {
|
|
|
1708
1725
|
? `activity_trail: ${formatActivityTrail(lastActivityTrail)}`
|
|
1709
1726
|
: "",
|
|
1710
1727
|
`total_transient_continuations: ${totalTransientContinuationAttempts}`,
|
|
1728
|
+
`loop_limit_continuations: ${loopLimitContinuations}`,
|
|
1711
1729
|
`background_follow_no_progress_detaches: ${backgroundFollowNoProgressDetaches}`,
|
|
1712
1730
|
`background_follow_consecutive_no_progress_detaches: ${backgroundFollowConsecutiveNoProgressDetaches}`,
|
|
1713
1731
|
backgroundFollowLastDetachReason
|
|
@@ -1738,7 +1756,7 @@ export function createAgentChatAdapter(options) {
|
|
|
1738
1756
|
: " the same action";
|
|
1739
1757
|
return `The agent got stuck preparing${tool} input and never started the tool, so I stopped the automatic retries. Try a smaller first step or a more compact version of the request.`;
|
|
1740
1758
|
}
|
|
1741
|
-
if (
|
|
1759
|
+
if (recoveryGaveUpOnContinuationBudget) {
|
|
1742
1760
|
return "This turn reached the limit on how many times it can be automatically continued, so I stopped it here. It may have still been making progress — retry as a single, narrower request so it can finish within fewer continuations.";
|
|
1743
1761
|
}
|
|
1744
1762
|
if (content.length === 0 &&
|
|
@@ -2815,15 +2833,6 @@ export function createAgentChatAdapter(options) {
|
|
|
2815
2833
|
const visibleContent = visibleContentForContinuation();
|
|
2816
2834
|
let currentPartialHistory = contentToContinuationHistory(visibleContent);
|
|
2817
2835
|
const madeContentProgress = hasContinuationProgress(visibleContent);
|
|
2818
|
-
// An action was streamed but has not returned yet (a tool_start with
|
|
2819
|
-
// no tool_done). Only a run_timeout in that window means the SERVER
|
|
2820
|
-
// is still executing it and a reconnect can still deliver the result;
|
|
2821
|
-
// every other reason means the stream is dead and nothing will
|
|
2822
|
-
// complete the tool.
|
|
2823
|
-
const hasInFlightTool = hasInFlightToolCall(visibleContent);
|
|
2824
|
-
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2825
|
-
const currentPreparingToolName = lastUnresolvedToolActivity(visibleContent) ??
|
|
2826
|
-
lastPreparingActionTool(signal.activityTrail);
|
|
2827
2836
|
// THE boundary. One question — did this continuation advance the
|
|
2828
2837
|
// turn? — decides every reason code, `loop_limit` included. It used
|
|
2829
2838
|
// to skip this block entirely AND reset two of the budgets, so a
|
|
@@ -2832,6 +2841,20 @@ export function createAgentChatAdapter(options) {
|
|
|
2832
2841
|
// no answer). The durable per-turn ledger lives server-side inside
|
|
2833
2842
|
// one run and never sees a client re-POST, so nothing else bounds it.
|
|
2834
2843
|
const advanceDelta = contentAfterContinuationPrefix(content, advanceCheckPrefix);
|
|
2844
|
+
// Everything that asks "what did THIS round do" reads the delta.
|
|
2845
|
+
// `visibleContent` accumulates across a `loop_limit` chain — its
|
|
2846
|
+
// prefix deliberately does not advance there — so one unresolved
|
|
2847
|
+
// "Preparing X" card from an earlier round would otherwise stall
|
|
2848
|
+
// every later round on the same name and kill a turn that was
|
|
2849
|
+
// streaming new prose the whole time.
|
|
2850
|
+
// An action streamed but not returned yet (a tool_start with no
|
|
2851
|
+
// tool_done) means the SERVER is still executing it, so a run_timeout
|
|
2852
|
+
// in that window can still deliver the result on reconnect; every
|
|
2853
|
+
// other reason means the stream is dead and nothing will complete it.
|
|
2854
|
+
const hasInFlightTool = hasInFlightToolCall(advanceDelta);
|
|
2855
|
+
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2856
|
+
const currentPreparingToolName = lastUnresolvedToolActivity(advanceDelta) ??
|
|
2857
|
+
lastPreparingActionTool(signal.activityTrail);
|
|
2835
2858
|
const advance = describeContinuationAdvance(advanceDelta, currentPreparingToolName);
|
|
2836
2859
|
const serverStillRunningTool = signal.reason === "run_timeout" && hasInFlightTool;
|
|
2837
2860
|
const advanced = serverStillRunningTool ||
|
|
@@ -2844,7 +2867,12 @@ export function createAgentChatAdapter(options) {
|
|
|
2844
2867
|
nonAdvancingContinuations = advanced
|
|
2845
2868
|
? 0
|
|
2846
2869
|
: nonAdvancingContinuations + 1;
|
|
2847
|
-
|
|
2870
|
+
if (isTransient) {
|
|
2871
|
+
totalTransientContinuationAttempts += 1;
|
|
2872
|
+
}
|
|
2873
|
+
else {
|
|
2874
|
+
loopLimitContinuations += 1;
|
|
2875
|
+
}
|
|
2848
2876
|
advanceCheckPrefix = snapshotContent(content);
|
|
2849
2877
|
// If a tool already completed, do not turn a missing closing
|
|
2850
2878
|
// sentence into a scary connection failure. Give the model one
|
|
@@ -2868,8 +2896,9 @@ export function createAgentChatAdapter(options) {
|
|
|
2868
2896
|
return { ok: false, resetVisibleContent: false };
|
|
2869
2897
|
}
|
|
2870
2898
|
if (totalTransientContinuationAttempts >
|
|
2871
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS
|
|
2872
|
-
|
|
2899
|
+
MAX_TOTAL_TRANSIENT_CONTINUATIONS ||
|
|
2900
|
+
loopLimitContinuations > MAX_LOOP_LIMIT_CONTINUATIONS) {
|
|
2901
|
+
recoveryGaveUpOnContinuationBudget = true;
|
|
2873
2902
|
return { ok: false, resetVisibleContent: false };
|
|
2874
2903
|
}
|
|
2875
2904
|
if (isTransient) {
|
|
@@ -534,6 +534,13 @@ function isAutoRecoverableError(ev, errMsg) {
|
|
|
534
534
|
}
|
|
535
535
|
if (ev.recoverable === true)
|
|
536
536
|
return true;
|
|
537
|
+
// An explicit flag outranks the message sniff below, which exists only for
|
|
538
|
+
// events that carry no flag at all. The repeat guards stop a turn with
|
|
539
|
+
// `recoverable: false` and a message that names the looping tool, so a stop
|
|
540
|
+
// on `list-workspace-connections` matched the "connection" sniff and
|
|
541
|
+
// auto-continued the very loop it was emitted to break.
|
|
542
|
+
if (ev.recoverable === false)
|
|
543
|
+
return false;
|
|
537
544
|
if (msg.includes("daily gateway request cap"))
|
|
538
545
|
return false;
|
|
539
546
|
// "gateway error" intentionally absent — that's the no-detail Builder
|
|
@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
|
|
|
62
62
|
error: string;
|
|
63
63
|
states?: undefined;
|
|
64
64
|
} | {
|
|
65
|
+
error?: undefined;
|
|
65
66
|
states: {
|
|
66
67
|
clientId: number;
|
|
67
68
|
state: string;
|
|
68
69
|
}[];
|
|
69
|
-
error?: undefined;
|
|
70
70
|
}>>;
|
|
71
71
|
/**
|
|
72
72
|
* GET /_agent-native/collab/:docId/users
|
|
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
|
|
|
77
77
|
error: string;
|
|
78
78
|
users?: undefined;
|
|
79
79
|
} | {
|
|
80
|
+
error?: undefined;
|
|
80
81
|
users: {
|
|
81
82
|
clientId: number;
|
|
82
83
|
lastSeen: number;
|
|
83
84
|
}[];
|
|
84
|
-
error?: undefined;
|
|
85
85
|
}>>;
|
package/dist/collab/routes.d.ts
CHANGED
|
@@ -26,8 +26,8 @@ export declare const getCollabState: import("h3").EventHandlerWithFetch<import("
|
|
|
26
26
|
* Body: { update: string (base64), requestSource?: string }
|
|
27
27
|
*/
|
|
28
28
|
export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
29
|
-
ok?: undefined;
|
|
30
29
|
error: string;
|
|
30
|
+
ok?: undefined;
|
|
31
31
|
} | {
|
|
32
32
|
error?: undefined;
|
|
33
33
|
ok: boolean;
|
package/dist/db/client.js
CHANGED
|
@@ -1091,7 +1091,7 @@ const _sharedDbPoolReplacementHooks = new Map();
|
|
|
1091
1091
|
* database access down with it.
|
|
1092
1092
|
*/
|
|
1093
1093
|
export function sharedDbPool(driver, url, create) {
|
|
1094
|
-
const key = `${driver}
|
|
1094
|
+
const key = `${driver}\u0000${url}`;
|
|
1095
1095
|
const existing = _sharedDbPools.get(key);
|
|
1096
1096
|
if (existing)
|
|
1097
1097
|
return existing;
|
|
@@ -1105,7 +1105,7 @@ export function sharedDbPool(driver, url, create) {
|
|
|
1105
1105
|
* handing out the discarded pool.
|
|
1106
1106
|
*/
|
|
1107
1107
|
export function replaceSharedDbPool(driver, url, previous, next) {
|
|
1108
|
-
const key = `${driver}
|
|
1108
|
+
const key = `${driver}\u0000${url}`;
|
|
1109
1109
|
if (_sharedDbPools.get(key) !== previous)
|
|
1110
1110
|
return;
|
|
1111
1111
|
_sharedDbPools.set(key, next);
|
|
@@ -4890,18 +4890,44 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
4890
4890
|
error: "Recurring-job sweep reached the synchronous server instead of the durable background worker.",
|
|
4891
4891
|
};
|
|
4892
4892
|
}
|
|
4893
|
+
// Stale reaping runs FIRST and site-wide, before the open-ended job
|
|
4894
|
+
// sweep can spend the platform wall. It is the durable driver the
|
|
4895
|
+
// in-process fast sweep below cannot be on serverless: that timer is
|
|
4896
|
+
// off wherever `shouldDisableInProcessSweeps` is on — i.e. every
|
|
4897
|
+
// production Lambda — and nothing replaced it, so a claimed run
|
|
4898
|
+
// whose producer died was only reaped when an unrelated request path
|
|
4899
|
+
// happened to look (prod: 1,216 runs, 12% of all runs, sitting
|
|
4900
|
+
// "running" for up to 59 minutes against a 15s window).
|
|
4901
|
+
//
|
|
4902
|
+
// One reap per site-tick instead of one per warm container is also
|
|
4903
|
+
// the point: the per-container timers are what issued 237k queries
|
|
4904
|
+
// in two minutes and earned that kill switch. `reapAllStaleRuns`
|
|
4905
|
+
// short-circuits on `hasRunningRuns()` (negative-cached), so an idle
|
|
4906
|
+
// app pays one probe.
|
|
4907
|
+
//
|
|
4908
|
+
// Reported, never swallowed, and never fatal to the job sweep: a
|
|
4909
|
+
// reap failure must not take recurring jobs down with it, and
|
|
4910
|
+
// `null` must stay distinguishable from "reaped nothing". The
|
|
4911
|
+
// result carries `failed` and `truncated` for the same reason one
|
|
4912
|
+
// level down — a pass where every row threw, and a pass that hit
|
|
4913
|
+
// the batch cap, both used to be reportable as a clean sweep.
|
|
4914
|
+
const { reapAllStaleRuns } = await import("../agent/run-store.js");
|
|
4915
|
+
const staleRunsReaped = await reapAllStaleRuns().catch((error) => {
|
|
4916
|
+
console.error("[agent-chat] durable stale-run reap failed:", error);
|
|
4917
|
+
return null;
|
|
4918
|
+
});
|
|
4893
4919
|
try {
|
|
4894
4920
|
// Jobs may request MCP tools, and `getActions` is synchronous —
|
|
4895
4921
|
// hydrate before the sweep so a serverless container that never
|
|
4896
4922
|
// eagerly initialized still resolves them.
|
|
4897
4923
|
await ensureMcpInitialized();
|
|
4898
4924
|
await processRecurringJobs(schedulerDeps);
|
|
4899
|
-
return { ok: true };
|
|
4925
|
+
return { ok: true, staleRunsReaped };
|
|
4900
4926
|
}
|
|
4901
4927
|
catch (error) {
|
|
4902
4928
|
console.error("[recurring-jobs] Sweep route failed:", error);
|
|
4903
4929
|
setResponseStatus(event, 500);
|
|
4904
|
-
return { error: "Recurring-job sweep failed" };
|
|
4930
|
+
return { error: "Recurring-job sweep failed", staleRunsReaped };
|
|
4905
4931
|
}
|
|
4906
4932
|
}));
|
|
4907
4933
|
if (disableRecurringJobsRuntime) {
|
|
@@ -5139,7 +5165,16 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
5139
5165
|
// against a 45s window). `reapAllStaleRuns` is per-row,
|
|
5140
5166
|
// idempotent, re-checks staleness at UPDATE time and honours
|
|
5141
5167
|
// the in-flight grace, so it is safe on this cadence.
|
|
5142
|
-
|
|
5168
|
+
//
|
|
5169
|
+
// This timer is the driver only where it actually runs — a
|
|
5170
|
+
// long-lived Node host. Wherever `sweepsDisabled` turns it off
|
|
5171
|
+
// (every production serverless function) the signed
|
|
5172
|
+
// RECURRING_JOBS_SWEEP_PATH route above owns the same reap,
|
|
5173
|
+
// driven by the platform scheduler. The two never both fire on
|
|
5174
|
+
// one host, which is why neither needs to coordinate.
|
|
5175
|
+
await reapAllStaleRuns().catch((error) => {
|
|
5176
|
+
console.error("[agent-chat] in-process stale-run reap failed:", error);
|
|
5177
|
+
});
|
|
5143
5178
|
let rows;
|
|
5144
5179
|
try {
|
|
5145
5180
|
rows = await listUnclaimedBackgroundRunRows();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.153.
|
|
3
|
+
"version": "0.153.8",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -421,8 +421,8 @@
|
|
|
421
421
|
"y-protocols": "^1.0.7",
|
|
422
422
|
"yjs": "^13.6.31",
|
|
423
423
|
"zod": "^4.3.6",
|
|
424
|
-
"@agent-native/
|
|
425
|
-
"@agent-native/
|
|
424
|
+
"@agent-native/recap-cli": "0.5.4",
|
|
425
|
+
"@agent-native/toolkit": "^0.14.1"
|
|
426
426
|
},
|
|
427
427
|
"devDependencies": {
|
|
428
428
|
"@ai-sdk/anthropic": "^3.0.71",
|