@agent-native/core 0.153.5 → 0.153.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/analytics/actions/get-data-program.ts +4 -1
- package/corpus/templates/analytics/server/lib/real-data-actions.ts +37 -13
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +2 -15
- package/dist/agent/engine/tool-call-journal-seed.js +18 -1
- package/dist/agent/production-agent.d.ts +13 -7
- package/dist/agent/production-agent.js +39 -15
- package/dist/agent/run-manager.js +17 -11
- package/dist/agent/run-store.d.ts +16 -0
- package/dist/agent/run-store.js +51 -9
- package/dist/agent/thread-data-builder.d.ts +1 -0
- package/dist/agent/thread-data-builder.js +15 -2
- package/dist/client/RouteTransitionIndicator.js +1 -1
- package/dist/client/agent-chat-adapter.js +58 -29
- package/dist/client/sse-event-processor.js +7 -0
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/db/client.js +2 -2
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +5 -5
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/agent-chat-plugin.js +35 -3
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/package.json +3 -3
|
@@ -22,7 +22,10 @@ export default defineAction({
|
|
|
22
22
|
}),
|
|
23
23
|
http: { method: "GET" },
|
|
24
24
|
readOnly: true,
|
|
25
|
-
grounding
|
|
25
|
+
// Not `grounding`: this reads the stored program plus its last-run summary,
|
|
26
|
+
// and even `includeRows` replays rows cached by an earlier run rather than
|
|
27
|
+
// querying the source. The flag means evidence retrieved at call time, which
|
|
28
|
+
// is why `get-analysis` is excluded too.
|
|
26
29
|
run: async (args) => {
|
|
27
30
|
const ctx = getCredentialContext();
|
|
28
31
|
if (!ctx) throw new Error("No authenticated context for get-data-program.");
|
|
@@ -91,27 +91,46 @@ function isToolName(name: string, expected: string): boolean {
|
|
|
91
91
|
let groundingActionNames: ReadonlySet<string> | null = null;
|
|
92
92
|
|
|
93
93
|
/**
|
|
94
|
-
*
|
|
94
|
+
* Read the action names whose definitions declare `grounding: true`.
|
|
95
95
|
*
|
|
96
|
-
*
|
|
97
|
-
* because that registry imports every action, and
|
|
98
|
-
* module — importing it back here would close an
|
|
99
|
-
* agent-chat plugin
|
|
96
|
+
* Takes the registry as an argument rather than importing
|
|
97
|
+
* `.generated/actions-registry` because that registry imports every action, and
|
|
98
|
+
* `save-analysis` imports this module — importing it back here would close an
|
|
99
|
+
* evaluation cycle. The agent-chat plugin passes it in once at module load.
|
|
100
100
|
*/
|
|
101
|
+
export function deriveGroundingActionNames(
|
|
102
|
+
registry: Record<string, unknown>,
|
|
103
|
+
): string[] {
|
|
104
|
+
return Object.entries(registry)
|
|
105
|
+
.filter(([, module]) => {
|
|
106
|
+
// Action modules reach the registry either as the module namespace or as
|
|
107
|
+
// an already-unwrapped definition, the same two shapes
|
|
108
|
+
// `loadActionsFromStaticRegistry` normalizes.
|
|
109
|
+
const candidate = module as
|
|
110
|
+
| { grounding?: boolean; default?: { grounding?: boolean } }
|
|
111
|
+
| undefined;
|
|
112
|
+
return (
|
|
113
|
+
candidate?.grounding === true || candidate?.default?.grounding === true
|
|
114
|
+
);
|
|
115
|
+
})
|
|
116
|
+
.map(([name]) => name);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Publish the derived set for the response guard to consult. */
|
|
101
120
|
export function registerGroundingActions(names: Iterable<string>): void {
|
|
102
|
-
|
|
121
|
+
groundingActionNames = new Set(
|
|
103
122
|
[...names].map((name) => normalizeActionToolName(name)),
|
|
104
123
|
);
|
|
105
124
|
// An empty set is never a real deployment: it means the installed core build
|
|
106
|
-
// predates `grounding` and dropped it from every definition.
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
125
|
+
// predates `grounding` and dropped it from every definition. Registration
|
|
126
|
+
// runs at plugin module scope, so throwing here would take the whole server
|
|
127
|
+
// down for a response-guard heuristic; the failure is raised at first use
|
|
128
|
+
// instead, where it costs one turn.
|
|
129
|
+
if (groundingActionNames.size === 0) {
|
|
130
|
+
console.error(
|
|
131
|
+
"[analytics] no action declares grounding: true; the installed @agent-native/core cannot carry the flag, so the response guard is unusable",
|
|
112
132
|
);
|
|
113
133
|
}
|
|
114
|
-
groundingActionNames = normalized;
|
|
115
134
|
}
|
|
116
135
|
|
|
117
136
|
function isGroundingActionName(name: string): boolean {
|
|
@@ -120,6 +139,11 @@ function isGroundingActionName(name: string): boolean {
|
|
|
120
139
|
"grounding actions were never registered: the analytics response guard cannot tell a grounded turn from an ungrounded one",
|
|
121
140
|
);
|
|
122
141
|
}
|
|
142
|
+
if (groundingActionNames.size === 0) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
"no action declares grounding: true; the installed @agent-native/core cannot carry the flag",
|
|
145
|
+
);
|
|
146
|
+
}
|
|
123
147
|
return groundingActionNames.has(normalizeActionToolName(name));
|
|
124
148
|
}
|
|
125
149
|
|
|
@@ -12,6 +12,7 @@ import { ANALYTICS_CONNECTOR_CATALOG } from "../lib/analytics-connector-catalog"
|
|
|
12
12
|
import { credentialProviderConfigs } from "../lib/credential-keys";
|
|
13
13
|
import { isProductionServerlessRuntime } from "../lib/production-serverless-runtime.js";
|
|
14
14
|
import {
|
|
15
|
+
deriveGroundingActionNames,
|
|
15
16
|
draftClaimsAnalyticsMetrics,
|
|
16
17
|
failedDataQueryAttemptMessage,
|
|
17
18
|
hasDashboardConstructionAttempt,
|
|
@@ -35,21 +36,7 @@ import {
|
|
|
35
36
|
// Which actions return real data-source evidence is a fact each action states
|
|
36
37
|
// on itself (`grounding: true`). Reading it off the definitions here is what
|
|
37
38
|
// keeps the response guard from drifting behind a newly shipped source action.
|
|
38
|
-
registerGroundingActions(
|
|
39
|
-
Object.entries(actionsRegistry)
|
|
40
|
-
.filter(([, module]) => {
|
|
41
|
-
// Action modules reach the registry either as the module namespace or as
|
|
42
|
-
// an already-unwrapped definition, the same two shapes
|
|
43
|
-
// `loadActionsFromStaticRegistry` normalizes.
|
|
44
|
-
const candidate = module as
|
|
45
|
-
| { grounding?: boolean; default?: { grounding?: boolean } }
|
|
46
|
-
| undefined;
|
|
47
|
-
return (
|
|
48
|
-
candidate?.grounding === true || candidate?.default?.grounding === true
|
|
49
|
-
);
|
|
50
|
-
})
|
|
51
|
-
.map(([name]) => name),
|
|
52
|
-
);
|
|
39
|
+
registerGroundingActions(deriveGroundingActionNames(actionsRegistry));
|
|
53
40
|
|
|
54
41
|
const ANALYTICS_BACKGROUND_RUN_SOFT_TIMEOUT_MS = 13 * 60_000;
|
|
55
42
|
// A background job may legitimately spend minutes inside a provider/tool call,
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
import { getCurrentTurnEventsForThread } from "../run-store.js";
|
|
2
2
|
import { classifyToolCallJournal, } from "../tool-call-journal.js";
|
|
3
|
+
const LEDGER_READ_RETRY_MS = 250;
|
|
4
|
+
/**
|
|
5
|
+
* `unreadable` ends the turn with a stop the user sees, so a single pool
|
|
6
|
+
* timeout or connection blip must not produce one. One retry, because the
|
|
7
|
+
* failures worth surviving are transient and the caller is holding the whole
|
|
8
|
+
* turn open while we wait.
|
|
9
|
+
*/
|
|
10
|
+
async function readCurrentTurnEventsWithRetry(threadId, turnId) {
|
|
11
|
+
try {
|
|
12
|
+
return await getCurrentTurnEventsForThread(threadId, turnId);
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
console.warn(`[tool-call-journal] per-turn ledger read failed for thread ${threadId}, retrying once: ${err instanceof Error ? err.message : String(err)}`);
|
|
16
|
+
await new Promise((resolve) => setTimeout(resolve, LEDGER_READ_RETRY_MS));
|
|
17
|
+
return await getCurrentTurnEventsForThread(threadId, turnId);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
3
20
|
/**
|
|
4
21
|
* Tool-call journal hard-block (resume safety). Snapshot the per-turn journal
|
|
5
22
|
* ONCE here, before any tool runs in this chunk, so it reflects only PRIOR
|
|
@@ -33,7 +50,7 @@ export async function loadPriorTurnToolCallJournal(threadId, turnId) {
|
|
|
33
50
|
}
|
|
34
51
|
let priorEvents;
|
|
35
52
|
try {
|
|
36
|
-
priorEvents = await
|
|
53
|
+
priorEvents = await readCurrentTurnEventsWithRetry(threadId, turnId);
|
|
37
54
|
}
|
|
38
55
|
catch (err) {
|
|
39
56
|
const error = err instanceof Error ? err.message : String(err);
|
|
@@ -618,12 +618,12 @@ export declare const MAX_IDENTICAL_TOOL_CALLS = 8;
|
|
|
618
618
|
/**
|
|
619
619
|
* A stop that ends the turn from inside the tool loop.
|
|
620
620
|
*
|
|
621
|
-
* `message` is the last thing the user reads, so it
|
|
622
|
-
*
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
621
|
+
* `message` is the last thing the user reads, so it names the remedy in plain
|
|
622
|
+
* words and does not quote a raw provider/tool error; the raw error goes in
|
|
623
|
+
* `details`. Both retry sniffers now honour the `recoverable: false` these
|
|
624
|
+
* stops are sent with, so a stop that happens to say "timeout" is no longer
|
|
625
|
+
* auto-continued — but the prose still has one reader who cannot ignore it,
|
|
626
|
+
* and a pasted stack trace is not a remedy.
|
|
627
627
|
*/
|
|
628
628
|
export interface TerminalActionStop {
|
|
629
629
|
message: string;
|
|
@@ -720,9 +720,15 @@ export declare function findApprovedStructuredToolCall(history: AgentChatStructu
|
|
|
720
720
|
* which is worse than one more retry.
|
|
721
721
|
*/
|
|
722
722
|
export declare function permanentPreconditionRemedy(message: string): string | null;
|
|
723
|
+
/**
|
|
724
|
+
* Deliberately carries no running call count. This decline is recorded as a
|
|
725
|
+
* tool error, and the breakers key on that text: a message that reports its own
|
|
726
|
+
* tally ("already made 13 call(s)") mints a fresh key on every decline, so the
|
|
727
|
+
* decline that exists to stop a spiral becomes the one thing nothing can count.
|
|
728
|
+
* The fixed threshold below says the same thing without varying.
|
|
729
|
+
*/
|
|
723
730
|
export declare function repeatedSourceSweepGuardMessage(opts: {
|
|
724
731
|
toolName: string;
|
|
725
|
-
priorCalls: number;
|
|
726
732
|
threshold?: number;
|
|
727
733
|
scope?: "tool" | "aggregate";
|
|
728
734
|
}): string;
|
|
@@ -2440,10 +2440,12 @@ async function waitForInterruptedToolLedgerEntry(opts) {
|
|
|
2440
2440
|
* sentence before it, and two failures that differ ONLY in the arguments they
|
|
2441
2441
|
* echo are the same failure.
|
|
2442
2442
|
*
|
|
2443
|
-
*
|
|
2444
|
-
*
|
|
2445
|
-
*
|
|
2446
|
-
*
|
|
2443
|
+
* Digits stay. Collapsing them merges failures that are genuinely different:
|
|
2444
|
+
* "Record 41 not found" and "Record 42 not found" are two missing records, so a
|
|
2445
|
+
* seven-item sweep where every item legitimately 404s would trip the
|
|
2446
|
+
* across-arguments breaker at item six. A guard message that varies its own
|
|
2447
|
+
* running tally is fixed where that message is written, not by blinding every
|
|
2448
|
+
* breaker to numbers.
|
|
2447
2449
|
*/
|
|
2448
2450
|
function normalizeToolErrorForBreaker(error) {
|
|
2449
2451
|
return (error
|
|
@@ -2451,7 +2453,6 @@ function normalizeToolErrorForBreaker(error) {
|
|
|
2451
2453
|
.replace(/Received:\s*[\s\S]*?\.\s(?=Expected:|The tool was not executed)/g, "")
|
|
2452
2454
|
// Bare JSON payloads some providers inline instead of a Received: span.
|
|
2453
2455
|
.replace(/\{[\s\S]{0,2000}?\}/g, "{}")
|
|
2454
|
-
.replace(/\d+/g, "#")
|
|
2455
2456
|
.replace(/\s+/g, " ")
|
|
2456
2457
|
.trim());
|
|
2457
2458
|
}
|
|
@@ -2494,16 +2495,27 @@ const PERMANENT_PRECONDITION_PATTERNS = [
|
|
|
2494
2495
|
/\b(?:api[ -]?keys?|access tokens?|credentials?|secrets?)\b[^.]{0,60}\bnot (?:configured|set|connected|available)\b/i,
|
|
2495
2496
|
/\bsave [A-Z][A-Z0-9_]{3,} in (?:the )?settings\b/i,
|
|
2496
2497
|
// "Connect Builder.io before indexing a design system from Figma or code."
|
|
2497
|
-
|
|
2498
|
-
//
|
|
2499
|
-
//
|
|
2500
|
-
|
|
2498
|
+
// Case-sensitive, sentence-initial, and followed by a capitalized service
|
|
2499
|
+
// name, because that is the only shape that separates the imperative remedy
|
|
2500
|
+
// from the retryable network failure: "failed to connect to the warehouse
|
|
2501
|
+
// before the deadline" and "Connect timed out, retry first" both satisfy
|
|
2502
|
+
// every looser reading of the same words.
|
|
2503
|
+
/(?:^|[.:!?]\s+)Connect [A-Z][\w.-]*[^;]{0,40}?\b(?:before|first|in settings)\b/,
|
|
2501
2504
|
// "Plan mode blocked `update-extension`. Switch to Act mode …" — cleared by
|
|
2502
2505
|
// the user approving the plan, never by the model trying again.
|
|
2503
2506
|
/\bplan mode blocked\b/i,
|
|
2507
|
+
// The templates' most-copied throw, and always `if (!getRequestUserEmail())`:
|
|
2508
|
+
// one synchronous AsyncLocalStorage read of a value fixed when the request
|
|
2509
|
+
// context was established. It cannot appear and disappear between tool calls
|
|
2510
|
+
// of the same chunk, so retrying inside this turn lands on it again.
|
|
2504
2511
|
/\bno authenticated user\b/i,
|
|
2505
2512
|
// "SSRF blocked: refusing to fetch private/internal address (…)"
|
|
2506
2513
|
/\bssrf blocked\b/i,
|
|
2514
|
+
// Deliberately absent: "only available in …". Retention windows ("only
|
|
2515
|
+
// available from the last 90 days") share its wording and are fixed by
|
|
2516
|
+
// narrowing the range, so it stopped turns that were one argument away from
|
|
2517
|
+
// succeeding. The count-based breaker still ends a genuine runtime gate
|
|
2518
|
+
// after six.
|
|
2507
2519
|
];
|
|
2508
2520
|
const SOURCE_SWEEP_TOOL_NAME = /\b(?:api|calls?|deals?|docs?|events?|issues?|messages?|metrics?|provider|query|records?|request|search|source|tickets?|transcripts?)\b/i;
|
|
2509
2521
|
const SOURCE_SWEEP_PROVIDER_TOKEN = /\b(?:amplitude|apollo|bigquery|commonroom|data-source|ga4|github|gong|grafana|hubspot|jira|mixpanel|notion|posthog|postgres|postgresql|pylon|sentry|slack|stripe)\b/i;
|
|
@@ -2610,14 +2622,20 @@ function restrictAgentTeamsAfterSourceSweep(tools) {
|
|
|
2610
2622
|
};
|
|
2611
2623
|
});
|
|
2612
2624
|
}
|
|
2625
|
+
/**
|
|
2626
|
+
* Deliberately carries no running call count. This decline is recorded as a
|
|
2627
|
+
* tool error, and the breakers key on that text: a message that reports its own
|
|
2628
|
+
* tally ("already made 13 call(s)") mints a fresh key on every decline, so the
|
|
2629
|
+
* decline that exists to stop a spiral becomes the one thing nothing can count.
|
|
2630
|
+
* The fixed threshold below says the same thing without varying.
|
|
2631
|
+
*/
|
|
2613
2632
|
export function repeatedSourceSweepGuardMessage(opts) {
|
|
2614
2633
|
const threshold = opts.threshold ?? SOURCE_SWEEP_TOOL_CALL_THRESHOLD;
|
|
2615
2634
|
const target = opts.scope === "aggregate"
|
|
2616
2635
|
? "read-only source/search tools"
|
|
2617
2636
|
: "the same read-only source/search tool";
|
|
2618
|
-
return (`Skipped ${opts.toolName}: this turn already
|
|
2619
|
-
|
|
2620
|
-
`${threshold}-call convergence budget. Stop calling ${opts.toolName} ` +
|
|
2637
|
+
return (`Skipped ${opts.toolName}: this turn already exhausted its ` +
|
|
2638
|
+
`${threshold}-call convergence budget for ${target}. Stop calling ${opts.toolName} ` +
|
|
2621
2639
|
`one item at a time and change strategy before answering. If a broader ` +
|
|
2622
2640
|
`read-only bulk/source mechanism is available, use it now: provider API ` +
|
|
2623
2641
|
`catalog/docs/request tools with pagination or staging, code execution ` +
|
|
@@ -2647,7 +2665,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
|
|
|
2647
2665
|
priorCalls: priorSourceSweepCalls,
|
|
2648
2666
|
message: repeatedSourceSweepGuardMessage({
|
|
2649
2667
|
toolName: opts.toolName,
|
|
2650
|
-
priorCalls: priorSourceSweepCalls,
|
|
2651
2668
|
threshold,
|
|
2652
2669
|
scope: "aggregate",
|
|
2653
2670
|
}),
|
|
@@ -2660,7 +2677,6 @@ export function shouldGuardRepeatedSourceSweep(opts) {
|
|
|
2660
2677
|
priorCalls,
|
|
2661
2678
|
message: repeatedSourceSweepGuardMessage({
|
|
2662
2679
|
toolName: opts.toolName,
|
|
2663
|
-
priorCalls,
|
|
2664
2680
|
threshold,
|
|
2665
2681
|
scope: "tool",
|
|
2666
2682
|
}),
|
|
@@ -4041,7 +4057,8 @@ export async function runAgentLoop(opts) {
|
|
|
4041
4057
|
const permanentRemedy = permanentPreconditionRemedy(sanitizedResult);
|
|
4042
4058
|
if (permanentRemedy) {
|
|
4043
4059
|
requestedActionStop ??= {
|
|
4044
|
-
message: `I stopped because ${toolCall.name}
|
|
4060
|
+
message: `I stopped because ${toolCall.name} needs a setup step outside this turn — a credential, a role, a connected account, or an approval — before it can run. ` +
|
|
4061
|
+
"Retrying would not have changed it, and anything completed before this is saved.",
|
|
4045
4062
|
errorCode: "permanent_precondition",
|
|
4046
4063
|
details: sanitizedResult,
|
|
4047
4064
|
};
|
|
@@ -5056,6 +5073,13 @@ function isRecoverableContinuationError(event) {
|
|
|
5056
5073
|
const message = event.error.toLowerCase();
|
|
5057
5074
|
if (code === "builder_gateway_error")
|
|
5058
5075
|
return false;
|
|
5076
|
+
// An explicit flag outranks the message sniff below, which exists for events
|
|
5077
|
+
// that carry no flag at all. Every guard stop sets `recoverable: false`, and
|
|
5078
|
+
// sniffing its prose is how a stop that merely NAMES a connection or a
|
|
5079
|
+
// timeout gets continued into the spiral it was raised to end. The client
|
|
5080
|
+
// applies the same precedence in `isAutoRecoverableError`.
|
|
5081
|
+
if (event.recoverable === false)
|
|
5082
|
+
return false;
|
|
5059
5083
|
return (event.recoverable === true ||
|
|
5060
5084
|
code === "builder_gateway_timeout" ||
|
|
5061
5085
|
code === "builder_gateway_network_error" ||
|
|
@@ -937,12 +937,15 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
937
937
|
// false-stale-reap zombie scenario where the reaper flipped the row while
|
|
938
938
|
// this isolate was briefly unable to heartbeat (DB latency / GC pause).
|
|
939
939
|
let lastAbortCheck = Date.now() - 3000;
|
|
940
|
-
// A read failure here
|
|
941
|
-
//
|
|
942
|
-
//
|
|
943
|
-
//
|
|
944
|
-
//
|
|
945
|
-
//
|
|
940
|
+
// A read failure here is not "not aborted" — but it is not grounds to kill
|
|
941
|
+
// the run either. This check is only a cross-isolate backstop: the outage
|
|
942
|
+
// that hides a Stop from us hides it from every other reader too, so
|
|
943
|
+
// self-aborting delivers nobody's Stop any sooner while destroying in-flight
|
|
944
|
+
// work in the far more common case where nobody pressed Stop at all. Fail
|
|
945
|
+
// open and stay bounded by the soft timeout, no-progress backstop, and
|
|
946
|
+
// iteration/token limits — none of which touch the DB. Report the outage to
|
|
947
|
+
// Sentry (once at the threshold, then ~once a minute) so a long unreadable
|
|
948
|
+
// window is visible and its duration is queryable instead of silent.
|
|
946
949
|
let consecutiveAbortCheckFailures = 0;
|
|
947
950
|
const checkSqlAbort = () => {
|
|
948
951
|
const now = Date.now();
|
|
@@ -970,7 +973,9 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
970
973
|
})
|
|
971
974
|
.catch((error) => {
|
|
972
975
|
consecutiveAbortCheckFailures += 1;
|
|
973
|
-
|
|
976
|
+
// 3 ticks ≈ 9s of unreadability, then every 20 ticks ≈ 60s.
|
|
977
|
+
if (consecutiveAbortCheckFailures === 3 ||
|
|
978
|
+
consecutiveAbortCheckFailures % 20 === 0) {
|
|
974
979
|
captureError(error, {
|
|
975
980
|
route: "/_agent-native/agent-chat",
|
|
976
981
|
tags: {
|
|
@@ -979,11 +984,12 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
979
984
|
consecutiveFailures: String(consecutiveAbortCheckFailures),
|
|
980
985
|
},
|
|
981
986
|
aiTraceId: runId,
|
|
982
|
-
extra: {
|
|
987
|
+
extra: {
|
|
988
|
+
runId,
|
|
989
|
+
threadId,
|
|
990
|
+
unreadableForMs: consecutiveAbortCheckFailures * 3000,
|
|
991
|
+
},
|
|
983
992
|
});
|
|
984
|
-
if (!abort.signal.aborted) {
|
|
985
|
-
abortInMemoryRun(run, "abort_check_unavailable");
|
|
986
|
-
}
|
|
987
993
|
}
|
|
988
994
|
});
|
|
989
995
|
};
|
|
@@ -44,6 +44,22 @@ export declare const STALE_RUN_ERROR_EVENT: {
|
|
|
44
44
|
readonly recoverable: true;
|
|
45
45
|
readonly details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.";
|
|
46
46
|
};
|
|
47
|
+
/**
|
|
48
|
+
* `agent_runs.terminal_reason` recorded by the stale reapers.
|
|
49
|
+
*
|
|
50
|
+
* Every other error path records `error:<code>` (`terminalReasonForEvent`), and
|
|
51
|
+
* `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
|
|
52
|
+
* replays the terminal event a reaper itself appended. The reapers used to write
|
|
53
|
+
* the bare code, which split one outcome across two permanent
|
|
54
|
+
* `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
|
|
55
|
+
* `error:stale_run`) purely by whether anything happened to read the row after
|
|
56
|
+
* the reap — and made the reconciliation convergence guard, which compares
|
|
57
|
+
* `terminal_reason` for equality, rewrite the row on every read forever.
|
|
58
|
+
*
|
|
59
|
+
* Rows reaped before this change keep the bare `stale_run`, so anything matching
|
|
60
|
+
* on the value must accept both forms for one retention window.
|
|
61
|
+
*/
|
|
62
|
+
export declare const STALE_RUN_TERMINAL_REASON: string;
|
|
47
63
|
/**
|
|
48
64
|
* Terminal error for a background-dispatched run whose worker NEVER claimed it
|
|
49
65
|
* (the foreground fired the self-dispatch, Netlify acked it async with a 202,
|
package/dist/agent/run-store.js
CHANGED
|
@@ -50,6 +50,22 @@ export const STALE_RUN_ERROR_EVENT = {
|
|
|
50
50
|
recoverable: true,
|
|
51
51
|
details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
|
|
52
52
|
};
|
|
53
|
+
/**
|
|
54
|
+
* `agent_runs.terminal_reason` recorded by the stale reapers.
|
|
55
|
+
*
|
|
56
|
+
* Every other error path records `error:<code>` (`terminalReasonForEvent`), and
|
|
57
|
+
* `reconcileTerminalRunFromEvents` writes that form for THIS failure too when it
|
|
58
|
+
* replays the terminal event a reaper itself appended. The reapers used to write
|
|
59
|
+
* the bare code, which split one outcome across two permanent
|
|
60
|
+
* `agent_run_outcome_daily` buckets (prod: 612 `stale_run` + 604
|
|
61
|
+
* `error:stale_run`) purely by whether anything happened to read the row after
|
|
62
|
+
* the reap — and made the reconciliation convergence guard, which compares
|
|
63
|
+
* `terminal_reason` for equality, rewrite the row on every read forever.
|
|
64
|
+
*
|
|
65
|
+
* Rows reaped before this change keep the bare `stale_run`, so anything matching
|
|
66
|
+
* on the value must accept both forms for one retention window.
|
|
67
|
+
*/
|
|
68
|
+
export const STALE_RUN_TERMINAL_REASON = `error:${STALE_RUN_ERROR_EVENT.errorCode}`;
|
|
53
69
|
/**
|
|
54
70
|
* Terminal error for a background-dispatched run whose worker NEVER claimed it
|
|
55
71
|
* (the foreground fired the self-dispatch, Netlify acked it async with a 202,
|
|
@@ -1306,6 +1322,22 @@ export async function recordRunDiagnostic(runId, stage, detail) {
|
|
|
1306
1322
|
// Diagnostics are best-effort; never let them break the run or the route.
|
|
1307
1323
|
}
|
|
1308
1324
|
}
|
|
1325
|
+
/**
|
|
1326
|
+
* Wall-clock budget for ONE heartbeat write.
|
|
1327
|
+
*
|
|
1328
|
+
* The default `DbExec` budget is `dbOpTimeoutMs()` (8s on serverless) across 3
|
|
1329
|
+
* connection attempts, so a single heartbeat can occupy up to 24s — against a
|
|
1330
|
+
* 15s `RUN_STALE_MS`. The run-manager single-flights the write, so that one
|
|
1331
|
+
* attempt suppresses every 1.5s tick behind it: a live foreground run in a quiet
|
|
1332
|
+
* phase becomes reapable while its own heartbeat is still in flight. The same
|
|
1333
|
+
* attempt also pins a pooler connection for those 24s, feeding the saturation
|
|
1334
|
+
* that starves heartbeats in the first place.
|
|
1335
|
+
*
|
|
1336
|
+
* A heartbeat that cannot land promptly has no value — the 1.5s tick IS the
|
|
1337
|
+
* retry — so it gets ONE attempt inside a third of the stale window, leaving
|
|
1338
|
+
* room for two more to land before the reap cutoff.
|
|
1339
|
+
*/
|
|
1340
|
+
const HEARTBEAT_WRITE_TIMEOUT_MS = Math.floor(RUN_STALE_MS / 3);
|
|
1309
1341
|
/** Update the run's liveness heartbeat. Called periodically by run-manager. */
|
|
1310
1342
|
export async function updateRunHeartbeat(runId) {
|
|
1311
1343
|
await ensureRunTables();
|
|
@@ -1324,6 +1356,8 @@ export async function updateRunHeartbeat(runId) {
|
|
|
1324
1356
|
END
|
|
1325
1357
|
WHERE id = ? AND status = 'running'`,
|
|
1326
1358
|
args: [Date.now(), currentRssMb(), currentRssMb(), runId],
|
|
1359
|
+
timeoutMs: HEARTBEAT_WRITE_TIMEOUT_MS,
|
|
1360
|
+
maxAttempts: 1,
|
|
1327
1361
|
});
|
|
1328
1362
|
}
|
|
1329
1363
|
/**
|
|
@@ -1612,9 +1646,15 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
|
|
|
1612
1646
|
: // First `?` is backgroundAwareStaleCutoffSql's CAST param, the next two
|
|
1613
1647
|
// are inFlightGraceSql's — all bound to the same "now".
|
|
1614
1648
|
[completedAt, completedAt, completedAt];
|
|
1649
|
+
// `completed_at` is when the run DIED, not when a reaper noticed. Stamping
|
|
1650
|
+
// reap time made every stale row report its own detection latency as run
|
|
1651
|
+
// duration (prod: 30-59 minute "runs" against a 15s window), which is the
|
|
1652
|
+
// measurement used to judge whether reaping is timely at all. The liveness
|
|
1653
|
+
// basis is the last proof the producer was alive, and it is what
|
|
1654
|
+
// `reconcileTerminalRunFromEvents` already records for the same failure.
|
|
1615
1655
|
const updateSql = `UPDATE agent_runs
|
|
1616
1656
|
SET status = 'errored',
|
|
1617
|
-
completed_at =
|
|
1657
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
1618
1658
|
error_code = ?,
|
|
1619
1659
|
error_detail = ?,
|
|
1620
1660
|
terminal_reason = ?
|
|
@@ -1623,10 +1663,9 @@ async function reapSingleStaleRun(runId, maxStaleMs) {
|
|
|
1623
1663
|
AND ${terminalRunEventExclusionSql()}
|
|
1624
1664
|
AND ${staleClause}`;
|
|
1625
1665
|
const updateArgs = [
|
|
1626
|
-
completedAt,
|
|
1627
1666
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
1628
1667
|
STALE_RUN_ERROR_EVENT.details,
|
|
1629
|
-
|
|
1668
|
+
STALE_RUN_TERMINAL_REASON,
|
|
1630
1669
|
runId,
|
|
1631
1670
|
...staleArgs,
|
|
1632
1671
|
];
|
|
@@ -2433,11 +2472,16 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2433
2472
|
await reconcileTerminalRunFromEvents(id);
|
|
2434
2473
|
}
|
|
2435
2474
|
}
|
|
2475
|
+
// Both UPDATEs below backdate `completed_at` to the liveness basis for the
|
|
2476
|
+
// reason given on `reapSingleStaleRun`'s UPDATE: this pass can fire a DAY
|
|
2477
|
+
// after the producer died, and stamping now would file that whole gap as run
|
|
2478
|
+
// duration. On the absolute-age branch the basis degrades to `started_at`,
|
|
2479
|
+
// which is still the truest thing known about a row that never heartbeat.
|
|
2436
2480
|
const completedAt = Date.now();
|
|
2437
2481
|
await client.execute({
|
|
2438
2482
|
sql: `UPDATE agent_runs
|
|
2439
2483
|
SET status = 'errored',
|
|
2440
|
-
completed_at =
|
|
2484
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
2441
2485
|
error_code = ?,
|
|
2442
2486
|
error_detail = ?,
|
|
2443
2487
|
terminal_reason = ?
|
|
@@ -2445,10 +2489,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2445
2489
|
AND ${terminalRunEventExclusionSql()}
|
|
2446
2490
|
AND started_at < ?`,
|
|
2447
2491
|
args: [
|
|
2448
|
-
completedAt,
|
|
2449
2492
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
2450
2493
|
STALE_RUN_ERROR_EVENT.details,
|
|
2451
|
-
|
|
2494
|
+
STALE_RUN_TERMINAL_REASON,
|
|
2452
2495
|
cutoff,
|
|
2453
2496
|
],
|
|
2454
2497
|
});
|
|
@@ -2463,7 +2506,7 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2463
2506
|
await client.execute({
|
|
2464
2507
|
sql: `UPDATE agent_runs
|
|
2465
2508
|
SET status = 'errored',
|
|
2466
|
-
completed_at =
|
|
2509
|
+
completed_at = COALESCE(completed_at, ${livenessBasisSql()}),
|
|
2467
2510
|
error_code = ?,
|
|
2468
2511
|
error_detail = ?,
|
|
2469
2512
|
terminal_reason = ?
|
|
@@ -2472,10 +2515,9 @@ export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
|
2472
2515
|
AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
|
|
2473
2516
|
AND ${inFlightGraceSql()}`,
|
|
2474
2517
|
args: [
|
|
2475
|
-
completedAt,
|
|
2476
2518
|
STALE_RUN_ERROR_EVENT.errorCode,
|
|
2477
2519
|
STALE_RUN_ERROR_EVENT.details,
|
|
2478
|
-
|
|
2520
|
+
STALE_RUN_TERMINAL_REASON,
|
|
2479
2521
|
completedAt,
|
|
2480
2522
|
completedAt,
|
|
2481
2523
|
completedAt,
|
|
@@ -243,18 +243,31 @@ export function buildAssistantMessage(events, runId, options = {}) {
|
|
|
243
243
|
metadata,
|
|
244
244
|
};
|
|
245
245
|
}
|
|
246
|
+
/**
|
|
247
|
+
* The rebuild half of the live client's `clearAssistantDraftContent`
|
|
248
|
+
* (client/sse-event-processor.ts). The two bodies are asserted identical by
|
|
249
|
+
* `keeps clearAssistantDraftContent identical to the live client copy` in
|
|
250
|
+
* thread-data-builder.spec.ts — a rebuild that clears more than the live stream
|
|
251
|
+
* did makes narration vanish on reload, which is worse than clearing nothing.
|
|
252
|
+
*/
|
|
246
253
|
function clearAssistantDraftContent(content) {
|
|
247
254
|
for (let index = content.length - 1; index >= 0; index--) {
|
|
248
255
|
const part = content[index];
|
|
249
256
|
if (!part)
|
|
250
257
|
continue;
|
|
258
|
+
if (part.type === "tool-call" &&
|
|
259
|
+
part.activity !== true &&
|
|
260
|
+
part.result !== undefined) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
251
263
|
if (part.type === "text" || part.type === "reasoning") {
|
|
252
264
|
content.splice(index, 1);
|
|
253
265
|
continue;
|
|
254
266
|
}
|
|
255
267
|
if (part.type === "tool-call" && part.result === undefined) {
|
|
256
|
-
//
|
|
257
|
-
//
|
|
268
|
+
// Only drop ephemeral placeholders. Materialized in-flight tool cards
|
|
269
|
+
// (real args from tool_start) stay mounted so a retry/auto-continue clear
|
|
270
|
+
// does not hide→show the same call when the next chunk re-emits it.
|
|
258
271
|
const isEphemeral = part.activity === true ||
|
|
259
272
|
part.argsText === "" ||
|
|
260
273
|
Object.keys(part.args ?? {}).length === 0;
|
|
@@ -37,5 +37,5 @@ export function RouteTransitionIndicator() {
|
|
|
37
37
|
}, [destination]);
|
|
38
38
|
if (!destination || visibleDestination !== destination)
|
|
39
39
|
return null;
|
|
40
|
-
return (_jsx("div", { "aria-label": "Loading
|
|
40
|
+
return (_jsx("div", { "aria-label": "Loading page...", "aria-live": "polite", className: "pointer-events-none fixed inset-x-0 top-0 z-50 h-0.5 overflow-hidden bg-primary/15", "data-route-transition-indicator": "true", "data-route-transition-target": destination, role: "status", children: _jsx("div", { className: "route-transition-indicator-bar h-full w-full bg-primary" }) }));
|
|
41
41
|
}
|
|
@@ -24,20 +24,29 @@ const AUTO_CONTINUE_COMPLETION_GUARD = "Before doing more work, inspect the prio
|
|
|
24
24
|
const MAX_RECONNECT_ATTEMPTS = 5;
|
|
25
25
|
const MAX_STARTUP_RECOVERY_ATTEMPTS = 8;
|
|
26
26
|
const MAX_QUEUED_CONFLICT_RETRIES = 120;
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
// five separate budgets — stale-run, stalled, empty, repeated-narration
|
|
30
|
-
// repeated-action-preparation — each of which existed because the rung above
|
|
27
|
+
// The consecutive non-advancing continuation (see `describeContinuationAdvance`)
|
|
28
|
+
// that ends the turn: the third one stops it, so two are tolerated. This
|
|
29
|
+
// replaced five separate budgets — stale-run, stalled, empty, repeated-narration
|
|
30
|
+
// and repeated-action-preparation — each of which existed because the rung above
|
|
31
31
|
// it had a hole the next one patched. One question answers all five: a round
|
|
32
32
|
// that produced nothing, or produced exactly what the round before it
|
|
33
33
|
// produced, did not advance.
|
|
34
34
|
const MAX_NON_ADVANCING_CONTINUATIONS = 3;
|
|
35
|
-
// Ceiling across the whole turn
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
// longest recovery we deliberately support.
|
|
35
|
+
// Ceiling across the whole turn for TRANSIENT continuations — rounds that
|
|
36
|
+
// followed a failure (timeout, dropped stream, stale run). Every attempt
|
|
37
|
+
// re-POSTs the full history plus attachments, so a high ceiling only burns
|
|
38
|
+
// tokens and wall time on a connection that keeps breaking.
|
|
40
39
|
const MAX_TOTAL_TRANSIENT_CONTINUATIONS = 12;
|
|
40
|
+
// Ceiling across the whole turn for WORK-boundary continuations. A `loop_limit`
|
|
41
|
+
// round is not a failure — the server spent a full iteration budget on real
|
|
42
|
+
// tool work and handed the turn back — so counting it against the transient
|
|
43
|
+
// ceiling killed progressing turns at round 13 that had never failed once.
|
|
44
|
+
// Sized against the server's own limits: at its default budget of 400
|
|
45
|
+
// iterations per run this is ~10,000 tool calls, two orders past the deepest
|
|
46
|
+
// legitimate production turn (117 tool calls) that sized that budget. It still
|
|
47
|
+
// has to exist, because the server's per-turn token backstop rides the request
|
|
48
|
+
// body between server-chained chunks and resets on every client re-POST.
|
|
49
|
+
const MAX_LOOP_LIMIT_CONTINUATIONS = 25;
|
|
41
50
|
const RETRY_BASE_DELAY_MS = 500;
|
|
42
51
|
const RETRY_MAX_DELAY_MS = 8_000;
|
|
43
52
|
const MAX_HISTORY_ATTACHMENT_CHARS = 60_000;
|
|
@@ -742,7 +751,11 @@ function estimateHistoryMessageCost(message) {
|
|
|
742
751
|
const argsText = tool.argsText ?? stableJson(tool.args ?? {});
|
|
743
752
|
cost += Math.min(argsText.length, argsCap);
|
|
744
753
|
if (tool.result !== undefined) {
|
|
745
|
-
cost += Math.min(
|
|
754
|
+
cost += Math.min(
|
|
755
|
+
// Price the string the request actually carries. `String(result)` is
|
|
756
|
+
// 15 chars ("[object Object]") for every object result, which is most
|
|
757
|
+
// of them.
|
|
758
|
+
toolResultContent(tool.result, tool.toolName).length, MAX_HISTORY_TOOL_RESULT_CHARS);
|
|
746
759
|
}
|
|
747
760
|
}
|
|
748
761
|
return Math.max(1, cost);
|
|
@@ -1634,6 +1647,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1634
1647
|
let startupRecoveryAttempts = 0;
|
|
1635
1648
|
let queuedConflictRetries = 0;
|
|
1636
1649
|
let totalTransientContinuationAttempts = 0;
|
|
1650
|
+
// Work-boundary continuations (`loop_limit`), counted apart from the
|
|
1651
|
+
// transient ones — see MAX_LOOP_LIMIT_CONTINUATIONS.
|
|
1652
|
+
let loopLimitContinuations = 0;
|
|
1637
1653
|
// Consecutive continuations whose advance signature matched the previous
|
|
1638
1654
|
// round's (or was empty). Reset by any real advance.
|
|
1639
1655
|
let nonAdvancingContinuations = 0;
|
|
@@ -1647,12 +1663,13 @@ export function createAgentChatAdapter(options) {
|
|
|
1647
1663
|
// continuation history; the advance check needs the per-round delta for
|
|
1648
1664
|
// every reason code, `loop_limit` included.
|
|
1649
1665
|
let advanceCheckPrefix = [];
|
|
1650
|
-
// Set when the turn is stopped by
|
|
1651
|
-
//
|
|
1652
|
-
//
|
|
1653
|
-
//
|
|
1654
|
-
// generic "connection kept
|
|
1655
|
-
|
|
1666
|
+
// Set when the turn is stopped by a whole-turn continuation ceiling
|
|
1667
|
+
// (MAX_TOTAL_TRANSIENT_CONTINUATIONS or MAX_LOOP_LIMIT_CONTINUATIONS)
|
|
1668
|
+
// rather than by a dropped connection. Either can trip on a turn that was
|
|
1669
|
+
// making real progress the entire time, so it needs its own honest
|
|
1670
|
+
// message instead of falling through to the generic "connection kept
|
|
1671
|
+
// failing" copy below.
|
|
1672
|
+
let recoveryGaveUpOnContinuationBudget = false;
|
|
1656
1673
|
const continuationHistoryFragments = [];
|
|
1657
1674
|
const structuredContinuationFragments = [];
|
|
1658
1675
|
let visibleContinuationPrefix = [];
|
|
@@ -1708,6 +1725,7 @@ export function createAgentChatAdapter(options) {
|
|
|
1708
1725
|
? `activity_trail: ${formatActivityTrail(lastActivityTrail)}`
|
|
1709
1726
|
: "",
|
|
1710
1727
|
`total_transient_continuations: ${totalTransientContinuationAttempts}`,
|
|
1728
|
+
`loop_limit_continuations: ${loopLimitContinuations}`,
|
|
1711
1729
|
`background_follow_no_progress_detaches: ${backgroundFollowNoProgressDetaches}`,
|
|
1712
1730
|
`background_follow_consecutive_no_progress_detaches: ${backgroundFollowConsecutiveNoProgressDetaches}`,
|
|
1713
1731
|
backgroundFollowLastDetachReason
|
|
@@ -1738,7 +1756,7 @@ export function createAgentChatAdapter(options) {
|
|
|
1738
1756
|
: " the same action";
|
|
1739
1757
|
return `The agent got stuck preparing${tool} input and never started the tool, so I stopped the automatic retries. Try a smaller first step or a more compact version of the request.`;
|
|
1740
1758
|
}
|
|
1741
|
-
if (
|
|
1759
|
+
if (recoveryGaveUpOnContinuationBudget) {
|
|
1742
1760
|
return "This turn reached the limit on how many times it can be automatically continued, so I stopped it here. It may have still been making progress — retry as a single, narrower request so it can finish within fewer continuations.";
|
|
1743
1761
|
}
|
|
1744
1762
|
if (content.length === 0 &&
|
|
@@ -2815,15 +2833,6 @@ export function createAgentChatAdapter(options) {
|
|
|
2815
2833
|
const visibleContent = visibleContentForContinuation();
|
|
2816
2834
|
let currentPartialHistory = contentToContinuationHistory(visibleContent);
|
|
2817
2835
|
const madeContentProgress = hasContinuationProgress(visibleContent);
|
|
2818
|
-
// An action was streamed but has not returned yet (a tool_start with
|
|
2819
|
-
// no tool_done). Only a run_timeout in that window means the SERVER
|
|
2820
|
-
// is still executing it and a reconnect can still deliver the result;
|
|
2821
|
-
// every other reason means the stream is dead and nothing will
|
|
2822
|
-
// complete the tool.
|
|
2823
|
-
const hasInFlightTool = hasInFlightToolCall(visibleContent);
|
|
2824
|
-
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2825
|
-
const currentPreparingToolName = lastUnresolvedToolActivity(visibleContent) ??
|
|
2826
|
-
lastPreparingActionTool(signal.activityTrail);
|
|
2827
2836
|
// THE boundary. One question — did this continuation advance the
|
|
2828
2837
|
// turn? — decides every reason code, `loop_limit` included. It used
|
|
2829
2838
|
// to skip this block entirely AND reset two of the budgets, so a
|
|
@@ -2832,6 +2841,20 @@ export function createAgentChatAdapter(options) {
|
|
|
2832
2841
|
// no answer). The durable per-turn ledger lives server-side inside
|
|
2833
2842
|
// one run and never sees a client re-POST, so nothing else bounds it.
|
|
2834
2843
|
const advanceDelta = contentAfterContinuationPrefix(content, advanceCheckPrefix);
|
|
2844
|
+
// Everything that asks "what did THIS round do" reads the delta.
|
|
2845
|
+
// `visibleContent` accumulates across a `loop_limit` chain — its
|
|
2846
|
+
// prefix deliberately does not advance there — so one unresolved
|
|
2847
|
+
// "Preparing X" card from an earlier round would otherwise stall
|
|
2848
|
+
// every later round on the same name and kill a turn that was
|
|
2849
|
+
// streaming new prose the whole time.
|
|
2850
|
+
// An action streamed but not returned yet (a tool_start with no
|
|
2851
|
+
// tool_done) means the SERVER is still executing it, so a run_timeout
|
|
2852
|
+
// in that window can still deliver the result on reconnect; every
|
|
2853
|
+
// other reason means the stream is dead and nothing will complete it.
|
|
2854
|
+
const hasInFlightTool = hasInFlightToolCall(advanceDelta);
|
|
2855
|
+
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2856
|
+
const currentPreparingToolName = lastUnresolvedToolActivity(advanceDelta) ??
|
|
2857
|
+
lastPreparingActionTool(signal.activityTrail);
|
|
2835
2858
|
const advance = describeContinuationAdvance(advanceDelta, currentPreparingToolName);
|
|
2836
2859
|
const serverStillRunningTool = signal.reason === "run_timeout" && hasInFlightTool;
|
|
2837
2860
|
const advanced = serverStillRunningTool ||
|
|
@@ -2844,7 +2867,12 @@ export function createAgentChatAdapter(options) {
|
|
|
2844
2867
|
nonAdvancingContinuations = advanced
|
|
2845
2868
|
? 0
|
|
2846
2869
|
: nonAdvancingContinuations + 1;
|
|
2847
|
-
|
|
2870
|
+
if (isTransient) {
|
|
2871
|
+
totalTransientContinuationAttempts += 1;
|
|
2872
|
+
}
|
|
2873
|
+
else {
|
|
2874
|
+
loopLimitContinuations += 1;
|
|
2875
|
+
}
|
|
2848
2876
|
advanceCheckPrefix = snapshotContent(content);
|
|
2849
2877
|
// If a tool already completed, do not turn a missing closing
|
|
2850
2878
|
// sentence into a scary connection failure. Give the model one
|
|
@@ -2868,8 +2896,9 @@ export function createAgentChatAdapter(options) {
|
|
|
2868
2896
|
return { ok: false, resetVisibleContent: false };
|
|
2869
2897
|
}
|
|
2870
2898
|
if (totalTransientContinuationAttempts >
|
|
2871
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS
|
|
2872
|
-
|
|
2899
|
+
MAX_TOTAL_TRANSIENT_CONTINUATIONS ||
|
|
2900
|
+
loopLimitContinuations > MAX_LOOP_LIMIT_CONTINUATIONS) {
|
|
2901
|
+
recoveryGaveUpOnContinuationBudget = true;
|
|
2873
2902
|
return { ok: false, resetVisibleContent: false };
|
|
2874
2903
|
}
|
|
2875
2904
|
if (isTransient) {
|
|
@@ -534,6 +534,13 @@ function isAutoRecoverableError(ev, errMsg) {
|
|
|
534
534
|
}
|
|
535
535
|
if (ev.recoverable === true)
|
|
536
536
|
return true;
|
|
537
|
+
// An explicit flag outranks the message sniff below, which exists only for
|
|
538
|
+
// events that carry no flag at all. The repeat guards stop a turn with
|
|
539
|
+
// `recoverable: false` and a message that names the looping tool, so a stop
|
|
540
|
+
// on `list-workspace-connections` matched the "connection" sniff and
|
|
541
|
+
// auto-continued the very loop it was emitted to break.
|
|
542
|
+
if (ev.recoverable === false)
|
|
543
|
+
return false;
|
|
537
544
|
if (msg.includes("daily gateway request cap"))
|
|
538
545
|
return false;
|
|
539
546
|
// "gateway error" intentionally absent — that's the no-detail Builder
|
|
@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
|
|
|
62
62
|
error: string;
|
|
63
63
|
states?: undefined;
|
|
64
64
|
} | {
|
|
65
|
+
error?: undefined;
|
|
65
66
|
states: {
|
|
66
67
|
clientId: number;
|
|
67
68
|
state: string;
|
|
68
69
|
}[];
|
|
69
|
-
error?: undefined;
|
|
70
70
|
}>>;
|
|
71
71
|
/**
|
|
72
72
|
* GET /_agent-native/collab/:docId/users
|
|
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
|
|
|
77
77
|
error: string;
|
|
78
78
|
users?: undefined;
|
|
79
79
|
} | {
|
|
80
|
+
error?: undefined;
|
|
80
81
|
users: {
|
|
81
82
|
clientId: number;
|
|
82
83
|
lastSeen: number;
|
|
83
84
|
}[];
|
|
84
|
-
error?: undefined;
|
|
85
85
|
}>>;
|
package/dist/collab/routes.d.ts
CHANGED
|
@@ -26,8 +26,8 @@ export declare const getCollabState: import("h3").EventHandlerWithFetch<import("
|
|
|
26
26
|
* Body: { update: string (base64), requestSource?: string }
|
|
27
27
|
*/
|
|
28
28
|
export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
29
|
-
ok?: undefined;
|
|
30
29
|
error: string;
|
|
30
|
+
ok?: undefined;
|
|
31
31
|
} | {
|
|
32
32
|
error?: undefined;
|
|
33
33
|
ok: boolean;
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* Body: { json: any, fieldName?: string, type?: "map"|"array", requestSource?: string }
|
|
14
14
|
*/
|
|
15
15
|
export declare const postCollabJson: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
16
|
-
error: string;
|
|
17
16
|
ok?: undefined;
|
|
17
|
+
error: string;
|
|
18
18
|
} | {
|
|
19
19
|
error?: undefined;
|
|
20
20
|
ok: boolean;
|
package/dist/db/client.js
CHANGED
|
@@ -1091,7 +1091,7 @@ const _sharedDbPoolReplacementHooks = new Map();
|
|
|
1091
1091
|
* database access down with it.
|
|
1092
1092
|
*/
|
|
1093
1093
|
export function sharedDbPool(driver, url, create) {
|
|
1094
|
-
const key = `${driver}
|
|
1094
|
+
const key = `${driver}\u0000${url}`;
|
|
1095
1095
|
const existing = _sharedDbPools.get(key);
|
|
1096
1096
|
if (existing)
|
|
1097
1097
|
return existing;
|
|
@@ -1105,7 +1105,7 @@ export function sharedDbPool(driver, url, create) {
|
|
|
1105
1105
|
* handing out the discarded pool.
|
|
1106
1106
|
*/
|
|
1107
1107
|
export function replaceSharedDbPool(driver, url, previous, next) {
|
|
1108
|
-
const key = `${driver}
|
|
1108
|
+
const key = `${driver}\u0000${url}`;
|
|
1109
1109
|
if (_sharedDbPools.get(key) !== previous)
|
|
1110
1110
|
return;
|
|
1111
1111
|
_sharedDbPools.set(key, next);
|
|
@@ -41,27 +41,27 @@ export declare function createObservabilityHandler(): import("h3").EventHandlerW
|
|
|
41
41
|
thumbsUpRate: number;
|
|
42
42
|
avgEvalScore: number;
|
|
43
43
|
} | {
|
|
44
|
+
error?: undefined;
|
|
45
|
+
ok?: undefined;
|
|
44
46
|
summary: import("./types.js").TraceSummary;
|
|
45
47
|
spans: import("./types.js").TraceSpan[];
|
|
46
48
|
id?: undefined;
|
|
49
|
+
} | {
|
|
47
50
|
error?: undefined;
|
|
48
51
|
ok?: undefined;
|
|
49
|
-
} | {
|
|
50
52
|
summary?: undefined;
|
|
51
53
|
spans?: undefined;
|
|
52
54
|
id: string;
|
|
53
|
-
error?: undefined;
|
|
54
|
-
ok?: undefined;
|
|
55
55
|
} | {
|
|
56
|
+
ok?: undefined;
|
|
56
57
|
summary?: undefined;
|
|
57
58
|
spans?: undefined;
|
|
58
59
|
id?: undefined;
|
|
59
60
|
error: any;
|
|
60
|
-
ok?: undefined;
|
|
61
61
|
} | {
|
|
62
|
+
error?: undefined;
|
|
62
63
|
summary?: undefined;
|
|
63
64
|
spans?: undefined;
|
|
64
65
|
id?: undefined;
|
|
65
66
|
ok: boolean;
|
|
66
|
-
error?: undefined;
|
|
67
67
|
}>>;
|
|
@@ -48,8 +48,8 @@ export declare function handleUpdateResource(event: any): Promise<import("./stor
|
|
|
48
48
|
}>;
|
|
49
49
|
/** DELETE /_agent-native/resources/:id — delete a resource */
|
|
50
50
|
export declare function handleDeleteResource(event: any): Promise<{
|
|
51
|
-
error: string;
|
|
52
51
|
ok?: undefined;
|
|
52
|
+
error: string;
|
|
53
53
|
} | {
|
|
54
54
|
error?: undefined;
|
|
55
55
|
ok: boolean;
|
package/dist/secrets/routes.d.ts
CHANGED
|
@@ -34,37 +34,37 @@ export declare function createListSecretsHandler(): import("h3").EventHandlerWit
|
|
|
34
34
|
/** POST /_agent-native/secrets/:key — write a secret. */
|
|
35
35
|
export declare function createWriteSecretHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
36
36
|
error: string;
|
|
37
|
-
status?: undefined;
|
|
38
37
|
ok?: undefined;
|
|
38
|
+
status?: undefined;
|
|
39
39
|
} | {
|
|
40
|
+
error?: undefined;
|
|
40
41
|
ok: boolean;
|
|
41
42
|
status: string;
|
|
42
|
-
error?: undefined;
|
|
43
43
|
} | {
|
|
44
|
+
ok?: undefined;
|
|
44
45
|
error: string;
|
|
45
46
|
removed?: undefined;
|
|
46
|
-
ok?: undefined;
|
|
47
47
|
} | {
|
|
48
|
+
error?: undefined;
|
|
48
49
|
ok: boolean;
|
|
49
50
|
removed: boolean;
|
|
50
|
-
error?: undefined;
|
|
51
51
|
}>>;
|
|
52
52
|
/**
|
|
53
53
|
* POST /_agent-native/secrets/:key/test — validate an optional candidate value
|
|
54
54
|
* or the current stored value without changing anything.
|
|
55
55
|
*/
|
|
56
56
|
export declare function createTestSecretHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
57
|
+
ok?: undefined;
|
|
57
58
|
error: string;
|
|
58
59
|
note?: undefined;
|
|
59
|
-
ok?: undefined;
|
|
60
60
|
} | {
|
|
61
|
+
error?: undefined;
|
|
61
62
|
ok: boolean;
|
|
62
63
|
note?: undefined;
|
|
63
|
-
error?: undefined;
|
|
64
64
|
} | {
|
|
65
|
+
error?: undefined;
|
|
65
66
|
ok: boolean;
|
|
66
67
|
note: string;
|
|
67
|
-
error?: undefined;
|
|
68
68
|
} | {
|
|
69
69
|
note?: undefined;
|
|
70
70
|
ok: boolean;
|
|
@@ -95,11 +95,11 @@ export interface AdHocSecretPayload {
|
|
|
95
95
|
export declare function createAdHocSecretHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<AdHocSecretPayload[] | {
|
|
96
96
|
error: string;
|
|
97
97
|
} | {
|
|
98
|
+
error?: undefined;
|
|
98
99
|
ok: boolean;
|
|
99
100
|
key: string;
|
|
100
|
-
error?: undefined;
|
|
101
101
|
} | {
|
|
102
|
+
error?: undefined;
|
|
102
103
|
ok: boolean;
|
|
103
104
|
removed: boolean;
|
|
104
|
-
error?: undefined;
|
|
105
105
|
}>>;
|
|
@@ -4890,18 +4890,41 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
4890
4890
|
error: "Recurring-job sweep reached the synchronous server instead of the durable background worker.",
|
|
4891
4891
|
};
|
|
4892
4892
|
}
|
|
4893
|
+
// Stale reaping runs FIRST and site-wide, before the open-ended job
|
|
4894
|
+
// sweep can spend the platform wall. It is the durable driver the
|
|
4895
|
+
// in-process fast sweep below cannot be on serverless: that timer is
|
|
4896
|
+
// off wherever `shouldDisableInProcessSweeps` is on — i.e. every
|
|
4897
|
+
// production Lambda — and nothing replaced it, so a claimed run
|
|
4898
|
+
// whose producer died was only reaped when an unrelated request path
|
|
4899
|
+
// happened to look (prod: 1,216 runs, 12% of all runs, sitting
|
|
4900
|
+
// "running" for up to 59 minutes against a 15s window).
|
|
4901
|
+
//
|
|
4902
|
+
// One reap per site-tick instead of one per warm container is also
|
|
4903
|
+
// the point: the per-container timers are what issued 237k queries
|
|
4904
|
+
// in two minutes and earned that kill switch. `reapAllStaleRuns`
|
|
4905
|
+
// short-circuits on `hasRunningRuns()` (negative-cached), so an idle
|
|
4906
|
+
// app pays one probe.
|
|
4907
|
+
//
|
|
4908
|
+
// Reported, never swallowed, and never fatal to the job sweep: a
|
|
4909
|
+
// reap failure must not take recurring jobs down with it, and
|
|
4910
|
+
// `null` must stay distinguishable from "reaped nothing".
|
|
4911
|
+
const { reapAllStaleRuns } = await import("../agent/run-store.js");
|
|
4912
|
+
const staleRunsReaped = await reapAllStaleRuns().catch((error) => {
|
|
4913
|
+
console.error("[agent-chat] durable stale-run reap failed:", error);
|
|
4914
|
+
return null;
|
|
4915
|
+
});
|
|
4893
4916
|
try {
|
|
4894
4917
|
// Jobs may request MCP tools, and `getActions` is synchronous —
|
|
4895
4918
|
// hydrate before the sweep so a serverless container that never
|
|
4896
4919
|
// eagerly initialized still resolves them.
|
|
4897
4920
|
await ensureMcpInitialized();
|
|
4898
4921
|
await processRecurringJobs(schedulerDeps);
|
|
4899
|
-
return { ok: true };
|
|
4922
|
+
return { ok: true, staleRunsReaped };
|
|
4900
4923
|
}
|
|
4901
4924
|
catch (error) {
|
|
4902
4925
|
console.error("[recurring-jobs] Sweep route failed:", error);
|
|
4903
4926
|
setResponseStatus(event, 500);
|
|
4904
|
-
return { error: "Recurring-job sweep failed" };
|
|
4927
|
+
return { error: "Recurring-job sweep failed", staleRunsReaped };
|
|
4905
4928
|
}
|
|
4906
4929
|
}));
|
|
4907
4930
|
if (disableRecurringJobsRuntime) {
|
|
@@ -5139,7 +5162,16 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
5139
5162
|
// against a 45s window). `reapAllStaleRuns` is per-row,
|
|
5140
5163
|
// idempotent, re-checks staleness at UPDATE time and honours
|
|
5141
5164
|
// the in-flight grace, so it is safe on this cadence.
|
|
5142
|
-
|
|
5165
|
+
//
|
|
5166
|
+
// This timer is the driver only where it actually runs — a
|
|
5167
|
+
// long-lived Node host. Wherever `sweepsDisabled` turns it off
|
|
5168
|
+
// (every production serverless function) the signed
|
|
5169
|
+
// RECURRING_JOBS_SWEEP_PATH route above owns the same reap,
|
|
5170
|
+
// driven by the platform scheduler. The two never both fire on
|
|
5171
|
+
// one host, which is why neither needs to coordinate.
|
|
5172
|
+
await reapAllStaleRuns().catch((error) => {
|
|
5173
|
+
console.error("[agent-chat] in-process stale-run reap failed:", error);
|
|
5174
|
+
});
|
|
5143
5175
|
let rows;
|
|
5144
5176
|
try {
|
|
5145
5177
|
rows = await listUnclaimedBackgroundRunRows();
|
|
@@ -27,10 +27,10 @@ export declare function resolveAgentEngineApiKeyWriteTarget(event: H3Event, scop
|
|
|
27
27
|
export declare function createAgentEngineApiKeyHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
28
28
|
error: any;
|
|
29
29
|
} | {
|
|
30
|
+
error?: undefined;
|
|
30
31
|
ok: boolean;
|
|
31
32
|
key: string;
|
|
32
33
|
baseUrlKey?: string;
|
|
33
34
|
scope: AgentEngineApiKeyScope;
|
|
34
|
-
error?: undefined;
|
|
35
35
|
}>>;
|
|
36
36
|
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.153.
|
|
3
|
+
"version": "0.153.6",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -421,8 +421,8 @@
|
|
|
421
421
|
"y-protocols": "^1.0.7",
|
|
422
422
|
"yjs": "^13.6.31",
|
|
423
423
|
"zod": "^4.3.6",
|
|
424
|
-
"@agent-native/
|
|
425
|
-
"@agent-native/
|
|
424
|
+
"@agent-native/recap-cli": "0.5.4",
|
|
425
|
+
"@agent-native/toolkit": "^0.14.1"
|
|
426
426
|
},
|
|
427
427
|
"devDependencies": {
|
|
428
428
|
"@ai-sdk/anthropic": "^3.0.71",
|