@agent-native/core 0.161.2 → 0.161.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/analytics/actions/update-dashboard.ts +8 -0
- package/corpus/templates/clips/actions/save-browser-transcript.ts +20 -4
- package/corpus/templates/clips/app/components/meetings/transcript-bubbles.tsx +167 -35
- package/corpus/templates/clips/desktop/src/lib/transcription-capture.ts +8 -1
- package/corpus/templates/clips/desktop/src/lib/transcription-engine.ts +39 -2
- package/corpus/templates/forms/server/lib/public-form-ssr.ts +3 -0
- package/corpus/templates/slides/actions/list-decks.ts +36 -1
- package/corpus/templates/slides/app/components/editor/SlideEditor.tsx +29 -17
- package/corpus/templates/slides/app/context/DeckContext.tsx +17 -6
- package/dist/agent/engine/builder-engine.js +30 -15
- package/dist/agent/engine/types.d.ts +19 -0
- package/dist/agent/engine/types.js +3 -0
- package/dist/agent/production-agent.d.ts +56 -1
- package/dist/agent/production-agent.js +130 -3
- package/dist/agent/run-manager.d.ts +15 -4
- package/dist/agent/run-manager.js +26 -0
- package/dist/agent/run-store.d.ts +5 -5
- package/dist/agent/run-store.js +52 -15
- package/dist/agent/thread-data-builder.js +7 -0
- package/dist/agent/types.d.ts +10 -0
- package/dist/cli/code-agent-connector.js +6 -1
- package/dist/client/AssistantChat.d.ts +1 -0
- package/dist/client/AssistantChat.js +10 -1
- package/dist/client/MultiTabAssistantChat.js +33 -1
- package/dist/client/sse-event-processor.js +11 -7
- package/dist/client/use-chat-threads.js +9 -9
- package/dist/db/client.js +10 -2
- package/dist/db/create-get-db.js +42 -0
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/onboarding-html.js +22 -77
- package/dist/server/poll.d.ts +5 -5
- package/dist/server/poll.js +19 -26
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/shared/auth-copy.d.ts +7 -0
- package/dist/shared/auth-copy.js +77 -0
- package/dist/shared/mcp-embed-headers.js +8 -4
- package/dist/vite/client.js +94 -3
- package/package.json +1 -1
|
@@ -226,6 +226,17 @@ class BuilderEngine {
|
|
|
226
226
|
}
|
|
227
227
|
: {}),
|
|
228
228
|
};
|
|
229
|
+
// Measured once, from the exact string that goes on the wire, and carried
|
|
230
|
+
// on every error stop below. A gateway rejection tells us nothing about
|
|
231
|
+
// what we sent, so without this an oversized or malformed request and a
|
|
232
|
+
// gateway outage are the same capture.
|
|
233
|
+
const payload = JSON.stringify(body);
|
|
234
|
+
const requestShape = {
|
|
235
|
+
model: opts.model,
|
|
236
|
+
payloadBytes: new TextEncoder().encode(payload).length,
|
|
237
|
+
toolCount: cachedTools.length,
|
|
238
|
+
messageCount: cachedMessages.length,
|
|
239
|
+
};
|
|
229
240
|
const gatewayBaseUrl = getBuilderGatewayBaseUrl();
|
|
230
241
|
const gatewayUrl = new URL("messages", gatewayBaseUrl.endsWith("/") ? gatewayBaseUrl : `${gatewayBaseUrl}/`);
|
|
231
242
|
gatewayUrl.searchParams.set("apiKey", spaceId);
|
|
@@ -246,7 +257,7 @@ class BuilderEngine {
|
|
|
246
257
|
...getBuilderGatewayRequestHeaders(),
|
|
247
258
|
...(builderUserId ? { "x-builder-user-id": builderUserId } : {}),
|
|
248
259
|
},
|
|
249
|
-
body:
|
|
260
|
+
body: payload,
|
|
250
261
|
signal: gatewayAbort.signal,
|
|
251
262
|
});
|
|
252
263
|
}
|
|
@@ -265,12 +276,12 @@ class BuilderEngine {
|
|
|
265
276
|
elapsedMs: Date.now() - tStart,
|
|
266
277
|
});
|
|
267
278
|
}
|
|
268
|
-
yield createBuilderGatewayTimeoutStop(err, timedOut, gatewayAbort.effectiveTimeoutMs(), creditsLane);
|
|
279
|
+
yield createBuilderGatewayTimeoutStop(err, timedOut, gatewayAbort.effectiveTimeoutMs(), creditsLane, requestShape);
|
|
269
280
|
return;
|
|
270
281
|
}
|
|
271
282
|
console.log(`[builder-engine] ← ${response.status} ${response.statusText} in ${Date.now() - tStart}ms`);
|
|
272
283
|
if (!response.ok) {
|
|
273
|
-
yield* emitHttpError(response, { creditsLane });
|
|
284
|
+
yield* emitHttpError(response, { creditsLane, requestShape });
|
|
274
285
|
return;
|
|
275
286
|
}
|
|
276
287
|
// A successful gateway call proves the connected credentials are valid
|
|
@@ -302,7 +313,7 @@ class BuilderEngine {
|
|
|
302
313
|
...(isTransientGatewayFailure(error, status)
|
|
303
314
|
? { providerRetryable: true }
|
|
304
315
|
: {}),
|
|
305
|
-
}, creditsLane);
|
|
316
|
+
}, creditsLane, requestShape);
|
|
306
317
|
return;
|
|
307
318
|
}
|
|
308
319
|
const reader = response.body?.getReader();
|
|
@@ -311,7 +322,7 @@ class BuilderEngine {
|
|
|
311
322
|
error: "Builder gateway response has no body",
|
|
312
323
|
errorCode: "builder_gateway_error",
|
|
313
324
|
statusCode: response.status,
|
|
314
|
-
}, creditsLane);
|
|
325
|
+
}, creditsLane, requestShape);
|
|
315
326
|
return;
|
|
316
327
|
}
|
|
317
328
|
yield* parseJsonlStream(reader, opts.model, {
|
|
@@ -322,6 +333,7 @@ class BuilderEngine {
|
|
|
322
333
|
onFirstEvent: gatewayAbort.markFirstEvent,
|
|
323
334
|
gatewayUrl,
|
|
324
335
|
requestStartedAt: tStart,
|
|
336
|
+
requestShape,
|
|
325
337
|
});
|
|
326
338
|
}
|
|
327
339
|
finally {
|
|
@@ -358,7 +370,7 @@ function isTransientGatewayFailure(rawMessage, status) {
|
|
|
358
370
|
* signals downstream may read: keyword coupling to the message turns a retryable
|
|
359
371
|
* throttle into a dead turn on credits sites alone.
|
|
360
372
|
*/
|
|
361
|
-
function gatewayErrorStop(details, creditsLane) {
|
|
373
|
+
function gatewayErrorStop(details, creditsLane, requestShape) {
|
|
362
374
|
const { error, errorCode, upgradeUrl, ...retry } = details;
|
|
363
375
|
return {
|
|
364
376
|
type: "stop",
|
|
@@ -373,6 +385,9 @@ function gatewayErrorStop(details, creditsLane) {
|
|
|
373
385
|
...(isContextOverflowMessage(error) || isContextOverflowCode(errorCode)
|
|
374
386
|
? { contextOverflow: true }
|
|
375
387
|
: {}),
|
|
388
|
+
// Absent before the request is built (missing credentials): a stop with no
|
|
389
|
+
// shape means nothing was sent, not that the payload measured zero.
|
|
390
|
+
...(requestShape ? { requestShape } : {}),
|
|
376
391
|
...retry,
|
|
377
392
|
};
|
|
378
393
|
}
|
|
@@ -394,7 +409,7 @@ async function* emitHttpError(response, opts) {
|
|
|
394
409
|
}
|
|
395
410
|
const code = errBody.code ?? `http_${status}`;
|
|
396
411
|
const message = errBody.message ?? `Builder gateway returned ${status}`;
|
|
397
|
-
const stop = (details) => gatewayErrorStop(details, opts.creditsLane);
|
|
412
|
+
const stop = (details) => gatewayErrorStop(details, opts.creditsLane, opts.requestShape);
|
|
398
413
|
// Belt-and-suspenders: 402 without a structured `credits-limit` code
|
|
399
414
|
// (e.g. bare proxy response) still means quota → show upgrade CTA.
|
|
400
415
|
if (code.startsWith("credits-limit") || status === 402) {
|
|
@@ -543,7 +558,7 @@ async function* parseJsonlStream(reader, model, captureContext = {}) {
|
|
|
543
558
|
errorCode: "http_502",
|
|
544
559
|
statusCode: 502,
|
|
545
560
|
providerRetryable: true,
|
|
546
|
-
}, captureContext.creditsLane);
|
|
561
|
+
}, captureContext.creditsLane, captureContext.requestShape);
|
|
547
562
|
return;
|
|
548
563
|
}
|
|
549
564
|
// Heartbeats are transport-level keepalives, not proof the model is
|
|
@@ -624,7 +639,7 @@ async function* parseJsonlStream(reader, model, captureContext = {}) {
|
|
|
624
639
|
yield* recoverUndeliveredToolCalls();
|
|
625
640
|
yield { type: "assistant-content", parts };
|
|
626
641
|
const reason = event.reason ?? "end_turn";
|
|
627
|
-
const stop = (details) => gatewayErrorStop(details, captureContext.creditsLane);
|
|
642
|
+
const stop = (details) => gatewayErrorStop(details, captureContext.creditsLane, captureContext.requestShape);
|
|
628
643
|
if (reason === "rate_limited") {
|
|
629
644
|
yield stop({
|
|
630
645
|
error: `rate_limit exceeded: ${event.error ?? "upstream provider rate limited"}`,
|
|
@@ -744,7 +759,7 @@ async function* parseJsonlStream(reader, model, captureContext = {}) {
|
|
|
744
759
|
yield gatewayErrorStop({
|
|
745
760
|
error: "Builder gateway stream ended without a stop event",
|
|
746
761
|
errorCode: BUILDER_GATEWAY_STREAM_ENDED_ERROR_CODE,
|
|
747
|
-
}, captureContext.creditsLane);
|
|
762
|
+
}, captureContext.creditsLane, captureContext.requestShape);
|
|
748
763
|
}
|
|
749
764
|
catch (err) {
|
|
750
765
|
const timedOut = captureContext.didGatewayTimeout?.() ?? false;
|
|
@@ -762,7 +777,7 @@ async function* parseJsonlStream(reader, model, captureContext = {}) {
|
|
|
762
777
|
: undefined,
|
|
763
778
|
});
|
|
764
779
|
}
|
|
765
|
-
yield createBuilderGatewayTimeoutStop(err, timedOut, gatewayTimeoutMs, captureContext.creditsLane);
|
|
780
|
+
yield createBuilderGatewayTimeoutStop(err, timedOut, gatewayTimeoutMs, captureContext.creditsLane, captureContext.requestShape);
|
|
766
781
|
}
|
|
767
782
|
finally {
|
|
768
783
|
// Release the reader on every exit path — early returns (invalid JSONL,
|
|
@@ -976,27 +991,27 @@ function normalizeBuilderGatewayFetchError(err, timedOut, timeoutMs) {
|
|
|
976
991
|
* the credits lane run-manager has no text left to classify at persistence time,
|
|
977
992
|
* and a run persisted as `unknown` reads as "do not attempt recovery".
|
|
978
993
|
*/
|
|
979
|
-
function createBuilderGatewayTimeoutStop(err, timedOut, timeoutMs, creditsLane) {
|
|
994
|
+
function createBuilderGatewayTimeoutStop(err, timedOut, timeoutMs, creditsLane, requestShape) {
|
|
980
995
|
const error = normalizeBuilderGatewayFetchError(err, timedOut, timeoutMs);
|
|
981
996
|
if (timedOut) {
|
|
982
997
|
// Deliberately no `providerRetryable`: the timeout spent the whole request
|
|
983
998
|
// budget, so the recovery is a fresh invocation (the client's
|
|
984
999
|
// `builder_gateway_timeout` continuation), never an in-call retry.
|
|
985
|
-
return gatewayErrorStop({ error, errorCode: "builder_gateway_timeout" }, creditsLane);
|
|
1000
|
+
return gatewayErrorStop({ error, errorCode: "builder_gateway_timeout" }, creditsLane, requestShape);
|
|
986
1001
|
}
|
|
987
1002
|
if (isBuilderGatewayNetworkError(err)) {
|
|
988
1003
|
return gatewayErrorStop({
|
|
989
1004
|
error,
|
|
990
1005
|
errorCode: BUILDER_GATEWAY_NETWORK_ERROR_CODE,
|
|
991
1006
|
providerRetryable: true,
|
|
992
|
-
}, creditsLane);
|
|
1007
|
+
}, creditsLane, requestShape);
|
|
993
1008
|
}
|
|
994
1009
|
const errorCode = classifyTerminalErrorCode(error);
|
|
995
1010
|
return gatewayErrorStop({
|
|
996
1011
|
error,
|
|
997
1012
|
...(errorCode ? { errorCode } : {}),
|
|
998
1013
|
...(isTransientGatewayFailure(error) ? { providerRetryable: true } : {}),
|
|
999
|
-
}, creditsLane);
|
|
1014
|
+
}, creditsLane, requestShape);
|
|
1000
1015
|
}
|
|
1001
1016
|
function formatTimeoutMs(timeoutMs) {
|
|
1002
1017
|
if (timeoutMs < 1000)
|
|
@@ -36,6 +36,8 @@ export declare class EngineError extends Error {
|
|
|
36
36
|
* one-shot trim-and-retry recovery.
|
|
37
37
|
*/
|
|
38
38
|
readonly contextOverflow?: boolean;
|
|
39
|
+
/** Sizes and counts of the failed request; see {@link EngineRequestShape}. */
|
|
40
|
+
readonly requestShape?: EngineRequestShape;
|
|
39
41
|
constructor(message: string, opts?: {
|
|
40
42
|
errorCode?: string;
|
|
41
43
|
upgradeUrl?: string;
|
|
@@ -43,6 +45,7 @@ export declare class EngineError extends Error {
|
|
|
43
45
|
providerRetryable?: boolean;
|
|
44
46
|
requestId?: string;
|
|
45
47
|
contextOverflow?: boolean;
|
|
48
|
+
requestShape?: EngineRequestShape;
|
|
46
49
|
});
|
|
47
50
|
}
|
|
48
51
|
/**
|
|
@@ -213,7 +216,23 @@ export type EngineEvent = {
|
|
|
213
216
|
* the time the agent decides whether to trim and retry.
|
|
214
217
|
*/
|
|
215
218
|
contextOverflow?: boolean;
|
|
219
|
+
/**
|
|
220
|
+
* Sizes and counts of the request that failed. Never prompt or user
|
|
221
|
+
* content — the point is to make "what did we send" answerable from a
|
|
222
|
+
* capture, which an opaque gateway 500 otherwise leaves unanswerable.
|
|
223
|
+
*/
|
|
224
|
+
requestShape?: EngineRequestShape;
|
|
216
225
|
};
|
|
226
|
+
/**
|
|
227
|
+
* Shape-only description of what an engine put on the wire. Every field is a
|
|
228
|
+
* size, a count, or a model id, so it is safe to attach to an error capture.
|
|
229
|
+
*/
|
|
230
|
+
export interface EngineRequestShape {
|
|
231
|
+
model: string;
|
|
232
|
+
payloadBytes: number;
|
|
233
|
+
toolCount: number;
|
|
234
|
+
messageCount: number;
|
|
235
|
+
}
|
|
217
236
|
export interface EngineCapabilities {
|
|
218
237
|
/** Extended / adaptive thinking support */
|
|
219
238
|
thinking: boolean;
|
|
@@ -35,6 +35,8 @@ export class EngineError extends Error {
|
|
|
35
35
|
* one-shot trim-and-retry recovery.
|
|
36
36
|
*/
|
|
37
37
|
contextOverflow;
|
|
38
|
+
/** Sizes and counts of the failed request; see {@link EngineRequestShape}. */
|
|
39
|
+
requestShape;
|
|
38
40
|
constructor(message, opts) {
|
|
39
41
|
super(message);
|
|
40
42
|
this.name = "EngineError";
|
|
@@ -44,5 +46,6 @@ export class EngineError extends Error {
|
|
|
44
46
|
this.providerRetryable = opts?.providerRetryable;
|
|
45
47
|
this.requestId = opts?.requestId;
|
|
46
48
|
this.contextOverflow = opts?.contextOverflow;
|
|
49
|
+
this.requestShape = opts?.requestShape;
|
|
47
50
|
}
|
|
48
51
|
}
|
|
@@ -935,6 +935,52 @@ export declare function runAgentLoopWithMainChatInternalContinuations(opts: Para
|
|
|
935
935
|
* background invocations forever, mirroring `MAX_AGENT_TEAM_CONTINUATIONS`.
|
|
936
936
|
*/
|
|
937
937
|
export declare const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
|
|
938
|
+
/**
|
|
939
|
+
* Consecutive chunks allowed to end on the SAME terminal error code having
|
|
940
|
+
* produced nothing before the chain stops.
|
|
941
|
+
*
|
|
942
|
+
* Two, because two independent recovery layers multiply here and neither can
|
|
943
|
+
* see the other: the engine already retried this identical request 3x with
|
|
944
|
+
* backoff before the error was ever emitted, and a recoverable error is also a
|
|
945
|
+
* continuation boundary, so every chunk that fails costs 4 gateway attempts
|
|
946
|
+
* and dispatches a fresh one. A production turn spent 27 background runs and
|
|
947
|
+
* 15 minutes on one message this way. The first repeat is the retry this path
|
|
948
|
+
* exists for; a second identical failure that moved nothing is evidence the
|
|
949
|
+
* retrying itself is what is broken, not the request.
|
|
950
|
+
*/
|
|
951
|
+
export declare const MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS = 2;
|
|
952
|
+
/** Consecutive-identical-failure state for one chunk of a background chain. */
|
|
953
|
+
export interface BackgroundNoProgressRepeat {
|
|
954
|
+
/** This chunk's terminal error code, when it ended having produced nothing. */
|
|
955
|
+
errorCode?: string;
|
|
956
|
+
/** Chunks in a row that ended on that code with no forward progress. */
|
|
957
|
+
count: number;
|
|
958
|
+
/** True once the streak reaches `MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS`. */
|
|
959
|
+
tripped: boolean;
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* Advance the no-progress streak across a chunk boundary. A chunk that emitted
|
|
963
|
+
* any text or tool activity resets it even when it still ended in an error —
|
|
964
|
+
* that turn IS moving, and cutting it off is what would weaken recovery for
|
|
965
|
+
* truncated streams. So does a different error code, and so does a boundary
|
|
966
|
+
* with no error at all (a soft-timeout `auto_continue` is the run-manager's
|
|
967
|
+
* no-progress backstop to bound, not this one).
|
|
968
|
+
*/
|
|
969
|
+
export declare function resolveBackgroundNoProgressRepeat(opts: {
|
|
970
|
+
run: ActiveRun;
|
|
971
|
+
priorErrorCode?: string;
|
|
972
|
+
priorCount?: number;
|
|
973
|
+
}): BackgroundNoProgressRepeat;
|
|
974
|
+
/**
|
|
975
|
+
* The single honest failure the breaker leaves behind. Keeps the underlying
|
|
976
|
+
* code and the gateway's own message (which carries its `ERROR ID:` reference)
|
|
977
|
+
* so the failure stays diagnosable, and marks it non-recoverable so neither
|
|
978
|
+
* this chain nor the client's continuation path re-enters it.
|
|
979
|
+
*/
|
|
980
|
+
export declare function backgroundNoProgressTerminalEvent(run: ActiveRun, repeat: BackgroundNoProgressRepeat): Extract<AgentChatEvent, {
|
|
981
|
+
type: "error";
|
|
982
|
+
}> | null;
|
|
983
|
+
export declare function installBackgroundNoProgressTerminalEvent(run: ActiveRun, repeat: BackgroundNoProgressRepeat): boolean;
|
|
938
984
|
/**
|
|
939
985
|
* Whether this run should self-fire the next server-driven continuation chunk
|
|
940
986
|
* instead of depending on the client to re-POST `auto_continue`. True for
|
|
@@ -949,7 +995,9 @@ export declare const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
|
|
|
949
995
|
* the durable background worker (`dispatchedToBackground` false) — a run
|
|
950
996
|
* already headed to the durable background path chains via the
|
|
951
997
|
* `isBackgroundWorker` branch above, never both.
|
|
952
|
-
* Aborted / user-stopped runs do NOT chain either way
|
|
998
|
+
* Aborted / user-stopped runs do NOT chain either way, and neither does a run
|
|
999
|
+
* whose no-progress streak has tripped
|
|
1000
|
+
* (`MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS`).
|
|
953
1001
|
*/
|
|
954
1002
|
export declare function shouldChainBackgroundContinuation(opts: {
|
|
955
1003
|
isBackgroundWorker: boolean;
|
|
@@ -970,6 +1018,10 @@ export declare function shouldChainBackgroundContinuation(opts: {
|
|
|
970
1018
|
* owned by the background circuit-breaker, not this path.
|
|
971
1019
|
*/
|
|
972
1020
|
dispatchedToBackground?: boolean;
|
|
1021
|
+
/** Streak state carried on the continuation marker — see
|
|
1022
|
+
* `resolveBackgroundNoProgressRepeat`. Absent on the first chunk. */
|
|
1023
|
+
priorNoProgressErrorCode?: string;
|
|
1024
|
+
priorNoProgressCount?: number;
|
|
973
1025
|
}): boolean;
|
|
974
1026
|
/**
|
|
975
1027
|
* Minimum remaining budget (ms) a synchronous self-chain continuation chunk
|
|
@@ -1275,6 +1327,9 @@ export declare function chainServerDrivenContinuation(opts: {
|
|
|
1275
1327
|
* is derived from it (marker stripped, `internalContinuation` set). */
|
|
1276
1328
|
requestBody: Record<string, unknown>;
|
|
1277
1329
|
backgroundContinuationCount: number;
|
|
1330
|
+
/** This chunk's no-progress streak, carried to the successor so the breaker
|
|
1331
|
+
* can see a repeat across the invocation boundary. */
|
|
1332
|
+
noProgressRepeat?: BackgroundNoProgressRepeat;
|
|
1278
1333
|
/**
|
|
1279
1334
|
* Input tokens this logical turn has consumed across every chunk so far,
|
|
1280
1335
|
* carried on the successor's body so the per-turn token ceiling is a real
|
|
@@ -3753,6 +3753,7 @@ export async function runAgentLoop(opts) {
|
|
|
3753
3753
|
providerRetryable: event.providerRetryable,
|
|
3754
3754
|
contextOverflow: event.contextOverflow,
|
|
3755
3755
|
requestId: event.requestId,
|
|
3756
|
+
requestShape: event.requestShape,
|
|
3756
3757
|
});
|
|
3757
3758
|
}
|
|
3758
3759
|
}
|
|
@@ -5421,6 +5422,87 @@ function endsAtContinuationBoundary(run) {
|
|
|
5421
5422
|
* background invocations forever, mirroring `MAX_AGENT_TEAM_CONTINUATIONS`.
|
|
5422
5423
|
*/
|
|
5423
5424
|
export const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
|
|
5425
|
+
/**
|
|
5426
|
+
* Consecutive chunks allowed to end on the SAME terminal error code having
|
|
5427
|
+
* produced nothing before the chain stops.
|
|
5428
|
+
*
|
|
5429
|
+
* Two, because two independent recovery layers multiply here and neither can
|
|
5430
|
+
* see the other: the engine already retried this identical request 3x with
|
|
5431
|
+
* backoff before the error was ever emitted, and a recoverable error is also a
|
|
5432
|
+
* continuation boundary, so every chunk that fails costs 4 gateway attempts
|
|
5433
|
+
* and dispatches a fresh one. A production turn spent 27 background runs and
|
|
5434
|
+
* 15 minutes on one message this way. The first repeat is the retry this path
|
|
5435
|
+
* exists for; a second identical failure that moved nothing is evidence the
|
|
5436
|
+
* retrying itself is what is broken, not the request.
|
|
5437
|
+
*/
|
|
5438
|
+
export const MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS = 2;
|
|
5439
|
+
/**
|
|
5440
|
+
* Forward progress inside ONE chunk, read from the events it actually emitted:
|
|
5441
|
+
* assistant text or tool activity. Same evidence the agent-teams no-progress
|
|
5442
|
+
* budget counts (`agent-teams.ts`), and the same events
|
|
5443
|
+
* `endsAfterCompletedToolWithoutAssistantFinal` reads to tell an unfinished
|
|
5444
|
+
* turn from a finished one.
|
|
5445
|
+
*/
|
|
5446
|
+
function chunkMadeForwardProgress(run) {
|
|
5447
|
+
return run.events.some(({ event }) => (event.type === "text" && event.text.trim().length > 0) ||
|
|
5448
|
+
event.type === "tool_start" ||
|
|
5449
|
+
event.type === "tool_done");
|
|
5450
|
+
}
|
|
5451
|
+
/**
|
|
5452
|
+
* Advance the no-progress streak across a chunk boundary. A chunk that emitted
|
|
5453
|
+
* any text or tool activity resets it even when it still ended in an error —
|
|
5454
|
+
* that turn IS moving, and cutting it off is what would weaken recovery for
|
|
5455
|
+
* truncated streams. So does a different error code, and so does a boundary
|
|
5456
|
+
* with no error at all (a soft-timeout `auto_continue` is the run-manager's
|
|
5457
|
+
* no-progress backstop to bound, not this one).
|
|
5458
|
+
*/
|
|
5459
|
+
export function resolveBackgroundNoProgressRepeat(opts) {
|
|
5460
|
+
const last = opts.run.events.at(-1)?.event;
|
|
5461
|
+
const errorCode = last?.type === "error" ? (last.errorCode ?? "").trim() : "";
|
|
5462
|
+
if (!errorCode || chunkMadeForwardProgress(opts.run)) {
|
|
5463
|
+
return { count: 0, tripped: false };
|
|
5464
|
+
}
|
|
5465
|
+
const prior = opts.priorErrorCode === errorCode &&
|
|
5466
|
+
typeof opts.priorCount === "number" &&
|
|
5467
|
+
Number.isFinite(opts.priorCount)
|
|
5468
|
+
? Math.max(0, Math.floor(opts.priorCount))
|
|
5469
|
+
: 0;
|
|
5470
|
+
const count = prior + 1;
|
|
5471
|
+
return {
|
|
5472
|
+
errorCode,
|
|
5473
|
+
count,
|
|
5474
|
+
tripped: count >= MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS,
|
|
5475
|
+
};
|
|
5476
|
+
}
|
|
5477
|
+
/**
|
|
5478
|
+
* The single honest failure the breaker leaves behind. Keeps the underlying
|
|
5479
|
+
* code and the gateway's own message (which carries its `ERROR ID:` reference)
|
|
5480
|
+
* so the failure stays diagnosable, and marks it non-recoverable so neither
|
|
5481
|
+
* this chain nor the client's continuation path re-enters it.
|
|
5482
|
+
*/
|
|
5483
|
+
export function backgroundNoProgressTerminalEvent(run, repeat) {
|
|
5484
|
+
const last = run.events.at(-1)?.event;
|
|
5485
|
+
if (last?.type !== "error")
|
|
5486
|
+
return null;
|
|
5487
|
+
return {
|
|
5488
|
+
...last,
|
|
5489
|
+
error: `${last.error}\n\nThis failed ${repeat.count} times in a row without ` +
|
|
5490
|
+
`making any progress, so I stopped instead of retrying again.`,
|
|
5491
|
+
recoverable: false,
|
|
5492
|
+
};
|
|
5493
|
+
}
|
|
5494
|
+
export function installBackgroundNoProgressTerminalEvent(run, repeat) {
|
|
5495
|
+
const terminalEvent = backgroundNoProgressTerminalEvent(run, repeat);
|
|
5496
|
+
const lastRunEvent = run.events.at(-1);
|
|
5497
|
+
if (!terminalEvent || lastRunEvent?.event.type !== "error")
|
|
5498
|
+
return false;
|
|
5499
|
+
run.events = [
|
|
5500
|
+
...run.events.slice(0, -1),
|
|
5501
|
+
{ ...lastRunEvent, event: terminalEvent },
|
|
5502
|
+
];
|
|
5503
|
+
run.continuationTerminalEvent = terminalEvent;
|
|
5504
|
+
return true;
|
|
5505
|
+
}
|
|
5424
5506
|
/**
|
|
5425
5507
|
* Whether this run should self-fire the next server-driven continuation chunk
|
|
5426
5508
|
* instead of depending on the client to re-POST `auto_continue`. True for
|
|
@@ -5435,7 +5517,9 @@ export const MAX_BACKGROUND_RUN_CONTINUATIONS = 20;
|
|
|
5435
5517
|
* the durable background worker (`dispatchedToBackground` false) — a run
|
|
5436
5518
|
* already headed to the durable background path chains via the
|
|
5437
5519
|
* `isBackgroundWorker` branch above, never both.
|
|
5438
|
-
* Aborted / user-stopped runs do NOT chain either way
|
|
5520
|
+
* Aborted / user-stopped runs do NOT chain either way, and neither does a run
|
|
5521
|
+
* whose no-progress streak has tripped
|
|
5522
|
+
* (`MAX_CONSECUTIVE_NO_PROGRESS_CONTINUATIONS`).
|
|
5439
5523
|
*/
|
|
5440
5524
|
export function shouldChainBackgroundContinuation(opts) {
|
|
5441
5525
|
const eligible = opts.isBackgroundWorker ||
|
|
@@ -5444,7 +5528,12 @@ export function shouldChainBackgroundContinuation(opts) {
|
|
|
5444
5528
|
return (eligible &&
|
|
5445
5529
|
opts.run.status !== "aborted" &&
|
|
5446
5530
|
endsAtContinuationBoundary(opts.run) &&
|
|
5447
|
-
opts.continuationCount < MAX_BACKGROUND_RUN_CONTINUATIONS
|
|
5531
|
+
opts.continuationCount < MAX_BACKGROUND_RUN_CONTINUATIONS &&
|
|
5532
|
+
!resolveBackgroundNoProgressRepeat({
|
|
5533
|
+
run: opts.run,
|
|
5534
|
+
priorErrorCode: opts.priorNoProgressErrorCode,
|
|
5535
|
+
priorCount: opts.priorNoProgressCount,
|
|
5536
|
+
}).tripped);
|
|
5448
5537
|
}
|
|
5449
5538
|
/**
|
|
5450
5539
|
* Minimum remaining budget (ms) a synchronous self-chain continuation chunk
|
|
@@ -5884,6 +5973,12 @@ export async function chainServerDrivenContinuation(opts) {
|
|
|
5884
5973
|
continuationCount: opts.backgroundContinuationCount + 1,
|
|
5885
5974
|
continuationReason,
|
|
5886
5975
|
...(actionPreparationTool ? { actionPreparationTool } : {}),
|
|
5976
|
+
...(opts.noProgressRepeat?.errorCode
|
|
5977
|
+
? {
|
|
5978
|
+
noProgressErrorCode: opts.noProgressRepeat.errorCode,
|
|
5979
|
+
noProgressCount: opts.noProgressRepeat.count,
|
|
5980
|
+
}
|
|
5981
|
+
: {}),
|
|
5887
5982
|
backgroundFunctionRuntimeExpected: continuationExpectsNetlifyBackgroundFunction,
|
|
5888
5983
|
};
|
|
5889
5984
|
// Strip this chunk's own marker before persisting/forwarding — the next
|
|
@@ -6236,6 +6331,15 @@ export function createProductionAgentHandler(options) {
|
|
|
6236
6331
|
Number.isFinite(backgroundRunMarker.continuationCount)
|
|
6237
6332
|
? Math.max(0, Math.floor(backgroundRunMarker.continuationCount))
|
|
6238
6333
|
: 0;
|
|
6334
|
+
// No-progress streak so far, carried on the marker: this invocation has no
|
|
6335
|
+
// other memory of what the previous chunk failed with.
|
|
6336
|
+
const priorNoProgressErrorCode = typeof backgroundRunMarker?.noProgressErrorCode === "string"
|
|
6337
|
+
? backgroundRunMarker.noProgressErrorCode
|
|
6338
|
+
: undefined;
|
|
6339
|
+
const priorNoProgressCount = typeof backgroundRunMarker?.noProgressCount === "number" &&
|
|
6340
|
+
Number.isFinite(backgroundRunMarker.noProgressCount)
|
|
6341
|
+
? Math.max(0, Math.floor(backgroundRunMarker.noProgressCount))
|
|
6342
|
+
: 0;
|
|
6239
6343
|
let backgroundRunClaimedEarly = false;
|
|
6240
6344
|
if (isBackgroundWorker && bgRunId) {
|
|
6241
6345
|
const earlyClaim = await claimBackgroundWorkerRunEarly({
|
|
@@ -7311,12 +7415,19 @@ export function createProductionAgentHandler(options) {
|
|
|
7311
7415
|
typeof threadId === "string" &&
|
|
7312
7416
|
threadId.trim().length > 0 &&
|
|
7313
7417
|
isAgentChatForegroundSelfChainEnabled();
|
|
7418
|
+
const noProgressRepeatForRun = (run) => resolveBackgroundNoProgressRepeat({
|
|
7419
|
+
run,
|
|
7420
|
+
priorErrorCode: priorNoProgressErrorCode,
|
|
7421
|
+
priorCount: priorNoProgressCount,
|
|
7422
|
+
});
|
|
7314
7423
|
const willChainBackgroundContinuation = (run) => shouldChainBackgroundContinuation({
|
|
7315
7424
|
isBackgroundWorker,
|
|
7316
7425
|
run,
|
|
7317
7426
|
continuationCount: backgroundContinuationCount,
|
|
7318
7427
|
foregroundSelfChainEligible,
|
|
7319
7428
|
dispatchedToBackground: dispatchToBackground,
|
|
7429
|
+
priorNoProgressErrorCode,
|
|
7430
|
+
priorNoProgressCount,
|
|
7320
7431
|
});
|
|
7321
7432
|
const completeTrackedProgressRun = async (run, completionError) => {
|
|
7322
7433
|
if (!trackedProgressRunId || !trackedProgressOwner)
|
|
@@ -7403,6 +7514,14 @@ export function createProductionAgentHandler(options) {
|
|
|
7403
7514
|
? `${errEvent.errorCode ?? ""} ${errEvent.error ?? ""}`.trim()
|
|
7404
7515
|
: "run ended in errored state").catch(() => { });
|
|
7405
7516
|
}
|
|
7517
|
+
const noProgressRepeat = noProgressRepeatForRun(run);
|
|
7518
|
+
if (noProgressRepeat.tripped) {
|
|
7519
|
+
// Install the replacement before the thread writer runs. The
|
|
7520
|
+
// writer builds durable thread_data from events, so changing
|
|
7521
|
+
// only continuationTerminalEvent afterwards leaves the original
|
|
7522
|
+
// recoverable error persisted and eligible for another retry.
|
|
7523
|
+
installBackgroundNoProgressTerminalEvent(run, noProgressRepeat);
|
|
7524
|
+
}
|
|
7406
7525
|
// Persist the (partial) assistant turn to thread_data FIRST — the
|
|
7407
7526
|
// server-driven continuation below rebuilds from it, so it must be
|
|
7408
7527
|
// committed before we re-fire.
|
|
@@ -7433,7 +7552,14 @@ export function createProductionAgentHandler(options) {
|
|
|
7433
7552
|
// succeeded — a dispatch fast-fail degrades to the inline
|
|
7434
7553
|
// foreground fallback, which is not a worker and rides the
|
|
7435
7554
|
// connected client's auto_continue instead.)
|
|
7436
|
-
if (
|
|
7555
|
+
if (noProgressRepeat.tripped) {
|
|
7556
|
+
if (run.continuationTerminalEvent?.type === "error") {
|
|
7557
|
+
console.error(`[agent-chat] stopping background chain: ${noProgressRepeat.errorCode} ` +
|
|
7558
|
+
`failed ${noProgressRepeat.count}x with no progress`, run.runId);
|
|
7559
|
+
await recordRunDiagnostic(run.runId, RUN_DIAG_STAGE.workerThrew, `chain_stopped_no_progress code=${noProgressRepeat.errorCode} count=${noProgressRepeat.count}`).catch(() => { });
|
|
7560
|
+
}
|
|
7561
|
+
}
|
|
7562
|
+
else if (willChainBackgroundContinuation(run)) {
|
|
7437
7563
|
// Full handoff discipline lives in
|
|
7438
7564
|
// `chainServerDrivenContinuation` (exported + unit-tested):
|
|
7439
7565
|
// per-turn SQL run budget, successor row PRE-INSERTED before
|
|
@@ -7446,6 +7572,7 @@ export function createProductionAgentHandler(options) {
|
|
|
7446
7572
|
effectiveTurnId,
|
|
7447
7573
|
requestBody: body,
|
|
7448
7574
|
backgroundContinuationCount,
|
|
7575
|
+
noProgressRepeat,
|
|
7449
7576
|
turnInputTokens,
|
|
7450
7577
|
// Re-evaluate the durable gate rather than keying off
|
|
7451
7578
|
// isBackgroundWorker: a successor chunk of a FOREGROUND
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { EngineRequestShape } from "./engine/types.js";
|
|
1
2
|
import type { AgentChatEvent, RunEvent, RunStatus } from "./types.js";
|
|
2
3
|
export interface ActiveRun {
|
|
3
4
|
runId: string;
|
|
@@ -10,13 +11,18 @@ export interface ActiveRun {
|
|
|
10
11
|
abort: AbortController;
|
|
11
12
|
abortReason?: string;
|
|
12
13
|
/**
|
|
13
|
-
* Terminal event
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* `send`
|
|
14
|
+
* Terminal event the completion callback installs in place of the one the
|
|
15
|
+
* loop stashed: `auto_continue` when a server-driven continuation has been
|
|
16
|
+
* handed off successfully (that continuation runs outside this process, so
|
|
17
|
+
* the loop-level auto_continue never goes through this run's `send`), or
|
|
18
|
+
* `error` when the callback decided the turn must stop here instead — the
|
|
19
|
+
* stashed error is recoverable by construction, so leaving it in place
|
|
20
|
+
* re-enters the very chain the callback just refused to continue.
|
|
17
21
|
*/
|
|
18
22
|
continuationTerminalEvent?: Extract<AgentChatEvent, {
|
|
19
23
|
type: "auto_continue";
|
|
24
|
+
} | {
|
|
25
|
+
type: "error";
|
|
20
26
|
}>;
|
|
21
27
|
startedAt: number;
|
|
22
28
|
}
|
|
@@ -233,6 +239,11 @@ export declare function resolveSqlSubscriptionPollMs(now: number, activePollUnti
|
|
|
233
239
|
*/
|
|
234
240
|
export declare function nextSqlSubscriptionEmptyPolls(current: number, hadEvents: boolean, now: number, activePollUntil: number): number;
|
|
235
241
|
export declare function resolveSqlSubscriptionRetryMs(consecutiveFailures: number): number;
|
|
242
|
+
/**
|
|
243
|
+
* Sentry tags are strings, and an absent shape must stay absent: a run that
|
|
244
|
+
* failed before the request was built did not send a zero-byte payload.
|
|
245
|
+
*/
|
|
246
|
+
export declare function engineRequestShapeTags(shape: EngineRequestShape | undefined): Record<string, string>;
|
|
236
247
|
export interface StartRunOptions {
|
|
237
248
|
/** Keep a request-scoped serverless invocation alive for this run. */
|
|
238
249
|
waitUntil?: (promise: Promise<unknown>) => void;
|
|
@@ -299,6 +299,20 @@ function getRunErrorCode(err) {
|
|
|
299
299
|
// only when the run row is persisted.
|
|
300
300
|
return classifyTerminalErrorCode(describeErrorWithCauses(err));
|
|
301
301
|
}
|
|
302
|
+
/**
|
|
303
|
+
* Sentry tags are strings, and an absent shape must stay absent: a run that
|
|
304
|
+
* failed before the request was built did not send a zero-byte payload.
|
|
305
|
+
*/
|
|
306
|
+
export function engineRequestShapeTags(shape) {
|
|
307
|
+
if (!shape)
|
|
308
|
+
return {};
|
|
309
|
+
return {
|
|
310
|
+
engineModel: shape.model,
|
|
311
|
+
enginePayloadBytes: String(shape.payloadBytes),
|
|
312
|
+
engineToolCount: String(shape.toolCount),
|
|
313
|
+
engineMessageCount: String(shape.messageCount),
|
|
314
|
+
};
|
|
315
|
+
}
|
|
302
316
|
function getEngineRunErrorDetails(err) {
|
|
303
317
|
if (err.statusCode === 429)
|
|
304
318
|
return err.message;
|
|
@@ -1078,6 +1092,11 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1078
1092
|
statusCode: engineError?.statusCode != null
|
|
1079
1093
|
? String(engineError.statusCode)
|
|
1080
1094
|
: undefined,
|
|
1095
|
+
// What we sent, in sizes and counts only. A gateway rejection describes
|
|
1096
|
+
// nothing about the request behind it, so without these an oversized
|
|
1097
|
+
// payload and an upstream outage produce the same capture — which is
|
|
1098
|
+
// how one gateway 500 cost a night of guessing.
|
|
1099
|
+
...engineRequestShapeTags(engineError?.requestShape),
|
|
1081
1100
|
},
|
|
1082
1101
|
extra: {
|
|
1083
1102
|
runId,
|
|
@@ -1263,6 +1282,13 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1263
1282
|
}
|
|
1264
1283
|
: run;
|
|
1265
1284
|
await onComplete(completionRun);
|
|
1285
|
+
// `completionRun` is a shallow COPY whenever the loop stashed a
|
|
1286
|
+
// terminal event, so a callback that installs its own terminal
|
|
1287
|
+
// event writes it to the copy and `resolveTerminalEventForCompletion`
|
|
1288
|
+
// below never sees it — the run then emits the pre-callback event
|
|
1289
|
+
// the callback was overriding.
|
|
1290
|
+
run.continuationTerminalEvent ??=
|
|
1291
|
+
completionRun.continuationTerminalEvent;
|
|
1266
1292
|
}
|
|
1267
1293
|
catch (err) {
|
|
1268
1294
|
completionError = err;
|
|
@@ -715,11 +715,11 @@ export declare function getRunOutcomeCounters(options?: {
|
|
|
715
715
|
terminalReason: string;
|
|
716
716
|
count: number;
|
|
717
717
|
}>>;
|
|
718
|
-
/**
|
|
719
|
-
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
|
|
718
|
+
/**
|
|
719
|
+
* Run cleanup is scheduled after every completed run, including completions
|
|
720
|
+
* from several concurrent requests in one isolate. Share one sweep locally;
|
|
721
|
+
* Postgres additionally serializes the durable prune across isolates.
|
|
722
|
+
*/
|
|
723
723
|
export declare function cleanupOldRuns(olderThanMs: number, erroredOlderThanMs?: number): Promise<void>;
|
|
724
724
|
/**
|
|
725
725
|
* List recent unsuccessful runs (errored, aborted, and truncated) for cut-off
|