@agent-native/core 0.168.12 → 0.169.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/clips/app/lib/capture-install-options.ts +20 -2
- package/corpus/templates/dispatch/app/root.tsx +9 -1
- package/dist/agent/engine/first-event-timeout.d.ts +8 -0
- package/dist/agent/engine/first-event-timeout.js +8 -0
- package/dist/agent/production-agent.d.ts +0 -30
- package/dist/agent/production-agent.js +17 -38
- package/dist/agent/run-loop-with-resume.d.ts +38 -25
- package/dist/agent/run-loop-with-resume.js +140 -55
- package/dist/agent/run-manager.d.ts +83 -68
- package/dist/agent/run-manager.js +280 -94
- package/dist/agent/run-store.d.ts +31 -0
- package/dist/agent/run-store.js +42 -12
- package/dist/app-config/agent.d.ts +2 -0
- package/dist/app-config/agent.js +33 -0
- package/dist/app-config/run-lifecycle-invariants.d.ts +248 -0
- package/dist/app-config/run-lifecycle-invariants.js +342 -0
- package/dist/app-config/schema.d.ts +2 -0
- package/dist/app-config/store.js +9 -1
- package/dist/client/EnvironmentBadge.d.ts +5 -4
- package/dist/client/EnvironmentBadge.js +19 -8
- package/dist/client/agent-chat-adapter.d.ts +0 -2
- package/dist/client/agent-chat-adapter.js +7 -23
- package/dist/client/app-providers.d.ts +3 -2
- package/dist/client/app-providers.js +3 -2
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/routes.d.ts +1 -1
- package/dist/jobs/background-automation-runner.d.ts +25 -0
- package/dist/jobs/background-automation-runner.js +104 -21
- package/dist/jobs/run-history.d.ts +7 -1
- package/dist/jobs/run-history.js +57 -14
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/traces.d.ts +13 -0
- package/dist/observability/traces.js +369 -317
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/server/agent-chat-plugin.js +2 -4
- package/dist/server/beta-opt-out-html.js +3 -2
- package/dist/server/onboarding-html.js +3 -2
- package/dist/server/realtime-token.d.ts +1 -1
- package/package.json +1 -1
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* Both paths route through `appendAgentLoopContinuation` so the agent sees a
|
|
21
21
|
* uniform "continue" instruction regardless of which recovery fired.
|
|
22
22
|
*/
|
|
23
|
+
import { MAX_BACKGROUND_RUN_LOOP_CONTINUATIONS, MAX_RUN_LOOP_CONTINUATIONS, } from "../app-config/run-lifecycle-invariants.js";
|
|
23
24
|
import { runAgentLoop, appendAgentLoopContinuation, isResumableEngineError, isTransientProviderRateLimitError, continuationReasonForResumableError, lastUnfinishedPreparingActionToolFromEvents, resolveFinalResponseGuardRequestText, SELF_CHAIN_MIN_CONTINUATION_BUDGET_MS, } from "./production-agent.js";
|
|
24
25
|
import { resolveRunSoftTimeoutMs } from "./run-manager.js";
|
|
25
26
|
import { getCurrentTurnEventsForThread } from "./run-store.js";
|
|
@@ -171,29 +172,6 @@ function internalContinuationReasonForAttempt(events) {
|
|
|
171
172
|
}
|
|
172
173
|
return undefined;
|
|
173
174
|
}
|
|
174
|
-
/**
|
|
175
|
-
* Cap on continuation iterations inside a single
|
|
176
|
-
* `runAgentLoopDirectWithSoftTimeout` invocation. The host's hard function
|
|
177
|
-
* timeout usually bounds this naturally — but a defensive cap prevents an
|
|
178
|
-
* instant-error spiral from looping forever inside hosting environments with a
|
|
179
|
-
* generous budget.
|
|
180
|
-
*
|
|
181
|
-
* 6 leaves room for: 1 normal completion + a few resume rounds for design
|
|
182
|
-
* generation (prompt + 3 variants ≈ 4 LLM calls), with a small safety margin.
|
|
183
|
-
*/
|
|
184
|
-
export const MAX_RUN_LOOP_CONTINUATIONS = 6;
|
|
185
|
-
/**
|
|
186
|
-
* A delegated turn that is proven to be running inside a durable background
|
|
187
|
-
* function has the same 15-minute host budget as main chat, but this wrapper
|
|
188
|
-
* historically kept the foreground-sized six-continuation cap. A healthy
|
|
189
|
-
* child A2A call can consume several minutes and the receiving model may then
|
|
190
|
-
* need more than six recovery/model-stream boundaries to finish its own tool
|
|
191
|
-
* work. Keep a hard cap, but give the proven background path the same bounded
|
|
192
|
-
* continuation allowance as the durable main-chat runner. The cumulative
|
|
193
|
-
* soft-timeout below still prevents these rounds from exceeding the one real
|
|
194
|
-
* background-function wall-clock budget.
|
|
195
|
-
*/
|
|
196
|
-
export const MAX_BACKGROUND_RUN_LOOP_CONTINUATIONS = 20;
|
|
197
175
|
/**
|
|
198
176
|
* The engine already performs its own short provider retries. After those are
|
|
199
177
|
* exhausted, a proven durable background A2A/MCP run gets one cooled-down
|
|
@@ -216,6 +194,45 @@ function waitForBackgroundRateLimitCooldown(signal) {
|
|
|
216
194
|
signal.addEventListener("abort", finish, { once: true });
|
|
217
195
|
});
|
|
218
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Abort reasons the SERVER sets on a run's own controller. Everything else —
|
|
199
|
+
* including any reason a client passes to the abort route — is a user Stop.
|
|
200
|
+
*
|
|
201
|
+
* Kept deliberately short. Each entry is a bound this package owns and can name
|
|
202
|
+
* in a terminal outcome; if you are adding a fourth, check first whether the
|
|
203
|
+
* bound belongs in `run-manager.ts` at all.
|
|
204
|
+
*
|
|
205
|
+
* Exported so the abort route can refuse these words from a client. That check
|
|
206
|
+
* belongs at the boundary where untrusted input enters, not here: by the time a
|
|
207
|
+
* reason reaches an `AbortSignal` it is just a string, and nothing downstream
|
|
208
|
+
* can tell who wrote it.
|
|
209
|
+
*/
|
|
210
|
+
export const SERVER_OWNED_ABORT_REASONS = new Set([
|
|
211
|
+
"no_progress",
|
|
212
|
+
"run_timeout",
|
|
213
|
+
"background_automation_hard_timeout",
|
|
214
|
+
]);
|
|
215
|
+
/**
|
|
216
|
+
* The abort reason to record for a client-initiated Stop.
|
|
217
|
+
*
|
|
218
|
+
* A caller reaching the abort route is a person pressing Stop, so it must not
|
|
219
|
+
* be able to name a bound only the server can reach: the terminal outcome keys
|
|
220
|
+
* off the abort reason, and a client sending `background_automation_hard_timeout`
|
|
221
|
+
* would file its own Stop as a server-side failure. Anything unrecognised,
|
|
222
|
+
* malformed, or reserved falls back to `"user"`.
|
|
223
|
+
*
|
|
224
|
+
* Normalised here rather than in the route because this is where the meaning of
|
|
225
|
+
* the string is decided — downstream it is just a string, and nothing can tell
|
|
226
|
+
* who wrote it.
|
|
227
|
+
*/
|
|
228
|
+
export function clientAbortReason(raw) {
|
|
229
|
+
if (typeof raw !== "string")
|
|
230
|
+
return "user";
|
|
231
|
+
const reason = raw.trim();
|
|
232
|
+
if (!/^[a-z0-9_-]{1,64}$/i.test(reason))
|
|
233
|
+
return "user";
|
|
234
|
+
return SERVER_OWNED_ABORT_REASONS.has(reason.toLowerCase()) ? "user" : reason;
|
|
235
|
+
}
|
|
219
236
|
/** Machine-readable code carried on the give-up terminal `error` event so the
|
|
220
237
|
* client renders a loud "stopped before finishing" terminal instead of an
|
|
221
238
|
* ambiguous silent stall. Deliberately NOT in the client's auto-recoverable
|
|
@@ -240,7 +257,15 @@ export const RUN_BUDGET_EXHAUSTED_MESSAGE = "I ran out of time before finishing
|
|
|
240
257
|
* an appropriate inner budget. Setting it to <= 0 disables both layers — the
|
|
241
258
|
* call goes straight to `runAgentLoop` with no wrapping.
|
|
242
259
|
*/
|
|
243
|
-
export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, timeoutOptions
|
|
260
|
+
export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, timeoutOptions,
|
|
261
|
+
/**
|
|
262
|
+
* Chunk control from `startRun`, for a caller that owns continuation inside
|
|
263
|
+
* this invocation. Without it `opts.signal` is the only signal there is, so a
|
|
264
|
+
* checkpoint fired from ABOVE this loop reads as a Stop and the recovery
|
|
265
|
+
* below — which already accepts `no_progress` and already has a 20-round
|
|
266
|
+
* background budget — is unreachable.
|
|
267
|
+
*/
|
|
268
|
+
control) {
|
|
244
269
|
const finalResponseGuardRequestText = opts.finalResponseGuardRequestText ??
|
|
245
270
|
resolveFinalResponseGuardRequestText(opts.messages);
|
|
246
271
|
const stableOpts = { ...opts, finalResponseGuardRequestText };
|
|
@@ -260,6 +285,40 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
260
285
|
// Disabling continuation recovery must not disable terminal classification.
|
|
261
286
|
// Keep the same outcome boundary around a direct loop so A2A/MCP callers
|
|
262
287
|
// never infer success from a rejected or canceled run.
|
|
288
|
+
const turnSignal = control?.turnSignal ?? opts.signal;
|
|
289
|
+
/**
|
|
290
|
+
* A turn someone pressed Stop on is `canceled`. A turn that ended because a
|
|
291
|
+
* SERVER bound fired is not: nobody cancelled it, it ran out of something,
|
|
292
|
+
* and the abort reason says which.
|
|
293
|
+
*
|
|
294
|
+
* Reporting both as `canceled` made a hard-timed-out automation
|
|
295
|
+
* byte-identical to a user Stop in every consumer, `$ai_error` included, and
|
|
296
|
+
* contradicted the no-timeout path above, which has always reported an
|
|
297
|
+
* unfinished reason as `failed` with that reason as its code.
|
|
298
|
+
*
|
|
299
|
+
* Allowlisted rather than "anything that isn't `user`", because the abort
|
|
300
|
+
* route accepts a client-supplied reason string: an inverted test would
|
|
301
|
+
* relabel a genuine Stop the moment a caller sent its own word for it.
|
|
302
|
+
*/
|
|
303
|
+
const turnAbortOutcome = () => {
|
|
304
|
+
const reason = typeof turnSignal.reason === "string" ? turnSignal.reason.trim() : "";
|
|
305
|
+
if (!SERVER_OWNED_ABORT_REASONS.has(reason)) {
|
|
306
|
+
return { state: "canceled", message: "Agent run was aborted." };
|
|
307
|
+
}
|
|
308
|
+
return {
|
|
309
|
+
state: "failed",
|
|
310
|
+
code: reason,
|
|
311
|
+
retryable: false,
|
|
312
|
+
message: `Agent run was aborted (${reason}).`,
|
|
313
|
+
};
|
|
314
|
+
};
|
|
315
|
+
let chunkSignal = control?.chunkSignal ?? opts.signal;
|
|
316
|
+
const recoverableChunkBoundary = () => {
|
|
317
|
+
if (!control || turnSignal.aborted)
|
|
318
|
+
return null;
|
|
319
|
+
const reason = control.chunkBoundaryReason();
|
|
320
|
+
return reason === "no_progress" || reason === "run_timeout" ? reason : null;
|
|
321
|
+
};
|
|
263
322
|
if (timeoutMs <= 0) {
|
|
264
323
|
const directEvents = [];
|
|
265
324
|
let directOutcome;
|
|
@@ -275,11 +334,8 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
275
334
|
},
|
|
276
335
|
});
|
|
277
336
|
const unfinishedReason = internalContinuationReasonForAttempt(directEvents);
|
|
278
|
-
if (
|
|
279
|
-
reportFinalOutcome(
|
|
280
|
-
state: "canceled",
|
|
281
|
-
message: "Agent run was aborted.",
|
|
282
|
-
});
|
|
337
|
+
if (turnSignal.aborted) {
|
|
338
|
+
reportFinalOutcome(turnAbortOutcome());
|
|
283
339
|
}
|
|
284
340
|
else if (unfinishedReason) {
|
|
285
341
|
reportFinalOutcome({
|
|
@@ -296,8 +352,8 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
296
352
|
}
|
|
297
353
|
catch (err) {
|
|
298
354
|
const candidate = err;
|
|
299
|
-
reportFinalOutcome(
|
|
300
|
-
?
|
|
355
|
+
reportFinalOutcome(turnSignal.aborted
|
|
356
|
+
? turnAbortOutcome()
|
|
301
357
|
: {
|
|
302
358
|
state: "failed",
|
|
303
359
|
code: typeof candidate?.errorCode === "string" && candidate.errorCode
|
|
@@ -311,7 +367,10 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
311
367
|
throw err;
|
|
312
368
|
}
|
|
313
369
|
}
|
|
314
|
-
|
|
370
|
+
// `turnSignal` answers "is this turn over?"; `chunkSignal` answers "is this
|
|
371
|
+
// ROUND over?". They are the same object for every caller that does not pass
|
|
372
|
+
// a control, which is what keeps the foreground/HTTP paths byte-for-byte
|
|
373
|
+
// unchanged.
|
|
315
374
|
const usage = {
|
|
316
375
|
inputTokens: 0,
|
|
317
376
|
outputTokens: 0,
|
|
@@ -333,6 +392,23 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
333
392
|
usage.firstEngineEventAtMs ??= next.firstEngineEventAtMs;
|
|
334
393
|
};
|
|
335
394
|
const localTurnEvents = [];
|
|
395
|
+
/**
|
|
396
|
+
* Recover a boundary the run manager decided from OUTSIDE this loop.
|
|
397
|
+
*
|
|
398
|
+
* Same treatment the loop's own `auto_continue` gets: drop partial text the
|
|
399
|
+
* client already saw (unless a side effect landed, whose tool card is the
|
|
400
|
+
* user's only proof), append the continuation context and tool-call journal,
|
|
401
|
+
* then open a fresh chunk. Not opening one would leave every later round
|
|
402
|
+
* running under an already-aborted signal, which fails instantly and looks
|
|
403
|
+
* exactly like the bug this replaces.
|
|
404
|
+
*/
|
|
405
|
+
const continueFromChunkBoundary = async (reason, attemptEvents) => {
|
|
406
|
+
if ((await completedSideEffectInCurrentTurn(opts.threadId, opts.turnId, localTurnEvents)) === "none") {
|
|
407
|
+
opts.send({ type: "clear" });
|
|
408
|
+
}
|
|
409
|
+
await appendContinuationAndJournal(opts.messages, reason, opts.threadId, opts.turnId, localTurnEvents, [...attemptEvents]);
|
|
410
|
+
chunkSignal = control?.beginChunk() ?? chunkSignal;
|
|
411
|
+
};
|
|
336
412
|
let attempts = 0;
|
|
337
413
|
// Every current hosted caller of this function (A2A/MCP delegated turns)
|
|
338
414
|
// runs inside ONE serverless invocation whose real platform hard-kill is
|
|
@@ -357,7 +433,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
357
433
|
// this is the silent give-up case: emit a loud terminal so the user sees an
|
|
358
434
|
// unambiguous "stopped before finishing" instead of a bare done/"…".
|
|
359
435
|
let lastAttemptWasUnfinishedContinuation = false;
|
|
360
|
-
while (!
|
|
436
|
+
while (!turnSignal.aborted && attempts < maxRunLoopContinuations) {
|
|
361
437
|
const roundTimeoutMs = attempts === 0 ? timeoutMs : timeoutMs - (Date.now() - loopEntryAt);
|
|
362
438
|
if (attempts > 0 &&
|
|
363
439
|
roundTimeoutMs < SELF_CHAIN_MIN_CONTINUATION_BUDGET_MS) {
|
|
@@ -372,11 +448,14 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
372
448
|
lastAttemptWasUnfinishedContinuation = false;
|
|
373
449
|
const controller = new AbortController();
|
|
374
450
|
const abortFromUpstream = () => controller.abort();
|
|
375
|
-
|
|
451
|
+
// Bound to the CURRENT chunk. A turn abort still reaches it — the run
|
|
452
|
+
// manager ends the live chunk whenever the turn ends.
|
|
453
|
+
const roundChunkSignal = chunkSignal;
|
|
454
|
+
if (roundChunkSignal.aborted) {
|
|
376
455
|
controller.abort();
|
|
377
456
|
}
|
|
378
457
|
else {
|
|
379
|
-
|
|
458
|
+
roundChunkSignal.addEventListener("abort", abortFromUpstream, {
|
|
380
459
|
once: true,
|
|
381
460
|
});
|
|
382
461
|
}
|
|
@@ -404,8 +483,14 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
404
483
|
});
|
|
405
484
|
addUsage(nextUsage);
|
|
406
485
|
const attemptEvents = localTurnEvents.slice(attemptStartIndex);
|
|
486
|
+
const chunkBoundaryReason = recoverableChunkBoundary();
|
|
487
|
+
if (chunkBoundaryReason) {
|
|
488
|
+
lastAttemptWasUnfinishedContinuation = true;
|
|
489
|
+
await continueFromChunkBoundary(chunkBoundaryReason, attemptEvents);
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
407
492
|
const internalContinuationReason = internalContinuationReasonForAttempt(attemptEvents);
|
|
408
|
-
if (internalContinuationReason && !
|
|
493
|
+
if (internalContinuationReason && !turnSignal.aborted) {
|
|
409
494
|
lastAttemptWasUnfinishedContinuation = true;
|
|
410
495
|
const continuationEvents = [...localTurnEvents];
|
|
411
496
|
if ((await completedSideEffectInCurrentTurn(opts.threadId, opts.turnId, continuationEvents)) === "none") {
|
|
@@ -414,18 +499,24 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
414
499
|
await appendContinuationAndJournal(opts.messages, internalContinuationReason, opts.threadId, opts.turnId, continuationEvents, attemptEvents);
|
|
415
500
|
continue;
|
|
416
501
|
}
|
|
417
|
-
if (softTimedOut && !
|
|
502
|
+
if (softTimedOut && !turnSignal.aborted) {
|
|
418
503
|
lastAttemptWasUnfinishedContinuation = true;
|
|
419
504
|
await appendContinuationAndJournal(opts.messages, "run_timeout", opts.threadId, opts.turnId, localTurnEvents, attemptEvents);
|
|
420
505
|
continue;
|
|
421
506
|
}
|
|
422
|
-
reportFinalOutcome(
|
|
423
|
-
?
|
|
507
|
+
reportFinalOutcome(turnSignal.aborted
|
|
508
|
+
? turnAbortOutcome()
|
|
424
509
|
: (attemptOutcome ?? { state: "completed" }));
|
|
425
510
|
return usage;
|
|
426
511
|
}
|
|
427
512
|
catch (err) {
|
|
428
|
-
|
|
513
|
+
const chunkBoundaryReason = recoverableChunkBoundary();
|
|
514
|
+
if (chunkBoundaryReason) {
|
|
515
|
+
lastAttemptWasUnfinishedContinuation = true;
|
|
516
|
+
await continueFromChunkBoundary(chunkBoundaryReason, localTurnEvents.slice(attemptStartIndex));
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (softTimedOut && !turnSignal.aborted) {
|
|
429
520
|
// Clear partial text the client received before the abort so the
|
|
430
521
|
// resumed model doesn't re-emit it and produce duplicated output.
|
|
431
522
|
lastAttemptWasUnfinishedContinuation = true;
|
|
@@ -443,7 +534,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
443
534
|
(Date.now() - loopEntryAt) -
|
|
444
535
|
BACKGROUND_RATE_LIMIT_CONTINUATION_DELAY_MS >=
|
|
445
536
|
SELF_CHAIN_MIN_CONTINUATION_BUDGET_MS;
|
|
446
|
-
if (!
|
|
537
|
+
if (!turnSignal.aborted &&
|
|
447
538
|
transientRateLimit &&
|
|
448
539
|
rateLimitRetryFitsBudget) {
|
|
449
540
|
lastAttemptWasUnfinishedContinuation = true;
|
|
@@ -452,7 +543,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
452
543
|
opts.send({ type: "clear" });
|
|
453
544
|
}
|
|
454
545
|
await appendContinuationAndJournal(opts.messages, "rate_limited", opts.threadId, opts.turnId, localTurnEvents, localTurnEvents.slice(attemptStartIndex));
|
|
455
|
-
await waitForBackgroundRateLimitCooldown(
|
|
546
|
+
await waitForBackgroundRateLimitCooldown(turnSignal);
|
|
456
547
|
continue;
|
|
457
548
|
}
|
|
458
549
|
// Resumable transport / gateway interruptions: the LLM call was cut off
|
|
@@ -468,7 +559,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
468
559
|
// from scratch and the fold produces duplicated text in one message
|
|
469
560
|
// (the partial text was already sent to the client but is now retained
|
|
470
561
|
// only as an internal checkpoint so the next attempt can finish it).
|
|
471
|
-
if (!
|
|
562
|
+
if (!turnSignal.aborted && isResumableEngineError(err)) {
|
|
472
563
|
lastAttemptWasUnfinishedContinuation = true;
|
|
473
564
|
if ((await completedSideEffectInCurrentTurn(opts.threadId, opts.turnId, localTurnEvents)) === "none") {
|
|
474
565
|
opts.send({ type: "clear" });
|
|
@@ -476,11 +567,8 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
476
567
|
await appendContinuationAndJournal(opts.messages, continuationReasonForResumableError(err), opts.threadId, opts.turnId, localTurnEvents, localTurnEvents.slice(attemptStartIndex));
|
|
477
568
|
continue;
|
|
478
569
|
}
|
|
479
|
-
if (
|
|
480
|
-
reportFinalOutcome(
|
|
481
|
-
state: "canceled",
|
|
482
|
-
message: "Agent run was aborted.",
|
|
483
|
-
});
|
|
570
|
+
if (turnSignal.aborted) {
|
|
571
|
+
reportFinalOutcome(turnAbortOutcome());
|
|
484
572
|
throw err;
|
|
485
573
|
}
|
|
486
574
|
const candidate = err;
|
|
@@ -498,7 +586,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
498
586
|
}
|
|
499
587
|
finally {
|
|
500
588
|
clearTimeout(timer);
|
|
501
|
-
|
|
589
|
+
roundChunkSignal.removeEventListener("abort", abortFromUpstream);
|
|
502
590
|
}
|
|
503
591
|
}
|
|
504
592
|
// The loop exited without a clean return. If the user aborted, that's a Stop —
|
|
@@ -508,7 +596,7 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
508
596
|
// give-up the run-manager would otherwise report as a clean `done`: emit a
|
|
509
597
|
// loud, non-auto-continuing terminal so the user knows the turn stopped
|
|
510
598
|
// before finishing and nothing was partially saved by the run itself.
|
|
511
|
-
if (!
|
|
599
|
+
if (!turnSignal.aborted && lastAttemptWasUnfinishedContinuation) {
|
|
512
600
|
// Discard any partial text already streamed for the unfinished attempt so
|
|
513
601
|
// the terminal message stands alone instead of trailing a half sentence.
|
|
514
602
|
// Preserve completed tool cards: they are the user's only durable proof
|
|
@@ -529,11 +617,8 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
|
|
|
529
617
|
message: RUN_BUDGET_EXHAUSTED_MESSAGE,
|
|
530
618
|
});
|
|
531
619
|
}
|
|
532
|
-
else if (
|
|
533
|
-
reportFinalOutcome(
|
|
534
|
-
state: "canceled",
|
|
535
|
-
message: "Agent run was aborted.",
|
|
536
|
-
});
|
|
620
|
+
else if (turnSignal.aborted) {
|
|
621
|
+
reportFinalOutcome(turnAbortOutcome());
|
|
537
622
|
}
|
|
538
623
|
return usage;
|
|
539
624
|
}
|
|
@@ -68,68 +68,6 @@ export declare const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40000;
|
|
|
68
68
|
* function (no ~60s wall, 15-min budget) and therefore can safely outlast 40s.
|
|
69
69
|
*/
|
|
70
70
|
export declare const HOSTED_SOFT_TIMEOUT_CEILING_MS = 40000;
|
|
71
|
-
/**
|
|
72
|
-
* Hard ceiling for the soft timeout when a run executes inside a Netlify
|
|
73
|
-
* background function (any deployed function whose name ends in `-background`).
|
|
74
|
-
* Background functions return 202 immediately and run detached for up to 15
|
|
75
|
-
* minutes, so the ~60s synchronous function wall that 40s defends against does
|
|
76
|
-
* NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard
|
|
77
|
-
* kill to abort, persist the partial turn, write the terminal event, and (for
|
|
78
|
-
* the rare >13-min turn) self-fire another background continuation.
|
|
79
|
-
*
|
|
80
|
-
* This ceiling is used ONLY when a caller explicitly opts in with
|
|
81
|
-
* `backgroundFunction: true`. It does not change the foreground/interactive
|
|
82
|
-
* ceiling and does not fire unless the durable-background path dispatched the
|
|
83
|
-
* run into a background function. Per the design doc Guardrail, the 40s
|
|
84
|
-
* interactive clamp stays correct for every non-background run.
|
|
85
|
-
*/
|
|
86
|
-
export declare const BACKGROUND_SOFT_TIMEOUT_CEILING_MS: number;
|
|
87
|
-
/**
|
|
88
|
-
* Default soft-timeout budget for a background-function run when the caller
|
|
89
|
-
* does not pass an explicit `softTimeoutMs`. Same value as the ceiling — we
|
|
90
|
-
* want a background turn to use nearly its whole 15-min budget before handing
|
|
91
|
-
* off to a chained background continuation.
|
|
92
|
-
*/
|
|
93
|
-
export declare const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS: number;
|
|
94
|
-
/**
|
|
95
|
-
* AUTHORITATIVE no-progress backstop for a run, enforced by the run manager
|
|
96
|
-
* itself (timer-driven, independent of any layer below).
|
|
97
|
-
*
|
|
98
|
-
* The finer-grained watchdogs inside the agent loop (model-stream and
|
|
99
|
-
* action-preparation no-progress, both 90s) only guard the model event stream
|
|
100
|
-
* — a stall in any segment OUTSIDE that guarded loop (engine-call
|
|
101
|
-
* establishment, worker setup between continuation chunks, a wedged transport
|
|
102
|
-
* that emits keepalives while the loop never runs) previously hung forever
|
|
103
|
-
* with the client watching keepalives. This backstop covers every segment by
|
|
104
|
-
* construction: if no REAL progress event (see `shouldBumpProgressForEvent`;
|
|
105
|
-
* keepalives and zero-byte prep activity don't count) lands for this long —
|
|
106
|
-
* and no unit of work is in flight (see `inFlightWorkDelta`: tool calls,
|
|
107
|
-
* cross-app calls, and the model stream all legitimately emit nothing for
|
|
108
|
-
* minutes and each carry a bound of their own) — the run manager emits
|
|
109
|
-
* `auto_continue { reason: "no_progress" }` and aborts the chunk, exactly
|
|
110
|
-
* like the soft timeout, so the normal continuation machinery recovers it.
|
|
111
|
-
*
|
|
112
|
-
* Being numerically larger than the in-loop watchdogs is NOT what keeps this
|
|
113
|
-
* from killing a healthy run, and treating it that way is what made it do so:
|
|
114
|
-
* this clock and the loop's `lastModelStreamProgressAt` measure DIFFERENT
|
|
115
|
-
* events. An extended-thinking phase bumps the inner clock on every engine
|
|
116
|
-
* frame while forwarding nothing, so the inner watchdog correctly stayed quiet
|
|
117
|
-
* and this one saw pure silence — runs whose worst gap crossed 150s died while
|
|
118
|
-
* still streaming, some by a single second. Ordering between two clocks only
|
|
119
|
-
* means something when they watch the same events; suspending on in-flight
|
|
120
|
-
* work is what actually makes the two agree.
|
|
121
|
-
*
|
|
122
|
-
* This is now only the CEILING, not the value: `resolveRunNoProgressTimeoutMs`
|
|
123
|
-
* clamps the foreground backstop to a fraction of the chunk's soft timeout
|
|
124
|
-
* (~30s at a 40s chunk), which is BELOW the 90s in-loop watchdogs rather than
|
|
125
|
-
* above them. That ordering is deliberate — the in-loop watchdogs could never
|
|
126
|
-
* fire inside a hosted foreground chunk anyway, since the serverless wall
|
|
127
|
-
* (~57-59s) arrives first. Proven durable-background chunks keep the full
|
|
128
|
-
* `DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS` so large outputs can use the
|
|
129
|
-
* background budget. Only armed when a soft-timeout regime is active (hosted
|
|
130
|
-
* runs); local dev stays unbounded.
|
|
131
|
-
*/
|
|
132
|
-
export declare const RUN_NO_PROGRESS_HARD_TIMEOUT_MS = 150000;
|
|
133
71
|
/**
|
|
134
72
|
* Default no-progress window for a run executing inside a proven durable
|
|
135
73
|
* background function. A background worker that is heartbeating but has no
|
|
@@ -296,9 +234,18 @@ export interface StartRunOptions {
|
|
|
296
234
|
*/
|
|
297
235
|
backgroundNoProgressTimeoutMs?: number;
|
|
298
236
|
/**
|
|
299
|
-
* Lifecycle metadata persisted to `agent_runs.dispatch_mode
|
|
300
|
-
* clients through `/runs/active
|
|
301
|
-
*
|
|
237
|
+
* Lifecycle metadata persisted to `agent_runs.dispatch_mode`, surfaced to
|
|
238
|
+
* clients through `/runs/active`, and carried on the terminal/boundary
|
|
239
|
+
* analytics events. This does not change run-manager behavior; callers use it
|
|
240
|
+
* to describe who owns continuation at hosted chunk boundaries.
|
|
241
|
+
*
|
|
242
|
+
* Unset is reported as ABSENT, never as `"foreground"`. The analytics events
|
|
243
|
+
* used to default it, and the default was wrong every single time it applied:
|
|
244
|
+
* the interactive handler is the one caller that passes this, so the default
|
|
245
|
+
* only ever labelled the callers that are NOT foreground — automations, agent
|
|
246
|
+
* teams, webhooks, harness runs. It made a 6-of-7 no-progress failure rate on
|
|
247
|
+
* the automation path indistinguishable from chat in the one place anybody
|
|
248
|
+
* would have looked.
|
|
302
249
|
*/
|
|
303
250
|
dispatchMode?: "foreground" | "foreground-self-chain" | "background";
|
|
304
251
|
/**
|
|
@@ -315,6 +262,53 @@ export interface StartRunOptions {
|
|
|
315
262
|
/** Continuation/redispatch attempt number for this logical turn, if the
|
|
316
263
|
* caller is tracking one. */
|
|
317
264
|
attemptCount?: number;
|
|
265
|
+
/**
|
|
266
|
+
* The `runFn` recovers chunk boundaries INSIDE this invocation — it threads
|
|
267
|
+
* the `RunChunkControl` it is handed into
|
|
268
|
+
* `runAgentLoopDirectWithSoftTimeout`.
|
|
269
|
+
*
|
|
270
|
+
* When true a checkpoint aborts only the current CHUNK; the turn-scoped
|
|
271
|
+
* controller (what a user Stop, a hard timeout, and the cross-isolate abort
|
|
272
|
+
* check use) is left alone so the loop can append its continuation context
|
|
273
|
+
* and keep going. Off by default, and it must stay off for every caller that
|
|
274
|
+
* hands continuation to a FRESH invocation: those need the turn to end here
|
|
275
|
+
* so the next invocation can pick it up.
|
|
276
|
+
*
|
|
277
|
+
* This is the fix for the in-process automation runner, whose checkpoints
|
|
278
|
+
* were aborting the turn for a continuation nobody was going to run.
|
|
279
|
+
*/
|
|
280
|
+
recoverChunkBoundaries?: boolean;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Handed to `runFn` so an in-invocation runner can tell a recoverable CHUNK
|
|
284
|
+
* boundary from a turn-ending abort.
|
|
285
|
+
*
|
|
286
|
+
* Without this distinction there is only one signal, and a checkpoint fired
|
|
287
|
+
* from above the agent loop is indistinguishable from a user pressing Stop —
|
|
288
|
+
* which is why `no_progress` was an accepted continuation reason with a
|
|
289
|
+
* 20-round budget that could never be reached.
|
|
290
|
+
*/
|
|
291
|
+
export interface RunChunkControl {
|
|
292
|
+
/**
|
|
293
|
+
* Aborted only when the TURN must end: user Stop, cross-isolate abort, the
|
|
294
|
+
* caller's own hard timeout, or a checkpoint on a run that did not opt into
|
|
295
|
+
* `recoverChunkBoundaries`. Never fires for a recoverable chunk boundary.
|
|
296
|
+
*/
|
|
297
|
+
readonly turnSignal: AbortSignal;
|
|
298
|
+
/** Signal for the chunk currently executing. Replaced by `beginChunk()`. */
|
|
299
|
+
readonly chunkSignal: AbortSignal;
|
|
300
|
+
/**
|
|
301
|
+
* Reason the CURRENT chunk was checkpointed, or `null` while it is live.
|
|
302
|
+
* Distinct from "the turn was aborted": a caller that cannot tell them apart
|
|
303
|
+
* turns every planned boundary into a terminal failure.
|
|
304
|
+
*/
|
|
305
|
+
chunkBoundaryReason(): string | null;
|
|
306
|
+
/**
|
|
307
|
+
* Open a fresh chunk after a recoverable boundary and return its signal.
|
|
308
|
+
* Returns the already-aborted turn signal when the turn is over, so a caller
|
|
309
|
+
* that races a Stop cannot accidentally start another chunk.
|
|
310
|
+
*/
|
|
311
|
+
beginChunk(): AbortSignal;
|
|
318
312
|
}
|
|
319
313
|
export interface ResolveRunSoftTimeoutOptions {
|
|
320
314
|
useHostedDefault?: boolean;
|
|
@@ -322,8 +316,9 @@ export interface ResolveRunSoftTimeoutOptions {
|
|
|
322
316
|
* Resolve the soft timeout for a run executing inside a Netlify background
|
|
323
317
|
* function. Lifts the hosted clamp to `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`
|
|
324
318
|
* (~13min) for this invocation only and, when no override/env is supplied,
|
|
325
|
-
* defaults to
|
|
326
|
-
*
|
|
319
|
+
* defaults to that same ceiling — a background turn should use nearly its
|
|
320
|
+
* whole budget before handing off to a chained continuation. Does NOT change
|
|
321
|
+
* the foreground ceiling. Off by default.
|
|
327
322
|
*/
|
|
328
323
|
backgroundFunction?: boolean;
|
|
329
324
|
}
|
|
@@ -339,6 +334,26 @@ export declare function isHostedRuntime(): boolean;
|
|
|
339
334
|
export declare function resolveRunSoftTimeoutMs(overrideMs?: number, options?: ResolveRunSoftTimeoutOptions): number;
|
|
340
335
|
export declare function resolveCompletedRunRetentionMs(): number;
|
|
341
336
|
export declare function resolveErroredRunRetentionMs(): number;
|
|
337
|
+
/**
|
|
338
|
+
* Hard abort for one in-process background automation run.
|
|
339
|
+
*
|
|
340
|
+
* This is the host's real function budget for scheduled work, which is exactly
|
|
341
|
+
* the kind of number that differs between deployments — so it is configuration,
|
|
342
|
+
* not a module constant nobody outside this package can see.
|
|
343
|
+
*/
|
|
344
|
+
export declare function resolveBackgroundRunHardTimeoutMs(): number;
|
|
345
|
+
/**
|
|
346
|
+
* Chunk budget for a background automation, derived from the runner's OWN hard
|
|
347
|
+
* abort rather than from the durable-chat background ceiling.
|
|
348
|
+
*
|
|
349
|
+
* The shipped build took the 13-minute chat ceiling for a path whose process is
|
|
350
|
+
* killed at 10 minutes, which made the recoverable soft-timeout boundary dead
|
|
351
|
+
* code and left the terminal no-progress backstop as the only boundary an
|
|
352
|
+
* automation could ever reach. Deriving from the hard abort keeps
|
|
353
|
+
* `soft timeout < hard abort` true by construction; the invariant check asserts
|
|
354
|
+
* the headroom still fits.
|
|
355
|
+
*/
|
|
356
|
+
export declare function resolveBackgroundAutomationSoftTimeoutMs(overrideMs?: number): number;
|
|
342
357
|
/**
|
|
343
358
|
* A completed tool with no later assistant text is an unfinished turn, not a
|
|
344
359
|
* successful terminal response. Keep this predicate beside the run-manager's
|
|
@@ -353,7 +368,7 @@ export declare function endsAfterCompletedToolWithoutAssistantFinal(run: ActiveR
|
|
|
353
368
|
*
|
|
354
369
|
* Events are persisted to SQL for cross-isolate access (Cloudflare Workers).
|
|
355
370
|
*/
|
|
356
|
-
export declare function startRun(runId: string, threadId: string, runFn: (send: (event: AgentChatEvent) => void, signal: AbortSignal) => Promise<void>, onComplete?: (run: ActiveRun) => void | Promise<void>, options?: StartRunOptions): StartedRun;
|
|
371
|
+
export declare function startRun(runId: string, threadId: string, runFn: (send: (event: AgentChatEvent) => void, signal: AbortSignal, control: RunChunkControl) => Promise<void>, onComplete?: (run: ActiveRun) => void | Promise<void>, options?: StartRunOptions): StartedRun;
|
|
357
372
|
/**
|
|
358
373
|
* Subscribe to a run's events starting from `fromSeq`.
|
|
359
374
|
* Returns a ReadableStream that replays buffered events then live-tails.
|