@agent-native/core 0.77.0 → 0.77.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/core/CHANGELOG.md +25 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +98 -5
- package/corpus/core/src/agent/run-store.ts +7 -1
- package/dist/agent/production-agent.d.ts +22 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +80 -3
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-store.d.ts +1 -0
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +4 -1
- package/dist/agent/run-store.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +3 -3
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/package.json +1 -1
package/corpus/core/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @agent-native/core
|
|
2
2
|
|
|
3
|
+
## 0.77.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- e3a084f: Durable background agent-chat: wait adaptively for a slow-but-alive worker to
|
|
8
|
+
claim a dispatched run, instead of abandoning it and recovering inline. The
|
|
9
|
+
foreground waits a base grace for the background worker to claim; heavy apps
|
|
10
|
+
(e.g. analytics) can take longer than that to build the system prompt and load
|
|
11
|
+
actions before claiming, so their worker lost the race every time and the
|
|
12
|
+
15-minute background budget went unused (observed in prod: the run stalls at
|
|
13
|
+
`auth_passed`, then recovers via `foreground_inline_recovery`).
|
|
14
|
+
|
|
15
|
+
The circuit-breaker now keeps polling past the base grace ONLY while the worker
|
|
16
|
+
is provably alive and still in setup — its `diag_stage` (parsed from the stored
|
|
17
|
+
JSON payload) is `auth_passed`/`worker_entered` but it has not claimed yet. A
|
|
18
|
+
dead handoff never records those stages, so it still recovers inline at the base
|
|
19
|
+
grace; a worker that recorded a pre-claim failure (`route_threw` / `worker_threw`
|
|
20
|
+
/ `auth_failed`) recovers inline immediately. The extension is bounded by the
|
|
21
|
+
unclaimed-run reaper's own window, measured from the run's liveness
|
|
22
|
+
(`COALESCE(heartbeat_at, started_at)`), so the foreground always claims the run
|
|
23
|
+
inline just before `reapUnclaimedBackgroundRun` could fire — immune to dispatch
|
|
24
|
+
latency between insert and the start of polling. The claim itself still happens
|
|
25
|
+
right before the agent loop, so all existing fast-recovery and duplicate-delivery
|
|
26
|
+
guarantees are preserved.
|
|
27
|
+
|
|
3
28
|
## 0.77.0
|
|
4
29
|
|
|
5
30
|
### Minor Changes
|
package/corpus/core/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.77.
|
|
3
|
+
"version": "0.77.1",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -121,6 +121,7 @@ import {
|
|
|
121
121
|
readBackgroundRunClaim,
|
|
122
122
|
recordRunDiagnostic,
|
|
123
123
|
RUN_DIAG_STAGE,
|
|
124
|
+
UNCLAIMED_BACKGROUND_RUN_GRACE_MS,
|
|
124
125
|
} from "./run-store.js";
|
|
125
126
|
import {
|
|
126
127
|
classifyToolCallJournal,
|
|
@@ -162,6 +163,14 @@ export { PROVIDER_TO_ENV };
|
|
|
162
163
|
* well within the foreground's ~40s soft-timeout.
|
|
163
164
|
*/
|
|
164
165
|
export const BACKGROUND_CLAIM_GRACE_MS = 15_000;
|
|
166
|
+
/**
|
|
167
|
+
* Safety margin subtracted from the unclaimed-reaper grace when deciding how
|
|
168
|
+
* long the foreground may keep waiting for a slow-but-live worker to claim. The
|
|
169
|
+
* foreground recovers the run inline this many ms BEFORE `reapUnclaimedBackgroundRun`
|
|
170
|
+
* would error an unclaimed row, so the foreground always wins the race to claim
|
|
171
|
+
* and the two never collide — see `resolveBackgroundDispatchOutcome`.
|
|
172
|
+
*/
|
|
173
|
+
export const BACKGROUND_REAPER_SAFETY_MARGIN_MS = 2_000;
|
|
165
174
|
export const BACKGROUND_CLAIM_POLL_MS = 400;
|
|
166
175
|
|
|
167
176
|
export type BackgroundDispatchOutcome =
|
|
@@ -172,6 +181,23 @@ export type BackgroundDispatchOutcome =
|
|
|
172
181
|
reason: "dispatch-failed" | "worker-never-claimed" | "no-row";
|
|
173
182
|
};
|
|
174
183
|
|
|
184
|
+
/**
|
|
185
|
+
* `diag_stage` is persisted as a JSON payload (`{ stage, detail?, at }`) by
|
|
186
|
+
* `recordRunDiagnostic`. Extract the bare stage name so it can be compared to
|
|
187
|
+
* `RUN_DIAG_STAGE` constants. Falls back to the raw value when it is not JSON
|
|
188
|
+
* (defensive — legacy rows or tests may store a bare stage).
|
|
189
|
+
*/
|
|
190
|
+
function parseRunDiagStage(raw: string | null | undefined): string | null {
|
|
191
|
+
if (!raw) return null;
|
|
192
|
+
try {
|
|
193
|
+
const parsed = JSON.parse(raw) as { stage?: unknown };
|
|
194
|
+
if (parsed && typeof parsed.stage === "string") return parsed.stage;
|
|
195
|
+
} catch {
|
|
196
|
+
// Not JSON — treat the raw value as the stage name.
|
|
197
|
+
}
|
|
198
|
+
return typeof raw === "string" ? raw : null;
|
|
199
|
+
}
|
|
200
|
+
|
|
175
201
|
/**
|
|
176
202
|
* Decide what the foreground should do after attempting a durable background
|
|
177
203
|
* dispatch. A Netlify async background function returns 202 the instant it
|
|
@@ -193,10 +219,25 @@ export async function resolveBackgroundDispatchOutcome(opts: {
|
|
|
193
219
|
backgroundRowInserted: boolean;
|
|
194
220
|
runId: string;
|
|
195
221
|
graceMs: number;
|
|
222
|
+
/**
|
|
223
|
+
* The unclaimed-run reaper's grace (`UNCLAIMED_BACKGROUND_RUN_GRACE_MS`). When
|
|
224
|
+
* provided, the foreground may keep waiting PAST `graceMs` while the worker is
|
|
225
|
+
* provably alive and still in setup — but it recovers inline before the run has
|
|
226
|
+
* been unclaimed this long (minus the safety margin), so it always claims
|
|
227
|
+
* before the reaper can fire. Omit to disable the extension (behaves exactly
|
|
228
|
+
* like the base grace).
|
|
229
|
+
*/
|
|
230
|
+
reaperGraceMs?: number;
|
|
231
|
+
/** Margin subtracted from `reaperGraceMs` (default `BACKGROUND_REAPER_SAFETY_MARGIN_MS`). */
|
|
232
|
+
reaperSafetyMarginMs?: number;
|
|
196
233
|
pollIntervalMs: number;
|
|
197
|
-
readClaim: (
|
|
198
|
-
|
|
199
|
-
|
|
234
|
+
readClaim: (runId: string) => Promise<{
|
|
235
|
+
dispatchMode: string | null;
|
|
236
|
+
status: string | null;
|
|
237
|
+
diagStage?: string | null;
|
|
238
|
+
/** COALESCE(heartbeat_at, started_at) — the reaper's liveness basis. */
|
|
239
|
+
lastLivenessAt?: number | null;
|
|
240
|
+
} | null>;
|
|
200
241
|
claim: (runId: string) => Promise<boolean>;
|
|
201
242
|
now?: () => number;
|
|
202
243
|
sleep?: (ms: number) => Promise<void>;
|
|
@@ -205,8 +246,32 @@ export async function resolveBackgroundDispatchOutcome(opts: {
|
|
|
205
246
|
const sleep =
|
|
206
247
|
opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
207
248
|
|
|
249
|
+
// Pre-claim diag stages that prove the worker is ALIVE and executing: it
|
|
250
|
+
// reached the route, passed HMAC auth, and is grinding through handler setup
|
|
251
|
+
// (system prompt build / action loading) on its way to `claimBackgroundRun`.
|
|
252
|
+
// A dead handoff — the generated wrapper never reached the route — never
|
|
253
|
+
// records these, so it is NOT eligible for the extended grace.
|
|
254
|
+
const ALIVE_IN_SETUP: ReadonlySet<string> = new Set([
|
|
255
|
+
RUN_DIAG_STAGE.authPassed,
|
|
256
|
+
RUN_DIAG_STAGE.workerEntered,
|
|
257
|
+
]);
|
|
258
|
+
// Pre-claim diag stages that prove the worker DIED before claiming — stop
|
|
259
|
+
// waiting and recover inline immediately instead of burning the rest of the
|
|
260
|
+
// grace on a worker that already failed.
|
|
261
|
+
const DIED_BEFORE_CLAIM: ReadonlySet<string> = new Set([
|
|
262
|
+
RUN_DIAG_STAGE.authFailed,
|
|
263
|
+
RUN_DIAG_STAGE.routeThrew,
|
|
264
|
+
RUN_DIAG_STAGE.workerThrew,
|
|
265
|
+
]);
|
|
266
|
+
|
|
208
267
|
if (opts.dispatched) {
|
|
209
|
-
|
|
268
|
+
// One now() at entry + one per iteration (so callers/tests that model a
|
|
269
|
+
// stepping clock stay deterministic).
|
|
270
|
+
const startedAt = now();
|
|
271
|
+
const baseDeadline = startedAt + opts.graceMs;
|
|
272
|
+
const reaperGraceMs = opts.reaperGraceMs;
|
|
273
|
+
const reaperMargin =
|
|
274
|
+
opts.reaperSafetyMarginMs ?? BACKGROUND_REAPER_SAFETY_MARGIN_MS;
|
|
210
275
|
for (;;) {
|
|
211
276
|
const claim = await opts.readClaim(opts.runId).catch(() => null);
|
|
212
277
|
if (
|
|
@@ -216,7 +281,34 @@ export async function resolveBackgroundDispatchOutcome(opts: {
|
|
|
216
281
|
) {
|
|
217
282
|
return { action: "stream" };
|
|
218
283
|
}
|
|
219
|
-
|
|
284
|
+
// `diag_stage` is stored as JSON ({stage, detail?, at}); compare on the
|
|
285
|
+
// bare stage name, not the raw payload.
|
|
286
|
+
const stage = parseRunDiagStage(claim?.diagStage);
|
|
287
|
+
// Worker recorded a pre-claim death — no point waiting out the grace.
|
|
288
|
+
if (stage && DIED_BEFORE_CLAIM.has(stage)) break;
|
|
289
|
+
const elapsedNow = now();
|
|
290
|
+
// The unclaimed-reaper errors any still-`background` row once it has been
|
|
291
|
+
// unclaimed for `reaperGraceMs`, measured from the row's OWN liveness
|
|
292
|
+
// (COALESCE(heartbeat_at, started_at)) — NOT from when we began polling.
|
|
293
|
+
// Recover inline just before that point so the foreground claims the run
|
|
294
|
+
// first; anchoring to the row's liveness makes this immune to dispatch
|
|
295
|
+
// latency between insertRun and the start of polling.
|
|
296
|
+
const reaperWillFireSoon =
|
|
297
|
+
reaperGraceMs != null &&
|
|
298
|
+
claim?.lastLivenessAt != null &&
|
|
299
|
+
elapsedNow - claim.lastLivenessAt >= reaperGraceMs - reaperMargin;
|
|
300
|
+
if (reaperWillFireSoon) break;
|
|
301
|
+
// ADAPTIVE GRACE: past the base window, keep polling ONLY while the worker
|
|
302
|
+
// is provably alive and still in setup (heavy cold start). A dead handoff
|
|
303
|
+
// never recorded an ALIVE_IN_SETUP stage, so it recovers inline at the base
|
|
304
|
+
// grace; the reaper-anchored break above bounds how long a live worker can
|
|
305
|
+
// extend. The extension is enabled only when a reaper grace was provided.
|
|
306
|
+
const aliveInSetup =
|
|
307
|
+
reaperGraceMs != null &&
|
|
308
|
+
claim?.status === "running" &&
|
|
309
|
+
!!stage &&
|
|
310
|
+
ALIVE_IN_SETUP.has(stage);
|
|
311
|
+
if (elapsedNow >= baseDeadline && !aliveInSetup) break;
|
|
220
312
|
await sleep(opts.pollIntervalMs);
|
|
221
313
|
}
|
|
222
314
|
}
|
|
@@ -4538,6 +4630,7 @@ export function createProductionAgentHandler(
|
|
|
4538
4630
|
backgroundRowInserted,
|
|
4539
4631
|
runId,
|
|
4540
4632
|
graceMs: BACKGROUND_CLAIM_GRACE_MS,
|
|
4633
|
+
reaperGraceMs: UNCLAIMED_BACKGROUND_RUN_GRACE_MS,
|
|
4541
4634
|
pollIntervalMs: BACKGROUND_CLAIM_POLL_MS,
|
|
4542
4635
|
readClaim: readBackgroundRunClaim,
|
|
4543
4636
|
claim: claimBackgroundRun,
|
|
@@ -392,11 +392,12 @@ export async function readBackgroundRunClaim(runId: string): Promise<{
|
|
|
392
392
|
dispatchMode: string | null;
|
|
393
393
|
status: string | null;
|
|
394
394
|
diagStage: string | null;
|
|
395
|
+
lastLivenessAt: number | null;
|
|
395
396
|
} | null> {
|
|
396
397
|
await ensureRunTables();
|
|
397
398
|
const client = getDbExec();
|
|
398
399
|
const { rows } = await client.execute({
|
|
399
|
-
sql: `SELECT dispatch_mode, status, diag_stage FROM agent_runs WHERE id = ? LIMIT 1`,
|
|
400
|
+
sql: `SELECT dispatch_mode, status, diag_stage, started_at, heartbeat_at FROM agent_runs WHERE id = ? LIMIT 1`,
|
|
400
401
|
args: [runId],
|
|
401
402
|
});
|
|
402
403
|
const row = rows?.[0] as
|
|
@@ -404,6 +405,8 @@ export async function readBackgroundRunClaim(runId: string): Promise<{
|
|
|
404
405
|
dispatch_mode?: string | null;
|
|
405
406
|
status?: string | null;
|
|
406
407
|
diag_stage?: string | null;
|
|
408
|
+
started_at?: number | null;
|
|
409
|
+
heartbeat_at?: number | null;
|
|
407
410
|
}
|
|
408
411
|
| undefined;
|
|
409
412
|
if (!row) return null;
|
|
@@ -411,6 +414,9 @@ export async function readBackgroundRunClaim(runId: string): Promise<{
|
|
|
411
414
|
dispatchMode: row.dispatch_mode ?? null,
|
|
412
415
|
status: row.status ?? null,
|
|
413
416
|
diagStage: row.diag_stage ?? null,
|
|
417
|
+
// Same liveness basis the unclaimed-reaper uses (COALESCE(heartbeat_at,
|
|
418
|
+
// started_at)), so the foreground can decide to recover BEFORE the reaper.
|
|
419
|
+
lastLivenessAt: row.heartbeat_at ?? row.started_at ?? null,
|
|
414
420
|
};
|
|
415
421
|
}
|
|
416
422
|
|
|
@@ -18,6 +18,14 @@ export { PROVIDER_TO_ENV };
|
|
|
18
18
|
* well within the foreground's ~40s soft-timeout.
|
|
19
19
|
*/
|
|
20
20
|
export declare const BACKGROUND_CLAIM_GRACE_MS = 15000;
|
|
21
|
+
/**
|
|
22
|
+
* Safety margin subtracted from the unclaimed-reaper grace when deciding how
|
|
23
|
+
* long the foreground may keep waiting for a slow-but-live worker to claim. The
|
|
24
|
+
* foreground recovers the run inline this many ms BEFORE `reapUnclaimedBackgroundRun`
|
|
25
|
+
* would error an unclaimed row, so the foreground always wins the race to claim
|
|
26
|
+
* and the two never collide — see `resolveBackgroundDispatchOutcome`.
|
|
27
|
+
*/
|
|
28
|
+
export declare const BACKGROUND_REAPER_SAFETY_MARGIN_MS = 2000;
|
|
21
29
|
export declare const BACKGROUND_CLAIM_POLL_MS = 400;
|
|
22
30
|
export type BackgroundDispatchOutcome = {
|
|
23
31
|
action: "stream";
|
|
@@ -48,10 +56,24 @@ export declare function resolveBackgroundDispatchOutcome(opts: {
|
|
|
48
56
|
backgroundRowInserted: boolean;
|
|
49
57
|
runId: string;
|
|
50
58
|
graceMs: number;
|
|
59
|
+
/**
|
|
60
|
+
* The unclaimed-run reaper's grace (`UNCLAIMED_BACKGROUND_RUN_GRACE_MS`). When
|
|
61
|
+
* provided, the foreground may keep waiting PAST `graceMs` while the worker is
|
|
62
|
+
* provably alive and still in setup — but it recovers inline before the run has
|
|
63
|
+
* been unclaimed this long (minus the safety margin), so it always claims
|
|
64
|
+
* before the reaper can fire. Omit to disable the extension (behaves exactly
|
|
65
|
+
* like the base grace).
|
|
66
|
+
*/
|
|
67
|
+
reaperGraceMs?: number;
|
|
68
|
+
/** Margin subtracted from `reaperGraceMs` (default `BACKGROUND_REAPER_SAFETY_MARGIN_MS`). */
|
|
69
|
+
reaperSafetyMarginMs?: number;
|
|
51
70
|
pollIntervalMs: number;
|
|
52
71
|
readClaim: (runId: string) => Promise<{
|
|
53
72
|
dispatchMode: string | null;
|
|
54
73
|
status: string | null;
|
|
74
|
+
diagStage?: string | null;
|
|
75
|
+
/** COALESCE(heartbeat_at, started_at) — the reaper's liveness basis. */
|
|
76
|
+
lastLivenessAt?: number | null;
|
|
55
77
|
} | null>;
|
|
56
78
|
claim: (runId: string) => Promise<boolean>;
|
|
57
79
|
now?: () => number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"production-agent.d.ts","sourceRoot":"","sources":["../../src/agent/production-agent.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,IAAI,CAAC;AAgCzD,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AA0BvC,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAMhE,OAAO,KAAK,EACV,WAAW,EACX,UAAU,EACV,aAAa,EACb,iBAAiB,EAGlB,MAAM,mBAAmB,CAAC;AAa3B,OAAO,EAIL,KAAK,SAAS,EACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,cAAc,EACd,qBAAqB,EACrB,0BAA0B,EAC1B,MAAM,EACN,QAAQ,EAET,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"production-agent.d.ts","sourceRoot":"","sources":["../../src/agent/production-agent.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,IAAI,CAAC;AAgCzD,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AA0BvC,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAMhE,OAAO,KAAK,EACV,WAAW,EACX,UAAU,EACV,aAAa,EACb,iBAAiB,EAGlB,MAAM,mBAAmB,CAAC;AAa3B,OAAO,EAIL,KAAK,SAAS,EACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,cAAc,EACd,qBAAqB,EACrB,0BAA0B,EAC1B,MAAM,EACN,QAAQ,EAET,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AA6BlD,OAAO,KAAK,EACV,UAAU,EAEV,mBAAmB,EAEnB,cAAc,EACd,kBAAkB,EAClB,0BAA0B,EAC3B,MAAM,YAAY,CAAC;AAKpB,OAAO,EAAE,eAAe,EAAE,CAAC;AAE3B;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,OAAQ,CAAC;AACxD,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,MAAM,MAAM,yBAAyB,GACjC;IAAE,MAAM,EAAE,QAAQ,CAAA;CAAE,GACpB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GACvB;IACE,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,iBAAiB,GAAG,sBAAsB,GAAG,QAAQ,CAAC;CAC/D,CAAC;AAmBN;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,gCAAgC,CAAC,IAAI,EAAE;IAC3D,UAAU,EAAE,OAAO,CAAC;IACpB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6FAA6F;IAC7F,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;QACpC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,wEAAwE;QACxE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,GAAG,IAAI,CAAC,CAAC;IACV,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,GAAG,OAAO,CAAC,yBAAyB,CAAC,CA6FrC;AA4CD;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8C7B;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE3D;AAaD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,oBAAoB,CACxC,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAsB7B;AAED,sEAAsE;AACtE,wBAAsB,uBAAuB,CAC3C,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAE7B;AAED;;;;GAIG;AACH,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEnE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,GAAG,EAAE,CACH,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE,OAAO,cAAc,EAAE,gBAAgB,KAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACxB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qFAAqF;IACrF,IAAI,CAAC,EAAE,OAAO,cAAc,EAAE,gBAAgB,GAAG,KAAK,CAAC;IACvD;;0EAEsE;IACtE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;sDAGkD;IAClD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;qFACiF;IACjF,WAAW,CAAC,EAAE,OAAO,cAAc,EAAE,uBAAuB,CAAC;IAC7D;4EACwE;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;oDAEgD;IAChD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;gDAK4C;IAC5C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;2EAEuE;IACvE,IAAI,CAAC,EAAE,OAAO,cAAc,EAAE,iBAAiB,CAAC;IAChD;;qCAEiC;IACjC,MAAM,CAAC,EAAE,OAAO,cAAc,EAAE,kBAAkB,CAAC;IACnD,2EAA2E;IAC3E,MAAM,CAAC,EAAE,OAAO,iBAAiB,EAAE,kBAAkB,CAAC;IACtD;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,aAAa,CAAC,EACV,OAAO,GACP,CAAC,CACC,IAAI,EAAE,GAAG,EACT,GAAG,CAAC,EAAE,OAAO,cAAc,EAAE,gBAAgB,KAC1C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;CACtC;AAED,4CAA4C;AAC5C,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC;AAEtC,MAAM,MAAM,kBAAkB,GAAG,KAAK,GAAG,MAAM,CAAC;AAEhD,eAAO,MAAM,uBAAuB,ivBAQ2H,CAAC;AAwFhK,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,WAAW,GACjB,OAAO,CAiBT;AA0ED,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACnC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAiC7B;AAED,MAAM,WAAW,sBAAsB;IACrC,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,0FAA0F;IAC1F,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,MAAM,CAAC,EACH,WAAW,GACX,MAAM,GACN;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC;IACtD,qEAAqE;IACrE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,6DAA6D;IAC7D,eAAe,CAAC,EAAE,aAAa,SAAS,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;IAC5D,uEAAuE;IACvE,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC;IACvE,mEAAmE;IACnE,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACrC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE;QACzB,KAAK,EAAE,GAAG,CAAC;QACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;QACnC,UAAU,EAAE,kBAAkB,EAAE,CAAC;QACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC;KAC1B,KACG,IAAI,GACJ;QACE,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACrC,GACD,OAAO,CAAC,IAAI,GAAG;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACrC,CAAC,CAAC;IACP;;;mDAG+C;IAC/C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4FAA4F;IAC5F,UAAU,CAAC,EAAE,CACX,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,EACrC,QAAQ,EAAE,MAAM,KACb,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChE,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7D;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,2BAA2B,CAAC;IACjD;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;;;OAIG;IACH,UAAU,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,mBAAmB,CAAC,EAC1D,KAAK,EAAE,GAAG,GACT,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAUxB;AAoFD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAoB3D;AAED,6CAA6C;AAC7C,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAwDtD;AAeD;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,aAAa,EAAE,EACzB,QAAQ,SAAyB,GAChC,aAAa,EAAE,GAAG,IAAI,CA0BxB;AAiGD,wBAAgB,+BAA+B,CAAC,IAAI,EAAE;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACrC,GAAG,iBAAiB,EAAE,CAiFtB;AAyBD,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,0BAA0B,EAAE,GAAG,SAAS,GAChD,aAAa,EAAE,GAAG,IAAI,CA2GxB;AAoBD,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,kBAAkB,GACtB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA8CxB;AAqED,qDAAqD;AACrD,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kCAAkC;IACjD,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,gBAAgB,EAAE,iBAAiB,EAAE,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,wBAAwB,EAAE,CAAC;IACtC,WAAW,EAAE,0BAA0B,EAAE,CAAC;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,MAAM,iCAAiC,GACzC,MAAM,GACN;IACE,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEN,MAAM,MAAM,2BAA2B,GAAG,CACxC,OAAO,EAAE,kCAAkC,KAEzC,iCAAiC,GACjC,IAAI,GACJ,SAAS,GACT,OAAO,CAAC,iCAAiC,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;AAYlE,eAAO,MAAM,8BAA8B,mOACuL,CAAC;AAEnO,MAAM,MAAM,2BAA2B,GACnC,aAAa,GACb,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,iBAAiB,GACjB,qBAAqB,CAAC;AAE1B,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,aAAa,EAAE,EACzB,MAAM,EAAE,2BAA2B,QAuBpC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAoC5D;AAED;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,GAAG,EAAE,OAAO,GACX,iBAAiB,GAAG,qBAAqB,CAa3C;AAqQD;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACnC,UAAU,EAAE,CAkBd;AAyOD,wBAAgB,+BAA+B,CAAC,IAAI,EAAE;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,MAAM,CAsBT;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,cAAc,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAgBnE;AA6FD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE;IACvC,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,cAAc,CAAC,EAAE,UAAU,EAAE,CAAC;IAC9B,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;IACtC,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,eAAe,CAAC,EAAE,GAAG,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,2BAA2B,CAAC;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7D;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;CAC1B,GAAG,OAAO,CAAC,cAAc,CAAC,CAsuC1B;AA4CD;;;;;;GAMG;AACH,eAAO,MAAM,gCAAgC,KAAK,CAAC;AAEnD;;;;;;GAMG;AACH,wBAAgB,iCAAiC,CAAC,IAAI,EAAE;IACtD,kBAAkB,EAAE,OAAO,CAAC;IAC5B,GAAG,EAAE,SAAS,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;CAC3B,GAAG,OAAO,CAOV;AA+BD,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,sBAAsB,GAC9B,cAAc,CAogDhB;AAED,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,MAAM,EACN,QAAQ,EACR,cAAc,GACf,CAAC"}
|
|
@@ -31,7 +31,7 @@ import { getDefaultMaxIterations, normalizeMaxIterations, readAgentLoopSettings,
|
|
|
31
31
|
import { maybeCompactThread, buildObservationalContext, hasObservationalMemory, serializeObservationalMemoryBlock, } from "./observational-memory/index.js";
|
|
32
32
|
import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js";
|
|
33
33
|
import { startRun, subscribeToRun, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, tryClaimRunSlot, } from "./run-manager.js";
|
|
34
|
-
import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, insertRun, updateRunHeartbeat, updateRunStatusIfRunning, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, RUN_DIAG_STAGE, } from "./run-store.js";
|
|
34
|
+
import { writeLedgerEntry, readLedgerEntry, clearLedgerForThread, getCurrentTurnEventsForThread, insertRun, updateRunHeartbeat, updateRunStatusIfRunning, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, } from "./run-store.js";
|
|
35
35
|
import { classifyToolCallJournal, findCompletedJournalEntry, } from "./tool-call-journal.js";
|
|
36
36
|
import { redactSensitiveFields, sanitizeToolErrorText, sanitizeToolErrorValue, } from "./tool-error-redaction.js";
|
|
37
37
|
import { createToolSearchEntry, TOOL_SEARCH_ACTION_NAME, } from "./tool-search.js";
|
|
@@ -49,7 +49,34 @@ export { PROVIDER_TO_ENV };
|
|
|
49
49
|
* well within the foreground's ~40s soft-timeout.
|
|
50
50
|
*/
|
|
51
51
|
export const BACKGROUND_CLAIM_GRACE_MS = 15_000;
|
|
52
|
+
/**
|
|
53
|
+
* Safety margin subtracted from the unclaimed-reaper grace when deciding how
|
|
54
|
+
* long the foreground may keep waiting for a slow-but-live worker to claim. The
|
|
55
|
+
* foreground recovers the run inline this many ms BEFORE `reapUnclaimedBackgroundRun`
|
|
56
|
+
* would error an unclaimed row, so the foreground always wins the race to claim
|
|
57
|
+
* and the two never collide — see `resolveBackgroundDispatchOutcome`.
|
|
58
|
+
*/
|
|
59
|
+
export const BACKGROUND_REAPER_SAFETY_MARGIN_MS = 2_000;
|
|
52
60
|
export const BACKGROUND_CLAIM_POLL_MS = 400;
|
|
61
|
+
/**
|
|
62
|
+
* `diag_stage` is persisted as a JSON payload (`{ stage, detail?, at }`) by
|
|
63
|
+
* `recordRunDiagnostic`. Extract the bare stage name so it can be compared to
|
|
64
|
+
* `RUN_DIAG_STAGE` constants. Falls back to the raw value when it is not JSON
|
|
65
|
+
* (defensive — legacy rows or tests may store a bare stage).
|
|
66
|
+
*/
|
|
67
|
+
function parseRunDiagStage(raw) {
|
|
68
|
+
if (!raw)
|
|
69
|
+
return null;
|
|
70
|
+
try {
|
|
71
|
+
const parsed = JSON.parse(raw);
|
|
72
|
+
if (parsed && typeof parsed.stage === "string")
|
|
73
|
+
return parsed.stage;
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Not JSON — treat the raw value as the stage name.
|
|
77
|
+
}
|
|
78
|
+
return typeof raw === "string" ? raw : null;
|
|
79
|
+
}
|
|
53
80
|
/**
|
|
54
81
|
* Decide what the foreground should do after attempting a durable background
|
|
55
82
|
* dispatch. A Netlify async background function returns 202 the instant it
|
|
@@ -69,8 +96,30 @@ export const BACKGROUND_CLAIM_POLL_MS = 400;
|
|
|
69
96
|
export async function resolveBackgroundDispatchOutcome(opts) {
|
|
70
97
|
const now = opts.now ?? (() => Date.now());
|
|
71
98
|
const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
99
|
+
// Pre-claim diag stages that prove the worker is ALIVE and executing: it
|
|
100
|
+
// reached the route, passed HMAC auth, and is grinding through handler setup
|
|
101
|
+
// (system prompt build / action loading) on its way to `claimBackgroundRun`.
|
|
102
|
+
// A dead handoff — the generated wrapper never reached the route — never
|
|
103
|
+
// records these, so it is NOT eligible for the extended grace.
|
|
104
|
+
const ALIVE_IN_SETUP = new Set([
|
|
105
|
+
RUN_DIAG_STAGE.authPassed,
|
|
106
|
+
RUN_DIAG_STAGE.workerEntered,
|
|
107
|
+
]);
|
|
108
|
+
// Pre-claim diag stages that prove the worker DIED before claiming — stop
|
|
109
|
+
// waiting and recover inline immediately instead of burning the rest of the
|
|
110
|
+
// grace on a worker that already failed.
|
|
111
|
+
const DIED_BEFORE_CLAIM = new Set([
|
|
112
|
+
RUN_DIAG_STAGE.authFailed,
|
|
113
|
+
RUN_DIAG_STAGE.routeThrew,
|
|
114
|
+
RUN_DIAG_STAGE.workerThrew,
|
|
115
|
+
]);
|
|
72
116
|
if (opts.dispatched) {
|
|
73
|
-
|
|
117
|
+
// One now() at entry + one per iteration (so callers/tests that model a
|
|
118
|
+
// stepping clock stay deterministic).
|
|
119
|
+
const startedAt = now();
|
|
120
|
+
const baseDeadline = startedAt + opts.graceMs;
|
|
121
|
+
const reaperGraceMs = opts.reaperGraceMs;
|
|
122
|
+
const reaperMargin = opts.reaperSafetyMarginMs ?? BACKGROUND_REAPER_SAFETY_MARGIN_MS;
|
|
74
123
|
for (;;) {
|
|
75
124
|
const claim = await opts.readClaim(opts.runId).catch(() => null);
|
|
76
125
|
if (claim &&
|
|
@@ -78,7 +127,34 @@ export async function resolveBackgroundDispatchOutcome(opts) {
|
|
|
78
127
|
(claim.status && claim.status !== "running"))) {
|
|
79
128
|
return { action: "stream" };
|
|
80
129
|
}
|
|
81
|
-
|
|
130
|
+
// `diag_stage` is stored as JSON ({stage, detail?, at}); compare on the
|
|
131
|
+
// bare stage name, not the raw payload.
|
|
132
|
+
const stage = parseRunDiagStage(claim?.diagStage);
|
|
133
|
+
// Worker recorded a pre-claim death — no point waiting out the grace.
|
|
134
|
+
if (stage && DIED_BEFORE_CLAIM.has(stage))
|
|
135
|
+
break;
|
|
136
|
+
const elapsedNow = now();
|
|
137
|
+
// The unclaimed-reaper errors any still-`background` row once it has been
|
|
138
|
+
// unclaimed for `reaperGraceMs`, measured from the row's OWN liveness
|
|
139
|
+
// (COALESCE(heartbeat_at, started_at)) — NOT from when we began polling.
|
|
140
|
+
// Recover inline just before that point so the foreground claims the run
|
|
141
|
+
// first; anchoring to the row's liveness makes this immune to dispatch
|
|
142
|
+
// latency between insertRun and the start of polling.
|
|
143
|
+
const reaperWillFireSoon = reaperGraceMs != null &&
|
|
144
|
+
claim?.lastLivenessAt != null &&
|
|
145
|
+
elapsedNow - claim.lastLivenessAt >= reaperGraceMs - reaperMargin;
|
|
146
|
+
if (reaperWillFireSoon)
|
|
147
|
+
break;
|
|
148
|
+
// ADAPTIVE GRACE: past the base window, keep polling ONLY while the worker
|
|
149
|
+
// is provably alive and still in setup (heavy cold start). A dead handoff
|
|
150
|
+
// never recorded an ALIVE_IN_SETUP stage, so it recovers inline at the base
|
|
151
|
+
// grace; the reaper-anchored break above bounds how long a live worker can
|
|
152
|
+
// extend. The extension is enabled only when a reaper grace was provided.
|
|
153
|
+
const aliveInSetup = reaperGraceMs != null &&
|
|
154
|
+
claim?.status === "running" &&
|
|
155
|
+
!!stage &&
|
|
156
|
+
ALIVE_IN_SETUP.has(stage);
|
|
157
|
+
if (elapsedNow >= baseDeadline && !aliveInSetup)
|
|
82
158
|
break;
|
|
83
159
|
await sleep(opts.pollIntervalMs);
|
|
84
160
|
}
|
|
@@ -3461,6 +3537,7 @@ export function createProductionAgentHandler(options) {
|
|
|
3461
3537
|
backgroundRowInserted,
|
|
3462
3538
|
runId,
|
|
3463
3539
|
graceMs: BACKGROUND_CLAIM_GRACE_MS,
|
|
3540
|
+
reaperGraceMs: UNCLAIMED_BACKGROUND_RUN_GRACE_MS,
|
|
3464
3541
|
pollIntervalMs: BACKGROUND_CLAIM_POLL_MS,
|
|
3465
3542
|
readClaim: readBackgroundRunClaim,
|
|
3466
3543
|
claim: claimBackgroundRun,
|