@ours.network/fleet 0.17.8 → 0.17.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +4 -4
- package/dist/cli.js +8 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +27 -2
- package/dist/harness/claude-code.js +198 -7
- package/dist/harness/codex.js +10 -1
- package/dist/harness/types.d.ts +50 -1
- package/dist/ops.js +1 -1
- package/dist/owner-channel/channel.d.ts +10 -0
- package/dist/owner-channel/channel.js +28 -13
- package/dist/owner-channel/notices.d.ts +7 -0
- package/dist/owner-channel/notices.js +9 -0
- package/dist/runner.d.ts +39 -0
- package/dist/runner.js +193 -89
- package/dist/session/acp.d.ts +46 -0
- package/dist/session/acp.js +62 -7
- package/dist/session/types.d.ts +11 -0
- package/dist/spawn.js +1 -1
- package/package.json +1 -1
|
@@ -715,11 +715,7 @@ export class OwnerChannel {
|
|
|
715
715
|
origin: { kind: 'owner', requestId,
|
|
716
716
|
...(group.caption ? { displayText: String(group.caption.text ?? '') } : {}) },
|
|
717
717
|
});
|
|
718
|
-
const accepted = this.
|
|
719
|
-
? ownerNotices.receivedInterrupting()
|
|
720
|
-
: queued.queuedBehind > 0
|
|
721
|
-
? ownerNotices.receivedQueued(queued.queuedBehind)
|
|
722
|
-
: ownerNotices.receivedStarted();
|
|
718
|
+
const accepted = this.acceptanceNotice(queued);
|
|
723
719
|
handledWireIds.forEach(wire => this.inFlight.add(wire));
|
|
724
720
|
const receipt = this.send(sender.id, accepted, originWireId).then(() => undefined).catch(error => {
|
|
725
721
|
this.logError(`attachment request ${requestId.slice(0, 12)} acceptance notice failed`, error);
|
|
@@ -860,14 +856,7 @@ export class OwnerChannel {
|
|
|
860
856
|
this.state.remember(wireId);
|
|
861
857
|
return true;
|
|
862
858
|
}
|
|
863
|
-
|
|
864
|
-
// accepted into the ACP queue. Never claim this request is running while
|
|
865
|
-
// the session itself says earlier work remains ahead of it.
|
|
866
|
-
const accepted = queued.queuedBehind > 0
|
|
867
|
-
? ownerNotices.receivedQueued(queued.queuedBehind)
|
|
868
|
-
: this.options.config.interrupt
|
|
869
|
-
? ownerNotices.receivedInterrupting()
|
|
870
|
-
: ownerNotices.receivedStarted();
|
|
859
|
+
const accepted = this.acceptanceNotice(queued);
|
|
871
860
|
this.inFlight.add(wireId);
|
|
872
861
|
const receipt = this.send(sender.id, accepted, wireId).then(() => undefined).catch(error => {
|
|
873
862
|
this.logError(`request ${requestId.slice(0, 12)} acceptance notice failed`, error);
|
|
@@ -1253,6 +1242,32 @@ export class OwnerChannel {
|
|
|
1253
1242
|
authorizationIntegrity() {
|
|
1254
1243
|
return this.options.config.agent ? { ok: true } : this.authorizations.integrity();
|
|
1255
1244
|
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Report what the session actually did with the prompt, not what the config
|
|
1247
|
+
* asked for. `interrupt: true` used to be reported as "your request
|
|
1248
|
+
* interrupted the previous task" unconditionally; the session now answers
|
|
1249
|
+
* whether anything was cancelled, whether the request is queued behind
|
|
1250
|
+
* earlier prompts, or whether it is held until the current task reaches a
|
|
1251
|
+
* safe stopping point. Backends that report no delivery state keep the old
|
|
1252
|
+
* queuedBehind-based wording.
|
|
1253
|
+
*/
|
|
1254
|
+
acceptanceNotice(queued) {
|
|
1255
|
+
switch (queued.delivery) {
|
|
1256
|
+
case 'interrupted': return ownerNotices.receivedInterrupting();
|
|
1257
|
+
case 'deferred': return ownerNotices.receivedDeferred();
|
|
1258
|
+
case 'queued': return ownerNotices.receivedQueued(Math.max(1, queued.queuedBehind));
|
|
1259
|
+
case 'started': return ownerNotices.receivedStarted();
|
|
1260
|
+
default:
|
|
1261
|
+
// Interrupting the live turn does not remove prompts which were already
|
|
1262
|
+
// accepted into the ACP queue. Never claim this request is running while
|
|
1263
|
+
// the session itself says earlier work remains ahead of it.
|
|
1264
|
+
return queued.queuedBehind > 0
|
|
1265
|
+
? ownerNotices.receivedQueued(queued.queuedBehind)
|
|
1266
|
+
: this.options.config.interrupt
|
|
1267
|
+
? ownerNotices.receivedInterrupting()
|
|
1268
|
+
: ownerNotices.receivedStarted();
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1256
1271
|
async complete(active, outbox, queued, activityCursor) {
|
|
1257
1272
|
const progressMs = this.options.config.progress_interval_ms;
|
|
1258
1273
|
let lastSeq = activityCursor;
|
|
@@ -22,6 +22,13 @@ export declare const ownerNotices: {
|
|
|
22
22
|
receivedStarted: () => string;
|
|
23
23
|
receivedQueued: (queuedBehind: number) => string;
|
|
24
24
|
receivedInterrupting: () => string;
|
|
25
|
+
/**
|
|
26
|
+
* The honest answer when the agent is mid-task and pre-empting it would have
|
|
27
|
+
* corrupted the conversation. Says "not started yet" rather than borrowing
|
|
28
|
+
* `receivedInterrupting`'s claim that something was cancelled for this
|
|
29
|
+
* request.
|
|
30
|
+
*/
|
|
31
|
+
receivedDeferred: () => string;
|
|
25
32
|
status: (role: string, snapshot: SessionSnapshot) => string;
|
|
26
33
|
interrupted: (role: string) => string;
|
|
27
34
|
/** The turn IS cancelled — say how, without implying the owner must retry. */
|
|
@@ -26,6 +26,15 @@ export const ownerNotices = {
|
|
|
26
26
|
receivedInterrupting: () => "ℹ️ Message received. The agent's previous task was interrupted to prioritize "
|
|
27
27
|
+ 'this request, and it is now working on a response. '
|
|
28
28
|
+ 'The response will arrive in this channel when ready.',
|
|
29
|
+
/**
|
|
30
|
+
* The honest answer when the agent is mid-task and pre-empting it would have
|
|
31
|
+
* corrupted the conversation. Says "not started yet" rather than borrowing
|
|
32
|
+
* `receivedInterrupting`'s claim that something was cancelled for this
|
|
33
|
+
* request.
|
|
34
|
+
*/
|
|
35
|
+
receivedDeferred: () => 'ℹ️ Message received and held. The agent is in the middle of a task that '
|
|
36
|
+
+ 'cannot be interrupted safely; this request starts as soon as that work '
|
|
37
|
+
+ 'reaches a stopping point. The response will arrive in this channel when ready.',
|
|
29
38
|
status: (role, snapshot) => `📊 ${role} status: ${snapshot.readiness}; session is ${snapshot.alive ? 'online' : 'offline'}.`,
|
|
30
39
|
interrupted: (role) => `🛑 Interrupt sent to ${role}'s active turn.`,
|
|
31
40
|
/** The turn IS cancelled — say how, without implying the owner must retry. */
|
package/dist/runner.d.ts
CHANGED
|
@@ -61,6 +61,23 @@ export declare function readExitRecord(path: string): ExitRecord | null;
|
|
|
61
61
|
export declare const RESTART_LEDGER_FILE = ".restart-ledger.json";
|
|
62
62
|
/** Consecutive immediate failures tolerated before the agent is held down. */
|
|
63
63
|
export declare const RESTART_FAIL_THRESHOLD = 5;
|
|
64
|
+
/**
|
|
65
|
+
* How the previous supervisor process ended.
|
|
66
|
+
*
|
|
67
|
+
* `abrupt` is the case the ledger used to miss entirely: an OOM-kill or any
|
|
68
|
+
* other external signal takes the supervisor down before it can write anything,
|
|
69
|
+
* the service manager restarts the unit, and every durable indicator still
|
|
70
|
+
* describes the run that died. A health check reading them reported "no
|
|
71
|
+
* restarts" for a role that had died and come back.
|
|
72
|
+
*/
|
|
73
|
+
export interface TerminationRecord {
|
|
74
|
+
class: 'clean' | 'abrupt' | 'unknown';
|
|
75
|
+
detail: string;
|
|
76
|
+
/** When the SURVIVING process observed it, not when it happened. */
|
|
77
|
+
observedAt: string;
|
|
78
|
+
/** Start time of the run that ended, when it was recorded. */
|
|
79
|
+
runStartedAt?: string;
|
|
80
|
+
}
|
|
64
81
|
export interface RestartLedger {
|
|
65
82
|
version: 1;
|
|
66
83
|
consecutiveImmediateFailures: number;
|
|
@@ -72,7 +89,29 @@ export interface RestartLedger {
|
|
|
72
89
|
updatedAt: string;
|
|
73
90
|
/** When the circuit opened, for the held-down status line. */
|
|
74
91
|
openedAt?: string;
|
|
92
|
+
/** How the previous supervisor process ended, including abnormal exits. */
|
|
93
|
+
lastTermination?: TerminationRecord;
|
|
94
|
+
/** Supervisor processes that died without closing their run marker. */
|
|
95
|
+
abruptTerminations?: number;
|
|
96
|
+
/** Start of the supervisor run that owns this state directory now. */
|
|
97
|
+
supervisorStartedAt?: string;
|
|
75
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Carried across a supervisor process's life so its successor can tell an
|
|
101
|
+
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
102
|
+
* running"; the next start finding one that is not its own is proof the
|
|
103
|
+
* previous process died without getting to write anything.
|
|
104
|
+
*/
|
|
105
|
+
export declare const RUN_MARKER_FILE = ".supervisor-run.json";
|
|
106
|
+
/**
|
|
107
|
+
* Claim this state directory for the current supervisor process and report how
|
|
108
|
+
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
109
|
+
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
110
|
+
* and an attempt can take minutes.
|
|
111
|
+
*/
|
|
112
|
+
export declare function claimSupervisorRun(dir: string, startedAt: string, pid?: number): TerminationRecord;
|
|
113
|
+
/** Orderly exit: the successor must not read this run as a kill. */
|
|
114
|
+
export declare function releaseSupervisorRun(dir: string): void;
|
|
76
115
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
77
116
|
export declare function backoffFor(consecutiveFailures: number): number;
|
|
78
117
|
/** Read a role's restart ledger; a missing or corrupt one starts clean. */
|
package/dist/runner.js
CHANGED
|
@@ -230,6 +230,66 @@ const emptyLedger = () => ({
|
|
|
230
230
|
circuit: 'closed',
|
|
231
231
|
updatedAt: new Date(0).toISOString(),
|
|
232
232
|
});
|
|
233
|
+
/**
|
|
234
|
+
* Carried across a supervisor process's life so its successor can tell an
|
|
235
|
+
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
236
|
+
* running"; the next start finding one that is not its own is proof the
|
|
237
|
+
* previous process died without getting to write anything.
|
|
238
|
+
*/
|
|
239
|
+
export const RUN_MARKER_FILE = '.supervisor-run.json';
|
|
240
|
+
function readRunMarker(dir) {
|
|
241
|
+
try {
|
|
242
|
+
const raw = JSON.parse(readFileSync(join(dir, RUN_MARKER_FILE), 'utf8'));
|
|
243
|
+
return raw.version === 1 && typeof raw.pid === 'number' && typeof raw.startedAt === 'string'
|
|
244
|
+
? raw : undefined;
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Claim this state directory for the current supervisor process and report how
|
|
252
|
+
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
253
|
+
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
254
|
+
* and an attempt can take minutes.
|
|
255
|
+
*/
|
|
256
|
+
export function claimSupervisorRun(dir, startedAt, pid = process.pid) {
|
|
257
|
+
const previous = readRunMarker(dir);
|
|
258
|
+
const termination = previous && previous.pid !== pid
|
|
259
|
+
? {
|
|
260
|
+
class: 'abrupt',
|
|
261
|
+
detail: `supervisor pid ${previous.pid} left an open run marker; `
|
|
262
|
+
+ 'it was terminated without an orderly exit (signal, OOM-kill, or host reset)',
|
|
263
|
+
observedAt: startedAt,
|
|
264
|
+
runStartedAt: previous.startedAt,
|
|
265
|
+
}
|
|
266
|
+
: previous
|
|
267
|
+
? { class: 'unknown', detail: 'run marker belongs to this process', observedAt: startedAt }
|
|
268
|
+
: { class: 'clean', detail: 'no previous run marker', observedAt: startedAt };
|
|
269
|
+
try {
|
|
270
|
+
mkdirSync(dir, { recursive: true });
|
|
271
|
+
writeFileSync(join(dir, RUN_MARKER_FILE), JSON.stringify({ version: 1, pid, startedAt }, null, 2) + '\n');
|
|
272
|
+
}
|
|
273
|
+
catch { /* diagnostics must never take the role down */ }
|
|
274
|
+
return termination;
|
|
275
|
+
}
|
|
276
|
+
/** Orderly exit: the successor must not read this run as a kill. */
|
|
277
|
+
export function releaseSupervisorRun(dir) {
|
|
278
|
+
try {
|
|
279
|
+
rmSync(join(dir, RUN_MARKER_FILE), { force: true });
|
|
280
|
+
}
|
|
281
|
+
catch { /* best effort */ }
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Fields that describe THIS process's history rather than the current failure
|
|
285
|
+
* streak. Clearing the streak (recovery, an operator `up`, an approved model
|
|
286
|
+
* transition) must not erase the record that the role died and came back.
|
|
287
|
+
*/
|
|
288
|
+
const carriedForward = (previous) => ({
|
|
289
|
+
...(previous.lastTermination ? { lastTermination: previous.lastTermination } : {}),
|
|
290
|
+
...(previous.abruptTerminations ? { abruptTerminations: previous.abruptTerminations } : {}),
|
|
291
|
+
...(previous.supervisorStartedAt ? { supervisorStartedAt: previous.supervisorStartedAt } : {}),
|
|
292
|
+
});
|
|
233
293
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
234
294
|
export function backoffFor(consecutiveFailures) {
|
|
235
295
|
if (consecutiveFailures <= 0)
|
|
@@ -264,7 +324,10 @@ export function writeRestartLedger(dir, ledger) {
|
|
|
264
324
|
export function resetRestartLedger(dir) {
|
|
265
325
|
if (!existsSync(dir))
|
|
266
326
|
return;
|
|
267
|
-
|
|
327
|
+
const previous = readRestartLedger(dir);
|
|
328
|
+
writeRestartLedger(dir, {
|
|
329
|
+
...emptyLedger(), ...carriedForward(previous), updatedAt: new Date().toISOString(),
|
|
330
|
+
});
|
|
268
331
|
}
|
|
269
332
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
270
333
|
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
@@ -414,8 +477,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
414
477
|
const exitFile = join(dir, '.exit-status');
|
|
415
478
|
const booted = existsSync(bootedFile);
|
|
416
479
|
const mode = booted && adapter.supportsResume ? 'resume' : 'fresh';
|
|
417
|
-
|
|
418
|
-
|
|
480
|
+
// Stamp EVERY attempt, not just the first. `.booted` used to be written only
|
|
481
|
+
// on the fresh path, so after a restart — including one the supervisor never
|
|
482
|
+
// saw, like an OOM-kill — its mtime still read the original boot and any
|
|
483
|
+
// health check reading it reported "no restarts". The existence test above
|
|
484
|
+
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
485
|
+
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
419
486
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
420
487
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
421
488
|
const sessionBackend = role.session ?? 'tmux';
|
|
@@ -541,6 +608,13 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
541
608
|
mode,
|
|
542
609
|
permissions: perms,
|
|
543
610
|
modeId: adapter.acpPermissionModeId?.(role),
|
|
611
|
+
// The role's declared MCP servers, and the bundled agent's `_meta`
|
|
612
|
+
// vocabulary for the options it takes no flag for. Both come from the
|
|
613
|
+
// ADAPTER and from `prep`: the ACP launch cannot carry `prep.argv`, so this
|
|
614
|
+
// is the route by which harness_options that used to be silently dropped
|
|
615
|
+
// for an ACP role actually reach the session.
|
|
616
|
+
mcpServers: adapter.acpMcpServers?.(role),
|
|
617
|
+
sessionMeta: adapter.acpSessionMeta?.(role, prep),
|
|
544
618
|
permissionMode: effectivePermissionMode(role),
|
|
545
619
|
// Provenance travels with the exact ACP launch. Keeping it out of a
|
|
546
620
|
// role-only adapter hook prevents a PATH fallback or resolver skew from
|
|
@@ -943,99 +1017,129 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
943
1017
|
mkdirSync(dir, { recursive: true });
|
|
944
1018
|
const shouldStop = deps.shouldStop ?? (() => false);
|
|
945
1019
|
const stamp = () => new Date(deps.now()).toISOString();
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
1020
|
+
// Record how the PREVIOUS supervisor process ended before doing anything
|
|
1021
|
+
// else. An external kill writes nothing itself, and the next ledger write is
|
|
1022
|
+
// an attempt away — which is why an OOM-kill used to leave every durable
|
|
1023
|
+
// indicator describing the run that died.
|
|
1024
|
+
const startedAt = stamp();
|
|
1025
|
+
const termination = claimSupervisorRun(dir, startedAt);
|
|
1026
|
+
{
|
|
1027
|
+
const previous = readRestartLedger(dir);
|
|
1028
|
+
const abrupt = (previous.abruptTerminations ?? 0) + (termination.class === 'abrupt' ? 1 : 0);
|
|
1029
|
+
writeRestartLedger(dir, {
|
|
1030
|
+
...previous,
|
|
1031
|
+
lastTermination: termination,
|
|
1032
|
+
abruptTerminations: abrupt,
|
|
1033
|
+
supervisorStartedAt: startedAt,
|
|
1034
|
+
updatedAt: startedAt,
|
|
1035
|
+
});
|
|
1036
|
+
if (termination.class === 'abrupt')
|
|
1037
|
+
deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
|
|
1038
|
+
+ `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
|
|
1039
|
+
}
|
|
1040
|
+
try {
|
|
1041
|
+
while (!shouldStop()) {
|
|
1042
|
+
let ledger = readRestartLedger(dir);
|
|
1043
|
+
try {
|
|
1044
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
1045
|
+
const role = findRole(loadConfig(configPath), name);
|
|
1046
|
+
reconcileModelRecovery(dir, role, stamp());
|
|
1047
|
+
if (modelRecoveryHeld(dir)) {
|
|
1048
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
catch {
|
|
1053
|
+
// Normal attempt path reports config errors through the restart circuit.
|
|
1054
|
+
}
|
|
1055
|
+
if (ledger.circuit === 'open') {
|
|
1056
|
+
// Held down. Stay alive — exiting would hand the role straight back to
|
|
1057
|
+
// the service manager — and watch for an operator reset.
|
|
953
1058
|
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
954
1059
|
continue;
|
|
955
1060
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
//
|
|
972
|
-
|
|
973
|
-
result
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1061
|
+
let result;
|
|
1062
|
+
try {
|
|
1063
|
+
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
1064
|
+
}
|
|
1065
|
+
catch (e) {
|
|
1066
|
+
// A session that could not even start is an immediate failure like any
|
|
1067
|
+
// other; it must count, or an unstartable role loops forever.
|
|
1068
|
+
result = {
|
|
1069
|
+
elapsedSecs: 0,
|
|
1070
|
+
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
1071
|
+
rotated: false,
|
|
1072
|
+
mode: 'fresh',
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1076
|
+
// have reset the ledger meanwhile.
|
|
1077
|
+
ledger = readRestartLedger(dir);
|
|
1078
|
+
if (result.modelRecovery === 'advance') {
|
|
1079
|
+
writeRestartLedger(dir, {
|
|
1080
|
+
...emptyLedger(),
|
|
1081
|
+
...carriedForward(ledger),
|
|
1082
|
+
lastReason: 'approved model-chain transition',
|
|
1083
|
+
updatedAt: stamp(),
|
|
1084
|
+
});
|
|
1085
|
+
continue;
|
|
1086
|
+
}
|
|
1087
|
+
if (result.modelRecovery === 'hold') {
|
|
1088
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
const fastFailSecs = fastFailSecsFor(name, opts.configPath);
|
|
1092
|
+
// The fast-fail boundary starts a recovery episode; it must not also be
|
|
1093
|
+
// the boundary that declares recovery successful. Otherwise alternating
|
|
1094
|
+
// 19s and 20s deaths erase one another forever. Require the configured
|
|
1095
|
+
// number of fast-fail windows to survive before closing an active streak.
|
|
1096
|
+
// This hysteresis stays adapter-relative (100s for the current 20s/5-attempt
|
|
1097
|
+
// policy) and still lets a genuinely sustained session reset the breaker.
|
|
1098
|
+
const stableRecoverySecs = fastFailSecs * RESTART_FAIL_THRESHOLD;
|
|
1099
|
+
const recoveryFailed = result.elapsedSecs < fastFailSecs
|
|
1100
|
+
|| (ledger.consecutiveImmediateFailures > 0 && result.elapsedSecs < stableRecoverySecs);
|
|
1101
|
+
if (!recoveryFailed) {
|
|
1102
|
+
// A session that ran for a while is not a restart loop, whatever ended it.
|
|
1103
|
+
writeRestartLedger(dir, {
|
|
1104
|
+
...emptyLedger(),
|
|
1105
|
+
...carriedForward(ledger),
|
|
1106
|
+
lastReason: result.exit.detail,
|
|
1107
|
+
updatedAt: stamp(),
|
|
1108
|
+
});
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
1112
|
+
const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
|
|
1113
|
+
const next = {
|
|
1114
|
+
version: 1,
|
|
1115
|
+
...carriedForward(ledger),
|
|
1116
|
+
consecutiveImmediateFailures: failures,
|
|
1117
|
+
lastReason: reason,
|
|
1118
|
+
nextDelayMs: backoffFor(failures),
|
|
1119
|
+
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1120
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
1010
1121
|
updatedAt: stamp(),
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
nextDelayMs: backoffFor(failures),
|
|
1021
|
-
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1022
|
-
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
1023
|
-
updatedAt: stamp(),
|
|
1024
|
-
};
|
|
1025
|
-
if (next.circuit === 'open') {
|
|
1026
|
-
next.openedAt = stamp();
|
|
1027
|
-
next.nextDelayMs = 0;
|
|
1122
|
+
};
|
|
1123
|
+
if (next.circuit === 'open') {
|
|
1124
|
+
next.openedAt = stamp();
|
|
1125
|
+
next.nextDelayMs = 0;
|
|
1126
|
+
writeRestartLedger(dir, next);
|
|
1127
|
+
deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
|
|
1128
|
+
`${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
|
|
1129
|
+
continue;
|
|
1130
|
+
}
|
|
1028
1131
|
writeRestartLedger(dir, next);
|
|
1029
|
-
deps.log(`[${name}]
|
|
1030
|
-
|
|
1031
|
-
|
|
1132
|
+
deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
|
|
1133
|
+
`-> backing off ${next.nextDelayMs}ms`);
|
|
1134
|
+
await deps.sleep(next.nextDelayMs);
|
|
1032
1135
|
}
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1136
|
+
return readRestartLedger(dir);
|
|
1137
|
+
}
|
|
1138
|
+
finally {
|
|
1139
|
+
// Only an orderly return through this loop clears the marker; a signal or
|
|
1140
|
+
// an OOM-kill leaves it, which is exactly how the successor detects them.
|
|
1141
|
+
releaseSupervisorRun(dir);
|
|
1037
1142
|
}
|
|
1038
|
-
return readRestartLedger(dir);
|
|
1039
1143
|
}
|
|
1040
1144
|
/**
|
|
1041
1145
|
* How short an attempt has to be to count as immediate. The role's harness
|
package/dist/session/acp.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as acp from '@agentclientprotocol/sdk';
|
|
2
2
|
import type { CommonPermissions } from '../config.js';
|
|
3
|
+
import type { AcpMcpServer } from '../harness/types.js';
|
|
3
4
|
import { ConversationEventStore } from './conversation-store.js';
|
|
4
5
|
import type { ConversationSnapshot, PromptOrigin, PromptReceipt, SubmitPromptCommand } from './conversation-types.js';
|
|
5
6
|
import type { ConversationHandlePage, ExitRecord, InterruptOutcome, QueuedPrompt, SessionEvent, RuntimeSelectorMetadata, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnOutcome, TurnResult } from './types.js';
|
|
@@ -37,6 +38,20 @@ export interface AcpSessionOptions {
|
|
|
37
38
|
permissionMode?: NonNullable<SessionSnapshot['permissionMode']>;
|
|
38
39
|
/** Adapter-authenticated request-metadata vocabulary; never inferred from ACP `_meta`. */
|
|
39
40
|
permissionMetadataSource?: 'codex-acp';
|
|
41
|
+
/**
|
|
42
|
+
* MCP servers the ROLE declares, for every session/new, resume and load. Empty
|
|
43
|
+
* or omitted sends `[]`, which is what fleet has always sent and leaves the
|
|
44
|
+
* agent's own configuration untouched.
|
|
45
|
+
*/
|
|
46
|
+
mcpServers?: AcpMcpServer[];
|
|
47
|
+
/**
|
|
48
|
+
* Adapter-supplied `_meta` for session/new — the only route by which a
|
|
49
|
+
* capability the CLI takes as a flag reaches an agent that accepts none.
|
|
50
|
+
* Per-agent vocabulary, so the ADAPTER decides whether there is anything to
|
|
51
|
+
* send; this layer only forwards it. Never sent on resume or load: it carries
|
|
52
|
+
* session-creation options the agent has already applied.
|
|
53
|
+
*/
|
|
54
|
+
sessionMeta?: Record<string, unknown>;
|
|
40
55
|
log(line: string): void;
|
|
41
56
|
/** Test seam for the cancel-escalation grace period; production uses the default. */
|
|
42
57
|
cancelGraceMs?: number;
|
|
@@ -172,6 +187,28 @@ export declare class AcpSession implements SessionHandle {
|
|
|
172
187
|
* for it is what turned a busy agent into a timeout and then into "dead".
|
|
173
188
|
*/
|
|
174
189
|
queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
|
|
190
|
+
/**
|
|
191
|
+
* Prepare the session for a prompt that asked to pre-empt current work.
|
|
192
|
+
*
|
|
193
|
+
* The old behaviour was one unconditional `session/cancel` notification
|
|
194
|
+
* followed immediately by `session/prompt`. That is what produced the owner's
|
|
195
|
+
* "request failed before completion":
|
|
196
|
+
*
|
|
197
|
+
* - `cancelActive` only awaits settlement when `this.activeTurn` is set, and
|
|
198
|
+
* a turn the ADAPTER started (steering's `startedNewTurn`) is never tracked
|
|
199
|
+
* here. So the cancel raced the adapter's own transcript repair and the new
|
|
200
|
+
* prompt landed while the last assistant message still held an unresolved
|
|
201
|
+
* `tool_use` — rejected with `stop_reason=tool_use`.
|
|
202
|
+
* - With nothing running at all, it still sent the cancel, and the prompt
|
|
203
|
+
* landed on a bare interrupted user message — rejected with
|
|
204
|
+
* `stop_reason=null`.
|
|
205
|
+
*
|
|
206
|
+
* So: never cancel across a tool boundary, and never cancel something whose
|
|
207
|
+
* settlement cannot be awaited. Everything else is queued, which the ACP queue
|
|
208
|
+
* already does correctly. The returned state is what the caller may claim to a
|
|
209
|
+
* human — `interrupted` only when a turn really was cancelled.
|
|
210
|
+
*/
|
|
211
|
+
private prepareInterruptingDelivery;
|
|
175
212
|
/**
|
|
176
213
|
* Durably record a prompt admission BEFORE acceptance is returned. Browser
|
|
177
214
|
* admissions are transactional — a prompt the ledger cannot hold is refused,
|
|
@@ -219,6 +256,15 @@ export declare class AcpSession implements SessionHandle {
|
|
|
219
256
|
private settlePendingAutomatically;
|
|
220
257
|
exitResult(): ExitRecord | null;
|
|
221
258
|
close(): Promise<void>;
|
|
259
|
+
/**
|
|
260
|
+
* The role's declared MCP servers, or `[]`.
|
|
261
|
+
*
|
|
262
|
+
* Sent on resume and load as well as on new: the agent builds its server set
|
|
263
|
+
* once per session, so a resumed session that omitted them would come back
|
|
264
|
+
* without the tools the role's config declares — which is exactly the shape of
|
|
265
|
+
* silent drop this plumbing exists to end.
|
|
266
|
+
*/
|
|
267
|
+
private declaredMcpServers;
|
|
222
268
|
private initialize;
|
|
223
269
|
private captureRuntimeMetadata;
|
|
224
270
|
private runPrompt;
|