@ours.network/fleet 0.18.0-nightly.6 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -43
- package/dist/application/fleet-query-service.js +12 -0
- package/dist/application/role-creation-service.js +3 -1
- package/dist/application/types.d.ts +11 -0
- package/dist/briefing.js +9 -2
- package/dist/build-info.json +7 -6
- package/dist/capabilities.d.ts +3 -1
- package/dist/capabilities.js +3 -0
- package/dist/cli.js +83 -7
- package/dist/config.d.ts +11 -3
- package/dist/config.js +40 -15
- package/dist/creation.d.ts +14 -15
- package/dist/creation.js +19 -13
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +117 -35
- package/dist/doctor.d.ts +1 -5
- package/dist/doctor.js +11 -18
- package/dist/fleet-proxy.d.ts +5 -0
- package/dist/harness/acp-agent.js +11 -6
- package/dist/harness/claude-code.js +204 -11
- package/dist/harness/codex.d.ts +4 -1
- package/dist/harness/codex.js +74 -12
- package/dist/harness/types.d.ts +54 -4
- package/dist/harness-plugins.d.ts +48 -0
- package/dist/harness-plugins.js +309 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/loops/manager.d.ts +30 -1
- package/dist/loops/manager.js +69 -6
- package/dist/loops/state.d.ts +18 -0
- package/dist/loops/state.js +4 -0
- package/dist/model-env.d.ts +71 -0
- package/dist/model-env.js +106 -0
- package/dist/monitor.js +1 -1
- package/dist/ops.js +1 -1
- package/dist/owner-channel/attachments.d.ts +2 -25
- package/dist/owner-channel/attachments.js +5 -61
- package/dist/owner-channel/channel.d.ts +30 -29
- package/dist/owner-channel/channel.js +291 -291
- package/dist/owner-channel/mcp.d.ts +24 -0
- package/dist/owner-channel/mcp.js +145 -0
- package/dist/owner-channel/notices.d.ts +7 -0
- package/dist/owner-channel/notices.js +9 -0
- package/dist/resolved-plan.js +1 -0
- package/dist/runner.d.ts +48 -0
- package/dist/runner.js +237 -85
- package/dist/session/acp.d.ts +104 -0
- package/dist/session/acp.js +213 -10
- package/dist/session/activity.d.ts +31 -0
- package/dist/session/activity.js +48 -0
- package/dist/session/conversation-normalizer.d.ts +6 -0
- package/dist/session/conversation-normalizer.js +153 -10
- package/dist/session/conversation-types.d.ts +23 -4
- package/dist/session/types.d.ts +35 -0
- package/dist/spawn.js +29 -17
- package/dist/supervisor/systemd.js +2 -29
- package/dist/watchdog/briefing.js +7 -0
- package/dist/web-app/assets/{TerminalView-BAVk1Bot.js → TerminalView-C_G1ID2P.js} +1 -1
- package/dist/web-app/assets/{index-C3S-xFRU.js → index-BCBK78hw.js} +5 -5
- package/dist/web-app/index.html +1 -1
- package/dist/worklog.d.ts +7 -1
- package/dist/worklog.js +191 -39
- package/package.json +1 -3
- package/dist/owner-channel/message-recovery.d.ts +0 -25
- package/dist/owner-channel/message-recovery.js +0 -114
- package/dist/owner-channel/ours-client.d.ts +0 -148
- package/dist/owner-channel/ours-client.js +0 -231
package/dist/runner.js
CHANGED
|
@@ -24,6 +24,7 @@ import { RoleTurnArbiter } from './session/arbiter.js';
|
|
|
24
24
|
import { ScheduledLoopManager, } from './loops/manager.js';
|
|
25
25
|
import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, inheritCallerSpawnDefaults, } from './fleet-proxy.js';
|
|
26
26
|
import { effectivePermissionMode } from './permissions.js';
|
|
27
|
+
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
27
28
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
28
29
|
const defaultDeps = () => ({
|
|
29
30
|
tmux: new Tmux(),
|
|
@@ -59,14 +60,32 @@ const defaultDeps = () => ({
|
|
|
59
60
|
},
|
|
60
61
|
});
|
|
61
62
|
const MONITOR_OWNER_FILE = '.monitor-owner';
|
|
63
|
+
/** Fleet roles consume the operator-owned daemon; a role session never starts it. */
|
|
64
|
+
const FLEET_OURS_AUTOSTART = '0';
|
|
62
65
|
/** Environment injected only into the managed harness process. */
|
|
63
66
|
export function managedFleetProxyEnv(role, stateDir) {
|
|
64
67
|
return {
|
|
65
68
|
...(role.env ?? {}),
|
|
69
|
+
// This must win over both inherited/configured auto-start. ACP agents run
|
|
70
|
+
// directly rather than through ours-codex, so the runner owns this fence.
|
|
71
|
+
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
66
72
|
[FLEET_PROXY_STATE_DIR_ENV]: stateDir,
|
|
67
73
|
[FLEET_PROXY_CALLER_ENV]: role.name,
|
|
68
74
|
};
|
|
69
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* The environment a managed harness child actually receives, checked at the one
|
|
78
|
+
* point where it is composed. `role.env` deliberately wins over harness prep,
|
|
79
|
+
* which is exactly how a stale fleet-wide model pin used to outrank the model
|
|
80
|
+
* the role was spawned with — so the model pin is verified here rather than
|
|
81
|
+
* trusted, and a disagreement stops the launch instead of being reported as a
|
|
82
|
+
* success (see src/model-env.ts).
|
|
83
|
+
*/
|
|
84
|
+
export function harnessChildEnv(role, launchEnv, stateDir) {
|
|
85
|
+
const env = { ...(launchEnv ?? {}), ...managedFleetProxyEnv(role, stateDir) };
|
|
86
|
+
assertModelPinReachesChild(role, env);
|
|
87
|
+
return env;
|
|
88
|
+
}
|
|
70
89
|
/**
|
|
71
90
|
* Execute a typed proxy request in the caller's supervisor. Dynamic imports
|
|
72
91
|
* avoid a runner↔spawn initialization cycle (spawn imports runner constants).
|
|
@@ -104,13 +123,19 @@ async function executeManagedSpawn(caller, configPath, requested, log) {
|
|
|
104
123
|
statePath,
|
|
105
124
|
harness: preview.harness,
|
|
106
125
|
session: preview.session,
|
|
107
|
-
|
|
126
|
+
// Read back from the resolved environment, not from the request: the banner
|
|
127
|
+
// must name the model the child will run, not the one that was asked for.
|
|
128
|
+
...(effectiveRoleModel(preview) ? { model: effectiveRoleModel(preview) } : {}),
|
|
108
129
|
monitor: { mode: preview.monitor.mode, interrupt: preview.monitor.interrupt },
|
|
130
|
+
permissionMode: effectivePermissionMode(preview),
|
|
109
131
|
inherited,
|
|
110
132
|
creationActionId,
|
|
111
133
|
};
|
|
112
134
|
log(`[${caller.name}] managed fleet proxy spawned ${result.lifetime} role ${result.role} `
|
|
113
|
-
+ `harness=${result.harness} session=${result.session}`
|
|
135
|
+
+ `harness=${result.harness} session=${result.session} `
|
|
136
|
+
+ `model=${result.model ?? '(harness default)'} `
|
|
137
|
+
+ `permission=${result.permissionMode.fleetMode} `
|
|
138
|
+
+ `native=${result.permissionMode.nativeMode}`);
|
|
114
139
|
return result;
|
|
115
140
|
}
|
|
116
141
|
/**
|
|
@@ -140,6 +165,10 @@ export function recordMonitorOwner(dir, owner) {
|
|
|
140
165
|
export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = launch.argv) {
|
|
141
166
|
const env = {
|
|
142
167
|
PATH: process.env.PATH ?? '', COLORTERM: 'truecolor', ...launch.env, ...(roleEnv ?? {}),
|
|
168
|
+
// Tmux roles have the same daemon-client boundary as ACP roles. Keep this
|
|
169
|
+
// last so neither harness preparation nor a role env block can take over
|
|
170
|
+
// the shared daemon lifecycle.
|
|
171
|
+
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
143
172
|
};
|
|
144
173
|
// Interactive panes should advertise colour even when the supervisor itself
|
|
145
174
|
// was launched with NO_COLOR. A role may still deliberately opt back in to
|
|
@@ -218,6 +247,66 @@ const emptyLedger = () => ({
|
|
|
218
247
|
circuit: 'closed',
|
|
219
248
|
updatedAt: new Date(0).toISOString(),
|
|
220
249
|
});
|
|
250
|
+
/**
|
|
251
|
+
* Carried across a supervisor process's life so its successor can tell an
|
|
252
|
+
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
253
|
+
* running"; the next start finding one that is not its own is proof the
|
|
254
|
+
* previous process died without getting to write anything.
|
|
255
|
+
*/
|
|
256
|
+
export const RUN_MARKER_FILE = '.supervisor-run.json';
|
|
257
|
+
function readRunMarker(dir) {
|
|
258
|
+
try {
|
|
259
|
+
const raw = JSON.parse(readFileSync(join(dir, RUN_MARKER_FILE), 'utf8'));
|
|
260
|
+
return raw.version === 1 && typeof raw.pid === 'number' && typeof raw.startedAt === 'string'
|
|
261
|
+
? raw : undefined;
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Claim this state directory for the current supervisor process and report how
|
|
269
|
+
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
270
|
+
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
271
|
+
* and an attempt can take minutes.
|
|
272
|
+
*/
|
|
273
|
+
export function claimSupervisorRun(dir, startedAt, pid = process.pid) {
|
|
274
|
+
const previous = readRunMarker(dir);
|
|
275
|
+
const termination = previous && previous.pid !== pid
|
|
276
|
+
? {
|
|
277
|
+
class: 'abrupt',
|
|
278
|
+
detail: `supervisor pid ${previous.pid} left an open run marker; `
|
|
279
|
+
+ 'it was terminated without an orderly exit (signal, OOM-kill, or host reset)',
|
|
280
|
+
observedAt: startedAt,
|
|
281
|
+
runStartedAt: previous.startedAt,
|
|
282
|
+
}
|
|
283
|
+
: previous
|
|
284
|
+
? { class: 'unknown', detail: 'run marker belongs to this process', observedAt: startedAt }
|
|
285
|
+
: { class: 'clean', detail: 'no previous run marker', observedAt: startedAt };
|
|
286
|
+
try {
|
|
287
|
+
mkdirSync(dir, { recursive: true });
|
|
288
|
+
writeFileSync(join(dir, RUN_MARKER_FILE), JSON.stringify({ version: 1, pid, startedAt }, null, 2) + '\n');
|
|
289
|
+
}
|
|
290
|
+
catch { /* diagnostics must never take the role down */ }
|
|
291
|
+
return termination;
|
|
292
|
+
}
|
|
293
|
+
/** Orderly exit: the successor must not read this run as a kill. */
|
|
294
|
+
export function releaseSupervisorRun(dir) {
|
|
295
|
+
try {
|
|
296
|
+
rmSync(join(dir, RUN_MARKER_FILE), { force: true });
|
|
297
|
+
}
|
|
298
|
+
catch { /* best effort */ }
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Fields that describe THIS process's history rather than the current failure
|
|
302
|
+
* streak. Clearing the streak (recovery, an operator `up`, an approved model
|
|
303
|
+
* transition) must not erase the record that the role died and came back.
|
|
304
|
+
*/
|
|
305
|
+
const carriedForward = (previous) => ({
|
|
306
|
+
...(previous.lastTermination ? { lastTermination: previous.lastTermination } : {}),
|
|
307
|
+
...(previous.abruptTerminations ? { abruptTerminations: previous.abruptTerminations } : {}),
|
|
308
|
+
...(previous.supervisorStartedAt ? { supervisorStartedAt: previous.supervisorStartedAt } : {}),
|
|
309
|
+
});
|
|
221
310
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
222
311
|
export function backoffFor(consecutiveFailures) {
|
|
223
312
|
if (consecutiveFailures <= 0)
|
|
@@ -252,7 +341,10 @@ export function writeRestartLedger(dir, ledger) {
|
|
|
252
341
|
export function resetRestartLedger(dir) {
|
|
253
342
|
if (!existsSync(dir))
|
|
254
343
|
return;
|
|
255
|
-
|
|
344
|
+
const previous = readRestartLedger(dir);
|
|
345
|
+
writeRestartLedger(dir, {
|
|
346
|
+
...emptyLedger(), ...carriedForward(previous), updatedAt: new Date().toISOString(),
|
|
347
|
+
});
|
|
256
348
|
}
|
|
257
349
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
258
350
|
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
@@ -383,11 +475,18 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
383
475
|
const effectiveModel = effectiveModelForRole(dir, role);
|
|
384
476
|
if (effectiveModel !== role.model) {
|
|
385
477
|
deps.log(`[${name}] model recovery drift: declared=${role.model ?? '(none)'} effective=${effectiveModel}`);
|
|
386
|
-
|
|
478
|
+
// The env pin has to move with it. A down-shift that changed only
|
|
479
|
+
// `role.model` was reported as a model change while the child kept running
|
|
480
|
+
// the model that had just failed, because the pin is what the harness reads.
|
|
481
|
+
role = { ...role, model: effectiveModel, env: repinModelEnv(role, effectiveModel) };
|
|
387
482
|
}
|
|
388
483
|
if (modelRecoveryHeld(dir))
|
|
389
484
|
throw new Error(`[${name}] model chain exhausted — held down until config changes or recovery reset`);
|
|
390
485
|
const adapter = getAdapter(role.harness);
|
|
486
|
+
// Say the running model out loud, once, from the resolved environment. The
|
|
487
|
+
// spawn banner is a claim made before the process exists; this is the log line
|
|
488
|
+
// that can be checked against the session afterwards.
|
|
489
|
+
deps.log(`[${name}] model: ${effectiveRoleModel(role) ?? '(harness default)'}`);
|
|
391
490
|
mkdirSync(dir, { recursive: true });
|
|
392
491
|
const rotation = rotateWorklog(join(dir, 'WORKLOG.md'), role.worklog);
|
|
393
492
|
if (rotation.deferred)
|
|
@@ -402,8 +501,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
402
501
|
const exitFile = join(dir, '.exit-status');
|
|
403
502
|
const booted = existsSync(bootedFile);
|
|
404
503
|
const mode = booted && adapter.supportsResume ? 'resume' : 'fresh';
|
|
405
|
-
|
|
406
|
-
|
|
504
|
+
// Stamp EVERY attempt, not just the first. `.booted` used to be written only
|
|
505
|
+
// on the fresh path, so after a restart — including one the supervisor never
|
|
506
|
+
// saw, like an OOM-kill — its mtime still read the original boot and any
|
|
507
|
+
// health check reading it reported "no restarts". The existence test above
|
|
508
|
+
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
509
|
+
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
407
510
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
408
511
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
409
512
|
const sessionBackend = role.session ?? 'tmux';
|
|
@@ -524,12 +627,23 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
524
627
|
name,
|
|
525
628
|
argv: wrappedArgv,
|
|
526
629
|
cwd: runCwd,
|
|
527
|
-
env:
|
|
630
|
+
env: harnessChildEnv(role, launch.env, dir),
|
|
528
631
|
stateDir: dir,
|
|
529
632
|
mode,
|
|
530
633
|
permissions: perms,
|
|
531
634
|
modeId: adapter.acpPermissionModeId?.(role),
|
|
635
|
+
// The role's declared MCP servers, and the bundled agent's `_meta`
|
|
636
|
+
// vocabulary for the options it takes no flag for. Both come from the
|
|
637
|
+
// ADAPTER and from `prep`: the ACP launch cannot carry `prep.argv`, so this
|
|
638
|
+
// is the route by which harness_options that used to be silently dropped
|
|
639
|
+
// for an ACP role actually reach the session.
|
|
640
|
+
mcpServers: adapter.acpMcpServers?.(role),
|
|
641
|
+
sessionMeta: adapter.acpSessionMeta?.(role, prep),
|
|
532
642
|
permissionMode: effectivePermissionMode(role),
|
|
643
|
+
// Provenance travels with the exact ACP launch. Keeping it out of a
|
|
644
|
+
// role-only adapter hook prevents a PATH fallback or resolver skew from
|
|
645
|
+
// claiming metadata trust for an argv it did not authenticate.
|
|
646
|
+
permissionMetadataSource: launch.permissionMetadataSource,
|
|
533
647
|
log: deps.log,
|
|
534
648
|
});
|
|
535
649
|
pid = acpSession.pid;
|
|
@@ -927,91 +1041,129 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
927
1041
|
mkdirSync(dir, { recursive: true });
|
|
928
1042
|
const shouldStop = deps.shouldStop ?? (() => false);
|
|
929
1043
|
const stamp = () => new Date(deps.now()).toISOString();
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1044
|
+
// Record how the PREVIOUS supervisor process ended before doing anything
|
|
1045
|
+
// else. An external kill writes nothing itself, and the next ledger write is
|
|
1046
|
+
// an attempt away — which is why an OOM-kill used to leave every durable
|
|
1047
|
+
// indicator describing the run that died.
|
|
1048
|
+
const startedAt = stamp();
|
|
1049
|
+
const termination = claimSupervisorRun(dir, startedAt);
|
|
1050
|
+
{
|
|
1051
|
+
const previous = readRestartLedger(dir);
|
|
1052
|
+
const abrupt = (previous.abruptTerminations ?? 0) + (termination.class === 'abrupt' ? 1 : 0);
|
|
1053
|
+
writeRestartLedger(dir, {
|
|
1054
|
+
...previous,
|
|
1055
|
+
lastTermination: termination,
|
|
1056
|
+
abruptTerminations: abrupt,
|
|
1057
|
+
supervisorStartedAt: startedAt,
|
|
1058
|
+
updatedAt: startedAt,
|
|
1059
|
+
});
|
|
1060
|
+
if (termination.class === 'abrupt')
|
|
1061
|
+
deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
|
|
1062
|
+
+ `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
|
|
1063
|
+
}
|
|
1064
|
+
try {
|
|
1065
|
+
while (!shouldStop()) {
|
|
1066
|
+
let ledger = readRestartLedger(dir);
|
|
1067
|
+
try {
|
|
1068
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
1069
|
+
const role = findRole(loadConfig(configPath), name);
|
|
1070
|
+
reconcileModelRecovery(dir, role, stamp());
|
|
1071
|
+
if (modelRecoveryHeld(dir)) {
|
|
1072
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
catch {
|
|
1077
|
+
// Normal attempt path reports config errors through the restart circuit.
|
|
1078
|
+
}
|
|
1079
|
+
if (ledger.circuit === 'open') {
|
|
1080
|
+
// Held down. Stay alive — exiting would hand the role straight back to
|
|
1081
|
+
// the service manager — and watch for an operator reset.
|
|
937
1082
|
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
938
1083
|
continue;
|
|
939
1084
|
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
//
|
|
956
|
-
|
|
957
|
-
result
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
1085
|
+
let result;
|
|
1086
|
+
try {
|
|
1087
|
+
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
1088
|
+
}
|
|
1089
|
+
catch (e) {
|
|
1090
|
+
// A session that could not even start is an immediate failure like any
|
|
1091
|
+
// other; it must count, or an unstartable role loops forever.
|
|
1092
|
+
result = {
|
|
1093
|
+
elapsedSecs: 0,
|
|
1094
|
+
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
1095
|
+
rotated: false,
|
|
1096
|
+
mode: 'fresh',
|
|
1097
|
+
};
|
|
1098
|
+
}
|
|
1099
|
+
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1100
|
+
// have reset the ledger meanwhile.
|
|
1101
|
+
ledger = readRestartLedger(dir);
|
|
1102
|
+
if (result.modelRecovery === 'advance') {
|
|
1103
|
+
writeRestartLedger(dir, {
|
|
1104
|
+
...emptyLedger(),
|
|
1105
|
+
...carriedForward(ledger),
|
|
1106
|
+
lastReason: 'approved model-chain transition',
|
|
1107
|
+
updatedAt: stamp(),
|
|
1108
|
+
});
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
if (result.modelRecovery === 'hold') {
|
|
1112
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
const fastFailSecs = fastFailSecsFor(name, opts.configPath);
|
|
1116
|
+
// The fast-fail boundary starts a recovery episode; it must not also be
|
|
1117
|
+
// the boundary that declares recovery successful. Otherwise alternating
|
|
1118
|
+
// 19s and 20s deaths erase one another forever. Require the configured
|
|
1119
|
+
// number of fast-fail windows to survive before closing an active streak.
|
|
1120
|
+
// This hysteresis stays adapter-relative (100s for the current 20s/5-attempt
|
|
1121
|
+
// policy) and still lets a genuinely sustained session reset the breaker.
|
|
1122
|
+
const stableRecoverySecs = fastFailSecs * RESTART_FAIL_THRESHOLD;
|
|
1123
|
+
const recoveryFailed = result.elapsedSecs < fastFailSecs
|
|
1124
|
+
|| (ledger.consecutiveImmediateFailures > 0 && result.elapsedSecs < stableRecoverySecs);
|
|
1125
|
+
if (!recoveryFailed) {
|
|
1126
|
+
// A session that ran for a while is not a restart loop, whatever ended it.
|
|
1127
|
+
writeRestartLedger(dir, {
|
|
1128
|
+
...emptyLedger(),
|
|
1129
|
+
...carriedForward(ledger),
|
|
1130
|
+
lastReason: result.exit.detail,
|
|
1131
|
+
updatedAt: stamp(),
|
|
1132
|
+
});
|
|
1133
|
+
continue;
|
|
1134
|
+
}
|
|
1135
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
1136
|
+
const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
|
|
1137
|
+
const next = {
|
|
1138
|
+
version: 1,
|
|
1139
|
+
...carriedForward(ledger),
|
|
1140
|
+
consecutiveImmediateFailures: failures,
|
|
1141
|
+
lastReason: reason,
|
|
1142
|
+
nextDelayMs: backoffFor(failures),
|
|
1143
|
+
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1144
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
986
1145
|
updatedAt: stamp(),
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
nextDelayMs: backoffFor(failures),
|
|
997
|
-
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
998
|
-
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
999
|
-
updatedAt: stamp(),
|
|
1000
|
-
};
|
|
1001
|
-
if (next.circuit === 'open') {
|
|
1002
|
-
next.openedAt = stamp();
|
|
1003
|
-
next.nextDelayMs = 0;
|
|
1146
|
+
};
|
|
1147
|
+
if (next.circuit === 'open') {
|
|
1148
|
+
next.openedAt = stamp();
|
|
1149
|
+
next.nextDelayMs = 0;
|
|
1150
|
+
writeRestartLedger(dir, next);
|
|
1151
|
+
deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
|
|
1152
|
+
`${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
|
|
1153
|
+
continue;
|
|
1154
|
+
}
|
|
1004
1155
|
writeRestartLedger(dir, next);
|
|
1005
|
-
deps.log(`[${name}]
|
|
1006
|
-
|
|
1007
|
-
|
|
1156
|
+
deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
|
|
1157
|
+
`-> backing off ${next.nextDelayMs}ms`);
|
|
1158
|
+
await deps.sleep(next.nextDelayMs);
|
|
1008
1159
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1160
|
+
return readRestartLedger(dir);
|
|
1161
|
+
}
|
|
1162
|
+
finally {
|
|
1163
|
+
// Only an orderly return through this loop clears the marker; a signal or
|
|
1164
|
+
// an OOM-kill leaves it, which is exactly how the successor detects them.
|
|
1165
|
+
releaseSupervisorRun(dir);
|
|
1013
1166
|
}
|
|
1014
|
-
return readRestartLedger(dir);
|
|
1015
1167
|
}
|
|
1016
1168
|
/**
|
|
1017
1169
|
* How short an attempt has to be to count as immediate. The role's harness
|
package/dist/session/acp.d.ts
CHANGED
|
@@ -1,10 +1,26 @@
|
|
|
1
1
|
import * as acp from '@agentclientprotocol/sdk';
|
|
2
2
|
import type { CommonPermissions } from '../config.js';
|
|
3
|
+
import type { AcpMcpServer } from '../harness/types.js';
|
|
3
4
|
import { ConversationEventStore } from './conversation-store.js';
|
|
4
5
|
import type { ConversationSnapshot, PromptOrigin, PromptReceipt, SubmitPromptCommand } from './conversation-types.js';
|
|
5
6
|
import type { ConversationHandlePage, ExitRecord, InterruptOutcome, QueuedPrompt, SessionEvent, RuntimeSelectorMetadata, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnOutcome, TurnResult } from './types.js';
|
|
6
7
|
/** Bound safe-boundary waiting without turning a hung tool into cancellation. */
|
|
7
8
|
export declare const AFTER_TOOL_BOUNDARY_TIMEOUT_MS = 120000;
|
|
9
|
+
/**
|
|
10
|
+
* How long a steering-started turn is presumed to still own the adapter after
|
|
11
|
+
* its last update. Such a turn has no prompt id, so it never reports a
|
|
12
|
+
* stopReason and there is no exact end to observe — silence is the only signal
|
|
13
|
+
* available, and this is the bound that turns it into a decision.
|
|
14
|
+
*
|
|
15
|
+
* Sized from the fleet's own scheduled-run history: across 1513 completed
|
|
16
|
+
* scheduled runs the longest silence WITHIN a working turn was 120.2 s (p99
|
|
17
|
+
* 41.0 s; 5 runs above 60 s). A shorter grace would release the lease while the
|
|
18
|
+
* adapter is still working and re-admit a prompt into a busy turn, which is the
|
|
19
|
+
* FLEET-003 failure itself. The costs are deliberately asymmetric: holding too
|
|
20
|
+
* long skips one best-effort maintenance tick, releasing too early SIGTERMs a
|
|
21
|
+
* live role.
|
|
22
|
+
*/
|
|
23
|
+
export declare const STEERING_OCCUPANCY_IDLE_MS = 150000;
|
|
8
24
|
/** Server-generated typed provenance followed by the exact human-authored body. */
|
|
9
25
|
export declare function promptContentBlocks(text: string, origin?: PromptOrigin): acp.ContentBlock[];
|
|
10
26
|
export declare function runtimeSelector(options: acp.SessionConfigOption[] | null | undefined, category: string): RuntimeSelectorMetadata | undefined;
|
|
@@ -20,6 +36,22 @@ export interface AcpSessionOptions {
|
|
|
20
36
|
modeId?: string;
|
|
21
37
|
/** Adapter-resolved live permission policy; separate from ACP agent-specific session modes. */
|
|
22
38
|
permissionMode?: NonNullable<SessionSnapshot['permissionMode']>;
|
|
39
|
+
/** Adapter-authenticated request-metadata vocabulary; never inferred from ACP `_meta`. */
|
|
40
|
+
permissionMetadataSource?: 'codex-acp';
|
|
41
|
+
/**
|
|
42
|
+
* MCP servers the ROLE declares, for every session/new, resume and load. Empty
|
|
43
|
+
* or omitted sends `[]`, which is what fleet has always sent and leaves the
|
|
44
|
+
* agent's own configuration untouched.
|
|
45
|
+
*/
|
|
46
|
+
mcpServers?: AcpMcpServer[];
|
|
47
|
+
/**
|
|
48
|
+
* Adapter-supplied `_meta` for session/new — the only route by which a
|
|
49
|
+
* capability the CLI takes as a flag reaches an agent that accepts none.
|
|
50
|
+
* Per-agent vocabulary, so the ADAPTER decides whether there is anything to
|
|
51
|
+
* send; this layer only forwards it. Never sent on resume or load: it carries
|
|
52
|
+
* session-creation options the agent has already applied.
|
|
53
|
+
*/
|
|
54
|
+
sessionMeta?: Record<string, unknown>;
|
|
23
55
|
log(line: string): void;
|
|
24
56
|
/** Test seam for the cancel-escalation grace period; production uses the default. */
|
|
25
57
|
cancelGraceMs?: number;
|
|
@@ -31,6 +63,8 @@ export interface AcpSessionOptions {
|
|
|
31
63
|
controllerGraceMs?: number;
|
|
32
64
|
/** Test seam; production uses AFTER_TOOL_BOUNDARY_TIMEOUT_MS. */
|
|
33
65
|
afterToolBoundaryTimeoutMs?: number;
|
|
66
|
+
/** Test seam; production uses STEERING_OCCUPANCY_IDLE_MS. */
|
|
67
|
+
steeringOccupancyIdleMs?: number;
|
|
34
68
|
}
|
|
35
69
|
/**
|
|
36
70
|
* Classify an ACP `stopReason` into a terminal outcome. A refusal and a
|
|
@@ -49,6 +83,8 @@ export declare class AcpSession implements SessionHandle {
|
|
|
49
83
|
private readonly child;
|
|
50
84
|
private readonly events;
|
|
51
85
|
private readonly conversation;
|
|
86
|
+
/** Cursor before this runner generation began; older durable events stay off the live console. */
|
|
87
|
+
private readonly conversationStartCursor?;
|
|
52
88
|
/** New on every runner start; permission/turn IDs from prior generations are stale. */
|
|
53
89
|
private readonly sessionGeneration;
|
|
54
90
|
/** True while `session/load` replays history as ordinary updates. */
|
|
@@ -58,6 +94,12 @@ export declare class AcpSession implements SessionHandle {
|
|
|
58
94
|
private connection;
|
|
59
95
|
private sessionId?;
|
|
60
96
|
private readiness;
|
|
97
|
+
/**
|
|
98
|
+
* Last non-replayed session update from the agent. `readiness` cannot answer
|
|
99
|
+
* "is this agent working" for a steered turn (FLEET-002), and this is the
|
|
100
|
+
* evidence that can.
|
|
101
|
+
*/
|
|
102
|
+
private lastUpdateAt?;
|
|
61
103
|
private lastError?;
|
|
62
104
|
private promptTail;
|
|
63
105
|
private queueDepth;
|
|
@@ -73,6 +115,13 @@ export declare class AcpSession implements SessionHandle {
|
|
|
73
115
|
private cancelEscalation?;
|
|
74
116
|
private cancelForceKill?;
|
|
75
117
|
private cancelRecoveryReason?;
|
|
118
|
+
/**
|
|
119
|
+
* Held while a steering-started turn is believed to own the adapter. It is a
|
|
120
|
+
* lease, not a latch: `steeringRelease` always fires, so the role can never be
|
|
121
|
+
* stranded busy by a wake whose turn ended without telling anyone.
|
|
122
|
+
*/
|
|
123
|
+
private steeringOccupied;
|
|
124
|
+
private steeringRelease?;
|
|
76
125
|
/**
|
|
77
126
|
* Rejects the moment the adapter process is gone. Every in-flight ACP request
|
|
78
127
|
* races it, so a dead adapter can never leave a turn — and therefore a
|
|
@@ -94,6 +143,20 @@ export declare class AcpSession implements SessionHandle {
|
|
|
94
143
|
*/
|
|
95
144
|
private recoverOpenPrompts;
|
|
96
145
|
isAlive(): boolean;
|
|
146
|
+
/**
|
|
147
|
+
* Take the occupancy lease for a turn the adapter started on its own behalf.
|
|
148
|
+
* Refreshed by every adapter update, so it tracks work actually happening
|
|
149
|
+
* rather than a fixed guess at how long a wake takes.
|
|
150
|
+
*/
|
|
151
|
+
private holdSteeringOccupancy;
|
|
152
|
+
private refreshSteeringOccupancy;
|
|
153
|
+
/**
|
|
154
|
+
* Every exit from occupancy comes through here, including the ones that are
|
|
155
|
+
* not the timer: a real turn boundary, close, and adapter exit. A lease that
|
|
156
|
+
* can leak is worse than the bug it fixes — it would leave the role reporting
|
|
157
|
+
* `running` forever and starve scheduled admission permanently.
|
|
158
|
+
*/
|
|
159
|
+
private releaseSteeringOccupancy;
|
|
97
160
|
snapshot(): SessionSnapshot;
|
|
98
161
|
private toolCall;
|
|
99
162
|
private reserveTool;
|
|
@@ -126,6 +189,28 @@ export declare class AcpSession implements SessionHandle {
|
|
|
126
189
|
* for it is what turned a busy agent into a timeout and then into "dead".
|
|
127
190
|
*/
|
|
128
191
|
queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
|
|
192
|
+
/**
|
|
193
|
+
* Prepare the session for a prompt that asked to pre-empt current work.
|
|
194
|
+
*
|
|
195
|
+
* The old behaviour was one unconditional `session/cancel` notification
|
|
196
|
+
* followed immediately by `session/prompt`. That is what produced the owner's
|
|
197
|
+
* "request failed before completion":
|
|
198
|
+
*
|
|
199
|
+
* - `cancelActive` only awaits settlement when `this.activeTurn` is set, and
|
|
200
|
+
* a turn the ADAPTER started (steering's `startedNewTurn`) is never tracked
|
|
201
|
+
* here. So the cancel raced the adapter's own transcript repair and the new
|
|
202
|
+
* prompt landed while the last assistant message still held an unresolved
|
|
203
|
+
* `tool_use` — rejected with `stop_reason=tool_use`.
|
|
204
|
+
* - With nothing running at all, it still sent the cancel, and the prompt
|
|
205
|
+
* landed on a bare interrupted user message — rejected with
|
|
206
|
+
* `stop_reason=null`.
|
|
207
|
+
*
|
|
208
|
+
* So: never cancel across a tool boundary, and never cancel something whose
|
|
209
|
+
* settlement cannot be awaited. Everything else is queued, which the ACP queue
|
|
210
|
+
* already does correctly. The returned state is what the caller may claim to a
|
|
211
|
+
* human — `interrupted` only when a turn really was cancelled.
|
|
212
|
+
*/
|
|
213
|
+
private prepareInterruptingDelivery;
|
|
129
214
|
/**
|
|
130
215
|
* Durably record a prompt admission BEFORE acceptance is returned. Browser
|
|
131
216
|
* admissions are transactional — a prompt the ledger cannot hold is refused,
|
|
@@ -173,6 +258,15 @@ export declare class AcpSession implements SessionHandle {
|
|
|
173
258
|
private settlePendingAutomatically;
|
|
174
259
|
exitResult(): ExitRecord | null;
|
|
175
260
|
close(): Promise<void>;
|
|
261
|
+
/**
|
|
262
|
+
* The role's declared MCP servers, or `[]`.
|
|
263
|
+
*
|
|
264
|
+
* Sent on resume and load as well as on new: the agent builds its server set
|
|
265
|
+
* once per session, so a resumed session that omitted them would come back
|
|
266
|
+
* without the tools the role's config declares — which is exactly the shape of
|
|
267
|
+
* silent drop this plumbing exists to end.
|
|
268
|
+
*/
|
|
269
|
+
private declaredMcpServers;
|
|
176
270
|
private initialize;
|
|
177
271
|
private captureRuntimeMetadata;
|
|
178
272
|
private runPrompt;
|
|
@@ -185,6 +279,15 @@ export declare class AcpSession implements SessionHandle {
|
|
|
185
279
|
*/
|
|
186
280
|
private settleAutomatically;
|
|
187
281
|
private withinAutomaticBoundary;
|
|
282
|
+
/**
|
|
283
|
+
* Codex ACP 1.1.7 marks its protected MCP elicitation bridge on a locationless
|
|
284
|
+
* execute request. The marker is meaningful only together with the runner's
|
|
285
|
+
* independently supplied, adapter-authenticated metadata vocabulary and effective
|
|
286
|
+
* mode: an arbitrary ACP process cannot gain this path by copying `_meta` alone.
|
|
287
|
+
* Exact option ids/kinds bind recognition to the protected-MCP shape and keep
|
|
288
|
+
* malformed requests on the ordinary fail-closed path.
|
|
289
|
+
*/
|
|
290
|
+
private isEffectiveCodexProtectedMcpApproval;
|
|
188
291
|
private recordUpdate;
|
|
189
292
|
/**
|
|
190
293
|
* Codex ACP's phase extension is the only currently supported visibility
|
|
@@ -199,5 +302,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
199
302
|
}): ConversationHandlePage;
|
|
200
303
|
conversationSnapshot(): ConversationSnapshot;
|
|
201
304
|
subscribeConversation(listener: Parameters<ConversationEventStore['subscribe']>[0]): () => void;
|
|
305
|
+
private isCurrentConversationEvent;
|
|
202
306
|
private fail;
|
|
203
307
|
}
|