@ours.network/fleet 0.18.1 → 0.19.0-nightly.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -91
- package/dist/application/fleet-query-service.js +0 -12
- package/dist/application/role-creation-service.js +10 -7
- package/dist/application/types.d.ts +0 -11
- package/dist/briefing.js +6 -15
- package/dist/build-info.json +5 -5
- package/dist/cli.js +12 -37
- package/dist/config.d.ts +8 -6
- package/dist/config.js +46 -30
- package/dist/creation.d.ts +38 -22
- package/dist/creation.js +111 -24
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +44 -95
- package/dist/doctor.d.ts +5 -1
- package/dist/doctor.js +18 -11
- package/dist/fleet-proxy.d.ts +0 -5
- package/dist/harness/acp-agent.js +6 -11
- package/dist/harness/claude-code.js +11 -200
- package/dist/harness/codex.d.ts +1 -4
- package/dist/harness/codex.js +12 -70
- package/dist/harness/types.d.ts +4 -54
- package/dist/loops/manager.d.ts +1 -30
- package/dist/loops/manager.js +6 -69
- package/dist/loops/state.d.ts +0 -18
- package/dist/loops/state.js +0 -4
- package/dist/monitor.js +1 -1
- package/dist/ops.d.ts +3 -0
- package/dist/ops.js +8 -3
- package/dist/owner-channel/attachments.d.ts +25 -2
- package/dist/owner-channel/attachments.js +61 -5
- package/dist/owner-channel/channel.d.ts +29 -30
- package/dist/owner-channel/channel.js +291 -291
- package/dist/owner-channel/commands.js +89 -0
- package/dist/owner-channel/message-recovery.d.ts +25 -0
- package/dist/owner-channel/message-recovery.js +114 -0
- package/dist/owner-channel/notices.d.ts +0 -7
- package/dist/owner-channel/notices.js +0 -9
- package/dist/owner-channel/ours-client.d.ts +148 -0
- package/dist/owner-channel/ours-client.js +231 -0
- package/dist/rooms-tasks/cli.d.ts +4 -0
- package/dist/rooms-tasks/cli.js +565 -0
- package/dist/rooms-tasks/config.d.ts +15 -0
- package/dist/rooms-tasks/config.js +171 -0
- package/dist/rooms-tasks/cowork-adapter.d.ts +80 -0
- package/dist/rooms-tasks/cowork-adapter.js +48 -0
- package/dist/rooms-tasks/index.d.ts +7 -0
- package/dist/rooms-tasks/index.js +7 -0
- package/dist/rooms-tasks/room-state.d.ts +22 -0
- package/dist/rooms-tasks/room-state.js +117 -0
- package/dist/rooms-tasks/task-state.d.ts +31 -0
- package/dist/rooms-tasks/task-state.js +173 -0
- package/dist/rooms-tasks/templates.d.ts +6 -0
- package/dist/rooms-tasks/templates.js +80 -0
- package/dist/rooms-tasks/types.d.ts +153 -0
- package/dist/rooms-tasks/types.js +20 -0
- package/dist/runner.d.ts +0 -48
- package/dist/runner.js +94 -236
- package/dist/session/acp.d.ts +0 -104
- package/dist/session/acp.js +10 -213
- package/dist/session/conversation-normalizer.d.ts +0 -6
- package/dist/session/conversation-normalizer.js +10 -153
- package/dist/session/conversation-types.d.ts +4 -23
- package/dist/session/types.d.ts +0 -35
- package/dist/spawn.js +26 -33
- package/dist/supervisor/systemd.js +29 -2
- package/dist/watchdog/briefing.js +0 -7
- package/dist/watchdog/run.js +3 -3
- package/dist/web-app/assets/{TerminalView-C_G1ID2P.js → TerminalView-BAVk1Bot.js} +1 -1
- package/dist/web-app/assets/{index-BCBK78hw.js → index-C3S-xFRU.js} +5 -5
- package/dist/web-app/index.html +1 -1
- package/dist/worklog.d.ts +1 -7
- package/dist/worklog.js +39 -191
- package/package.json +3 -1
- package/dist/model-env.d.ts +0 -71
- package/dist/model-env.js +0 -106
- package/dist/owner-channel/mcp.d.ts +0 -24
- package/dist/owner-channel/mcp.js +0 -145
- package/dist/session/activity.d.ts +0 -31
- package/dist/session/activity.js +0 -48
package/dist/runner.js
CHANGED
|
@@ -18,13 +18,13 @@ import { TmuxSession } from './session/tmux.js';
|
|
|
18
18
|
import { ACP_CANCEL_DEADLINE_EXCEEDED, classifyShellStatus } from './session/types.js';
|
|
19
19
|
import { effectiveModelForRole, modelRecoveryHeld, reconcileModelRecovery, recordModelFailure, classifyFailureText, } from './model-recovery.js';
|
|
20
20
|
import { rotateWorklog } from './worklog.js';
|
|
21
|
+
import { reconcilePermanentRoleIdentities } from './creation.js';
|
|
21
22
|
import { OwnerChannel } from './owner-channel/channel.js';
|
|
22
23
|
import { acquireOwnerBinderLease, OwnerBinderHandoffTimeoutError, } from './owner-channel/binder.js';
|
|
23
24
|
import { RoleTurnArbiter } from './session/arbiter.js';
|
|
24
25
|
import { ScheduledLoopManager, } from './loops/manager.js';
|
|
25
26
|
import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, inheritCallerSpawnDefaults, } from './fleet-proxy.js';
|
|
26
27
|
import { effectivePermissionMode } from './permissions.js';
|
|
27
|
-
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
28
28
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
29
29
|
const defaultDeps = () => ({
|
|
30
30
|
tmux: new Tmux(),
|
|
@@ -60,32 +60,14 @@ const defaultDeps = () => ({
|
|
|
60
60
|
},
|
|
61
61
|
});
|
|
62
62
|
const MONITOR_OWNER_FILE = '.monitor-owner';
|
|
63
|
-
/** Fleet roles consume the operator-owned daemon; a role session never starts it. */
|
|
64
|
-
const FLEET_OURS_AUTOSTART = '0';
|
|
65
63
|
/** Environment injected only into the managed harness process. */
|
|
66
64
|
export function managedFleetProxyEnv(role, stateDir) {
|
|
67
65
|
return {
|
|
68
66
|
...(role.env ?? {}),
|
|
69
|
-
// This must win over both inherited/configured auto-start. ACP agents run
|
|
70
|
-
// directly rather than through ours-codex, so the runner owns this fence.
|
|
71
|
-
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
72
67
|
[FLEET_PROXY_STATE_DIR_ENV]: stateDir,
|
|
73
68
|
[FLEET_PROXY_CALLER_ENV]: role.name,
|
|
74
69
|
};
|
|
75
70
|
}
|
|
76
|
-
/**
|
|
77
|
-
* The environment a managed harness child actually receives, checked at the one
|
|
78
|
-
* point where it is composed. `role.env` deliberately wins over harness prep,
|
|
79
|
-
* which is exactly how a stale fleet-wide model pin used to outrank the model
|
|
80
|
-
* the role was spawned with — so the model pin is verified here rather than
|
|
81
|
-
* trusted, and a disagreement stops the launch instead of being reported as a
|
|
82
|
-
* success (see src/model-env.ts).
|
|
83
|
-
*/
|
|
84
|
-
export function harnessChildEnv(role, launchEnv, stateDir) {
|
|
85
|
-
const env = { ...(launchEnv ?? {}), ...managedFleetProxyEnv(role, stateDir) };
|
|
86
|
-
assertModelPinReachesChild(role, env);
|
|
87
|
-
return env;
|
|
88
|
-
}
|
|
89
71
|
/**
|
|
90
72
|
* Execute a typed proxy request in the caller's supervisor. Dynamic imports
|
|
91
73
|
* avoid a runner↔spawn initialization cycle (spawn imports runner constants).
|
|
@@ -123,19 +105,13 @@ async function executeManagedSpawn(caller, configPath, requested, log) {
|
|
|
123
105
|
statePath,
|
|
124
106
|
harness: preview.harness,
|
|
125
107
|
session: preview.session,
|
|
126
|
-
|
|
127
|
-
// must name the model the child will run, not the one that was asked for.
|
|
128
|
-
...(effectiveRoleModel(preview) ? { model: effectiveRoleModel(preview) } : {}),
|
|
108
|
+
...(preview.model ? { model: preview.model } : {}),
|
|
129
109
|
monitor: { mode: preview.monitor.mode, interrupt: preview.monitor.interrupt },
|
|
130
|
-
permissionMode: effectivePermissionMode(preview),
|
|
131
110
|
inherited,
|
|
132
111
|
creationActionId,
|
|
133
112
|
};
|
|
134
113
|
log(`[${caller.name}] managed fleet proxy spawned ${result.lifetime} role ${result.role} `
|
|
135
|
-
+ `harness=${result.harness} session=${result.session}
|
|
136
|
-
+ `model=${result.model ?? '(harness default)'} `
|
|
137
|
-
+ `permission=${result.permissionMode.fleetMode} `
|
|
138
|
-
+ `native=${result.permissionMode.nativeMode}`);
|
|
114
|
+
+ `harness=${result.harness} session=${result.session}`);
|
|
139
115
|
return result;
|
|
140
116
|
}
|
|
141
117
|
/**
|
|
@@ -165,10 +141,6 @@ export function recordMonitorOwner(dir, owner) {
|
|
|
165
141
|
export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = launch.argv) {
|
|
166
142
|
const env = {
|
|
167
143
|
PATH: process.env.PATH ?? '', COLORTERM: 'truecolor', ...launch.env, ...(roleEnv ?? {}),
|
|
168
|
-
// Tmux roles have the same daemon-client boundary as ACP roles. Keep this
|
|
169
|
-
// last so neither harness preparation nor a role env block can take over
|
|
170
|
-
// the shared daemon lifecycle.
|
|
171
|
-
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
172
144
|
};
|
|
173
145
|
// Interactive panes should advertise colour even when the supervisor itself
|
|
174
146
|
// was launched with NO_COLOR. A role may still deliberately opt back in to
|
|
@@ -247,66 +219,6 @@ const emptyLedger = () => ({
|
|
|
247
219
|
circuit: 'closed',
|
|
248
220
|
updatedAt: new Date(0).toISOString(),
|
|
249
221
|
});
|
|
250
|
-
/**
|
|
251
|
-
* Carried across a supervisor process's life so its successor can tell an
|
|
252
|
-
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
253
|
-
* running"; the next start finding one that is not its own is proof the
|
|
254
|
-
* previous process died without getting to write anything.
|
|
255
|
-
*/
|
|
256
|
-
export const RUN_MARKER_FILE = '.supervisor-run.json';
|
|
257
|
-
function readRunMarker(dir) {
|
|
258
|
-
try {
|
|
259
|
-
const raw = JSON.parse(readFileSync(join(dir, RUN_MARKER_FILE), 'utf8'));
|
|
260
|
-
return raw.version === 1 && typeof raw.pid === 'number' && typeof raw.startedAt === 'string'
|
|
261
|
-
? raw : undefined;
|
|
262
|
-
}
|
|
263
|
-
catch {
|
|
264
|
-
return undefined;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* Claim this state directory for the current supervisor process and report how
|
|
269
|
-
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
270
|
-
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
271
|
-
* and an attempt can take minutes.
|
|
272
|
-
*/
|
|
273
|
-
export function claimSupervisorRun(dir, startedAt, pid = process.pid) {
|
|
274
|
-
const previous = readRunMarker(dir);
|
|
275
|
-
const termination = previous && previous.pid !== pid
|
|
276
|
-
? {
|
|
277
|
-
class: 'abrupt',
|
|
278
|
-
detail: `supervisor pid ${previous.pid} left an open run marker; `
|
|
279
|
-
+ 'it was terminated without an orderly exit (signal, OOM-kill, or host reset)',
|
|
280
|
-
observedAt: startedAt,
|
|
281
|
-
runStartedAt: previous.startedAt,
|
|
282
|
-
}
|
|
283
|
-
: previous
|
|
284
|
-
? { class: 'unknown', detail: 'run marker belongs to this process', observedAt: startedAt }
|
|
285
|
-
: { class: 'clean', detail: 'no previous run marker', observedAt: startedAt };
|
|
286
|
-
try {
|
|
287
|
-
mkdirSync(dir, { recursive: true });
|
|
288
|
-
writeFileSync(join(dir, RUN_MARKER_FILE), JSON.stringify({ version: 1, pid, startedAt }, null, 2) + '\n');
|
|
289
|
-
}
|
|
290
|
-
catch { /* diagnostics must never take the role down */ }
|
|
291
|
-
return termination;
|
|
292
|
-
}
|
|
293
|
-
/** Orderly exit: the successor must not read this run as a kill. */
|
|
294
|
-
export function releaseSupervisorRun(dir) {
|
|
295
|
-
try {
|
|
296
|
-
rmSync(join(dir, RUN_MARKER_FILE), { force: true });
|
|
297
|
-
}
|
|
298
|
-
catch { /* best effort */ }
|
|
299
|
-
}
|
|
300
|
-
/**
|
|
301
|
-
* Fields that describe THIS process's history rather than the current failure
|
|
302
|
-
* streak. Clearing the streak (recovery, an operator `up`, an approved model
|
|
303
|
-
* transition) must not erase the record that the role died and came back.
|
|
304
|
-
*/
|
|
305
|
-
const carriedForward = (previous) => ({
|
|
306
|
-
...(previous.lastTermination ? { lastTermination: previous.lastTermination } : {}),
|
|
307
|
-
...(previous.abruptTerminations ? { abruptTerminations: previous.abruptTerminations } : {}),
|
|
308
|
-
...(previous.supervisorStartedAt ? { supervisorStartedAt: previous.supervisorStartedAt } : {}),
|
|
309
|
-
});
|
|
310
222
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
311
223
|
export function backoffFor(consecutiveFailures) {
|
|
312
224
|
if (consecutiveFailures <= 0)
|
|
@@ -341,10 +253,7 @@ export function writeRestartLedger(dir, ledger) {
|
|
|
341
253
|
export function resetRestartLedger(dir) {
|
|
342
254
|
if (!existsSync(dir))
|
|
343
255
|
return;
|
|
344
|
-
|
|
345
|
-
writeRestartLedger(dir, {
|
|
346
|
-
...emptyLedger(), ...carriedForward(previous), updatedAt: new Date().toISOString(),
|
|
347
|
-
});
|
|
256
|
+
writeRestartLedger(dir, { ...emptyLedger(), updatedAt: new Date().toISOString() });
|
|
348
257
|
}
|
|
349
258
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
350
259
|
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
@@ -475,18 +384,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
475
384
|
const effectiveModel = effectiveModelForRole(dir, role);
|
|
476
385
|
if (effectiveModel !== role.model) {
|
|
477
386
|
deps.log(`[${name}] model recovery drift: declared=${role.model ?? '(none)'} effective=${effectiveModel}`);
|
|
478
|
-
|
|
479
|
-
// `role.model` was reported as a model change while the child kept running
|
|
480
|
-
// the model that had just failed, because the pin is what the harness reads.
|
|
481
|
-
role = { ...role, model: effectiveModel, env: repinModelEnv(role, effectiveModel) };
|
|
387
|
+
role = { ...role, model: effectiveModel };
|
|
482
388
|
}
|
|
483
389
|
if (modelRecoveryHeld(dir))
|
|
484
390
|
throw new Error(`[${name}] model chain exhausted — held down until config changes or recovery reset`);
|
|
485
391
|
const adapter = getAdapter(role.harness);
|
|
486
|
-
// Say the running model out loud, once, from the resolved environment. The
|
|
487
|
-
// spawn banner is a claim made before the process exists; this is the log line
|
|
488
|
-
// that can be checked against the session afterwards.
|
|
489
|
-
deps.log(`[${name}] model: ${effectiveRoleModel(role) ?? '(harness default)'}`);
|
|
490
392
|
mkdirSync(dir, { recursive: true });
|
|
491
393
|
const rotation = rotateWorklog(join(dir, 'WORKLOG.md'), role.worklog);
|
|
492
394
|
if (rotation.deferred)
|
|
@@ -501,12 +403,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
501
403
|
const exitFile = join(dir, '.exit-status');
|
|
502
404
|
const booted = existsSync(bootedFile);
|
|
503
405
|
const mode = booted && adapter.supportsResume ? 'resume' : 'fresh';
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
// saw, like an OOM-kill — its mtime still read the original boot and any
|
|
507
|
-
// health check reading it reported "no restarts". The existence test above
|
|
508
|
-
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
509
|
-
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
406
|
+
if (mode === 'fresh')
|
|
407
|
+
writeFileSync(bootedFile, '');
|
|
510
408
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
511
409
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
512
410
|
const sessionBackend = role.session ?? 'tmux';
|
|
@@ -627,23 +525,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
627
525
|
name,
|
|
628
526
|
argv: wrappedArgv,
|
|
629
527
|
cwd: runCwd,
|
|
630
|
-
env:
|
|
528
|
+
env: { ...launch.env, ...managedFleetProxyEnv(role, dir) },
|
|
631
529
|
stateDir: dir,
|
|
632
530
|
mode,
|
|
633
531
|
permissions: perms,
|
|
634
532
|
modeId: adapter.acpPermissionModeId?.(role),
|
|
635
|
-
// The role's declared MCP servers, and the bundled agent's `_meta`
|
|
636
|
-
// vocabulary for the options it takes no flag for. Both come from the
|
|
637
|
-
// ADAPTER and from `prep`: the ACP launch cannot carry `prep.argv`, so this
|
|
638
|
-
// is the route by which harness_options that used to be silently dropped
|
|
639
|
-
// for an ACP role actually reach the session.
|
|
640
|
-
mcpServers: adapter.acpMcpServers?.(role),
|
|
641
|
-
sessionMeta: adapter.acpSessionMeta?.(role, prep),
|
|
642
533
|
permissionMode: effectivePermissionMode(role),
|
|
643
|
-
// Provenance travels with the exact ACP launch. Keeping it out of a
|
|
644
|
-
// role-only adapter hook prevents a PATH fallback or resolver skew from
|
|
645
|
-
// claiming metadata trust for an argv it did not authenticate.
|
|
646
|
-
permissionMetadataSource: launch.permissionMetadataSource,
|
|
647
534
|
log: deps.log,
|
|
648
535
|
});
|
|
649
536
|
pid = acpSession.pid;
|
|
@@ -1041,129 +928,100 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1041
928
|
mkdirSync(dir, { recursive: true });
|
|
1042
929
|
const shouldStop = deps.shouldStop ?? (() => false);
|
|
1043
930
|
const stamp = () => new Date(deps.now()).toISOString();
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
const previous = readRestartLedger(dir);
|
|
1052
|
-
const abrupt = (previous.abruptTerminations ?? 0) + (termination.class === 'abrupt' ? 1 : 0);
|
|
1053
|
-
writeRestartLedger(dir, {
|
|
1054
|
-
...previous,
|
|
1055
|
-
lastTermination: termination,
|
|
1056
|
-
abruptTerminations: abrupt,
|
|
1057
|
-
supervisorStartedAt: startedAt,
|
|
1058
|
-
updatedAt: startedAt,
|
|
1059
|
-
});
|
|
1060
|
-
if (termination.class === 'abrupt')
|
|
1061
|
-
deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
|
|
1062
|
-
+ `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
|
|
1063
|
-
}
|
|
1064
|
-
try {
|
|
1065
|
-
while (!shouldStop()) {
|
|
1066
|
-
let ledger = readRestartLedger(dir);
|
|
1067
|
-
try {
|
|
1068
|
-
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
1069
|
-
const role = findRole(loadConfig(configPath), name);
|
|
1070
|
-
reconcileModelRecovery(dir, role, stamp());
|
|
1071
|
-
if (modelRecoveryHeld(dir)) {
|
|
1072
|
-
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1073
|
-
continue;
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
|
-
catch {
|
|
1077
|
-
// Normal attempt path reports config errors through the restart circuit.
|
|
1078
|
-
}
|
|
1079
|
-
if (ledger.circuit === 'open') {
|
|
1080
|
-
// Held down. Stay alive — exiting would hand the role straight back to
|
|
1081
|
-
// the service manager — and watch for an operator reset.
|
|
1082
|
-
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1083
|
-
continue;
|
|
1084
|
-
}
|
|
1085
|
-
let result;
|
|
1086
|
-
try {
|
|
1087
|
-
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
1088
|
-
}
|
|
1089
|
-
catch (e) {
|
|
1090
|
-
// A session that could not even start is an immediate failure like any
|
|
1091
|
-
// other; it must count, or an unstartable role loops forever.
|
|
1092
|
-
result = {
|
|
1093
|
-
elapsedSecs: 0,
|
|
1094
|
-
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
1095
|
-
rotated: false,
|
|
1096
|
-
mode: 'fresh',
|
|
1097
|
-
};
|
|
1098
|
-
}
|
|
1099
|
-
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1100
|
-
// have reset the ledger meanwhile.
|
|
1101
|
-
ledger = readRestartLedger(dir);
|
|
1102
|
-
if (result.modelRecovery === 'advance') {
|
|
1103
|
-
writeRestartLedger(dir, {
|
|
1104
|
-
...emptyLedger(),
|
|
1105
|
-
...carriedForward(ledger),
|
|
1106
|
-
lastReason: 'approved model-chain transition',
|
|
1107
|
-
updatedAt: stamp(),
|
|
1108
|
-
});
|
|
1109
|
-
continue;
|
|
1110
|
-
}
|
|
1111
|
-
if (result.modelRecovery === 'hold') {
|
|
931
|
+
while (!shouldStop()) {
|
|
932
|
+
let ledger = readRestartLedger(dir);
|
|
933
|
+
try {
|
|
934
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
935
|
+
const role = findRole(loadConfig(configPath), name);
|
|
936
|
+
reconcileModelRecovery(dir, role, stamp());
|
|
937
|
+
if (modelRecoveryHeld(dir)) {
|
|
1112
938
|
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1113
939
|
continue;
|
|
1114
940
|
}
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
//
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
//
|
|
1121
|
-
//
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
941
|
+
}
|
|
942
|
+
catch {
|
|
943
|
+
// Normal attempt path reports config errors through the restart circuit.
|
|
944
|
+
}
|
|
945
|
+
if (ledger.circuit === 'open') {
|
|
946
|
+
// Held down. Stay alive — exiting would hand the role straight back to
|
|
947
|
+
// the service manager — and watch for an operator reset.
|
|
948
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
949
|
+
continue;
|
|
950
|
+
}
|
|
951
|
+
let result;
|
|
952
|
+
try {
|
|
953
|
+
// Service-manager boot and automatic retry bypass the operator-facing
|
|
954
|
+
// up/restart commands. Reconcile again immediately before every real
|
|
955
|
+
// permanent harness attempt; the operation is idempotent and releases its
|
|
956
|
+
// provisioning lease before the agent or owner channel binds.
|
|
957
|
+
if (attempt === runOnce) {
|
|
958
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
959
|
+
const role = findRole(loadConfig(configPath), name);
|
|
960
|
+
await reconcilePermanentRoleIdentities(role, undefined, deps.log);
|
|
1134
961
|
}
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
updatedAt: stamp(),
|
|
962
|
+
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
963
|
+
}
|
|
964
|
+
catch (e) {
|
|
965
|
+
// A session that could not even start is an immediate failure like any
|
|
966
|
+
// other; it must count, or an unstartable role loops forever.
|
|
967
|
+
result = {
|
|
968
|
+
elapsedSecs: 0,
|
|
969
|
+
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
970
|
+
rotated: false,
|
|
971
|
+
mode: 'fresh',
|
|
1146
972
|
};
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
973
|
+
}
|
|
974
|
+
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
975
|
+
// have reset the ledger meanwhile.
|
|
976
|
+
ledger = readRestartLedger(dir);
|
|
977
|
+
if (result.modelRecovery === 'advance') {
|
|
978
|
+
writeRestartLedger(dir, {
|
|
979
|
+
...emptyLedger(),
|
|
980
|
+
lastReason: 'approved model-chain transition',
|
|
981
|
+
updatedAt: stamp(),
|
|
982
|
+
});
|
|
983
|
+
continue;
|
|
984
|
+
}
|
|
985
|
+
if (result.modelRecovery === 'hold') {
|
|
986
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
const fastFailSecs = fastFailSecsFor(name, opts.configPath);
|
|
990
|
+
const immediate = result.elapsedSecs < fastFailSecs;
|
|
991
|
+
if (!immediate) {
|
|
992
|
+
// A session that ran for a while is not a restart loop, whatever ended it.
|
|
993
|
+
writeRestartLedger(dir, {
|
|
994
|
+
...emptyLedger(),
|
|
995
|
+
lastReason: result.exit.detail,
|
|
996
|
+
updatedAt: stamp(),
|
|
997
|
+
});
|
|
998
|
+
continue;
|
|
999
|
+
}
|
|
1000
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
1001
|
+
const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
|
|
1002
|
+
const next = {
|
|
1003
|
+
version: 1,
|
|
1004
|
+
consecutiveImmediateFailures: failures,
|
|
1005
|
+
lastReason: reason,
|
|
1006
|
+
nextDelayMs: backoffFor(failures),
|
|
1007
|
+
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1008
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
1009
|
+
updatedAt: stamp(),
|
|
1010
|
+
};
|
|
1011
|
+
if (next.circuit === 'open') {
|
|
1012
|
+
next.openedAt = stamp();
|
|
1013
|
+
next.nextDelayMs = 0;
|
|
1155
1014
|
writeRestartLedger(dir, next);
|
|
1156
|
-
deps.log(`[${name}]
|
|
1157
|
-
|
|
1158
|
-
|
|
1015
|
+
deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
|
|
1016
|
+
`${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
|
|
1017
|
+
continue;
|
|
1159
1018
|
}
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
// an OOM-kill leaves it, which is exactly how the successor detects them.
|
|
1165
|
-
releaseSupervisorRun(dir);
|
|
1019
|
+
writeRestartLedger(dir, next);
|
|
1020
|
+
deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
|
|
1021
|
+
`-> backing off ${next.nextDelayMs}ms`);
|
|
1022
|
+
await deps.sleep(next.nextDelayMs);
|
|
1166
1023
|
}
|
|
1024
|
+
return readRestartLedger(dir);
|
|
1167
1025
|
}
|
|
1168
1026
|
/**
|
|
1169
1027
|
* How short an attempt has to be to count as immediate. The role's harness
|
package/dist/session/acp.d.ts
CHANGED
|
@@ -1,26 +1,10 @@
|
|
|
1
1
|
import * as acp from '@agentclientprotocol/sdk';
|
|
2
2
|
import type { CommonPermissions } from '../config.js';
|
|
3
|
-
import type { AcpMcpServer } from '../harness/types.js';
|
|
4
3
|
import { ConversationEventStore } from './conversation-store.js';
|
|
5
4
|
import type { ConversationSnapshot, PromptOrigin, PromptReceipt, SubmitPromptCommand } from './conversation-types.js';
|
|
6
5
|
import type { ConversationHandlePage, ExitRecord, InterruptOutcome, QueuedPrompt, SessionEvent, RuntimeSelectorMetadata, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnOutcome, TurnResult } from './types.js';
|
|
7
6
|
/** Bound safe-boundary waiting without turning a hung tool into cancellation. */
|
|
8
7
|
export declare const AFTER_TOOL_BOUNDARY_TIMEOUT_MS = 120000;
|
|
9
|
-
/**
|
|
10
|
-
* How long a steering-started turn is presumed to still own the adapter after
|
|
11
|
-
* its last update. Such a turn has no prompt id, so it never reports a
|
|
12
|
-
* stopReason and there is no exact end to observe — silence is the only signal
|
|
13
|
-
* available, and this is the bound that turns it into a decision.
|
|
14
|
-
*
|
|
15
|
-
* Sized from the fleet's own scheduled-run history: across 1513 completed
|
|
16
|
-
* scheduled runs the longest silence WITHIN a working turn was 120.2 s (p99
|
|
17
|
-
* 41.0 s; 5 runs above 60 s). A shorter grace would release the lease while the
|
|
18
|
-
* adapter is still working and re-admit a prompt into a busy turn, which is the
|
|
19
|
-
* FLEET-003 failure itself. The costs are deliberately asymmetric: holding too
|
|
20
|
-
* long skips one best-effort maintenance tick, releasing too early SIGTERMs a
|
|
21
|
-
* live role.
|
|
22
|
-
*/
|
|
23
|
-
export declare const STEERING_OCCUPANCY_IDLE_MS = 150000;
|
|
24
8
|
/** Server-generated typed provenance followed by the exact human-authored body. */
|
|
25
9
|
export declare function promptContentBlocks(text: string, origin?: PromptOrigin): acp.ContentBlock[];
|
|
26
10
|
export declare function runtimeSelector(options: acp.SessionConfigOption[] | null | undefined, category: string): RuntimeSelectorMetadata | undefined;
|
|
@@ -36,22 +20,6 @@ export interface AcpSessionOptions {
|
|
|
36
20
|
modeId?: string;
|
|
37
21
|
/** Adapter-resolved live permission policy; separate from ACP agent-specific session modes. */
|
|
38
22
|
permissionMode?: NonNullable<SessionSnapshot['permissionMode']>;
|
|
39
|
-
/** Adapter-authenticated request-metadata vocabulary; never inferred from ACP `_meta`. */
|
|
40
|
-
permissionMetadataSource?: 'codex-acp';
|
|
41
|
-
/**
|
|
42
|
-
* MCP servers the ROLE declares, for every session/new, resume and load. Empty
|
|
43
|
-
* or omitted sends `[]`, which is what fleet has always sent and leaves the
|
|
44
|
-
* agent's own configuration untouched.
|
|
45
|
-
*/
|
|
46
|
-
mcpServers?: AcpMcpServer[];
|
|
47
|
-
/**
|
|
48
|
-
* Adapter-supplied `_meta` for session/new — the only route by which a
|
|
49
|
-
* capability the CLI takes as a flag reaches an agent that accepts none.
|
|
50
|
-
* Per-agent vocabulary, so the ADAPTER decides whether there is anything to
|
|
51
|
-
* send; this layer only forwards it. Never sent on resume or load: it carries
|
|
52
|
-
* session-creation options the agent has already applied.
|
|
53
|
-
*/
|
|
54
|
-
sessionMeta?: Record<string, unknown>;
|
|
55
23
|
log(line: string): void;
|
|
56
24
|
/** Test seam for the cancel-escalation grace period; production uses the default. */
|
|
57
25
|
cancelGraceMs?: number;
|
|
@@ -63,8 +31,6 @@ export interface AcpSessionOptions {
|
|
|
63
31
|
controllerGraceMs?: number;
|
|
64
32
|
/** Test seam; production uses AFTER_TOOL_BOUNDARY_TIMEOUT_MS. */
|
|
65
33
|
afterToolBoundaryTimeoutMs?: number;
|
|
66
|
-
/** Test seam; production uses STEERING_OCCUPANCY_IDLE_MS. */
|
|
67
|
-
steeringOccupancyIdleMs?: number;
|
|
68
34
|
}
|
|
69
35
|
/**
|
|
70
36
|
* Classify an ACP `stopReason` into a terminal outcome. A refusal and a
|
|
@@ -83,8 +49,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
83
49
|
private readonly child;
|
|
84
50
|
private readonly events;
|
|
85
51
|
private readonly conversation;
|
|
86
|
-
/** Cursor before this runner generation began; older durable events stay off the live console. */
|
|
87
|
-
private readonly conversationStartCursor?;
|
|
88
52
|
/** New on every runner start; permission/turn IDs from prior generations are stale. */
|
|
89
53
|
private readonly sessionGeneration;
|
|
90
54
|
/** True while `session/load` replays history as ordinary updates. */
|
|
@@ -94,12 +58,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
94
58
|
private connection;
|
|
95
59
|
private sessionId?;
|
|
96
60
|
private readiness;
|
|
97
|
-
/**
|
|
98
|
-
* Last non-replayed session update from the agent. `readiness` cannot answer
|
|
99
|
-
* "is this agent working" for a steered turn (FLEET-002), and this is the
|
|
100
|
-
* evidence that can.
|
|
101
|
-
*/
|
|
102
|
-
private lastUpdateAt?;
|
|
103
61
|
private lastError?;
|
|
104
62
|
private promptTail;
|
|
105
63
|
private queueDepth;
|
|
@@ -115,13 +73,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
115
73
|
private cancelEscalation?;
|
|
116
74
|
private cancelForceKill?;
|
|
117
75
|
private cancelRecoveryReason?;
|
|
118
|
-
/**
|
|
119
|
-
* Held while a steering-started turn is believed to own the adapter. It is a
|
|
120
|
-
* lease, not a latch: `steeringRelease` always fires, so the role can never be
|
|
121
|
-
* stranded busy by a wake whose turn ended without telling anyone.
|
|
122
|
-
*/
|
|
123
|
-
private steeringOccupied;
|
|
124
|
-
private steeringRelease?;
|
|
125
76
|
/**
|
|
126
77
|
* Rejects the moment the adapter process is gone. Every in-flight ACP request
|
|
127
78
|
* races it, so a dead adapter can never leave a turn — and therefore a
|
|
@@ -143,20 +94,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
143
94
|
*/
|
|
144
95
|
private recoverOpenPrompts;
|
|
145
96
|
isAlive(): boolean;
|
|
146
|
-
/**
|
|
147
|
-
* Take the occupancy lease for a turn the adapter started on its own behalf.
|
|
148
|
-
* Refreshed by every adapter update, so it tracks work actually happening
|
|
149
|
-
* rather than a fixed guess at how long a wake takes.
|
|
150
|
-
*/
|
|
151
|
-
private holdSteeringOccupancy;
|
|
152
|
-
private refreshSteeringOccupancy;
|
|
153
|
-
/**
|
|
154
|
-
* Every exit from occupancy comes through here, including the ones that are
|
|
155
|
-
* not the timer: a real turn boundary, close, and adapter exit. A lease that
|
|
156
|
-
* can leak is worse than the bug it fixes — it would leave the role reporting
|
|
157
|
-
* `running` forever and starve scheduled admission permanently.
|
|
158
|
-
*/
|
|
159
|
-
private releaseSteeringOccupancy;
|
|
160
97
|
snapshot(): SessionSnapshot;
|
|
161
98
|
private toolCall;
|
|
162
99
|
private reserveTool;
|
|
@@ -189,28 +126,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
189
126
|
* for it is what turned a busy agent into a timeout and then into "dead".
|
|
190
127
|
*/
|
|
191
128
|
queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
|
|
192
|
-
/**
|
|
193
|
-
* Prepare the session for a prompt that asked to pre-empt current work.
|
|
194
|
-
*
|
|
195
|
-
* The old behaviour was one unconditional `session/cancel` notification
|
|
196
|
-
* followed immediately by `session/prompt`. That is what produced the owner's
|
|
197
|
-
* "request failed before completion":
|
|
198
|
-
*
|
|
199
|
-
* - `cancelActive` only awaits settlement when `this.activeTurn` is set, and
|
|
200
|
-
* a turn the ADAPTER started (steering's `startedNewTurn`) is never tracked
|
|
201
|
-
* here. So the cancel raced the adapter's own transcript repair and the new
|
|
202
|
-
* prompt landed while the last assistant message still held an unresolved
|
|
203
|
-
* `tool_use` — rejected with `stop_reason=tool_use`.
|
|
204
|
-
* - With nothing running at all, it still sent the cancel, and the prompt
|
|
205
|
-
* landed on a bare interrupted user message — rejected with
|
|
206
|
-
* `stop_reason=null`.
|
|
207
|
-
*
|
|
208
|
-
* So: never cancel across a tool boundary, and never cancel something whose
|
|
209
|
-
* settlement cannot be awaited. Everything else is queued, which the ACP queue
|
|
210
|
-
* already does correctly. The returned state is what the caller may claim to a
|
|
211
|
-
* human — `interrupted` only when a turn really was cancelled.
|
|
212
|
-
*/
|
|
213
|
-
private prepareInterruptingDelivery;
|
|
214
129
|
/**
|
|
215
130
|
* Durably record a prompt admission BEFORE acceptance is returned. Browser
|
|
216
131
|
* admissions are transactional — a prompt the ledger cannot hold is refused,
|
|
@@ -258,15 +173,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
258
173
|
private settlePendingAutomatically;
|
|
259
174
|
exitResult(): ExitRecord | null;
|
|
260
175
|
close(): Promise<void>;
|
|
261
|
-
/**
|
|
262
|
-
* The role's declared MCP servers, or `[]`.
|
|
263
|
-
*
|
|
264
|
-
* Sent on resume and load as well as on new: the agent builds its server set
|
|
265
|
-
* once per session, so a resumed session that omitted them would come back
|
|
266
|
-
* without the tools the role's config declares — which is exactly the shape of
|
|
267
|
-
* silent drop this plumbing exists to end.
|
|
268
|
-
*/
|
|
269
|
-
private declaredMcpServers;
|
|
270
176
|
private initialize;
|
|
271
177
|
private captureRuntimeMetadata;
|
|
272
178
|
private runPrompt;
|
|
@@ -279,15 +185,6 @@ export declare class AcpSession implements SessionHandle {
|
|
|
279
185
|
*/
|
|
280
186
|
private settleAutomatically;
|
|
281
187
|
private withinAutomaticBoundary;
|
|
282
|
-
/**
|
|
283
|
-
* Codex ACP 1.1.7 marks its protected MCP elicitation bridge on a locationless
|
|
284
|
-
* execute request. The marker is meaningful only together with the runner's
|
|
285
|
-
* independently supplied, adapter-authenticated metadata vocabulary and effective
|
|
286
|
-
* mode: an arbitrary ACP process cannot gain this path by copying `_meta` alone.
|
|
287
|
-
* Exact option ids/kinds bind recognition to the protected-MCP shape and keep
|
|
288
|
-
* malformed requests on the ordinary fail-closed path.
|
|
289
|
-
*/
|
|
290
|
-
private isEffectiveCodexProtectedMcpApproval;
|
|
291
188
|
private recordUpdate;
|
|
292
189
|
/**
|
|
293
190
|
* Codex ACP's phase extension is the only currently supported visibility
|
|
@@ -302,6 +199,5 @@ export declare class AcpSession implements SessionHandle {
|
|
|
302
199
|
}): ConversationHandlePage;
|
|
303
200
|
conversationSnapshot(): ConversationSnapshot;
|
|
304
201
|
subscribeConversation(listener: Parameters<ConversationEventStore['subscribe']>[0]): () => void;
|
|
305
|
-
private isCurrentConversationEvent;
|
|
306
202
|
private fail;
|
|
307
203
|
}
|