@ours.network/fleet 0.9.4 → 0.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +148 -30
  2. package/dist/atomic-file.d.ts +30 -0
  3. package/dist/atomic-file.js +86 -0
  4. package/dist/briefing.d.ts +6 -0
  5. package/dist/briefing.js +41 -11
  6. package/dist/cli.js +238 -26
  7. package/dist/config.d.ts +39 -1
  8. package/dist/config.js +126 -3
  9. package/dist/creation.d.ts +179 -0
  10. package/dist/creation.js +254 -0
  11. package/dist/docs.d.ts +34 -0
  12. package/dist/docs.js +309 -0
  13. package/dist/doctor.js +123 -21
  14. package/dist/harness/acp-agent.d.ts +11 -0
  15. package/dist/harness/acp-agent.js +27 -0
  16. package/dist/harness/claude-code.d.ts +39 -3
  17. package/dist/harness/claude-code.js +145 -13
  18. package/dist/harness/codex.d.ts +7 -1
  19. package/dist/harness/codex.js +89 -4
  20. package/dist/harness/registry.d.ts +2 -0
  21. package/dist/harness/registry.js +19 -0
  22. package/dist/harness/types.d.ts +59 -1
  23. package/dist/index.d.ts +6 -3
  24. package/dist/index.js +3 -1
  25. package/dist/isolation/bubblewrap.js +7 -1
  26. package/dist/isolation/policy.d.ts +34 -5
  27. package/dist/isolation/policy.js +114 -7
  28. package/dist/isolation/resources.d.ts +6 -3
  29. package/dist/isolation/resources.js +6 -3
  30. package/dist/isolation/types.d.ts +19 -1
  31. package/dist/monitor.d.ts +44 -2
  32. package/dist/monitor.js +177 -42
  33. package/dist/ops.d.ts +15 -2
  34. package/dist/ops.js +32 -9
  35. package/dist/permissions.d.ts +70 -0
  36. package/dist/permissions.js +97 -0
  37. package/dist/runner.d.ts +65 -2
  38. package/dist/runner.js +307 -32
  39. package/dist/session/acp.d.ts +70 -0
  40. package/dist/session/acp.js +364 -0
  41. package/dist/session/control.d.ts +89 -0
  42. package/dist/session/control.js +322 -0
  43. package/dist/session/events.d.ts +14 -0
  44. package/dist/session/events.js +67 -0
  45. package/dist/session/tmux.d.ts +27 -0
  46. package/dist/session/tmux.js +76 -0
  47. package/dist/session/types.d.ts +138 -0
  48. package/dist/session/types.js +42 -0
  49. package/dist/spawn.d.ts +32 -2
  50. package/dist/spawn.js +177 -16
  51. package/dist/supervisor/launchd.d.ts +50 -0
  52. package/dist/supervisor/launchd.js +121 -4
  53. package/dist/supervisor/none.js +22 -4
  54. package/dist/supervisor/systemd.d.ts +8 -1
  55. package/dist/supervisor/systemd.js +94 -4
  56. package/dist/supervisor/types.d.ts +36 -3
  57. package/dist/tmux.d.ts +34 -2
  58. package/dist/tmux.js +48 -11
  59. package/package.json +7 -2
@@ -0,0 +1,97 @@
1
+ import { getAdapter } from './harness/registry.js';
2
+ /**
3
+ * The capability floor every unattended role must clear. These are not
4
+ * nice-to-haves: an agent that cannot read its briefing, append its worklog,
5
+ * bind its identity, arm its monitor, edit its workspace, or run the status
6
+ * commands its briefing prescribes cannot carry out the job it was spawned for
7
+ * — and, being unattended, will report no error while failing to.
8
+ */
9
+ export const UNATTENDED_FLOOR = [
10
+ 'read-state', 'write-state', 'messaging', 'monitor', 'workspace-edit', 'status-commands',
11
+ ];
12
+ /** Which floor capabilities a set of granted capabilities fails to cover. */
13
+ export function checkUnattendedFloor(granted) {
14
+ const missing = UNATTENDED_FLOOR.filter(c => !granted.includes(c));
15
+ return { meets: missing.length === 0, missing };
16
+ }
17
+ /**
18
+ * Find native settings that contradict the neutral block. Only fires when the
19
+ * operator wrote BOTH — a role that states its intent once, neutrally or
20
+ * natively, has nothing to contradict and stays quiet. `harness_options` wins
21
+ * at launch, which is precisely why a silent disagreement is dangerous: the
22
+ * neutral block reads like the source of truth and is not.
23
+ */
24
+ function findConflicts(role, fromNeutral, fromNative) {
25
+ if (!role.permissionsDeclared)
26
+ return [];
27
+ const conflicts = [];
28
+ for (const [key, nativeValue] of Object.entries(fromNative)) {
29
+ const neutralValue = fromNeutral[key];
30
+ if (neutralValue === undefined || String(neutralValue) === String(nativeValue))
31
+ continue;
32
+ conflicts.push({
33
+ key,
34
+ fromNeutral: String(neutralValue),
35
+ fromNative: String(nativeValue),
36
+ warning: `role '${role.name}': harness_options.${key}=${String(nativeValue)} contradicts the `
37
+ + `permissions block, which translates to ${key}=${String(neutralValue)} — `
38
+ + `harness_options.${key}=${String(nativeValue)} wins`,
39
+ });
40
+ }
41
+ return conflicts;
42
+ }
43
+ /** Resolve one role's permissions through its adapter. Never throws. */
44
+ export function analyzeRolePermissions(role) {
45
+ const base = { role: role.name, harness: role.harness, permissions: role.permissions };
46
+ let adapter;
47
+ try {
48
+ adapter = getAdapter(role.harness);
49
+ }
50
+ catch (e) {
51
+ return { ...base, supported: false, warnings: [`role '${role.name}': ${e.message}`] };
52
+ }
53
+ const translation = adapter.translatePermissions(role.permissions);
54
+ if (!translation.supported) {
55
+ return {
56
+ ...base, supported: false,
57
+ warnings: [`role '${role.name}': harness '${role.harness}' cannot express neutral ` +
58
+ `permissions — ${translation.reason}`],
59
+ };
60
+ }
61
+ const conflicts = findConflicts(role, translation.native, adapter.nativePermissionOverrides(role.harness_options));
62
+ const floor = checkUnattendedFloor(translation.capabilities);
63
+ const floorSeverity = role.permissions.unattended === 'deny' ? 'fail' : 'warn';
64
+ return {
65
+ ...base,
66
+ supported: true,
67
+ native: translation.native,
68
+ exact: translation.exact,
69
+ capabilities: translation.capabilities,
70
+ floor,
71
+ floorSeverity,
72
+ conflicts,
73
+ floorWarning: floor.meets ? undefined : (`role '${role.name}': resolved ${role.harness} permissions do not meet the unattended ` +
74
+ `capability floor — missing ${floor.missing.join(', ')} ` +
75
+ `(${formatNative(translation.native)}; unattended=${role.permissions.unattended} means these ` +
76
+ `requests will ${role.permissions.unattended === 'deny' ? 'be denied silently' : 'block the turn'})`),
77
+ warnings: translation.warnings.map(w => `role '${role.name}': ${w}`),
78
+ };
79
+ }
80
+ /** Every line a command should show for a role: translation, conflicts, floor. */
81
+ export function allWarnings(a) {
82
+ return [
83
+ ...a.warnings,
84
+ ...(a.conflicts ?? []).map(c => c.warning),
85
+ ...(a.floorWarning ? [a.floorWarning] : []),
86
+ ];
87
+ }
88
+ /** Resolve every role's permissions, in config order. */
89
+ export function analyzeFleetPermissions(roles) {
90
+ return roles.map(analyzeRolePermissions);
91
+ }
92
+ /** Render an analysis's native settings compactly, for one-line reporting. */
93
+ export function formatNative(native) {
94
+ if (!native || !Object.keys(native).length)
95
+ return '(none)';
96
+ return Object.entries(native).map(([k, v]) => `${k}=${String(v)}`).join(' ');
97
+ }
package/dist/runner.d.ts CHANGED
@@ -3,6 +3,7 @@ import type { Launch } from './harness/types.js';
3
3
  import { Tmux } from './tmux.js';
4
4
  import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
5
5
  import { type Exec } from './exec.js';
6
+ import type { ExitRecord } from './session/types.js';
6
7
  export interface RunnerDeps {
7
8
  tmux: Tmux;
8
9
  exec: Exec;
@@ -15,6 +16,8 @@ export interface RunnerDeps {
15
16
  fetch: FetchLike;
16
17
  /** Construct the supervisor mail monitor (injectable so tests stub it out). */
17
18
  createMonitor(opts: MonitorOpts): MonitorHandle;
19
+ /** Lets a test (or a shutdown path) end the supervised restart loop. */
20
+ shouldStop?(): boolean;
18
21
  }
19
22
  /**
20
23
  * Compose the tmux pane shell command: env prefix + argv + exit-status capture.
@@ -24,6 +27,41 @@ export interface RunnerDeps {
24
27
  * still sees the real exit code.
25
28
  */
26
29
  export declare function buildPaneCommand(launch: Launch, roleEnv: Record<string, string> | undefined, exitStatusPath: string, paneArgv?: string[]): string;
30
+ /**
31
+ * Read the pane's `.exit-status`. Three shapes are accepted: the structured
32
+ * record written above, a bare number left by a pre-upgrade pane (so an
33
+ * in-place upgrade does not misread a real exit), and anything else — which is
34
+ * `unknown`, never an invented failure. A missing file returns null so the
35
+ * caller can distinguish "no record" from "a record saying unknown".
36
+ */
37
+ export declare function readExitRecord(path: string): ExitRecord | null;
38
+ export declare const RESTART_LEDGER_FILE = ".restart-ledger.json";
39
+ /** Consecutive immediate failures tolerated before the agent is held down. */
40
+ export declare const RESTART_FAIL_THRESHOLD = 5;
41
+ export interface RestartLedger {
42
+ version: 1;
43
+ consecutiveImmediateFailures: number;
44
+ lastReason: string;
45
+ nextDelayMs: number;
46
+ /** Whether this failure sequence has already thrown away resume state. */
47
+ resumeDiscarded: boolean;
48
+ circuit: 'closed' | 'open';
49
+ updatedAt: string;
50
+ /** When the circuit opened, for the held-down status line. */
51
+ openedAt?: string;
52
+ }
53
+ /** Bounded exponential backoff for the nth consecutive immediate failure. */
54
+ export declare function backoffFor(consecutiveFailures: number): number;
55
+ /** Read a role's restart ledger; a missing or corrupt one starts clean. */
56
+ export declare function readRestartLedger(dir: string): RestartLedger;
57
+ export declare function writeRestartLedger(dir: string, ledger: RestartLedger): void;
58
+ /**
59
+ * Close the circuit and forget the failure streak. Called by an explicit
60
+ * operator `up`/`restart`, which is the only thing that may release a held-down
61
+ * role — a held-down runner polls this file, so a role can be released without
62
+ * bouncing its unit.
63
+ */
64
+ export declare function resetRestartLedger(dir: string): void;
27
65
  /** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
28
66
  export declare const START_STAGGER_FILE = ".start-stagger-ms";
29
67
  /**
@@ -37,10 +75,35 @@ export declare const START_STAGGER_FILE = ".start-stagger-ms";
37
75
  export declare function reserveLaunchSlot(root: string, staggerMs: number, deps: Pick<RunnerDeps, 'now' | 'sleep' | 'log'>): Promise<number>;
38
76
  /** Read a temp role's config snapshot written by spawnTemp. */
39
77
  export declare function loadTempRole(name: string): ResolvedRole;
40
- /** One supervised session lifecycle. The supervisor re-invokes us after we return. */
78
+ /** What one child session did, so the supervising loop can decide what follows. */
79
+ export interface AttemptResult {
80
+ elapsedSecs: number;
81
+ exit: ExitRecord;
82
+ /** Whether this attempt threw away resume state to start fresh. */
83
+ rotated: boolean;
84
+ mode: 'fresh' | 'resume';
85
+ }
86
+ /** One session lifecycle. `runSupervised` (or a one-shot caller) drives it. */
41
87
  export declare function runOnce(name: string, opts?: {
42
88
  temp?: boolean;
43
89
  configPath?: string;
44
- }, partialDeps?: Partial<RunnerDeps>): Promise<void>;
90
+ allowResumeRotation?: boolean;
91
+ }, partialDeps?: Partial<RunnerDeps>): Promise<AttemptResult>;
92
+ /**
93
+ * The persistent supervisor for one permanent role: run child sessions in a
94
+ * loop, count consecutive immediate failures across them, back off between
95
+ * attempts, and after `RESTART_FAIL_THRESHOLD` hold the agent down while
96
+ * staying alive — so the service manager has nothing to restart and cannot
97
+ * resume the two-second loop behind our back.
98
+ *
99
+ * `attempt` is injectable so the policy can be tested against a fake clock and
100
+ * fake child instead of real sessions.
101
+ */
102
+ export declare function runSupervised(name: string, opts?: {
103
+ configPath?: string;
104
+ }, partialDeps?: Partial<RunnerDeps>, attempt?: (n: string, o: {
105
+ configPath?: string;
106
+ allowResumeRotation?: boolean;
107
+ }, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
45
108
  /** Temp-agent entrypoint: run one session, then remove the temp dir. */
46
109
  export declare function runTemp(name: string, deps?: Partial<RunnerDeps>): Promise<void>;
package/dist/runner.js CHANGED
@@ -2,8 +2,8 @@ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node
2
2
  import { join } from 'node:path';
3
3
  import { randomUUID } from 'node:crypto';
4
4
  import { parse } from 'yaml';
5
- import { agentDir, home, stateRoot } from './paths.js';
6
- import { loadConfig, findRole } from './config.js';
5
+ import { agentDir, stateRoot } from './paths.js';
6
+ import { loadConfig, findRole, isolationContextFor, resolvePermissions, } from './config.js';
7
7
  import { getAdapter } from './harness/registry.js';
8
8
  import { Tmux } from './tmux.js';
9
9
  import { createMonitor } from './monitor.js';
@@ -11,6 +11,10 @@ import { realExec, shq } from './exec.js';
11
11
  import { resolveIsolation } from './isolation/policy.js';
12
12
  import { selectIsolationBackend } from './isolation/registry.js';
13
13
  import { resourceArgs, cpuControllerDelegated } from './isolation/resources.js';
14
+ import { AcpSession } from './session/acp.js';
15
+ import { RoleControlServer } from './session/control.js';
16
+ import { TmuxSession } from './session/tmux.js';
17
+ import { classifyShellStatus } from './session/types.js';
14
18
  const defaultDeps = () => ({
15
19
  tmux: new Tmux(),
16
20
  exec: realExec,
@@ -39,7 +43,12 @@ export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = lau
39
43
  const env = { PATH: process.env.PATH ?? '', ...launch.env, ...(roleEnv ?? {}) };
40
44
  const envPfx = 'env ' + Object.entries(env).map(([k, v]) => `${k}=${shq(v)}`).join(' ');
41
45
  const cmd = paneArgv.map(shq).join(' ');
42
- return `${envPfx} ${cmd}; echo $? > ${shq(exitStatusPath)}`;
46
+ // Write a structured record, not a bare number: the wait status alone cannot
47
+ // say whether the file is missing because the program never exited or because
48
+ // nothing ever wrote it. `printf` is POSIX; no shell branching is needed
49
+ // because classification happens in one place, in TypeScript.
50
+ const record = `'{"version":1,"backend":"tmux","status":'"$__ofs"'}'`;
51
+ return `${envPfx} ${cmd}; __ofs=$?; printf %s ${record} > ${shq(exitStatusPath)}`;
43
52
  }
44
53
  /** Adapt runner deps and the role's daemon-profile overrides for the monitor. */
45
54
  function monitorDeps(deps, roleEnv) {
@@ -56,6 +65,89 @@ function monitorDeps(deps, roleEnv) {
56
65
  timers: { set: (fn, ms) => setTimeout(fn, ms), clear: t => clearTimeout(t) },
57
66
  };
58
67
  }
68
+ /**
69
+ * Read the pane's `.exit-status`. Three shapes are accepted: the structured
70
+ * record written above, a bare number left by a pre-upgrade pane (so an
71
+ * in-place upgrade does not misread a real exit), and anything else — which is
72
+ * `unknown`, never an invented failure. A missing file returns null so the
73
+ * caller can distinguish "no record" from "a record saying unknown".
74
+ */
75
+ export function readExitRecord(path) {
76
+ if (!existsSync(path))
77
+ return null;
78
+ const raw = readFileSync(path, 'utf8').trim();
79
+ if (!raw)
80
+ return { version: 1, class: 'unknown', detail: 'the pane left an empty exit record' };
81
+ if (/^-?\d+$/.test(raw))
82
+ return classifyShellStatus(Number(raw)); // legacy `echo $?`
83
+ try {
84
+ const parsed = JSON.parse(raw);
85
+ if (typeof parsed.status === 'number')
86
+ return classifyShellStatus(parsed.status);
87
+ }
88
+ catch { /* fall through to unknown */ }
89
+ return { version: 1, class: 'unknown', detail: `unreadable exit record: ${raw.slice(0, 120)}` };
90
+ }
91
+ // ─── Restart-loop containment (3.2) ──────────────────────────────────────────
92
+ //
93
+ // The child-session restart loop used to BE the service manager: systemd's
94
+ // `Restart=always RestartSec=2` and launchd's `KeepAlive`. Neither can count,
95
+ // so a program that dies instantly was relaunched every two seconds forever,
96
+ // and each relaunch was a fresh process with no memory of the previous one.
97
+ // The count now lives with the role, in its state directory, so it survives the
98
+ // runner being restarted and is consistent across both service managers.
99
+ export const RESTART_LEDGER_FILE = '.restart-ledger.json';
100
+ /** Consecutive immediate failures tolerated before the agent is held down. */
101
+ export const RESTART_FAIL_THRESHOLD = 5;
102
+ const RESTART_BACKOFF_BASE_MS = 2_000;
103
+ const RESTART_BACKOFF_MAX_MS = 60_000;
104
+ /** How often a held-down runner re-reads its ledger, so `up` can release it. */
105
+ const HELD_DOWN_POLL_MS = 5_000;
106
+ const emptyLedger = () => ({
107
+ version: 1,
108
+ consecutiveImmediateFailures: 0,
109
+ lastReason: '',
110
+ nextDelayMs: 0,
111
+ resumeDiscarded: false,
112
+ circuit: 'closed',
113
+ updatedAt: new Date(0).toISOString(),
114
+ });
115
+ /** Bounded exponential backoff for the nth consecutive immediate failure. */
116
+ export function backoffFor(consecutiveFailures) {
117
+ if (consecutiveFailures <= 0)
118
+ return 0;
119
+ return Math.min(RESTART_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESTART_BACKOFF_MAX_MS);
120
+ }
121
+ /** Read a role's restart ledger; a missing or corrupt one starts clean. */
122
+ export function readRestartLedger(dir) {
123
+ try {
124
+ const raw = JSON.parse(readFileSync(join(dir, RESTART_LEDGER_FILE), 'utf8'));
125
+ if (raw.version !== 1)
126
+ return emptyLedger();
127
+ return { ...emptyLedger(), ...raw, version: 1 };
128
+ }
129
+ catch {
130
+ return emptyLedger();
131
+ }
132
+ }
133
+ export function writeRestartLedger(dir, ledger) {
134
+ try {
135
+ mkdirSync(dir, { recursive: true });
136
+ writeFileSync(join(dir, RESTART_LEDGER_FILE), JSON.stringify(ledger, null, 2) + '\n');
137
+ }
138
+ catch { /* diagnostics must never take the role down */ }
139
+ }
140
+ /**
141
+ * Close the circuit and forget the failure streak. Called by an explicit
142
+ * operator `up`/`restart`, which is the only thing that may release a held-down
143
+ * role — a held-down runner polls this file, so a role can be released without
144
+ * bouncing its unit.
145
+ */
146
+ export function resetRestartLedger(dir) {
147
+ if (!existsSync(dir))
148
+ return;
149
+ writeRestartLedger(dir, { ...emptyLedger(), updatedAt: new Date().toISOString() });
150
+ }
59
151
  /** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
60
152
  export const START_STAGGER_FILE = '.start-stagger-ms';
61
153
  /** Read the start-stagger a temp agent was spawned with (0 if none / unreadable). */
@@ -148,7 +240,7 @@ function resolveConfigPath(dir, explicit) {
148
240
  return undefined;
149
241
  return readFileSync(marker, 'utf8').trim() || undefined;
150
242
  }
151
- /** One supervised session lifecycle. The supervisor re-invokes us after we return. */
243
+ /** One session lifecycle. `runSupervised` (or a one-shot caller) drives it. */
152
244
  export async function runOnce(name, opts = {}, partialDeps = {}) {
153
245
  const deps = { ...defaultDeps(), ...partialDeps };
154
246
  const temp = opts.temp === true;
@@ -182,17 +274,22 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
182
274
  writeFileSync(bootedFile, '');
183
275
  const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
184
276
  const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
185
- const launch = adapter.buildLaunch(role, mode, { sessionId }, prep);
277
+ const sessionBackend = role.session ?? 'tmux';
278
+ const launch = sessionBackend === 'acp'
279
+ ? (() => {
280
+ if (!adapter.buildAcpLaunch)
281
+ throw new Error(`harness '${role.harness}' does not support the ACP session backend`);
282
+ return adapter.buildAcpLaunch(role, prep);
283
+ })()
284
+ : adapter.buildLaunch(role, mode, { sessionId }, prep);
186
285
  // Isolation is additive: only roles that declare `isolation:` are wrapped. The
187
286
  // env prefix + exit capture in buildPaneCommand stay host-side (see §5.3).
188
- let paneArgv = launch.argv;
287
+ let wrappedArgv = launch.argv;
189
288
  if (role.isolation) {
190
- const addDirs = role.harness === 'codex'
191
- ? (role.harness_options?.add_dirs ?? [])
192
- : [];
193
- const ctx = {
194
- stateDir: dir, runCwd, home: home(), harness: role.harness, additionalWriteDirs: addDirs,
195
- };
289
+ // The SAME context config validation and doctor judged (5.2): a policy
290
+ // checked against a different mount set than the one that launches is not a
291
+ // check at all.
292
+ const ctx = { ...isolationContextFor(role), stateDir: dir, runCwd };
196
293
  const policy = resolveIsolation(role.isolation, ctx);
197
294
  const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
198
295
  const degradedMarker = join(dir, '.isolation-degraded');
@@ -204,14 +301,14 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
204
301
  deps.log(`[${name}] isolation: ${sel.backend.id} (net=${policy.network}) ${sel.detail}`);
205
302
  rmSync(degradedMarker, { force: true });
206
303
  }
207
- paneArgv = sel.backend.wrap(launch.argv, policy, ctx);
304
+ wrappedArgv = sel.backend.wrap(launch.argv, policy, ctx);
208
305
  // Resource caps wrap the sandbox from OUTSIDE, at the pane's own cgroup scope
209
306
  // (§5.4). Applies even when the sandbox degraded to none.
210
307
  const { argv: rprefix, warnings } = resourceArgs(policy.resources, deps.cpuDelegated());
211
308
  for (const w of warnings)
212
309
  deps.log(`[${name}] WARNING ${w}`);
213
310
  if (rprefix.length)
214
- paneArgv = [...rprefix, ...paneArgv];
311
+ wrappedArgv = [...rprefix, ...wrappedArgv];
215
312
  }
216
313
  // Start-stagger: space this launch at least start_stagger_ms after the previous
217
314
  // agent launch across the whole host, so a burst of boots (systemd starts every
@@ -232,47 +329,225 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
232
329
  // (backlog before the tip is the SessionStart hook's job). Disabled roles keep
233
330
  // the legacy in-session watch. Temp snapshots predating `monitor:` are treated
234
331
  // as disabled (monitor may be undefined on an old role.yaml).
332
+ const resolvedMonitorDeps = monitorDeps(deps, role.env);
235
333
  const monitor = role.monitor?.enabled ? deps.createMonitor({
236
- name, agentDir: dir, cfg: role.monitor,
237
- deps: monitorDeps(deps, role.env),
334
+ name, identity: role.identity, agentDir: dir, cfg: role.monitor,
335
+ deps: resolvedMonitorDeps,
238
336
  }) : null;
239
337
  if (monitor)
240
338
  await monitor.prime();
241
339
  rmSync(exitFile, { force: true });
242
- await deps.tmux.kill(name);
243
- await deps.tmux.newSession(name, runCwd, buildPaneCommand(launch, role.env, exitFile, paneArgv));
244
- let pid = null;
245
- for (let i = 0; i < 40 && pid === null; i++) {
246
- pid = await deps.tmux.panePid(name);
247
- if (pid === null)
248
- await deps.sleep(250);
340
+ let pid;
341
+ let sessionHandle;
342
+ let acpSession;
343
+ let control;
344
+ if (sessionBackend === 'acp') {
345
+ const perms = role.permissions ?? resolvePermissions(undefined, undefined);
346
+ // Say once, at startup, that this role will decide permission requests by
347
+ // itself. Without it the only trace of an auto-denied tool call is a turn
348
+ // that quietly did less than it was asked to.
349
+ if (perms.unattended === 'deny')
350
+ deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
351
+ `permission requests are automatically denied once each (reject_once) and the turn continues`);
352
+ acpSession = await AcpSession.start({
353
+ name,
354
+ argv: wrappedArgv,
355
+ cwd: runCwd,
356
+ env: { ...launch.env, ...(role.env ?? {}) },
357
+ stateDir: dir,
358
+ mode,
359
+ permissions: perms,
360
+ log: deps.log,
361
+ });
362
+ pid = acpSession.pid;
363
+ sessionHandle = acpSession;
364
+ control = new RoleControlServer(dir, acpSession, deps.log);
365
+ await control.start();
366
+ resolvedMonitorDeps.delivery = {
367
+ // A wake is only delivered when its turn TERMINATES successfully. A
368
+ // refusal or a cancellation reached the agent and was not acted on, so
369
+ // the monitor must keep its cursor and try again.
370
+ submit: async (text) => {
371
+ const result = await acpSession.submitPrompt(text);
372
+ return { succeeded: result.succeeded, outcome: result.outcome, detail: result.detail };
373
+ },
374
+ };
375
+ const firstPrompt = mode === 'fresh'
376
+ ? `Read and follow ${join(dir, 'briefing.md')} now.`
377
+ : adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
378
+ // Wait for the first turn's TERMINAL result. An agent that accepts the
379
+ // startup prompt and then refuses it has not started; logging the role as
380
+ // up would hide a role that never read its briefing.
381
+ const started = await acpSession.submitPrompt(firstPrompt);
382
+ if (!started.succeeded) {
383
+ monitor?.stop();
384
+ await control.close();
385
+ await acpSession.close();
386
+ throw new Error(`[${name}] ACP startup prompt ${started.outcome}` +
387
+ `${started.detail ? `: ${started.detail}` : ''}`);
388
+ }
249
389
  }
250
- if (pid === null)
251
- throw new Error(`[${name}] could not resolve tmux pane pid`);
252
- deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} mode=${mode}`);
390
+ else {
391
+ await deps.tmux.kill(name);
392
+ await deps.tmux.newSession(name, runCwd, buildPaneCommand(launch, role.env, exitFile, wrappedArgv));
393
+ let panePid = null;
394
+ for (let i = 0; i < 40 && panePid === null; i++) {
395
+ panePid = await deps.tmux.panePid(name);
396
+ if (panePid === null)
397
+ await deps.sleep(250);
398
+ }
399
+ if (panePid === null)
400
+ throw new Error(`[${name}] could not resolve tmux pane pid`);
401
+ pid = panePid;
402
+ sessionHandle = new TmuxSession(name, pid, deps.tmux, deps.isAlive);
403
+ }
404
+ deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
253
405
  // The monitor loop lives exactly as long as the session: it starts once the
254
406
  // pane pid is known and is stopped when that pid dies (task dies with runner).
255
407
  const monitorLoop = monitor?.run(pid);
256
408
  const start = deps.now();
257
- while (deps.isAlive(pid))
409
+ while (sessionHandle.isAlive())
258
410
  await deps.sleep(2000);
259
411
  if (monitor) {
260
412
  monitor.stop();
261
413
  await monitorLoop;
262
414
  }
415
+ if (control)
416
+ await control.close();
417
+ if (acpSession)
418
+ await acpSession.close();
263
419
  const elapsed = (deps.now() - start) / 1000;
264
- const code = existsSync(exitFile) ? readFileSync(exitFile, 'utf8').trim() : 'crash';
420
+ // Establish what actually happened before deciding anything. Absence of a
421
+ // record is `unknown` — except when the console itself is gone, which is a
422
+ // different event with a different consequence.
423
+ const exitRecord = acpSession
424
+ ? acpSession.exitResult()
425
+ ?? { version: 1, class: 'unknown', detail: 'the ACP agent stopped without reporting an exit' }
426
+ : readExitRecord(exitFile)
427
+ ?? (await deps.tmux.has(name)
428
+ ? { version: 1, class: 'unknown', detail: 'the pane process ended without writing an exit record' }
429
+ : { version: 1, class: 'session-destroyed', detail: `the tmux session '${name}' no longer exists` });
430
+ writeFileSync(exitFile, JSON.stringify({
431
+ ...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
432
+ }) + '\n');
433
+ let rotated = false;
265
434
  const rotate = (why) => {
266
435
  writeFileSync(sidFile, randomUUID() + '\n');
267
436
  rmSync(bootedFile, { force: true });
437
+ rotated = true;
268
438
  deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
269
439
  };
270
- if (code === '0' && adapter.exitPolicy.cleanExitIsFresh)
440
+ if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
271
441
  rotate(`clean exit (code 0)`);
272
- else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs)
273
- rotate(`resume failed fast (${elapsed.toFixed(0)}s, code ${code})`);
442
+ else if (exitRecord.class === 'session-destroyed')
443
+ // Someone tore the console down; the agent did not fail. Rotating here
444
+ // would discard a live conversation for an operator action.
445
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
446
+ else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
447
+ // Self-heal a poisoned resume — but only once per failure sequence. Rotating
448
+ // on every attempt would discard the conversation again and again while the
449
+ // real cause (a broken command, a missing binary) went unaddressed.
450
+ if (opts.allowResumeRotation === false)
451
+ deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
452
+ `-> resume state was already discarded once; keeping it`);
453
+ else
454
+ rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
455
+ }
274
456
  else
275
- deps.log(`[${name}] exited (code ${code}, ${elapsed.toFixed(0)}s) -> next start RESUMES context`);
457
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
458
+ return { elapsedSecs: elapsed, exit: exitRecord, rotated, mode };
459
+ }
460
+ /**
461
+ * The persistent supervisor for one permanent role: run child sessions in a
462
+ * loop, count consecutive immediate failures across them, back off between
463
+ * attempts, and after `RESTART_FAIL_THRESHOLD` hold the agent down while
464
+ * staying alive — so the service manager has nothing to restart and cannot
465
+ * resume the two-second loop behind our back.
466
+ *
467
+ * `attempt` is injectable so the policy can be tested against a fake clock and
468
+ * fake child instead of real sessions.
469
+ */
470
+ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt = runOnce) {
471
+ const deps = { ...defaultDeps(), ...partialDeps };
472
+ const dir = agentDir(name);
473
+ mkdirSync(dir, { recursive: true });
474
+ const shouldStop = deps.shouldStop ?? (() => false);
475
+ const stamp = () => new Date(deps.now()).toISOString();
476
+ while (!shouldStop()) {
477
+ let ledger = readRestartLedger(dir);
478
+ if (ledger.circuit === 'open') {
479
+ // Held down. Stay alive — exiting would hand the role straight back to
480
+ // the service manager — and watch for an operator reset.
481
+ await deps.sleep(HELD_DOWN_POLL_MS);
482
+ continue;
483
+ }
484
+ let result;
485
+ try {
486
+ result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
487
+ }
488
+ catch (e) {
489
+ // A session that could not even start is an immediate failure like any
490
+ // other; it must count, or an unstartable role loops forever.
491
+ result = {
492
+ elapsedSecs: 0,
493
+ exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
494
+ rotated: false,
495
+ mode: 'fresh',
496
+ };
497
+ }
498
+ // Re-read: the attempt itself may have taken minutes, and an operator may
499
+ // have reset the ledger meanwhile.
500
+ ledger = readRestartLedger(dir);
501
+ const fastFailSecs = fastFailSecsFor(name, opts.configPath);
502
+ const immediate = result.elapsedSecs < fastFailSecs;
503
+ if (!immediate) {
504
+ // A session that ran for a while is not a restart loop, whatever ended it.
505
+ writeRestartLedger(dir, {
506
+ ...emptyLedger(),
507
+ lastReason: result.exit.detail,
508
+ updatedAt: stamp(),
509
+ });
510
+ continue;
511
+ }
512
+ const failures = ledger.consecutiveImmediateFailures + 1;
513
+ const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
514
+ const next = {
515
+ version: 1,
516
+ consecutiveImmediateFailures: failures,
517
+ lastReason: reason,
518
+ nextDelayMs: backoffFor(failures),
519
+ resumeDiscarded: ledger.resumeDiscarded || result.rotated,
520
+ circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
521
+ updatedAt: stamp(),
522
+ };
523
+ if (next.circuit === 'open') {
524
+ next.openedAt = stamp();
525
+ next.nextDelayMs = 0;
526
+ writeRestartLedger(dir, next);
527
+ deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
528
+ `${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
529
+ continue;
530
+ }
531
+ writeRestartLedger(dir, next);
532
+ deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
533
+ `-> backing off ${next.nextDelayMs}ms`);
534
+ await deps.sleep(next.nextDelayMs);
535
+ }
536
+ return readRestartLedger(dir);
537
+ }
538
+ /**
539
+ * How short an attempt has to be to count as immediate. The role's harness
540
+ * decides; an unreadable config falls back to the common 20s so a broken config
541
+ * cannot disable the breaker.
542
+ */
543
+ function fastFailSecsFor(name, configPath) {
544
+ try {
545
+ const role = findRole(loadConfig(configPath), name);
546
+ return getAdapter(role.harness).exitPolicy.fastFailSecs;
547
+ }
548
+ catch {
549
+ return 20;
550
+ }
276
551
  }
277
552
  /** Temp-agent entrypoint: run one session, then remove the temp dir. */
278
553
  export async function runTemp(name, deps = {}) {