@ours.network/fleet 0.9.4 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -30
- package/dist/atomic-file.d.ts +30 -0
- package/dist/atomic-file.js +86 -0
- package/dist/briefing.d.ts +6 -0
- package/dist/briefing.js +41 -11
- package/dist/cli.js +238 -26
- package/dist/config.d.ts +39 -1
- package/dist/config.js +126 -3
- package/dist/creation.d.ts +179 -0
- package/dist/creation.js +254 -0
- package/dist/docs.d.ts +34 -0
- package/dist/docs.js +309 -0
- package/dist/doctor.js +123 -21
- package/dist/harness/acp-agent.d.ts +11 -0
- package/dist/harness/acp-agent.js +27 -0
- package/dist/harness/claude-code.d.ts +39 -3
- package/dist/harness/claude-code.js +145 -13
- package/dist/harness/codex.d.ts +7 -1
- package/dist/harness/codex.js +89 -4
- package/dist/harness/registry.d.ts +2 -0
- package/dist/harness/registry.js +19 -0
- package/dist/harness/types.d.ts +59 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +3 -1
- package/dist/isolation/bubblewrap.js +7 -1
- package/dist/isolation/policy.d.ts +34 -5
- package/dist/isolation/policy.js +114 -7
- package/dist/isolation/resources.d.ts +6 -3
- package/dist/isolation/resources.js +6 -3
- package/dist/isolation/types.d.ts +19 -1
- package/dist/monitor.d.ts +44 -2
- package/dist/monitor.js +177 -42
- package/dist/ops.d.ts +15 -2
- package/dist/ops.js +32 -9
- package/dist/permissions.d.ts +70 -0
- package/dist/permissions.js +97 -0
- package/dist/runner.d.ts +65 -2
- package/dist/runner.js +307 -32
- package/dist/session/acp.d.ts +70 -0
- package/dist/session/acp.js +364 -0
- package/dist/session/control.d.ts +89 -0
- package/dist/session/control.js +322 -0
- package/dist/session/events.d.ts +14 -0
- package/dist/session/events.js +67 -0
- package/dist/session/tmux.d.ts +27 -0
- package/dist/session/tmux.js +76 -0
- package/dist/session/types.d.ts +138 -0
- package/dist/session/types.js +42 -0
- package/dist/spawn.d.ts +32 -2
- package/dist/spawn.js +177 -16
- package/dist/supervisor/launchd.d.ts +50 -0
- package/dist/supervisor/launchd.js +121 -4
- package/dist/supervisor/none.js +22 -4
- package/dist/supervisor/systemd.d.ts +8 -1
- package/dist/supervisor/systemd.js +94 -4
- package/dist/supervisor/types.d.ts +36 -3
- package/dist/tmux.d.ts +34 -2
- package/dist/tmux.js +48 -11
- package/package.json +7 -2
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { getAdapter } from './harness/registry.js';
|
|
2
|
+
/**
|
|
3
|
+
* The capability floor every unattended role must clear. These are not
|
|
4
|
+
* nice-to-haves: an agent that cannot read its briefing, append its worklog,
|
|
5
|
+
* bind its identity, arm its monitor, edit its workspace, or run the status
|
|
6
|
+
* commands its briefing prescribes cannot carry out the job it was spawned for
|
|
7
|
+
* — and, being unattended, will report no error while failing to.
|
|
8
|
+
*/
|
|
9
|
+
export const UNATTENDED_FLOOR = [
|
|
10
|
+
'read-state', 'write-state', 'messaging', 'monitor', 'workspace-edit', 'status-commands',
|
|
11
|
+
];
|
|
12
|
+
/** Which floor capabilities a set of granted capabilities fails to cover. */
|
|
13
|
+
export function checkUnattendedFloor(granted) {
|
|
14
|
+
const missing = UNATTENDED_FLOOR.filter(c => !granted.includes(c));
|
|
15
|
+
return { meets: missing.length === 0, missing };
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Find native settings that contradict the neutral block. Only fires when the
|
|
19
|
+
* operator wrote BOTH — a role that states its intent once, neutrally or
|
|
20
|
+
* natively, has nothing to contradict and stays quiet. `harness_options` wins
|
|
21
|
+
* at launch, which is precisely why a silent disagreement is dangerous: the
|
|
22
|
+
* neutral block reads like the source of truth and is not.
|
|
23
|
+
*/
|
|
24
|
+
function findConflicts(role, fromNeutral, fromNative) {
|
|
25
|
+
if (!role.permissionsDeclared)
|
|
26
|
+
return [];
|
|
27
|
+
const conflicts = [];
|
|
28
|
+
for (const [key, nativeValue] of Object.entries(fromNative)) {
|
|
29
|
+
const neutralValue = fromNeutral[key];
|
|
30
|
+
if (neutralValue === undefined || String(neutralValue) === String(nativeValue))
|
|
31
|
+
continue;
|
|
32
|
+
conflicts.push({
|
|
33
|
+
key,
|
|
34
|
+
fromNeutral: String(neutralValue),
|
|
35
|
+
fromNative: String(nativeValue),
|
|
36
|
+
warning: `role '${role.name}': harness_options.${key}=${String(nativeValue)} contradicts the `
|
|
37
|
+
+ `permissions block, which translates to ${key}=${String(neutralValue)} — `
|
|
38
|
+
+ `harness_options.${key}=${String(nativeValue)} wins`,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
return conflicts;
|
|
42
|
+
}
|
|
43
|
+
/** Resolve one role's permissions through its adapter. Never throws. */
|
|
44
|
+
export function analyzeRolePermissions(role) {
|
|
45
|
+
const base = { role: role.name, harness: role.harness, permissions: role.permissions };
|
|
46
|
+
let adapter;
|
|
47
|
+
try {
|
|
48
|
+
adapter = getAdapter(role.harness);
|
|
49
|
+
}
|
|
50
|
+
catch (e) {
|
|
51
|
+
return { ...base, supported: false, warnings: [`role '${role.name}': ${e.message}`] };
|
|
52
|
+
}
|
|
53
|
+
const translation = adapter.translatePermissions(role.permissions);
|
|
54
|
+
if (!translation.supported) {
|
|
55
|
+
return {
|
|
56
|
+
...base, supported: false,
|
|
57
|
+
warnings: [`role '${role.name}': harness '${role.harness}' cannot express neutral ` +
|
|
58
|
+
`permissions — ${translation.reason}`],
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const conflicts = findConflicts(role, translation.native, adapter.nativePermissionOverrides(role.harness_options));
|
|
62
|
+
const floor = checkUnattendedFloor(translation.capabilities);
|
|
63
|
+
const floorSeverity = role.permissions.unattended === 'deny' ? 'fail' : 'warn';
|
|
64
|
+
return {
|
|
65
|
+
...base,
|
|
66
|
+
supported: true,
|
|
67
|
+
native: translation.native,
|
|
68
|
+
exact: translation.exact,
|
|
69
|
+
capabilities: translation.capabilities,
|
|
70
|
+
floor,
|
|
71
|
+
floorSeverity,
|
|
72
|
+
conflicts,
|
|
73
|
+
floorWarning: floor.meets ? undefined : (`role '${role.name}': resolved ${role.harness} permissions do not meet the unattended ` +
|
|
74
|
+
`capability floor — missing ${floor.missing.join(', ')} ` +
|
|
75
|
+
`(${formatNative(translation.native)}; unattended=${role.permissions.unattended} means these ` +
|
|
76
|
+
`requests will ${role.permissions.unattended === 'deny' ? 'be denied silently' : 'block the turn'})`),
|
|
77
|
+
warnings: translation.warnings.map(w => `role '${role.name}': ${w}`),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/** Every line a command should show for a role: translation, conflicts, floor. */
|
|
81
|
+
export function allWarnings(a) {
|
|
82
|
+
return [
|
|
83
|
+
...a.warnings,
|
|
84
|
+
...(a.conflicts ?? []).map(c => c.warning),
|
|
85
|
+
...(a.floorWarning ? [a.floorWarning] : []),
|
|
86
|
+
];
|
|
87
|
+
}
|
|
88
|
+
/** Resolve every role's permissions, in config order. */
|
|
89
|
+
export function analyzeFleetPermissions(roles) {
|
|
90
|
+
return roles.map(analyzeRolePermissions);
|
|
91
|
+
}
|
|
92
|
+
/** Render an analysis's native settings compactly, for one-line reporting. */
|
|
93
|
+
export function formatNative(native) {
|
|
94
|
+
if (!native || !Object.keys(native).length)
|
|
95
|
+
return '(none)';
|
|
96
|
+
return Object.entries(native).map(([k, v]) => `${k}=${String(v)}`).join(' ');
|
|
97
|
+
}
|
package/dist/runner.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { Launch } from './harness/types.js';
|
|
|
3
3
|
import { Tmux } from './tmux.js';
|
|
4
4
|
import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
|
|
5
5
|
import { type Exec } from './exec.js';
|
|
6
|
+
import type { ExitRecord } from './session/types.js';
|
|
6
7
|
export interface RunnerDeps {
|
|
7
8
|
tmux: Tmux;
|
|
8
9
|
exec: Exec;
|
|
@@ -15,6 +16,8 @@ export interface RunnerDeps {
|
|
|
15
16
|
fetch: FetchLike;
|
|
16
17
|
/** Construct the supervisor mail monitor (injectable so tests stub it out). */
|
|
17
18
|
createMonitor(opts: MonitorOpts): MonitorHandle;
|
|
19
|
+
/** Lets a test (or a shutdown path) end the supervised restart loop. */
|
|
20
|
+
shouldStop?(): boolean;
|
|
18
21
|
}
|
|
19
22
|
/**
|
|
20
23
|
* Compose the tmux pane shell command: env prefix + argv + exit-status capture.
|
|
@@ -24,6 +27,41 @@ export interface RunnerDeps {
|
|
|
24
27
|
* still sees the real exit code.
|
|
25
28
|
*/
|
|
26
29
|
export declare function buildPaneCommand(launch: Launch, roleEnv: Record<string, string> | undefined, exitStatusPath: string, paneArgv?: string[]): string;
|
|
30
|
+
/**
|
|
31
|
+
* Read the pane's `.exit-status`. Three shapes are accepted: the structured
|
|
32
|
+
* record written above, a bare number left by a pre-upgrade pane (so an
|
|
33
|
+
* in-place upgrade does not misread a real exit), and anything else — which is
|
|
34
|
+
* `unknown`, never an invented failure. A missing file returns null so the
|
|
35
|
+
* caller can distinguish "no record" from "a record saying unknown".
|
|
36
|
+
*/
|
|
37
|
+
export declare function readExitRecord(path: string): ExitRecord | null;
|
|
38
|
+
export declare const RESTART_LEDGER_FILE = ".restart-ledger.json";
|
|
39
|
+
/** Consecutive immediate failures tolerated before the agent is held down. */
|
|
40
|
+
export declare const RESTART_FAIL_THRESHOLD = 5;
|
|
41
|
+
export interface RestartLedger {
|
|
42
|
+
version: 1;
|
|
43
|
+
consecutiveImmediateFailures: number;
|
|
44
|
+
lastReason: string;
|
|
45
|
+
nextDelayMs: number;
|
|
46
|
+
/** Whether this failure sequence has already thrown away resume state. */
|
|
47
|
+
resumeDiscarded: boolean;
|
|
48
|
+
circuit: 'closed' | 'open';
|
|
49
|
+
updatedAt: string;
|
|
50
|
+
/** When the circuit opened, for the held-down status line. */
|
|
51
|
+
openedAt?: string;
|
|
52
|
+
}
|
|
53
|
+
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
54
|
+
export declare function backoffFor(consecutiveFailures: number): number;
|
|
55
|
+
/** Read a role's restart ledger; a missing or corrupt one starts clean. */
|
|
56
|
+
export declare function readRestartLedger(dir: string): RestartLedger;
|
|
57
|
+
export declare function writeRestartLedger(dir: string, ledger: RestartLedger): void;
|
|
58
|
+
/**
|
|
59
|
+
* Close the circuit and forget the failure streak. Called by an explicit
|
|
60
|
+
* operator `up`/`restart`, which is the only thing that may release a held-down
|
|
61
|
+
* role — a held-down runner polls this file, so a role can be released without
|
|
62
|
+
* bouncing its unit.
|
|
63
|
+
*/
|
|
64
|
+
export declare function resetRestartLedger(dir: string): void;
|
|
27
65
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
28
66
|
export declare const START_STAGGER_FILE = ".start-stagger-ms";
|
|
29
67
|
/**
|
|
@@ -37,10 +75,35 @@ export declare const START_STAGGER_FILE = ".start-stagger-ms";
|
|
|
37
75
|
export declare function reserveLaunchSlot(root: string, staggerMs: number, deps: Pick<RunnerDeps, 'now' | 'sleep' | 'log'>): Promise<number>;
|
|
38
76
|
/** Read a temp role's config snapshot written by spawnTemp. */
|
|
39
77
|
export declare function loadTempRole(name: string): ResolvedRole;
|
|
40
|
-
/**
|
|
78
|
+
/** What one child session did, so the supervising loop can decide what follows. */
|
|
79
|
+
export interface AttemptResult {
|
|
80
|
+
elapsedSecs: number;
|
|
81
|
+
exit: ExitRecord;
|
|
82
|
+
/** Whether this attempt threw away resume state to start fresh. */
|
|
83
|
+
rotated: boolean;
|
|
84
|
+
mode: 'fresh' | 'resume';
|
|
85
|
+
}
|
|
86
|
+
/** One session lifecycle. `runSupervised` (or a one-shot caller) drives it. */
|
|
41
87
|
export declare function runOnce(name: string, opts?: {
|
|
42
88
|
temp?: boolean;
|
|
43
89
|
configPath?: string;
|
|
44
|
-
|
|
90
|
+
allowResumeRotation?: boolean;
|
|
91
|
+
}, partialDeps?: Partial<RunnerDeps>): Promise<AttemptResult>;
|
|
92
|
+
/**
|
|
93
|
+
* The persistent supervisor for one permanent role: run child sessions in a
|
|
94
|
+
* loop, count consecutive immediate failures across them, back off between
|
|
95
|
+
* attempts, and after `RESTART_FAIL_THRESHOLD` hold the agent down while
|
|
96
|
+
* staying alive — so the service manager has nothing to restart and cannot
|
|
97
|
+
* resume the two-second loop behind our back.
|
|
98
|
+
*
|
|
99
|
+
* `attempt` is injectable so the policy can be tested against a fake clock and
|
|
100
|
+
* fake child instead of real sessions.
|
|
101
|
+
*/
|
|
102
|
+
export declare function runSupervised(name: string, opts?: {
|
|
103
|
+
configPath?: string;
|
|
104
|
+
}, partialDeps?: Partial<RunnerDeps>, attempt?: (n: string, o: {
|
|
105
|
+
configPath?: string;
|
|
106
|
+
allowResumeRotation?: boolean;
|
|
107
|
+
}, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
|
|
45
108
|
/** Temp-agent entrypoint: run one session, then remove the temp dir. */
|
|
46
109
|
export declare function runTemp(name: string, deps?: Partial<RunnerDeps>): Promise<void>;
|
package/dist/runner.js
CHANGED
|
@@ -2,8 +2,8 @@ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
4
|
import { parse } from 'yaml';
|
|
5
|
-
import { agentDir,
|
|
6
|
-
import { loadConfig, findRole } from './config.js';
|
|
5
|
+
import { agentDir, stateRoot } from './paths.js';
|
|
6
|
+
import { loadConfig, findRole, isolationContextFor, resolvePermissions, } from './config.js';
|
|
7
7
|
import { getAdapter } from './harness/registry.js';
|
|
8
8
|
import { Tmux } from './tmux.js';
|
|
9
9
|
import { createMonitor } from './monitor.js';
|
|
@@ -11,6 +11,10 @@ import { realExec, shq } from './exec.js';
|
|
|
11
11
|
import { resolveIsolation } from './isolation/policy.js';
|
|
12
12
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
13
13
|
import { resourceArgs, cpuControllerDelegated } from './isolation/resources.js';
|
|
14
|
+
import { AcpSession } from './session/acp.js';
|
|
15
|
+
import { RoleControlServer } from './session/control.js';
|
|
16
|
+
import { TmuxSession } from './session/tmux.js';
|
|
17
|
+
import { classifyShellStatus } from './session/types.js';
|
|
14
18
|
const defaultDeps = () => ({
|
|
15
19
|
tmux: new Tmux(),
|
|
16
20
|
exec: realExec,
|
|
@@ -39,7 +43,12 @@ export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = lau
|
|
|
39
43
|
const env = { PATH: process.env.PATH ?? '', ...launch.env, ...(roleEnv ?? {}) };
|
|
40
44
|
const envPfx = 'env ' + Object.entries(env).map(([k, v]) => `${k}=${shq(v)}`).join(' ');
|
|
41
45
|
const cmd = paneArgv.map(shq).join(' ');
|
|
42
|
-
|
|
46
|
+
// Write a structured record, not a bare number: the wait status alone cannot
|
|
47
|
+
// say whether the file is missing because the program never exited or because
|
|
48
|
+
// nothing ever wrote it. `printf` is POSIX; no shell branching is needed
|
|
49
|
+
// because classification happens in one place, in TypeScript.
|
|
50
|
+
const record = `'{"version":1,"backend":"tmux","status":'"$__ofs"'}'`;
|
|
51
|
+
return `${envPfx} ${cmd}; __ofs=$?; printf %s ${record} > ${shq(exitStatusPath)}`;
|
|
43
52
|
}
|
|
44
53
|
/** Adapt runner deps and the role's daemon-profile overrides for the monitor. */
|
|
45
54
|
function monitorDeps(deps, roleEnv) {
|
|
@@ -56,6 +65,89 @@ function monitorDeps(deps, roleEnv) {
|
|
|
56
65
|
timers: { set: (fn, ms) => setTimeout(fn, ms), clear: t => clearTimeout(t) },
|
|
57
66
|
};
|
|
58
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Read the pane's `.exit-status`. Three shapes are accepted: the structured
|
|
70
|
+
* record written above, a bare number left by a pre-upgrade pane (so an
|
|
71
|
+
* in-place upgrade does not misread a real exit), and anything else — which is
|
|
72
|
+
* `unknown`, never an invented failure. A missing file returns null so the
|
|
73
|
+
* caller can distinguish "no record" from "a record saying unknown".
|
|
74
|
+
*/
|
|
75
|
+
export function readExitRecord(path) {
|
|
76
|
+
if (!existsSync(path))
|
|
77
|
+
return null;
|
|
78
|
+
const raw = readFileSync(path, 'utf8').trim();
|
|
79
|
+
if (!raw)
|
|
80
|
+
return { version: 1, class: 'unknown', detail: 'the pane left an empty exit record' };
|
|
81
|
+
if (/^-?\d+$/.test(raw))
|
|
82
|
+
return classifyShellStatus(Number(raw)); // legacy `echo $?`
|
|
83
|
+
try {
|
|
84
|
+
const parsed = JSON.parse(raw);
|
|
85
|
+
if (typeof parsed.status === 'number')
|
|
86
|
+
return classifyShellStatus(parsed.status);
|
|
87
|
+
}
|
|
88
|
+
catch { /* fall through to unknown */ }
|
|
89
|
+
return { version: 1, class: 'unknown', detail: `unreadable exit record: ${raw.slice(0, 120)}` };
|
|
90
|
+
}
|
|
91
|
+
// ─── Restart-loop containment (3.2) ──────────────────────────────────────────
|
|
92
|
+
//
|
|
93
|
+
// The child-session restart loop used to BE the service manager: systemd's
|
|
94
|
+
// `Restart=always RestartSec=2` and launchd's `KeepAlive`. Neither can count,
|
|
95
|
+
// so a program that dies instantly was relaunched every two seconds forever,
|
|
96
|
+
// and each relaunch was a fresh process with no memory of the previous one.
|
|
97
|
+
// The count now lives with the role, in its state directory, so it survives the
|
|
98
|
+
// runner being restarted and is consistent across both service managers.
|
|
99
|
+
export const RESTART_LEDGER_FILE = '.restart-ledger.json';
|
|
100
|
+
/** Consecutive immediate failures tolerated before the agent is held down. */
|
|
101
|
+
export const RESTART_FAIL_THRESHOLD = 5;
|
|
102
|
+
const RESTART_BACKOFF_BASE_MS = 2_000;
|
|
103
|
+
const RESTART_BACKOFF_MAX_MS = 60_000;
|
|
104
|
+
/** How often a held-down runner re-reads its ledger, so `up` can release it. */
|
|
105
|
+
const HELD_DOWN_POLL_MS = 5_000;
|
|
106
|
+
const emptyLedger = () => ({
|
|
107
|
+
version: 1,
|
|
108
|
+
consecutiveImmediateFailures: 0,
|
|
109
|
+
lastReason: '',
|
|
110
|
+
nextDelayMs: 0,
|
|
111
|
+
resumeDiscarded: false,
|
|
112
|
+
circuit: 'closed',
|
|
113
|
+
updatedAt: new Date(0).toISOString(),
|
|
114
|
+
});
|
|
115
|
+
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
116
|
+
export function backoffFor(consecutiveFailures) {
|
|
117
|
+
if (consecutiveFailures <= 0)
|
|
118
|
+
return 0;
|
|
119
|
+
return Math.min(RESTART_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESTART_BACKOFF_MAX_MS);
|
|
120
|
+
}
|
|
121
|
+
/** Read a role's restart ledger; a missing or corrupt one starts clean. */
|
|
122
|
+
export function readRestartLedger(dir) {
|
|
123
|
+
try {
|
|
124
|
+
const raw = JSON.parse(readFileSync(join(dir, RESTART_LEDGER_FILE), 'utf8'));
|
|
125
|
+
if (raw.version !== 1)
|
|
126
|
+
return emptyLedger();
|
|
127
|
+
return { ...emptyLedger(), ...raw, version: 1 };
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return emptyLedger();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export function writeRestartLedger(dir, ledger) {
|
|
134
|
+
try {
|
|
135
|
+
mkdirSync(dir, { recursive: true });
|
|
136
|
+
writeFileSync(join(dir, RESTART_LEDGER_FILE), JSON.stringify(ledger, null, 2) + '\n');
|
|
137
|
+
}
|
|
138
|
+
catch { /* diagnostics must never take the role down */ }
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Close the circuit and forget the failure streak. Called by an explicit
|
|
142
|
+
* operator `up`/`restart`, which is the only thing that may release a held-down
|
|
143
|
+
* role — a held-down runner polls this file, so a role can be released without
|
|
144
|
+
* bouncing its unit.
|
|
145
|
+
*/
|
|
146
|
+
export function resetRestartLedger(dir) {
|
|
147
|
+
if (!existsSync(dir))
|
|
148
|
+
return;
|
|
149
|
+
writeRestartLedger(dir, { ...emptyLedger(), updatedAt: new Date().toISOString() });
|
|
150
|
+
}
|
|
59
151
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
60
152
|
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
61
153
|
/** Read the start-stagger a temp agent was spawned with (0 if none / unreadable). */
|
|
@@ -148,7 +240,7 @@ function resolveConfigPath(dir, explicit) {
|
|
|
148
240
|
return undefined;
|
|
149
241
|
return readFileSync(marker, 'utf8').trim() || undefined;
|
|
150
242
|
}
|
|
151
|
-
/** One
|
|
243
|
+
/** One session lifecycle. `runSupervised` (or a one-shot caller) drives it. */
|
|
152
244
|
export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
153
245
|
const deps = { ...defaultDeps(), ...partialDeps };
|
|
154
246
|
const temp = opts.temp === true;
|
|
@@ -182,17 +274,22 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
182
274
|
writeFileSync(bootedFile, '');
|
|
183
275
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
184
276
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
185
|
-
const
|
|
277
|
+
const sessionBackend = role.session ?? 'tmux';
|
|
278
|
+
const launch = sessionBackend === 'acp'
|
|
279
|
+
? (() => {
|
|
280
|
+
if (!adapter.buildAcpLaunch)
|
|
281
|
+
throw new Error(`harness '${role.harness}' does not support the ACP session backend`);
|
|
282
|
+
return adapter.buildAcpLaunch(role, prep);
|
|
283
|
+
})()
|
|
284
|
+
: adapter.buildLaunch(role, mode, { sessionId }, prep);
|
|
186
285
|
// Isolation is additive: only roles that declare `isolation:` are wrapped. The
|
|
187
286
|
// env prefix + exit capture in buildPaneCommand stay host-side (see §5.3).
|
|
188
|
-
let
|
|
287
|
+
let wrappedArgv = launch.argv;
|
|
189
288
|
if (role.isolation) {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
const ctx = {
|
|
194
|
-
stateDir: dir, runCwd, home: home(), harness: role.harness, additionalWriteDirs: addDirs,
|
|
195
|
-
};
|
|
289
|
+
// The SAME context config validation and doctor judged (5.2): a policy
|
|
290
|
+
// checked against a different mount set than the one that launches is not a
|
|
291
|
+
// check at all.
|
|
292
|
+
const ctx = { ...isolationContextFor(role), stateDir: dir, runCwd };
|
|
196
293
|
const policy = resolveIsolation(role.isolation, ctx);
|
|
197
294
|
const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
|
|
198
295
|
const degradedMarker = join(dir, '.isolation-degraded');
|
|
@@ -204,14 +301,14 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
204
301
|
deps.log(`[${name}] isolation: ${sel.backend.id} (net=${policy.network}) ${sel.detail}`);
|
|
205
302
|
rmSync(degradedMarker, { force: true });
|
|
206
303
|
}
|
|
207
|
-
|
|
304
|
+
wrappedArgv = sel.backend.wrap(launch.argv, policy, ctx);
|
|
208
305
|
// Resource caps wrap the sandbox from OUTSIDE, at the pane's own cgroup scope
|
|
209
306
|
// (§5.4). Applies even when the sandbox degraded to none.
|
|
210
307
|
const { argv: rprefix, warnings } = resourceArgs(policy.resources, deps.cpuDelegated());
|
|
211
308
|
for (const w of warnings)
|
|
212
309
|
deps.log(`[${name}] WARNING ${w}`);
|
|
213
310
|
if (rprefix.length)
|
|
214
|
-
|
|
311
|
+
wrappedArgv = [...rprefix, ...wrappedArgv];
|
|
215
312
|
}
|
|
216
313
|
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
217
314
|
// agent launch across the whole host, so a burst of boots (systemd starts every
|
|
@@ -232,47 +329,225 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
232
329
|
// (backlog before the tip is the SessionStart hook's job). Disabled roles keep
|
|
233
330
|
// the legacy in-session watch. Temp snapshots predating `monitor:` are treated
|
|
234
331
|
// as disabled (monitor may be undefined on an old role.yaml).
|
|
332
|
+
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
235
333
|
const monitor = role.monitor?.enabled ? deps.createMonitor({
|
|
236
|
-
name, agentDir: dir, cfg: role.monitor,
|
|
237
|
-
deps:
|
|
334
|
+
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
335
|
+
deps: resolvedMonitorDeps,
|
|
238
336
|
}) : null;
|
|
239
337
|
if (monitor)
|
|
240
338
|
await monitor.prime();
|
|
241
339
|
rmSync(exitFile, { force: true });
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
let
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
340
|
+
let pid;
|
|
341
|
+
let sessionHandle;
|
|
342
|
+
let acpSession;
|
|
343
|
+
let control;
|
|
344
|
+
if (sessionBackend === 'acp') {
|
|
345
|
+
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
346
|
+
// Say once, at startup, that this role will decide permission requests by
|
|
347
|
+
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
348
|
+
// that quietly did less than it was asked to.
|
|
349
|
+
if (perms.unattended === 'deny')
|
|
350
|
+
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
351
|
+
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
352
|
+
acpSession = await AcpSession.start({
|
|
353
|
+
name,
|
|
354
|
+
argv: wrappedArgv,
|
|
355
|
+
cwd: runCwd,
|
|
356
|
+
env: { ...launch.env, ...(role.env ?? {}) },
|
|
357
|
+
stateDir: dir,
|
|
358
|
+
mode,
|
|
359
|
+
permissions: perms,
|
|
360
|
+
log: deps.log,
|
|
361
|
+
});
|
|
362
|
+
pid = acpSession.pid;
|
|
363
|
+
sessionHandle = acpSession;
|
|
364
|
+
control = new RoleControlServer(dir, acpSession, deps.log);
|
|
365
|
+
await control.start();
|
|
366
|
+
resolvedMonitorDeps.delivery = {
|
|
367
|
+
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
368
|
+
// refusal or a cancellation reached the agent and was not acted on, so
|
|
369
|
+
// the monitor must keep its cursor and try again.
|
|
370
|
+
submit: async (text) => {
|
|
371
|
+
const result = await acpSession.submitPrompt(text);
|
|
372
|
+
return { succeeded: result.succeeded, outcome: result.outcome, detail: result.detail };
|
|
373
|
+
},
|
|
374
|
+
};
|
|
375
|
+
const firstPrompt = mode === 'fresh'
|
|
376
|
+
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
377
|
+
: adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
|
|
378
|
+
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
379
|
+
// startup prompt and then refuses it has not started; logging the role as
|
|
380
|
+
// up would hide a role that never read its briefing.
|
|
381
|
+
const started = await acpSession.submitPrompt(firstPrompt);
|
|
382
|
+
if (!started.succeeded) {
|
|
383
|
+
monitor?.stop();
|
|
384
|
+
await control.close();
|
|
385
|
+
await acpSession.close();
|
|
386
|
+
throw new Error(`[${name}] ACP startup prompt ${started.outcome}` +
|
|
387
|
+
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
388
|
+
}
|
|
249
389
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
390
|
+
else {
|
|
391
|
+
await deps.tmux.kill(name);
|
|
392
|
+
await deps.tmux.newSession(name, runCwd, buildPaneCommand(launch, role.env, exitFile, wrappedArgv));
|
|
393
|
+
let panePid = null;
|
|
394
|
+
for (let i = 0; i < 40 && panePid === null; i++) {
|
|
395
|
+
panePid = await deps.tmux.panePid(name);
|
|
396
|
+
if (panePid === null)
|
|
397
|
+
await deps.sleep(250);
|
|
398
|
+
}
|
|
399
|
+
if (panePid === null)
|
|
400
|
+
throw new Error(`[${name}] could not resolve tmux pane pid`);
|
|
401
|
+
pid = panePid;
|
|
402
|
+
sessionHandle = new TmuxSession(name, pid, deps.tmux, deps.isAlive);
|
|
403
|
+
}
|
|
404
|
+
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
253
405
|
// The monitor loop lives exactly as long as the session: it starts once the
|
|
254
406
|
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
255
407
|
const monitorLoop = monitor?.run(pid);
|
|
256
408
|
const start = deps.now();
|
|
257
|
-
while (
|
|
409
|
+
while (sessionHandle.isAlive())
|
|
258
410
|
await deps.sleep(2000);
|
|
259
411
|
if (monitor) {
|
|
260
412
|
monitor.stop();
|
|
261
413
|
await monitorLoop;
|
|
262
414
|
}
|
|
415
|
+
if (control)
|
|
416
|
+
await control.close();
|
|
417
|
+
if (acpSession)
|
|
418
|
+
await acpSession.close();
|
|
263
419
|
const elapsed = (deps.now() - start) / 1000;
|
|
264
|
-
|
|
420
|
+
// Establish what actually happened before deciding anything. Absence of a
|
|
421
|
+
// record is `unknown` — except when the console itself is gone, which is a
|
|
422
|
+
// different event with a different consequence.
|
|
423
|
+
const exitRecord = acpSession
|
|
424
|
+
? acpSession.exitResult()
|
|
425
|
+
?? { version: 1, class: 'unknown', detail: 'the ACP agent stopped without reporting an exit' }
|
|
426
|
+
: readExitRecord(exitFile)
|
|
427
|
+
?? (await deps.tmux.has(name)
|
|
428
|
+
? { version: 1, class: 'unknown', detail: 'the pane process ended without writing an exit record' }
|
|
429
|
+
: { version: 1, class: 'session-destroyed', detail: `the tmux session '${name}' no longer exists` });
|
|
430
|
+
writeFileSync(exitFile, JSON.stringify({
|
|
431
|
+
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
432
|
+
}) + '\n');
|
|
433
|
+
let rotated = false;
|
|
265
434
|
const rotate = (why) => {
|
|
266
435
|
writeFileSync(sidFile, randomUUID() + '\n');
|
|
267
436
|
rmSync(bootedFile, { force: true });
|
|
437
|
+
rotated = true;
|
|
268
438
|
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
269
439
|
};
|
|
270
|
-
if (
|
|
440
|
+
if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
271
441
|
rotate(`clean exit (code 0)`);
|
|
272
|
-
else if (
|
|
273
|
-
|
|
442
|
+
else if (exitRecord.class === 'session-destroyed')
|
|
443
|
+
// Someone tore the console down; the agent did not fail. Rotating here
|
|
444
|
+
// would discard a live conversation for an operator action.
|
|
445
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
446
|
+
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
447
|
+
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
448
|
+
// on every attempt would discard the conversation again and again while the
|
|
449
|
+
// real cause (a broken command, a missing binary) went unaddressed.
|
|
450
|
+
if (opts.allowResumeRotation === false)
|
|
451
|
+
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
452
|
+
`-> resume state was already discarded once; keeping it`);
|
|
453
|
+
else
|
|
454
|
+
rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
|
|
455
|
+
}
|
|
274
456
|
else
|
|
275
|
-
deps.log(`[${name}]
|
|
457
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
458
|
+
return { elapsedSecs: elapsed, exit: exitRecord, rotated, mode };
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* The persistent supervisor for one permanent role: run child sessions in a
|
|
462
|
+
* loop, count consecutive immediate failures across them, back off between
|
|
463
|
+
* attempts, and after `RESTART_FAIL_THRESHOLD` hold the agent down while
|
|
464
|
+
* staying alive — so the service manager has nothing to restart and cannot
|
|
465
|
+
* resume the two-second loop behind our back.
|
|
466
|
+
*
|
|
467
|
+
* `attempt` is injectable so the policy can be tested against a fake clock and
|
|
468
|
+
* fake child instead of real sessions.
|
|
469
|
+
*/
|
|
470
|
+
export async function runSupervised(name, opts = {}, partialDeps = {}, attempt = runOnce) {
|
|
471
|
+
const deps = { ...defaultDeps(), ...partialDeps };
|
|
472
|
+
const dir = agentDir(name);
|
|
473
|
+
mkdirSync(dir, { recursive: true });
|
|
474
|
+
const shouldStop = deps.shouldStop ?? (() => false);
|
|
475
|
+
const stamp = () => new Date(deps.now()).toISOString();
|
|
476
|
+
while (!shouldStop()) {
|
|
477
|
+
let ledger = readRestartLedger(dir);
|
|
478
|
+
if (ledger.circuit === 'open') {
|
|
479
|
+
// Held down. Stay alive — exiting would hand the role straight back to
|
|
480
|
+
// the service manager — and watch for an operator reset.
|
|
481
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
let result;
|
|
485
|
+
try {
|
|
486
|
+
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
487
|
+
}
|
|
488
|
+
catch (e) {
|
|
489
|
+
// A session that could not even start is an immediate failure like any
|
|
490
|
+
// other; it must count, or an unstartable role loops forever.
|
|
491
|
+
result = {
|
|
492
|
+
elapsedSecs: 0,
|
|
493
|
+
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
494
|
+
rotated: false,
|
|
495
|
+
mode: 'fresh',
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
499
|
+
// have reset the ledger meanwhile.
|
|
500
|
+
ledger = readRestartLedger(dir);
|
|
501
|
+
const fastFailSecs = fastFailSecsFor(name, opts.configPath);
|
|
502
|
+
const immediate = result.elapsedSecs < fastFailSecs;
|
|
503
|
+
if (!immediate) {
|
|
504
|
+
// A session that ran for a while is not a restart loop, whatever ended it.
|
|
505
|
+
writeRestartLedger(dir, {
|
|
506
|
+
...emptyLedger(),
|
|
507
|
+
lastReason: result.exit.detail,
|
|
508
|
+
updatedAt: stamp(),
|
|
509
|
+
});
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
513
|
+
const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
|
|
514
|
+
const next = {
|
|
515
|
+
version: 1,
|
|
516
|
+
consecutiveImmediateFailures: failures,
|
|
517
|
+
lastReason: reason,
|
|
518
|
+
nextDelayMs: backoffFor(failures),
|
|
519
|
+
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
520
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
521
|
+
updatedAt: stamp(),
|
|
522
|
+
};
|
|
523
|
+
if (next.circuit === 'open') {
|
|
524
|
+
next.openedAt = stamp();
|
|
525
|
+
next.nextDelayMs = 0;
|
|
526
|
+
writeRestartLedger(dir, next);
|
|
527
|
+
deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
|
|
528
|
+
`${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
writeRestartLedger(dir, next);
|
|
532
|
+
deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
|
|
533
|
+
`-> backing off ${next.nextDelayMs}ms`);
|
|
534
|
+
await deps.sleep(next.nextDelayMs);
|
|
535
|
+
}
|
|
536
|
+
return readRestartLedger(dir);
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* How short an attempt has to be to count as immediate. The role's harness
|
|
540
|
+
* decides; an unreadable config falls back to the common 20s so a broken config
|
|
541
|
+
* cannot disable the breaker.
|
|
542
|
+
*/
|
|
543
|
+
function fastFailSecsFor(name, configPath) {
|
|
544
|
+
try {
|
|
545
|
+
const role = findRole(loadConfig(configPath), name);
|
|
546
|
+
return getAdapter(role.harness).exitPolicy.fastFailSecs;
|
|
547
|
+
}
|
|
548
|
+
catch {
|
|
549
|
+
return 20;
|
|
550
|
+
}
|
|
276
551
|
}
|
|
277
552
|
/** Temp-agent entrypoint: run one session, then remove the temp dir. */
|
|
278
553
|
export async function runTemp(name, deps = {}) {
|