@ours.network/fleet 0.19.0-nightly.9 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -11
- package/dist/application/fleet-query-service.d.ts +1 -1
- package/dist/application/fleet-query-service.js +12 -0
- package/dist/application/role-creation-service.js +3 -1
- package/dist/application/types.d.ts +11 -0
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +103 -10
- package/dist/build-info.json +5 -5
- package/dist/cli.js +41 -11
- package/dist/config.d.ts +21 -7
- package/dist/config.js +32 -18
- package/dist/creation.d.ts +1 -1
- package/dist/docs.d.ts +2 -2
- package/dist/docs.js +117 -24
- package/dist/doctor.js +10 -10
- package/dist/duration.js +1 -1
- package/dist/fleet-proxy.d.ts +5 -0
- package/dist/harness/acp-agent.js +11 -6
- package/dist/harness/claude-code.js +201 -8
- package/dist/harness/codex.d.ts +4 -1
- package/dist/harness/codex.js +72 -13
- package/dist/harness/types.d.ts +57 -6
- package/dist/isolation/bubblewrap.d.ts +4 -4
- package/dist/isolation/bubblewrap.js +4 -4
- package/dist/isolation/policy.d.ts +1 -1
- package/dist/isolation/policy.js +2 -2
- package/dist/isolation/registry.d.ts +3 -3
- package/dist/isolation/registry.js +5 -5
- package/dist/isolation/resources.d.ts +1 -1
- package/dist/isolation/resources.js +1 -1
- package/dist/isolation/types.d.ts +3 -3
- package/dist/loops/manager.d.ts +30 -1
- package/dist/loops/manager.js +69 -6
- package/dist/loops/state.d.ts +18 -0
- package/dist/loops/state.js +4 -0
- package/dist/model-env.d.ts +71 -0
- package/dist/model-env.js +106 -0
- package/dist/monitor.d.ts +2 -2
- package/dist/monitor.js +4 -4
- package/dist/ops.d.ts +2 -2
- package/dist/ops.js +9 -9
- package/dist/owner-channel/channel.d.ts +17 -0
- package/dist/owner-channel/channel.js +104 -23
- package/dist/owner-channel/commands.d.ts +9 -0
- package/dist/owner-channel/commands.js +226 -99
- package/dist/owner-channel/notices.d.ts +7 -0
- package/dist/owner-channel/notices.js +9 -0
- package/dist/rooms-tasks/cli.js +731 -219
- package/dist/rooms-tasks/close.d.ts +35 -0
- package/dist/rooms-tasks/close.js +190 -0
- package/dist/rooms-tasks/cowork-adapter.d.ts +24 -0
- package/dist/rooms-tasks/cowork-adapter.js +120 -0
- package/dist/rooms-tasks/external-worker.d.ts +2 -0
- package/dist/rooms-tasks/external-worker.js +40 -0
- package/dist/rooms-tasks/index.d.ts +2 -0
- package/dist/rooms-tasks/index.js +2 -0
- package/dist/rooms-tasks/markdown.d.ts +49 -0
- package/dist/rooms-tasks/markdown.js +206 -0
- package/dist/rooms-tasks/member-startup.d.ts +51 -0
- package/dist/rooms-tasks/member-startup.js +151 -0
- package/dist/rooms-tasks/provision.d.ts +8 -0
- package/dist/rooms-tasks/provision.js +418 -97
- package/dist/rooms-tasks/room-state.d.ts +13 -1
- package/dist/rooms-tasks/room-state.js +149 -1
- package/dist/rooms-tasks/task-state.d.ts +13 -2
- package/dist/rooms-tasks/task-state.js +131 -3
- package/dist/rooms-tasks/terminal.d.ts +23 -0
- package/dist/rooms-tasks/terminal.js +56 -0
- package/dist/rooms-tasks/types.d.ts +108 -2
- package/dist/runner.d.ts +48 -0
- package/dist/runner.js +252 -100
- package/dist/session/acp.d.ts +106 -2
- package/dist/session/acp.js +217 -14
- package/dist/session/activity.d.ts +31 -0
- package/dist/session/activity.js +48 -0
- package/dist/session/control.d.ts +1 -1
- package/dist/session/conversation-normalizer.d.ts +7 -1
- package/dist/session/conversation-normalizer.js +154 -11
- package/dist/session/conversation-store.js +1 -1
- package/dist/session/conversation-types.d.ts +25 -6
- package/dist/session/types.d.ts +35 -0
- package/dist/spawn.d.ts +4 -2
- package/dist/spawn.js +39 -26
- package/dist/supervisor/launchd.d.ts +2 -2
- package/dist/supervisor/launchd.js +4 -4
- package/dist/supervisor/systemd.js +4 -4
- package/dist/supervisor/types.d.ts +1 -1
- package/dist/temp-lifecycle.d.ts +14 -2
- package/dist/temp-lifecycle.js +90 -0
- package/dist/watchdog/alerts.d.ts +2 -2
- package/dist/watchdog/alerts.js +2 -2
- package/dist/watchdog/briefing.d.ts +5 -5
- package/dist/watchdog/briefing.js +13 -6
- package/dist/watchdog/config.js +1 -1
- package/dist/watchdog/query.d.ts +2 -2
- package/dist/watchdog/query.js +2 -2
- package/dist/watchdog/report.js +2 -2
- package/dist/watchdog/run.d.ts +2 -2
- package/dist/watchdog/run.js +6 -6
- package/dist/watchdog/scheduler.d.ts +7 -7
- package/dist/watchdog/scheduler.js +10 -10
- package/dist/watchdog/service.d.ts +4 -4
- package/dist/watchdog/service.js +5 -6
- package/dist/watchdog/store.d.ts +4 -4
- package/dist/watchdog/store.js +4 -4
- package/dist/web/runtime.js +1 -1
- package/dist/web/server.js +1 -1
- package/dist/web/topology-promote.js +1 -1
- package/dist/web-app/assets/{TerminalView-BAVk1Bot.js → TerminalView-C_G1ID2P.js} +1 -1
- package/dist/web-app/assets/{index-C3S-xFRU.js → index-BCBK78hw.js} +5 -5
- package/dist/web-app/index.html +1 -1
- package/dist/worklog.d.ts +7 -1
- package/dist/worklog.js +191 -39
- package/package.json +1 -1
package/dist/runner.d.ts
CHANGED
|
@@ -35,6 +35,15 @@ export interface RunnerDeps {
|
|
|
35
35
|
}
|
|
36
36
|
/** Environment injected only into the managed harness process. */
|
|
37
37
|
export declare function managedFleetProxyEnv(role: ResolvedRole, stateDir: string): Record<string, string>;
|
|
38
|
+
/**
|
|
39
|
+
* The environment a managed harness child actually receives, checked at the one
|
|
40
|
+
* point where it is composed. `role.env` deliberately wins over harness prep,
|
|
41
|
+
* which is exactly how a stale fleet-wide model pin used to outrank the model
|
|
42
|
+
* the role was spawned with — so the model pin is verified here rather than
|
|
43
|
+
* trusted, and a disagreement stops the launch instead of being reported as a
|
|
44
|
+
* success (see src/model-env.ts).
|
|
45
|
+
*/
|
|
46
|
+
export declare function harnessChildEnv(role: ResolvedRole, launchEnv: Record<string, string> | undefined, stateDir: string): Record<string, string>;
|
|
38
47
|
/**
|
|
39
48
|
* Record who owns wake delivery for this run. Returning true means a fleet
|
|
40
49
|
* monitor is taking ownership back from a native harness and must start at the
|
|
@@ -61,6 +70,23 @@ export declare function readExitRecord(path: string): ExitRecord | null;
|
|
|
61
70
|
export declare const RESTART_LEDGER_FILE = ".restart-ledger.json";
|
|
62
71
|
/** Consecutive immediate failures tolerated before the agent is held down. */
|
|
63
72
|
export declare const RESTART_FAIL_THRESHOLD = 5;
|
|
73
|
+
/**
|
|
74
|
+
* How the previous supervisor process ended.
|
|
75
|
+
*
|
|
76
|
+
* `abrupt` is the case the ledger used to miss entirely: an OOM-kill or any
|
|
77
|
+
* other external signal takes the supervisor down before it can write anything,
|
|
78
|
+
* the service manager restarts the unit, and every durable indicator still
|
|
79
|
+
* describes the run that died. A health check reading them reported "no
|
|
80
|
+
* restarts" for a role that had died and come back.
|
|
81
|
+
*/
|
|
82
|
+
export interface TerminationRecord {
|
|
83
|
+
class: 'clean' | 'abrupt' | 'unknown';
|
|
84
|
+
detail: string;
|
|
85
|
+
/** When the SURVIVING process observed it, not when it happened. */
|
|
86
|
+
observedAt: string;
|
|
87
|
+
/** Start time of the run that ended, when it was recorded. */
|
|
88
|
+
runStartedAt?: string;
|
|
89
|
+
}
|
|
64
90
|
export interface RestartLedger {
|
|
65
91
|
version: 1;
|
|
66
92
|
consecutiveImmediateFailures: number;
|
|
@@ -72,7 +98,29 @@ export interface RestartLedger {
|
|
|
72
98
|
updatedAt: string;
|
|
73
99
|
/** When the circuit opened, for the held-down status line. */
|
|
74
100
|
openedAt?: string;
|
|
101
|
+
/** How the previous supervisor process ended, including abnormal exits. */
|
|
102
|
+
lastTermination?: TerminationRecord;
|
|
103
|
+
/** Supervisor processes that died without closing their run marker. */
|
|
104
|
+
abruptTerminations?: number;
|
|
105
|
+
/** Start of the supervisor run that owns this state directory now. */
|
|
106
|
+
supervisorStartedAt?: string;
|
|
75
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Carried across a supervisor process's life so its successor can tell an
|
|
110
|
+
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
111
|
+
* running"; the next start finding one that is not its own is proof the
|
|
112
|
+
* previous process died without getting to write anything.
|
|
113
|
+
*/
|
|
114
|
+
export declare const RUN_MARKER_FILE = ".supervisor-run.json";
|
|
115
|
+
/**
|
|
116
|
+
* Claim this state directory for the current supervisor process and report how
|
|
117
|
+
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
118
|
+
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
119
|
+
* and an attempt can take minutes.
|
|
120
|
+
*/
|
|
121
|
+
export declare function claimSupervisorRun(dir: string, startedAt: string, pid?: number): TerminationRecord;
|
|
122
|
+
/** Orderly exit: the successor must not read this run as a kill. */
|
|
123
|
+
export declare function releaseSupervisorRun(dir: string): void;
|
|
76
124
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
77
125
|
export declare function backoffFor(consecutiveFailures: number): number;
|
|
78
126
|
/** Read a role's restart ledger; a missing or corrupt one starts clean. */
|
package/dist/runner.js
CHANGED
|
@@ -25,6 +25,7 @@ import { RoleTurnArbiter } from './session/arbiter.js';
|
|
|
25
25
|
import { ScheduledLoopManager, } from './loops/manager.js';
|
|
26
26
|
import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, inheritCallerSpawnDefaults, } from './fleet-proxy.js';
|
|
27
27
|
import { effectivePermissionMode } from './permissions.js';
|
|
28
|
+
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
28
29
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
29
30
|
const defaultDeps = () => ({
|
|
30
31
|
tmux: new Tmux(),
|
|
@@ -60,14 +61,32 @@ const defaultDeps = () => ({
|
|
|
60
61
|
},
|
|
61
62
|
});
|
|
62
63
|
const MONITOR_OWNER_FILE = '.monitor-owner';
|
|
64
|
+
/** Fleet roles consume the operator-owned daemon; a role session never starts it. */
|
|
65
|
+
const FLEET_OURS_AUTOSTART = '0';
|
|
63
66
|
/** Environment injected only into the managed harness process. */
|
|
64
67
|
export function managedFleetProxyEnv(role, stateDir) {
|
|
65
68
|
return {
|
|
66
69
|
...(role.env ?? {}),
|
|
70
|
+
// This must win over both inherited/configured auto-start. ACP agents run
|
|
71
|
+
// directly rather than through ours-codex, so the runner owns this fence.
|
|
72
|
+
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
67
73
|
[FLEET_PROXY_STATE_DIR_ENV]: stateDir,
|
|
68
74
|
[FLEET_PROXY_CALLER_ENV]: role.name,
|
|
69
75
|
};
|
|
70
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* The environment a managed harness child actually receives, checked at the one
|
|
79
|
+
* point where it is composed. `role.env` deliberately wins over harness prep,
|
|
80
|
+
* which is exactly how a stale fleet-wide model pin used to outrank the model
|
|
81
|
+
* the role was spawned with — so the model pin is verified here rather than
|
|
82
|
+
* trusted, and a disagreement stops the launch instead of being reported as a
|
|
83
|
+
* success (see src/model-env.ts).
|
|
84
|
+
*/
|
|
85
|
+
export function harnessChildEnv(role, launchEnv, stateDir) {
|
|
86
|
+
const env = { ...(launchEnv ?? {}), ...managedFleetProxyEnv(role, stateDir) };
|
|
87
|
+
assertModelPinReachesChild(role, env);
|
|
88
|
+
return env;
|
|
89
|
+
}
|
|
71
90
|
/**
|
|
72
91
|
* Execute a typed proxy request in the caller's supervisor. Dynamic imports
|
|
73
92
|
* avoid a runner↔spawn initialization cycle (spawn imports runner constants).
|
|
@@ -105,13 +124,19 @@ async function executeManagedSpawn(caller, configPath, requested, log) {
|
|
|
105
124
|
statePath,
|
|
106
125
|
harness: preview.harness,
|
|
107
126
|
session: preview.session,
|
|
108
|
-
|
|
127
|
+
// Read back from the resolved environment, not from the request: the banner
|
|
128
|
+
// must name the model the child will run, not the one that was asked for.
|
|
129
|
+
...(effectiveRoleModel(preview) ? { model: effectiveRoleModel(preview) } : {}),
|
|
109
130
|
monitor: { mode: preview.monitor.mode, interrupt: preview.monitor.interrupt },
|
|
131
|
+
permissionMode: effectivePermissionMode(preview),
|
|
110
132
|
inherited,
|
|
111
133
|
creationActionId,
|
|
112
134
|
};
|
|
113
135
|
log(`[${caller.name}] managed fleet proxy spawned ${result.lifetime} role ${result.role} `
|
|
114
|
-
+ `harness=${result.harness} session=${result.session}`
|
|
136
|
+
+ `harness=${result.harness} session=${result.session} `
|
|
137
|
+
+ `model=${result.model ?? '(harness default)'} `
|
|
138
|
+
+ `permission=${result.permissionMode.fleetMode} `
|
|
139
|
+
+ `native=${result.permissionMode.nativeMode}`);
|
|
115
140
|
return result;
|
|
116
141
|
}
|
|
117
142
|
/**
|
|
@@ -141,6 +166,10 @@ export function recordMonitorOwner(dir, owner) {
|
|
|
141
166
|
export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = launch.argv) {
|
|
142
167
|
const env = {
|
|
143
168
|
PATH: process.env.PATH ?? '', COLORTERM: 'truecolor', ...launch.env, ...(roleEnv ?? {}),
|
|
169
|
+
// Tmux roles have the same daemon-client boundary as ACP roles. Keep this
|
|
170
|
+
// last so neither harness preparation nor a role env block can take over
|
|
171
|
+
// the shared daemon lifecycle.
|
|
172
|
+
OURS_AUTOSTART: FLEET_OURS_AUTOSTART,
|
|
144
173
|
};
|
|
145
174
|
// Interactive panes should advertise colour even when the supervisor itself
|
|
146
175
|
// was launched with NO_COLOR. A role may still deliberately opt back in to
|
|
@@ -195,7 +224,7 @@ export function readExitRecord(path) {
|
|
|
195
224
|
catch { /* fall through to unknown */ }
|
|
196
225
|
return { version: 1, class: 'unknown', detail: `unreadable exit record: ${raw.slice(0, 120)}` };
|
|
197
226
|
}
|
|
198
|
-
// ─── Restart-loop containment
|
|
227
|
+
// ─── Restart-loop containment ──────────────────────────────────────────
|
|
199
228
|
//
|
|
200
229
|
// The child-session restart loop used to BE the service manager: systemd's
|
|
201
230
|
// `Restart=always RestartSec=2` and launchd's `KeepAlive`. Neither can count,
|
|
@@ -219,6 +248,66 @@ const emptyLedger = () => ({
|
|
|
219
248
|
circuit: 'closed',
|
|
220
249
|
updatedAt: new Date(0).toISOString(),
|
|
221
250
|
});
|
|
251
|
+
/**
|
|
252
|
+
* Carried across a supervisor process's life so its successor can tell an
|
|
253
|
+
* orderly exit from a kill. Present on disk == "a supervisor believed it was
|
|
254
|
+
* running"; the next start finding one that is not its own is proof the
|
|
255
|
+
* previous process died without getting to write anything.
|
|
256
|
+
*/
|
|
257
|
+
export const RUN_MARKER_FILE = '.supervisor-run.json';
|
|
258
|
+
function readRunMarker(dir) {
|
|
259
|
+
try {
|
|
260
|
+
const raw = JSON.parse(readFileSync(join(dir, RUN_MARKER_FILE), 'utf8'));
|
|
261
|
+
return raw.version === 1 && typeof raw.pid === 'number' && typeof raw.startedAt === 'string'
|
|
262
|
+
? raw : undefined;
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
return undefined;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Claim this state directory for the current supervisor process and report how
|
|
270
|
+
* the previous one ended. Runs BEFORE the first attempt, which is the whole
|
|
271
|
+
* point: after an abrupt kill nothing else writes until an attempt finishes,
|
|
272
|
+
* and an attempt can take minutes.
|
|
273
|
+
*/
|
|
274
|
+
export function claimSupervisorRun(dir, startedAt, pid = process.pid) {
|
|
275
|
+
const previous = readRunMarker(dir);
|
|
276
|
+
const termination = previous && previous.pid !== pid
|
|
277
|
+
? {
|
|
278
|
+
class: 'abrupt',
|
|
279
|
+
detail: `supervisor pid ${previous.pid} left an open run marker; `
|
|
280
|
+
+ 'it was terminated without an orderly exit (signal, OOM-kill, or host reset)',
|
|
281
|
+
observedAt: startedAt,
|
|
282
|
+
runStartedAt: previous.startedAt,
|
|
283
|
+
}
|
|
284
|
+
: previous
|
|
285
|
+
? { class: 'unknown', detail: 'run marker belongs to this process', observedAt: startedAt }
|
|
286
|
+
: { class: 'clean', detail: 'no previous run marker', observedAt: startedAt };
|
|
287
|
+
try {
|
|
288
|
+
mkdirSync(dir, { recursive: true });
|
|
289
|
+
writeFileSync(join(dir, RUN_MARKER_FILE), JSON.stringify({ version: 1, pid, startedAt }, null, 2) + '\n');
|
|
290
|
+
}
|
|
291
|
+
catch { /* diagnostics must never take the role down */ }
|
|
292
|
+
return termination;
|
|
293
|
+
}
|
|
294
|
+
/** Orderly exit: the successor must not read this run as a kill. */
|
|
295
|
+
export function releaseSupervisorRun(dir) {
|
|
296
|
+
try {
|
|
297
|
+
rmSync(join(dir, RUN_MARKER_FILE), { force: true });
|
|
298
|
+
}
|
|
299
|
+
catch { /* best effort */ }
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Fields that describe THIS process's history rather than the current failure
|
|
303
|
+
* streak. Clearing the streak (recovery, an operator `up`, an approved model
|
|
304
|
+
* transition) must not erase the record that the role died and came back.
|
|
305
|
+
*/
|
|
306
|
+
const carriedForward = (previous) => ({
|
|
307
|
+
...(previous.lastTermination ? { lastTermination: previous.lastTermination } : {}),
|
|
308
|
+
...(previous.abruptTerminations ? { abruptTerminations: previous.abruptTerminations } : {}),
|
|
309
|
+
...(previous.supervisorStartedAt ? { supervisorStartedAt: previous.supervisorStartedAt } : {}),
|
|
310
|
+
});
|
|
222
311
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
223
312
|
export function backoffFor(consecutiveFailures) {
|
|
224
313
|
if (consecutiveFailures <= 0)
|
|
@@ -253,7 +342,10 @@ export function writeRestartLedger(dir, ledger) {
|
|
|
253
342
|
export function resetRestartLedger(dir) {
|
|
254
343
|
if (!existsSync(dir))
|
|
255
344
|
return;
|
|
256
|
-
|
|
345
|
+
const previous = readRestartLedger(dir);
|
|
346
|
+
writeRestartLedger(dir, {
|
|
347
|
+
...emptyLedger(), ...carriedForward(previous), updatedAt: new Date().toISOString(),
|
|
348
|
+
});
|
|
257
349
|
}
|
|
258
350
|
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
259
351
|
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
@@ -384,11 +476,18 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
384
476
|
const effectiveModel = effectiveModelForRole(dir, role);
|
|
385
477
|
if (effectiveModel !== role.model) {
|
|
386
478
|
deps.log(`[${name}] model recovery drift: declared=${role.model ?? '(none)'} effective=${effectiveModel}`);
|
|
387
|
-
|
|
479
|
+
// The env pin has to move with it. A down-shift that changed only
|
|
480
|
+
// `role.model` was reported as a model change while the child kept running
|
|
481
|
+
// the model that had just failed, because the pin is what the harness reads.
|
|
482
|
+
role = { ...role, model: effectiveModel, env: repinModelEnv(role, effectiveModel) };
|
|
388
483
|
}
|
|
389
484
|
if (modelRecoveryHeld(dir))
|
|
390
485
|
throw new Error(`[${name}] model chain exhausted — held down until config changes or recovery reset`);
|
|
391
486
|
const adapter = getAdapter(role.harness);
|
|
487
|
+
// Say the running model out loud, once, from the resolved environment. The
|
|
488
|
+
// spawn banner is a claim made before the process exists; this is the log line
|
|
489
|
+
// that can be checked against the session afterwards.
|
|
490
|
+
deps.log(`[${name}] model: ${effectiveRoleModel(role) ?? '(harness default)'}`);
|
|
392
491
|
mkdirSync(dir, { recursive: true });
|
|
393
492
|
const rotation = rotateWorklog(join(dir, 'WORKLOG.md'), role.worklog);
|
|
394
493
|
if (rotation.deferred)
|
|
@@ -403,8 +502,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
403
502
|
const exitFile = join(dir, '.exit-status');
|
|
404
503
|
const booted = existsSync(bootedFile);
|
|
405
504
|
const mode = booted && adapter.supportsResume ? 'resume' : 'fresh';
|
|
406
|
-
|
|
407
|
-
|
|
505
|
+
// Stamp EVERY attempt, not just the first. `.booted` used to be written only
|
|
506
|
+
// on the fresh path, so after a restart — including one the supervisor never
|
|
507
|
+
// saw, like an OOM-kill — its mtime still read the original boot and any
|
|
508
|
+
// health check reading it reported "no restarts". The existence test above
|
|
509
|
+
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
510
|
+
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
408
511
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
409
512
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
410
513
|
const sessionBackend = role.session ?? 'tmux';
|
|
@@ -416,11 +519,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
416
519
|
})()
|
|
417
520
|
: adapter.buildLaunch(role, mode, { sessionId }, prep);
|
|
418
521
|
// Isolation is additive: only roles that declare `isolation:` are wrapped. The
|
|
419
|
-
// env prefix + exit capture in buildPaneCommand stay host-side
|
|
522
|
+
// env prefix + exit capture in buildPaneCommand stay host-side.
|
|
420
523
|
let wrappedArgv = launch.argv;
|
|
421
524
|
if (role.isolation) {
|
|
422
|
-
// Start with the
|
|
423
|
-
//
|
|
525
|
+
// Start with the same durable context that config validation and doctor judged,
|
|
526
|
+
// then add the selected launch's exact runtime closure. Those paths
|
|
424
527
|
// still pass through resolveIsolation's canonical blocklist enforcement.
|
|
425
528
|
const runtime = resolveLaunchRuntime(launch.argv);
|
|
426
529
|
launch = { ...launch, argv: runtime.argv };
|
|
@@ -440,8 +543,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
440
543
|
rmSync(degradedMarker, { force: true });
|
|
441
544
|
}
|
|
442
545
|
wrappedArgv = sel.backend.wrap(launch.argv, policy, ctx);
|
|
443
|
-
// Resource caps wrap the sandbox from
|
|
444
|
-
//
|
|
546
|
+
// Resource caps wrap the sandbox from outside, at the pane's own cgroup scope.
|
|
547
|
+
// This applies even when the sandbox degraded to none.
|
|
445
548
|
const { argv: rprefix, warnings } = resourceArgs(policy.resources, deps.cpuDelegated());
|
|
446
549
|
for (const w of warnings)
|
|
447
550
|
deps.log(`[${name}] WARNING ${w}`);
|
|
@@ -462,7 +565,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
462
565
|
await deps.sleep(wait);
|
|
463
566
|
}
|
|
464
567
|
}
|
|
465
|
-
//
|
|
568
|
+
// Prime the supervisor mail monitor's notification cursor at the
|
|
466
569
|
// stream tip BEFORE the session launches so no arrival is missed during boot
|
|
467
570
|
// (backlog before the tip is the SessionStart hook's job). Native-mode roles
|
|
468
571
|
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
@@ -525,12 +628,23 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
525
628
|
name,
|
|
526
629
|
argv: wrappedArgv,
|
|
527
630
|
cwd: runCwd,
|
|
528
|
-
env:
|
|
631
|
+
env: harnessChildEnv(role, launch.env, dir),
|
|
529
632
|
stateDir: dir,
|
|
530
633
|
mode,
|
|
531
634
|
permissions: perms,
|
|
532
635
|
modeId: adapter.acpPermissionModeId?.(role),
|
|
636
|
+
// The role's declared MCP servers, and the bundled agent's `_meta`
|
|
637
|
+
// vocabulary for the options it takes no flag for. Both come from the
|
|
638
|
+
// ADAPTER and from `prep`: the ACP launch cannot carry `prep.argv`, so this
|
|
639
|
+
// is the route by which harness_options that used to be silently dropped
|
|
640
|
+
// for an ACP role actually reach the session.
|
|
641
|
+
mcpServers: adapter.acpMcpServers?.(role),
|
|
642
|
+
sessionMeta: adapter.acpSessionMeta?.(role, prep),
|
|
533
643
|
permissionMode: effectivePermissionMode(role),
|
|
644
|
+
// Provenance travels with the exact ACP launch. Keeping it out of a
|
|
645
|
+
// role-only adapter hook prevents a PATH fallback or resolver skew from
|
|
646
|
+
// claiming metadata trust for an argv it did not authenticate.
|
|
647
|
+
permissionMetadataSource: launch.permissionMetadataSource,
|
|
534
648
|
log: deps.log,
|
|
535
649
|
});
|
|
536
650
|
pid = acpSession.pid;
|
|
@@ -928,100 +1042,138 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
928
1042
|
mkdirSync(dir, { recursive: true });
|
|
929
1043
|
const shouldStop = deps.shouldStop ?? (() => false);
|
|
930
1044
|
const stamp = () => new Date(deps.now()).toISOString();
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1045
|
+
// Record how the PREVIOUS supervisor process ended before doing anything
|
|
1046
|
+
// else. An external kill writes nothing itself, and the next ledger write is
|
|
1047
|
+
// an attempt away — which is why an OOM-kill used to leave every durable
|
|
1048
|
+
// indicator describing the run that died.
|
|
1049
|
+
const startedAt = stamp();
|
|
1050
|
+
const termination = claimSupervisorRun(dir, startedAt);
|
|
1051
|
+
{
|
|
1052
|
+
const previous = readRestartLedger(dir);
|
|
1053
|
+
const abrupt = (previous.abruptTerminations ?? 0) + (termination.class === 'abrupt' ? 1 : 0);
|
|
1054
|
+
writeRestartLedger(dir, {
|
|
1055
|
+
...previous,
|
|
1056
|
+
lastTermination: termination,
|
|
1057
|
+
abruptTerminations: abrupt,
|
|
1058
|
+
supervisorStartedAt: startedAt,
|
|
1059
|
+
updatedAt: startedAt,
|
|
1060
|
+
});
|
|
1061
|
+
if (termination.class === 'abrupt')
|
|
1062
|
+
deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
|
|
1063
|
+
+ `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
|
|
1064
|
+
}
|
|
1065
|
+
try {
|
|
1066
|
+
while (!shouldStop()) {
|
|
1067
|
+
let ledger = readRestartLedger(dir);
|
|
1068
|
+
try {
|
|
1069
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
1070
|
+
const role = findRole(loadConfig(configPath), name);
|
|
1071
|
+
reconcileModelRecovery(dir, role, stamp());
|
|
1072
|
+
if (modelRecoveryHeld(dir)) {
|
|
1073
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1074
|
+
continue;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
catch {
|
|
1078
|
+
// Normal attempt path reports config errors through the restart circuit.
|
|
1079
|
+
}
|
|
1080
|
+
if (ledger.circuit === 'open') {
|
|
1081
|
+
// Held down. Stay alive — exiting would hand the role straight back to
|
|
1082
|
+
// the service manager — and watch for an operator reset.
|
|
938
1083
|
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
939
1084
|
continue;
|
|
940
1085
|
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
// Service-manager boot and automatic retry bypass the operator-facing
|
|
954
|
-
// up/restart commands. Reconcile again immediately before every real
|
|
955
|
-
// permanent harness attempt; the operation is idempotent and releases its
|
|
956
|
-
// provisioning lease before the agent or owner channel binds.
|
|
957
|
-
if (attempt === runOnce) {
|
|
958
|
-
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
959
|
-
const role = findRole(loadConfig(configPath), name);
|
|
960
|
-
await reconcilePermanentRoleIdentities(role, undefined, deps.log);
|
|
1086
|
+
let result;
|
|
1087
|
+
try {
|
|
1088
|
+
// Service-manager boot and automatic retry bypass the operator-facing
|
|
1089
|
+
// up/restart commands. Reconcile again immediately before every real
|
|
1090
|
+
// permanent harness attempt; the operation is idempotent and releases its
|
|
1091
|
+
// provisioning lease before the agent or owner channel binds.
|
|
1092
|
+
if (attempt === runOnce) {
|
|
1093
|
+
const configPath = resolveConfigPath(dir, opts.configPath);
|
|
1094
|
+
const role = findRole(loadConfig(configPath), name);
|
|
1095
|
+
await reconcilePermanentRoleIdentities(role, undefined, deps.log);
|
|
1096
|
+
}
|
|
1097
|
+
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
961
1098
|
}
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
//
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
1099
|
+
catch (e) {
|
|
1100
|
+
// A session that could not even start is an immediate failure like any
|
|
1101
|
+
// other; it must count, or an unstartable role loops forever.
|
|
1102
|
+
result = {
|
|
1103
|
+
elapsedSecs: 0,
|
|
1104
|
+
exit: { version: 1, class: 'unknown', detail: e instanceof Error ? e.message : String(e) },
|
|
1105
|
+
rotated: false,
|
|
1106
|
+
mode: 'fresh',
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1110
|
+
// have reset the ledger meanwhile.
|
|
1111
|
+
ledger = readRestartLedger(dir);
|
|
1112
|
+
if (result.modelRecovery === 'advance') {
|
|
1113
|
+
writeRestartLedger(dir, {
|
|
1114
|
+
...emptyLedger(),
|
|
1115
|
+
...carriedForward(ledger),
|
|
1116
|
+
lastReason: 'approved model-chain transition',
|
|
1117
|
+
updatedAt: stamp(),
|
|
1118
|
+
});
|
|
1119
|
+
continue;
|
|
1120
|
+
}
|
|
1121
|
+
if (result.modelRecovery === 'hold') {
|
|
1122
|
+
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
const fastFailSecs = fastFailSecsFor(name, opts.configPath);
|
|
1126
|
+
// The fast-fail boundary starts a recovery episode; it must not also be
|
|
1127
|
+
// the boundary that declares recovery successful. Otherwise alternating
|
|
1128
|
+
// 19s and 20s deaths erase one another forever. Require the configured
|
|
1129
|
+
// number of fast-fail windows to survive before closing an active streak.
|
|
1130
|
+
// This hysteresis stays adapter-relative (100s for the current 20s/5-attempt
|
|
1131
|
+
// policy) and still lets a genuinely sustained session reset the breaker.
|
|
1132
|
+
const stableRecoverySecs = fastFailSecs * RESTART_FAIL_THRESHOLD;
|
|
1133
|
+
const recoveryFailed = result.elapsedSecs < fastFailSecs
|
|
1134
|
+
|| (ledger.consecutiveImmediateFailures > 0 && result.elapsedSecs < stableRecoverySecs);
|
|
1135
|
+
if (!recoveryFailed) {
|
|
1136
|
+
// A session that ran for a while is not a restart loop, whatever ended it.
|
|
1137
|
+
writeRestartLedger(dir, {
|
|
1138
|
+
...emptyLedger(),
|
|
1139
|
+
...carriedForward(ledger),
|
|
1140
|
+
lastReason: result.exit.detail,
|
|
1141
|
+
updatedAt: stamp(),
|
|
1142
|
+
});
|
|
1143
|
+
continue;
|
|
1144
|
+
}
|
|
1145
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
1146
|
+
const reason = `${result.exit.detail} after ${result.elapsedSecs.toFixed(1)}s`;
|
|
1147
|
+
const next = {
|
|
1148
|
+
version: 1,
|
|
1149
|
+
...carriedForward(ledger),
|
|
1150
|
+
consecutiveImmediateFailures: failures,
|
|
1151
|
+
lastReason: reason,
|
|
1152
|
+
nextDelayMs: backoffFor(failures),
|
|
1153
|
+
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1154
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
996
1155
|
updatedAt: stamp(),
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
nextDelayMs: backoffFor(failures),
|
|
1007
|
-
resumeDiscarded: ledger.resumeDiscarded || result.rotated,
|
|
1008
|
-
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed',
|
|
1009
|
-
updatedAt: stamp(),
|
|
1010
|
-
};
|
|
1011
|
-
if (next.circuit === 'open') {
|
|
1012
|
-
next.openedAt = stamp();
|
|
1013
|
-
next.nextDelayMs = 0;
|
|
1156
|
+
};
|
|
1157
|
+
if (next.circuit === 'open') {
|
|
1158
|
+
next.openedAt = stamp();
|
|
1159
|
+
next.nextDelayMs = 0;
|
|
1160
|
+
writeRestartLedger(dir, next);
|
|
1161
|
+
deps.log(`[${name}] HELD DOWN after ${failures} immediate failures at ${next.openedAt} — ` +
|
|
1162
|
+
`${reason}; the agent will not be restarted until: ours-fleet restart ${name}`);
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1014
1165
|
writeRestartLedger(dir, next);
|
|
1015
|
-
deps.log(`[${name}]
|
|
1016
|
-
|
|
1017
|
-
|
|
1166
|
+
deps.log(`[${name}] immediate failure ${failures}/${RESTART_FAIL_THRESHOLD} (${reason}) ` +
|
|
1167
|
+
`-> backing off ${next.nextDelayMs}ms`);
|
|
1168
|
+
await deps.sleep(next.nextDelayMs);
|
|
1018
1169
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1170
|
+
return readRestartLedger(dir);
|
|
1171
|
+
}
|
|
1172
|
+
finally {
|
|
1173
|
+
// Only an orderly return through this loop clears the marker; a signal or
|
|
1174
|
+
// an OOM-kill leaves it, which is exactly how the successor detects them.
|
|
1175
|
+
releaseSupervisorRun(dir);
|
|
1023
1176
|
}
|
|
1024
|
-
return readRestartLedger(dir);
|
|
1025
1177
|
}
|
|
1026
1178
|
/**
|
|
1027
1179
|
* How short an attempt has to be to count as immediate. The role's harness
|