@ours.network/fleet 1.2.0-nightly.54 → 1.2.0-nightly.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -12
- package/dist/agent-ours/service.d.ts +4 -1
- package/dist/agent-ours/service.js +31 -7
- package/dist/application/log-service.js +8 -8
- package/dist/application/role-creation-service.js +2 -1
- package/dist/application/role-removal-service.js +7 -1
- package/dist/application/role-repository.js +17 -0
- package/dist/application/task-layouts.js +12 -2
- package/dist/application/task-room-service.js +19 -17
- package/dist/application/types.d.ts +2 -0
- package/dist/build-info.json +4 -4
- package/dist/cli.js +31 -3
- package/dist/config.d.ts +2 -0
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +2 -2
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +1 -1
- package/dist/harness/codex-session.js +2 -2
- package/dist/managed-recovery.d.ts +13 -0
- package/dist/managed-recovery.js +31 -0
- package/dist/ops.d.ts +2 -0
- package/dist/ops.js +15 -2
- package/dist/rooms-tasks/cli.js +21 -0
- package/dist/rooms-tasks/close.d.ts +2 -0
- package/dist/rooms-tasks/close.js +66 -1
- package/dist/rooms-tasks/deletion.js +10 -0
- package/dist/rooms-tasks/erasure.js +4 -2
- package/dist/rooms-tasks/layout-member-retirement.d.ts +10 -0
- package/dist/rooms-tasks/layout-member-retirement.js +90 -0
- package/dist/rooms-tasks/layout-service.d.ts +1 -0
- package/dist/rooms-tasks/layout-service.js +31 -13
- package/dist/rooms-tasks/layout.d.ts +7 -0
- package/dist/rooms-tasks/layout.js +35 -0
- package/dist/rooms-tasks/legacy-task-member.d.ts +19 -0
- package/dist/rooms-tasks/legacy-task-member.js +116 -0
- package/dist/rooms-tasks/member-ownership.d.ts +5 -0
- package/dist/rooms-tasks/member-ownership.js +40 -0
- package/dist/rooms-tasks/provision.js +26 -1
- package/dist/rooms-tasks/recovery.d.ts +19 -0
- package/dist/rooms-tasks/recovery.js +110 -0
- package/dist/rooms-tasks/task-service-retirement.d.ts +11 -0
- package/dist/rooms-tasks/task-service-retirement.js +21 -0
- package/dist/rooms-tasks/task-state.d.ts +1 -0
- package/dist/rooms-tasks/task-state.js +16 -1
- package/dist/rooms-tasks/types.d.ts +3 -0
- package/dist/rooms-tasks/workspace.d.ts +2 -0
- package/dist/rooms-tasks/workspace.js +5 -0
- package/dist/runner.d.ts +6 -0
- package/dist/runner.js +160 -22
- package/dist/session/acp.d.ts +1 -0
- package/dist/session/acp.js +7 -0
- package/dist/session/arbiter.d.ts +5 -0
- package/dist/session/arbiter.js +16 -2
- package/dist/session/codex-app-server.d.ts +1 -0
- package/dist/session/codex-app-server.js +5 -0
- package/dist/session/types.d.ts +2 -0
- package/dist/spawn.d.ts +5 -1
- package/dist/spawn.js +32 -3
- package/dist/startup-readiness.d.ts +1 -1
- package/dist/startup-readiness.js +3 -3
- package/dist/supervisor/adoption.d.ts +8 -0
- package/dist/supervisor/adoption.js +78 -0
- package/dist/supervisor/catalog.d.ts +62 -0
- package/dist/supervisor/catalog.js +254 -0
- package/dist/supervisor/fleet.d.ts +8 -0
- package/dist/supervisor/fleet.js +89 -0
- package/dist/supervisor/index.js +3 -6
- package/dist/supervisor/launchd.d.ts +1 -1
- package/dist/supervisor/launchd.js +24 -12
- package/dist/supervisor/legacy.d.ts +35 -0
- package/dist/supervisor/legacy.js +524 -0
- package/dist/supervisor/manager.d.ts +15 -0
- package/dist/supervisor/manager.js +258 -0
- package/dist/supervisor/scope.d.ts +4 -0
- package/dist/supervisor/scope.js +10 -0
- package/dist/supervisor/systemd.d.ts +2 -2
- package/dist/supervisor/systemd.js +14 -14
- package/dist/task-supervision.d.ts +9 -0
- package/dist/task-supervision.js +145 -0
- package/dist/temp-lifecycle.d.ts +17 -7
- package/dist/temp-lifecycle.js +85 -53
- package/dist/temp-supervisor-recovery.d.ts +1 -1
- package/dist/temp-supervisor-recovery.js +2 -2
- package/docs/fleet-migration.md +54 -0
- package/docs/task-workspaces.md +26 -0
- package/package.json +4 -1
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { closeSync, existsSync, openSync, rmSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { acquireOwnerBinderLease } from '../owner-channel/binder.js';
|
|
6
|
+
import { stateRoot } from '../paths.js';
|
|
7
|
+
import { readRestartLedger, writeRestartLedger, backoffFor, RESTART_FAIL_THRESHOLD } from '../runner.js';
|
|
8
|
+
import { realExec } from '../exec.js';
|
|
9
|
+
import { listMemberKeys, managedMemberEnvironment, memberPath, memberProcess, readMember, withMemberLock, writeMember } from './catalog.js';
|
|
10
|
+
/** One parent owns process creation for every kind of Fleet member. */
|
|
11
|
+
export async function runFleetManager(entrypoint, deps = {}) {
|
|
12
|
+
const lease = await acquireOwnerBinderLease(join(stateRoot(), 'supervisor'), 'fleet-manager', 'fleet-manager');
|
|
13
|
+
const children = new Map();
|
|
14
|
+
const now = deps.now ?? Date.now;
|
|
15
|
+
const log = deps.log ?? (message => console.error(message));
|
|
16
|
+
const exec = deps.exec ?? realExec;
|
|
17
|
+
const sleep = deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
18
|
+
let stopping = false;
|
|
19
|
+
const stop = () => { stopping = true; };
|
|
20
|
+
process.on('SIGTERM', stop);
|
|
21
|
+
process.on('SIGINT', stop);
|
|
22
|
+
const spawnChild = deps.spawnChild ?? ((member, generation) => {
|
|
23
|
+
if (!existsSync(member.dir))
|
|
24
|
+
throw Error('Fleet member state missing');
|
|
25
|
+
const fd = openSync(join(member.dir, 'supervisor.log'), 'a', 0o600);
|
|
26
|
+
try {
|
|
27
|
+
return spawn(process.execPath, [entrypoint, '_run-managed', member.key, generation], {
|
|
28
|
+
// A process group permits bounded cleanup of a hung runner and its harness.
|
|
29
|
+
// IPC couples its lifetime to this parent even if the parent is SIGKILLed.
|
|
30
|
+
detached: true, stdio: ['ignore', fd, fd, 'ipc'], env: managedMemberEnvironment(member),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
closeSync(fd);
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
async function reconcile(key) {
|
|
38
|
+
await withMemberLock(key, async () => {
|
|
39
|
+
const member = readMember(key);
|
|
40
|
+
const owned = children.get(key);
|
|
41
|
+
if (!member) {
|
|
42
|
+
if (owned && !owned.stoppingAt) {
|
|
43
|
+
owned.stoppingAt = Date.now();
|
|
44
|
+
owned.child.kill('SIGTERM');
|
|
45
|
+
}
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
// Our ChildProcess supplies exit notification for steady owned members.
|
|
49
|
+
// Probe argv only for predecessor discovery or before an actual stop;
|
|
50
|
+
// macOS otherwise spawned one ps process per member every 250ms.
|
|
51
|
+
if (owned && owned.generation === member.generation && member.desired === 'running' && !stopping && !owned.stoppingAt)
|
|
52
|
+
return;
|
|
53
|
+
const live = await memberProcess(member, exec);
|
|
54
|
+
if (member.desired === 'stopped' || stopping) {
|
|
55
|
+
if (live.state === 'running' && live.pid) {
|
|
56
|
+
if (owned && owned.generation === member.generation) {
|
|
57
|
+
if (!owned.stoppingAt) {
|
|
58
|
+
owned.stoppingAt = Date.now();
|
|
59
|
+
owned.child.kill('SIGTERM');
|
|
60
|
+
}
|
|
61
|
+
else if (Date.now() - owned.stoppingAt >= 15_000) {
|
|
62
|
+
// Re-check exact generation immediately before a group signal.
|
|
63
|
+
try {
|
|
64
|
+
process.kill(-live.pid, 'SIGKILL');
|
|
65
|
+
}
|
|
66
|
+
catch { /* next probe proves absence */ }
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
// A predecessor's child has lost IPC and is shutting down. Never adopt
|
|
71
|
+
// or launch alongside it; its worker also holds the member lifetime lease.
|
|
72
|
+
log(`[fleet] waiting for predecessor ${key}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
if (live.state !== 'stopped' || owned || (member.retryAt ?? 0) > Date.now() || readRestartLedger(member.dir).circuit === 'open')
|
|
78
|
+
return;
|
|
79
|
+
const generation = randomUUID();
|
|
80
|
+
writeMember({ ...member, generation, pid: undefined });
|
|
81
|
+
let child;
|
|
82
|
+
try {
|
|
83
|
+
child = spawnChild(member, generation);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
if (!existsSync(member.dir)) {
|
|
87
|
+
writeMember({ ...member, generation: undefined, pid: undefined, retryAt: Date.now() + 30_000 });
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const ledger = readRestartLedger(member.dir);
|
|
91
|
+
const failures = ledger.consecutiveImmediateFailures + 1;
|
|
92
|
+
writeRestartLedger(member.dir, { ...ledger, consecutiveImmediateFailures: failures, circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed', nextDelayMs: backoffFor(failures), lastReason: 'Fleet child spawn failed', updatedAt: new Date().toISOString() });
|
|
93
|
+
writeMember({ ...member, generation: undefined, pid: undefined, retryAt: Date.now() + backoffFor(failures) });
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
const childState = { child, generation, startedAt: now() };
|
|
97
|
+
children.set(key, childState);
|
|
98
|
+
writeMember({ ...member, generation, pid: child.pid });
|
|
99
|
+
let finished = false;
|
|
100
|
+
const finish = (failed) => {
|
|
101
|
+
if (finished)
|
|
102
|
+
return;
|
|
103
|
+
finished = true;
|
|
104
|
+
if (children.get(key) === childState)
|
|
105
|
+
children.delete(key);
|
|
106
|
+
void withMemberLock(key, async () => {
|
|
107
|
+
const current = readMember(key);
|
|
108
|
+
if (!current || current.generation !== generation)
|
|
109
|
+
return;
|
|
110
|
+
if (current.kind === 'temporary' && !existsSync(current.dir)) {
|
|
111
|
+
rmSync(memberPath(key));
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
let taskCompleted = false;
|
|
115
|
+
if (!failed && !stopping && !childState.stoppingAt && current.kind === 'task') {
|
|
116
|
+
try {
|
|
117
|
+
taskCompleted = !(await import('../task-supervision.js')).taskSupervisorMayRun(current.name);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
failed = true;
|
|
121
|
+
} // uncertainty consumes the existing bounded failure budget
|
|
122
|
+
}
|
|
123
|
+
const ledger = readRestartLedger(current.dir);
|
|
124
|
+
let previousFailures = ledger.consecutiveImmediateFailures;
|
|
125
|
+
if (failed && !stopping && !childState.stoppingAt) {
|
|
126
|
+
const { stableSupervisorWindow } = await import('../runner.js');
|
|
127
|
+
if (now() - childState.startedAt >= stableSupervisorWindow(current.name, current.kind !== 'permanent', current.configPath) * 1000)
|
|
128
|
+
previousFailures = 0;
|
|
129
|
+
}
|
|
130
|
+
const failures = failed && !stopping && !childState.stoppingAt ? previousFailures + 1 : ledger.consecutiveImmediateFailures;
|
|
131
|
+
if (failed && !stopping && !childState.stoppingAt && existsSync(current.dir))
|
|
132
|
+
writeRestartLedger(current.dir, {
|
|
133
|
+
...ledger, consecutiveImmediateFailures: failures, lastReason: 'Fleet member runner exited unexpectedly',
|
|
134
|
+
circuit: failures >= RESTART_FAIL_THRESHOLD ? 'open' : 'closed', nextDelayMs: backoffFor(failures),
|
|
135
|
+
updatedAt: new Date().toISOString(), ...(failures >= RESTART_FAIL_THRESHOLD ? { openedAt: new Date().toISOString() } : {}),
|
|
136
|
+
});
|
|
137
|
+
writeMember({ ...current, pid: undefined, generation: undefined,
|
|
138
|
+
retryAt: Date.now() + backoffFor(failures),
|
|
139
|
+
// A completed task/transient worker must not become an endless launch loop.
|
|
140
|
+
desired: !failed && !stopping && !childState.stoppingAt && (current.kind === 'temporary' || taskCompleted) ? 'stopped' : current.desired });
|
|
141
|
+
}).catch(() => log(`[fleet] child completion record unavailable: ${key}`));
|
|
142
|
+
};
|
|
143
|
+
child.once('error', () => finish(true));
|
|
144
|
+
child.once('exit', (code, signal) => {
|
|
145
|
+
// The tracked child created this private process group; reclaim any harness
|
|
146
|
+
// descendants even when the runner itself crashed before cleaning them.
|
|
147
|
+
if (child.pid) {
|
|
148
|
+
try {
|
|
149
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
150
|
+
}
|
|
151
|
+
catch { /* already absent */ }
|
|
152
|
+
}
|
|
153
|
+
finish(code !== 0 || signal !== null);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
try {
|
|
158
|
+
try {
|
|
159
|
+
const { resumeFleetTransfers } = await import('./adoption.js');
|
|
160
|
+
await resumeFleetTransfers(entrypoint, exec);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
// Existing catalog members continue; uncertain transfers remain held with
|
|
164
|
+
// their receipts. Operator init/up reports the same actionable failure.
|
|
165
|
+
log(`[fleet] transfer pending: ${error instanceof Error ? error.message : 'proof unavailable'}`);
|
|
166
|
+
}
|
|
167
|
+
while (!stopping && !deps.shouldStop?.()) {
|
|
168
|
+
for (const key of listMemberKeys()) {
|
|
169
|
+
try {
|
|
170
|
+
await reconcile(key);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
log(`[fleet] reconciliation held: ${key}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
await sleep(250);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
stopping = true;
|
|
181
|
+
// Shutdown preserves desired intent; the successor resumes the same catalog.
|
|
182
|
+
const deadline = Date.now() + 17_000;
|
|
183
|
+
while (children.size && Date.now() < deadline) {
|
|
184
|
+
for (const key of children.keys())
|
|
185
|
+
await reconcile(key);
|
|
186
|
+
await sleep(100);
|
|
187
|
+
}
|
|
188
|
+
process.off('SIGTERM', stop);
|
|
189
|
+
process.off('SIGINT', stop);
|
|
190
|
+
lease.release();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/** Workers cannot outlive their parent or overlap another generation of this member. */
|
|
194
|
+
export async function runManagedMember(key, generation) {
|
|
195
|
+
let stopping = false;
|
|
196
|
+
const stop = () => { stopping = true; };
|
|
197
|
+
const disconnected = () => {
|
|
198
|
+
stopping = true;
|
|
199
|
+
process.kill(process.pid, 'SIGTERM');
|
|
200
|
+
setTimeout(() => process.kill(-process.pid, 'SIGKILL'), 15_000).unref();
|
|
201
|
+
};
|
|
202
|
+
process.on('SIGTERM', stop);
|
|
203
|
+
process.on('SIGINT', stop);
|
|
204
|
+
process.on('disconnect', disconnected);
|
|
205
|
+
if (!process.connected)
|
|
206
|
+
throw Error('FLEET_MEMBER_PARENT_MISSING');
|
|
207
|
+
const lease = await acquireOwnerBinderLease(join(stateRoot(), 'supervisor', 'workers', key), 'fleet-member', key);
|
|
208
|
+
try {
|
|
209
|
+
const member = readMember(key);
|
|
210
|
+
if (!member || member.generation !== generation || member.desired !== 'running' || stopping)
|
|
211
|
+
return;
|
|
212
|
+
const { runSupervised } = await import('../runner.js');
|
|
213
|
+
if (member.kind === 'permanent') {
|
|
214
|
+
const { waitForRoleDaemon } = await import('../startup-readiness.js');
|
|
215
|
+
// Waiting for a daemon is readiness, not a failed harness attempt.
|
|
216
|
+
while (!stopping) {
|
|
217
|
+
try {
|
|
218
|
+
await waitForRoleDaemon(member.name, member.configPath);
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
if (!stopping)
|
|
223
|
+
await new Promise(resolve => setTimeout(resolve, 5000));
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (!stopping)
|
|
227
|
+
await runSupervised(member.name, { configPath: member.configPath }, { shouldStop: () => stopping });
|
|
228
|
+
}
|
|
229
|
+
else if (member.kind === 'watchdogs') {
|
|
230
|
+
const { runScheduler } = await import('../watchdog/scheduler.js');
|
|
231
|
+
await runScheduler(member.configPath, {
|
|
232
|
+
now: () => new Date(), sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
233
|
+
log: message => console.log(message), binPath: process.argv[1], shouldStop: () => stopping,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
const { runTempSupervisor } = await import('../temp-supervisor-recovery.js');
|
|
238
|
+
const { runTemp } = await import('../runner.js');
|
|
239
|
+
await runTempSupervisor(member.name, process.argv[1], () => runTemp(member.name, {
|
|
240
|
+
shouldStop: () => stopping, suspendOnStop: true,
|
|
241
|
+
}), true);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
finally {
|
|
245
|
+
lease.release();
|
|
246
|
+
process.off('SIGTERM', stop);
|
|
247
|
+
process.off('SIGINT', stop);
|
|
248
|
+
process.off('disconnect', disconnected);
|
|
249
|
+
if (process.connected)
|
|
250
|
+
process.disconnect();
|
|
251
|
+
else if (stopping) {
|
|
252
|
+
try {
|
|
253
|
+
process.kill(-process.pid, 'SIGKILL');
|
|
254
|
+
}
|
|
255
|
+
catch { /* no private group */ }
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type Exec } from '../exec.js';
|
|
2
|
+
/** OS user services have one namespace. Isolated homes must not hijack it.
|
|
3
|
+
* Injected executors exercise isolated definitions without touching a user manager. */
|
|
4
|
+
export declare function assertNativeFleetScope(exec: Exec): void;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { home } from '../paths.js';
|
|
4
|
+
import { realExec } from '../exec.js';
|
|
5
|
+
/** OS user services have one namespace. Isolated homes must not hijack it.
|
|
6
|
+
* Injected executors exercise isolated definitions without touching a user manager. */
|
|
7
|
+
export function assertNativeFleetScope(exec) {
|
|
8
|
+
if (exec === realExec && resolve(home()) !== resolve(homedir()))
|
|
9
|
+
throw Error('FLEET_SERVICE_HOME_CONFLICT: managed OS supervision requires the OS user home; use OURS_FLEET_SUPERVISOR=none for isolated fleets');
|
|
10
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Exec } from '../exec.js';
|
|
2
2
|
import type { LivenessState, SupervisorBackend } from './types.js';
|
|
3
|
-
export declare const UNIT_TEMPLATE = "ours-fleet
|
|
3
|
+
export declare const UNIT_TEMPLATE = "ours-fleet.service";
|
|
4
4
|
/**
|
|
5
5
|
* Actionable hint when systemctl cannot reach the user bus. After the cli.ts
|
|
6
6
|
* XDG_RUNTIME_DIR fallback this branch stays reachable only when
|
|
@@ -8,7 +8,7 @@ export declare const UNIT_TEMPLATE = "ours-fleet-agent@.service";
|
|
|
8
8
|
* active — so pointing at linger is the correct first hint. (#9)
|
|
9
9
|
*/
|
|
10
10
|
export declare const busHint: (stderr: string) => string;
|
|
11
|
-
export declare const unitFor: (
|
|
11
|
+
export declare const unitFor: (_name: string) => string;
|
|
12
12
|
/**
|
|
13
13
|
* systemd's own ActiveState vocabulary, classified. `activating` covers
|
|
14
14
|
* `auto-restart` — the unit is mid-restart, not stopped, so its context stands.
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { assertNativeFleetScope } from './scope.js';
|
|
1
2
|
import { clientConfigPath } from '../client-profile.js';
|
|
2
3
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
3
4
|
import { delimiter, dirname, join } from 'node:path';
|
|
4
5
|
import { userInfo } from 'node:os';
|
|
5
6
|
import { home } from '../paths.js';
|
|
6
7
|
import { realExec } from '../exec.js';
|
|
7
|
-
export const UNIT_TEMPLATE = 'ours-fleet
|
|
8
|
+
export const UNIT_TEMPLATE = 'ours-fleet.service';
|
|
8
9
|
/**
|
|
9
10
|
* Actionable hint when systemctl cannot reach the user bus. After the cli.ts
|
|
10
11
|
* XDG_RUNTIME_DIR fallback this branch stays reachable only when
|
|
@@ -15,7 +16,7 @@ export const busHint = (stderr) => /user scope bus|XDG_RUNTIME_DIR/.test(stderr)
|
|
|
15
16
|
? `\nhint: no user runtime dir — enable linger: sudo loginctl enable-linger ${userInfo().username}` +
|
|
16
17
|
`\n (if linger is already on: export XDG_RUNTIME_DIR=/run/user/$(id -u))`
|
|
17
18
|
: '';
|
|
18
|
-
export const unitFor = (
|
|
19
|
+
export const unitFor = (_name) => UNIT_TEMPLATE;
|
|
19
20
|
/** Quote a systemd unit argument and escape its specifier marker. */
|
|
20
21
|
const unitArg = (value) => `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/%/g, '%%')}"`;
|
|
21
22
|
/**
|
|
@@ -57,10 +58,11 @@ async function probeLiveness(ctl, name) {
|
|
|
57
58
|
};
|
|
58
59
|
}
|
|
59
60
|
export function makeSystemdBackend(exec = realExec) {
|
|
60
|
-
const ctl = (...args) => exec('systemctl', ['--user', ...args]);
|
|
61
|
+
const ctl = (...args) => { assertNativeFleetScope(exec); return exec('systemctl', ['--user', ...args]); };
|
|
61
62
|
return {
|
|
62
63
|
id: 'systemd',
|
|
63
64
|
async init(binPath) {
|
|
65
|
+
assertNativeFleetScope(exec);
|
|
64
66
|
const msgs = [];
|
|
65
67
|
const unitDir = join(home(), '.config', 'systemd', 'user');
|
|
66
68
|
// Lingering user units often start before a login shell imports its PATH.
|
|
@@ -74,30 +76,28 @@ export function makeSystemdBackend(exec = realExec) {
|
|
|
74
76
|
// are picked up by every fresh process, including lingering user services.
|
|
75
77
|
const unitEnv = ['PATH=' + servicePath];
|
|
76
78
|
unitEnv.push('OURS_CONFIG=' + clientConfigPath(process.env));
|
|
79
|
+
for (const key of ['OURS_FLEET_HOME', 'OURS_FLEET_SOCKET_ROOT', 'CODEX_HOME', 'CODEX_PATH']) {
|
|
80
|
+
if (process.env[key] !== undefined)
|
|
81
|
+
unitEnv.push(key + '=' + process.env[key]);
|
|
82
|
+
}
|
|
77
83
|
const environmentLines = unitEnv
|
|
78
84
|
.map(value => `Environment="${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/%/g, '%%')}"`)
|
|
79
85
|
.join('\n');
|
|
80
86
|
mkdirSync(unitDir, { recursive: true });
|
|
81
87
|
writeFileSync(join(unitDir, UNIT_TEMPLATE), `[Unit]
|
|
82
|
-
Description=ours-fleet
|
|
88
|
+
Description=ours-fleet process manager
|
|
83
89
|
After=default.target
|
|
84
90
|
StartLimitIntervalSec=0
|
|
85
91
|
|
|
86
92
|
[Service]
|
|
87
93
|
Type=simple
|
|
88
94
|
${environmentLines}
|
|
89
|
-
|
|
90
|
-
#
|
|
91
|
-
|
|
92
|
-
TimeoutStartSec=270
|
|
93
|
-
ExecStart=${unitArg(process.execPath)} ${unitArg(binPath)} _run %i
|
|
94
|
-
# The RUNNER owns the child-session restart loop, with a counted, backed-off
|
|
95
|
-
# circuit breaker. systemd must only recover the runner PROCESS crashing —
|
|
96
|
-
# Restart=always here would resume the uncounted two-second relaunch loop, and
|
|
97
|
-
# would also restart a runner that is deliberately holding a failing agent down.
|
|
95
|
+
ExecStart=${unitArg(process.execPath)} ${unitArg(binPath)} _run-fleet
|
|
96
|
+
# Fleet owns agent startup and restart; the OS recovers only the Fleet parent.
|
|
97
|
+
KillMode=control-group
|
|
98
98
|
Restart=on-failure
|
|
99
99
|
RestartSec=5
|
|
100
|
-
TimeoutStopSec=
|
|
100
|
+
TimeoutStopSec=25
|
|
101
101
|
|
|
102
102
|
[Install]
|
|
103
103
|
WantedBy=default.target
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Read-only ownership fence: absence/mismatch never permits a new identity or launch. */
|
|
2
|
+
export declare function taskSupervisorMayRun(name: string): boolean;
|
|
3
|
+
/** Resolve the externally recorded CID/action for explicit terminal retirement.
|
|
4
|
+
* Run only after the same launch passed taskSupervisorMayRun's ownership fence.
|
|
5
|
+
* An unpublished failed launch stays for its seat's retry/retirement saga. */
|
|
6
|
+
export declare function taskMemberRetirementProof(name: string): {
|
|
7
|
+
cid: string;
|
|
8
|
+
action: string;
|
|
9
|
+
};
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { readFileSync, lstatSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { parse } from 'yaml';
|
|
4
|
+
import { agentDir, stateRoot } from './paths.js';
|
|
5
|
+
import { readTempSupervisor } from './temp-lifecycle.js';
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { readProvenance } from './creation.js';
|
|
8
|
+
import { getTask } from './rooms-tasks/task-state.js';
|
|
9
|
+
import { getRoomRecord } from './rooms-tasks/room-state.js';
|
|
10
|
+
import { assertWorkspacePresent } from './rooms-tasks/workspace.js';
|
|
11
|
+
import { privateRuntimeRoot } from './agent-ours/service.js';
|
|
12
|
+
import { binderKey, RuntimeJournal } from './agent-ours/state.js';
|
|
13
|
+
import { assertMemberNotPermanent } from './rooms-tasks/member-ownership.js';
|
|
14
|
+
import { readClientProfile } from './client-profile.js';
|
|
15
|
+
import { firstManagedSessionMayStart } from './managed-recovery.js';
|
|
16
|
+
function readProof(path) {
|
|
17
|
+
const stat = lstatSync(path);
|
|
18
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 256 * 1024)
|
|
19
|
+
throw Error('TASK_SUPERVISOR_UNSAFE_PROOF');
|
|
20
|
+
return readFileSync(path, 'utf8');
|
|
21
|
+
}
|
|
22
|
+
/** An admitted task member must resume its retained instance, never allocate a successor. */
|
|
23
|
+
function assertRetainedRuntime(role, action, cid, launch, session) {
|
|
24
|
+
const profile = readClientProfile({ ...process.env, ...role.env });
|
|
25
|
+
if (!profile)
|
|
26
|
+
throw Error('TASK_RUNTIME_PROFILE_MISSING');
|
|
27
|
+
const dir = join(privateRuntimeRoot(), binderKey(profile.expectedInstanceId, role.identity));
|
|
28
|
+
const instance = JSON.parse(readProof(join(dir, 'instance.json')));
|
|
29
|
+
const owner = JSON.parse(readProof(join(dir, 'owner.json')));
|
|
30
|
+
const pin = JSON.parse(readProof(join(dir, 'identity-pin.json')));
|
|
31
|
+
readProof(join(dir, 'state.json'));
|
|
32
|
+
const state = new RuntimeJournal(dir).read();
|
|
33
|
+
const creation = JSON.parse(readProof(join(privateRuntimeRoot(), 'launches', binderKey('temporary', role.name) + '.json')));
|
|
34
|
+
const sessionDir = agentDir(role.name, true);
|
|
35
|
+
let retainedSession = '';
|
|
36
|
+
try {
|
|
37
|
+
retainedSession = readProof(join(sessionDir, role.session === 'codex-app-server' ? '.session-id' : '.acp-session-id')).trim();
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
if (error.code !== 'ENOENT' || session || !firstManagedSessionMayStart(sessionDir))
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
readProof(join(sessionDir, '.booted')); // presence is the runner's resume marker, including legacy empty files
|
|
44
|
+
if (creation.role !== role.name || creation.identity !== role.identity || creation.action !== action
|
|
45
|
+
|| !state || instance.role !== role.name || instance.temporary !== true || !instance.instance
|
|
46
|
+
|| owner.instance !== instance.instance || typeof owner.token !== 'string' || !owner.token
|
|
47
|
+
|| pin.daemon !== profile.expectedInstanceId || pin.name !== role.identity || pin.cid !== cid
|
|
48
|
+
|| state.daemon !== profile.expectedInstanceId || state.name !== role.identity || state.cid !== cid
|
|
49
|
+
|| state.instance !== instance.instance || state.lifetime !== 'temporary' || state.action !== action
|
|
50
|
+
|| ['QUIESCING', 'TERMINAL_INTENT', 'RELEASED', 'CLEANUP_PENDING', 'FAILED'].includes(state.phase)
|
|
51
|
+
|| (!retainedSession && !firstManagedSessionMayStart(sessionDir)) || (launch && launch !== instance.instance) || (session && session !== retainedSession))
|
|
52
|
+
throw Error('TASK_RETAINED_RUNTIME_MISMATCH');
|
|
53
|
+
}
|
|
54
|
+
/** Read-only ownership fence: absence/mismatch never permits a new identity or launch. */
|
|
55
|
+
export function taskSupervisorMayRun(name) {
|
|
56
|
+
assertMemberNotPermanent(name);
|
|
57
|
+
const dir = agentDir(name, true);
|
|
58
|
+
readProof(join(dir, '.temp-supervisor.json'));
|
|
59
|
+
readProof(join(dir, 'creation.json'));
|
|
60
|
+
const supervisor = readTempSupervisor(dir), owner = supervisor?.taskOwner;
|
|
61
|
+
if (!owner || supervisor.role !== name || !owner.taskId || !owner.creationActionId)
|
|
62
|
+
throw Error('TASK_SUPERVISOR_OWNER_MISSING');
|
|
63
|
+
const role = parse(readProof(join(dir, 'role.yaml')));
|
|
64
|
+
const startup = role.roomMemberStartup, provenance = readProvenance(dir);
|
|
65
|
+
const task = getTask(owner.taskId);
|
|
66
|
+
if (role.name !== name || role.identity !== name || provenance?.role !== name
|
|
67
|
+
|| provenance.creationActionId !== owner.creationActionId)
|
|
68
|
+
throw Error('TASK_SUPERVISOR_OWNERSHIP_MISMATCH');
|
|
69
|
+
if (!task.deletion && !task.terminal_intent && ['provisioning', 'active', 'review'].includes(task.state) && task.workspace) {
|
|
70
|
+
if (role.cwd !== task.workspace.path)
|
|
71
|
+
throw Error('TASK_WORKSPACE_MISMATCH');
|
|
72
|
+
assertWorkspacePresent(task.workspace, 'task', task.task_id);
|
|
73
|
+
}
|
|
74
|
+
if (owner.layout) {
|
|
75
|
+
const { runId, participant } = owner.layout;
|
|
76
|
+
const expectedName = `layout-${createHash('sha256').update(runId + ':' + participant).digest('hex').slice(0, 16)}`;
|
|
77
|
+
if (!task.layout || task.layout.run_id !== runId || runId !== `task-${owner.taskId}`
|
|
78
|
+
|| !/^[A-Za-z0-9_-]+$/.test(runId) || expectedName !== name
|
|
79
|
+
|| owner.creationActionId !== `${runId}:${participant}`)
|
|
80
|
+
throw Error('TASK_LAYOUT_OWNER_MISMATCH');
|
|
81
|
+
const state = JSON.parse(readProof(join(stateRoot(), 'layouts', `${runId}.json`)));
|
|
82
|
+
const member = state.participants[participant];
|
|
83
|
+
if (!member?.owned || member.instance?.remote || (member.instance?.agent && member.instance.agent !== name)
|
|
84
|
+
|| (member.instance && member.instance.temporary !== true))
|
|
85
|
+
throw Error('TASK_LAYOUT_PARTICIPANT_MISMATCH');
|
|
86
|
+
if (task.deletion || task.terminal_intent || state.closed || state.closing || member.retired
|
|
87
|
+
|| !['provisioning', 'active', 'review'].includes(task.state))
|
|
88
|
+
return false;
|
|
89
|
+
if (member.instance)
|
|
90
|
+
assertRetainedRuntime(role, owner.creationActionId, member.instance.cid, member.instance.launch, member.instance.session);
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
if (!owner.roomId || !owner.roomIdentityCid)
|
|
94
|
+
throw Error('TASK_SUPERVISOR_OWNER_MISSING');
|
|
95
|
+
const room = getRoomRecord(owner.roomId);
|
|
96
|
+
const seat = room?.member_seats.find(member => member.role_name === name);
|
|
97
|
+
if (!task || !room || room.task_id !== task.task_id || room.room_identity_cid !== owner.roomIdentityCid
|
|
98
|
+
|| (task.room_id !== undefined && task.room_id !== owner.roomId)
|
|
99
|
+
|| (task.room_identity_cid !== undefined && task.room_identity_cid !== owner.roomIdentityCid)
|
|
100
|
+
|| role.name !== name || role.identity !== name || startup?.task_id !== owner.taskId
|
|
101
|
+
|| startup.room_id !== owner.roomId || startup.room_identity_cid !== owner.roomIdentityCid
|
|
102
|
+
|| startup.identity_name !== name || !seat || seat.invite_id !== startup.invite_id
|
|
103
|
+
|| seat.cowork_role !== startup.role || provenance?.role !== name
|
|
104
|
+
|| provenance.creationActionId !== owner.creationActionId
|
|
105
|
+
|| seat.launch?.action_id !== owner.creationActionId
|
|
106
|
+
|| (seat.launch.launch_id !== undefined && seat.launch.launch_id !== supervisor.launchId))
|
|
107
|
+
throw Error('TASK_SUPERVISOR_OWNERSHIP_MISMATCH');
|
|
108
|
+
if (task.deletion || task.terminal_intent || room.close || seat.retirement
|
|
109
|
+
|| !['provisioning', 'active', 'review'].includes(task.state)
|
|
110
|
+
|| !['provisioning', 'active'].includes(room.state))
|
|
111
|
+
return false;
|
|
112
|
+
if (task.workspace && (role.cwd !== task.workspace.path
|
|
113
|
+
|| JSON.stringify(startup.workspace) !== JSON.stringify(task.workspace)
|
|
114
|
+
|| JSON.stringify(room.workspace) !== JSON.stringify(task.workspace)))
|
|
115
|
+
throw Error('TASK_WORKSPACE_MISMATCH');
|
|
116
|
+
if (seat.identity_cid)
|
|
117
|
+
assertRetainedRuntime(role, owner.creationActionId, seat.identity_cid);
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
/** Resolve the externally recorded CID/action for explicit terminal retirement.
|
|
121
|
+
* Run only after the same launch passed taskSupervisorMayRun's ownership fence.
|
|
122
|
+
* An unpublished failed launch stays for its seat's retry/retirement saga. */
|
|
123
|
+
export function taskMemberRetirementProof(name) {
|
|
124
|
+
const dir = agentDir(name, true), metadata = readTempSupervisor(dir), owner = metadata?.taskOwner;
|
|
125
|
+
if (!owner || metadata.role !== name)
|
|
126
|
+
throw Error('TASK_SUPERVISOR_OWNER_MISSING');
|
|
127
|
+
let cid;
|
|
128
|
+
if (owner.layout) {
|
|
129
|
+
const state = JSON.parse(readProof(join(stateRoot(), 'layouts', `${owner.layout.runId}.json`)));
|
|
130
|
+
const participant = state.participants[owner.layout.participant];
|
|
131
|
+
if (!participant?.owned || participant.instance?.agent !== name || participant.instance?.remote
|
|
132
|
+
|| participant.instance?.temporary !== true)
|
|
133
|
+
throw Error('TASK_LAYOUT_PARTICIPANT_MISMATCH');
|
|
134
|
+
cid = participant.instance.cid;
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
const seat = owner.roomId && getRoomRecord(owner.roomId)?.member_seats.find(member => member.role_name === name);
|
|
138
|
+
if (!seat || seat.launch?.action_id !== owner.creationActionId || seat.launch.launch_id !== metadata.launchId)
|
|
139
|
+
throw Error('TASK_SUPERVISOR_OWNERSHIP_MISMATCH');
|
|
140
|
+
cid = seat.identity_cid;
|
|
141
|
+
}
|
|
142
|
+
if (!cid)
|
|
143
|
+
throw Error('TASK_MEMBER_RETIREMENT_CID_MISSING');
|
|
144
|
+
return { cid, action: owner.creationActionId };
|
|
145
|
+
}
|
package/dist/temp-lifecycle.d.ts
CHANGED
|
@@ -4,8 +4,18 @@ export declare const TEMP_TERMINATION_FILE = "termination.jsonl";
|
|
|
4
4
|
export declare const TEMP_STOP_REQUEST_FILE = ".temp-stop-request.json";
|
|
5
5
|
export declare const TEMP_RECLAIM_BATCH = 32;
|
|
6
6
|
export declare const TEMP_LAUNCH_GRACE_MS = 60000;
|
|
7
|
-
export type TempSupervisorKind = 'systemd-transient' | 'launchd-transient' | 'detached';
|
|
7
|
+
export type TempSupervisorKind = 'fleet-managed' | 'systemd-transient' | 'launchd-transient' | 'detached';
|
|
8
8
|
export type TempTerminationReason = 'idle-timeout' | 'identity-closed' | 'session-ended' | 'operator-stop' | 'supervisor-signal' | 'supervisor-recycle' | 'startup-failure' | 'stale-supervisor';
|
|
9
|
+
export interface TaskSupervisorOwner {
|
|
10
|
+
taskId: string;
|
|
11
|
+
roomId?: string;
|
|
12
|
+
roomIdentityCid?: string;
|
|
13
|
+
layout?: {
|
|
14
|
+
runId: string;
|
|
15
|
+
participant: string;
|
|
16
|
+
};
|
|
17
|
+
creationActionId: string;
|
|
18
|
+
}
|
|
9
19
|
export interface TempSupervisorRecord {
|
|
10
20
|
version: 1;
|
|
11
21
|
role: string;
|
|
@@ -16,6 +26,7 @@ export interface TempSupervisorRecord {
|
|
|
16
26
|
target?: string;
|
|
17
27
|
pid?: number;
|
|
18
28
|
binPath?: string;
|
|
29
|
+
taskOwner?: TaskSupervisorOwner;
|
|
19
30
|
}
|
|
20
31
|
export interface TempTerminationRecord {
|
|
21
32
|
version: 1;
|
|
@@ -29,13 +40,12 @@ export interface TempTerminationRecord {
|
|
|
29
40
|
export type SupervisorLauncher = (binPath: string, args: string[], dir: string) => void | Promise<void>;
|
|
30
41
|
export declare const tempSystemdUnit: (name: string) => string;
|
|
31
42
|
export declare const tempLaunchdLabel: (name: string) => string;
|
|
32
|
-
export declare function prepareTempSupervisor(dir: string, role: string): TempSupervisorRecord;
|
|
43
|
+
export declare function prepareTempSupervisor(dir: string, role: string, taskOwner?: TaskSupervisorOwner): TempSupervisorRecord;
|
|
33
44
|
export declare function readTempSupervisor(dir: string): TempSupervisorRecord | undefined;
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
*/
|
|
45
|
+
/** Broad scans retain unsupported entries instead of blocking other members. */
|
|
46
|
+
export declare function readTempSupervisorForScan(dir: string): TempSupervisorRecord | undefined;
|
|
47
|
+
export declare function updateTempSupervisor(dir: string, update: Partial<Omit<TempSupervisorRecord, 'version' | 'role' | 'launchId' | 'createdAt'>>): Promise<TempSupervisorRecord | undefined>;
|
|
48
|
+
/** Register managed members under Fleet; explicit manual mode uses a detached child. */
|
|
39
49
|
export declare function makeTempSupervisorLauncher(options?: {
|
|
40
50
|
exec?: Exec;
|
|
41
51
|
platform?: NodeJS.Platform;
|