@aiwg/cli 2026.10.10 → 2026.10.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/model-capabilities.v1.json +78 -14
- package/dist/src/cli/handlers/sessions.js +198 -15
- package/dist/src/cli/handlers/use.js +12 -4
- package/dist/src/config/worker-context.js +337 -0
- package/dist/src/mcp/tools/worker-invoke.mjs +108 -10
- package/dist/src/models/model-capabilities.v1.json +78 -14
- package/dist/src/models/provider-policy.js +7 -0
- package/dist/src/policy/authorization.js +29 -1
- package/dist/src/process/aiwg-probe-command.d.mts +15 -0
- package/dist/src/process/aiwg-probe-command.mjs +37 -0
- package/dist/src/sessions/adapters/grokbot.js +7 -0
- package/dist/src/sessions/batch-import.js +34 -6
- package/dist/src/sessions/contracts.js +19 -0
- package/dist/src/sessions/coverage-preflight.js +7 -1
- package/dist/src/sessions/fortemi-export-mapping.js +33 -3
- package/dist/src/sessions/host-catalog.js +157 -0
- package/dist/src/sessions/image-blobs.js +141 -0
- package/dist/src/sessions/importer.js +23 -3
- package/dist/src/sessions/index.js +2 -0
- package/dist/src/sessions/refresh.js +25 -2
- package/dist/src/sessions/repository.js +164 -22
- package/dist/src/sessions/workspace-discovery.js +5 -2
- package/package.json +7 -1
- package/tools/ralph-external/lib/muse-adapter.mjs +10 -0
- package/tools/ralph-external/lib/worker-context.mjs +237 -0
- package/tools/ralph-external/lib/worker-invocation-ancestry.mjs +132 -0
- package/tools/ralph-external/lib/worker-invocation-processes.mjs +123 -0
- package/tools/ralph-external/lib/worker-isolation-status.mjs +48 -0
- package/tools/ralph-external/lib/worker-isolation.mjs +17 -29
- package/tools/ralph-external/lib/worker-launch-cli.mjs +3 -1
- package/tools/ralph-external/lib/worker-overflow.mjs +48 -13
- package/tools/ralph-external/lib/worker-retry-evidence.mjs +103 -0
- package/tools/ralph-external/lib/worker-termination.mjs +82 -0
- package/tools/ralph-external/session-launcher.mjs +51 -96
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// Classify unreadable same-uid processes as possible escaped worker descendants or unrelated trees.
|
|
2
|
+
import { constants } from 'node:fs';
|
|
3
|
+
|
|
4
|
+
// Ancestry walks stay bounded: a pathological ppid chain can never turn this
|
|
5
|
+
// proof into an unbounded scan. Depth and the caller's time bound both apply.
|
|
6
|
+
export const MAX_WORKER_ANCESTRY_DEPTH = 32;
|
|
7
|
+
|
|
8
|
+
const vanished = error => ['ENOENT', 'ESRCH'].includes(error?.code);
|
|
9
|
+
|
|
10
|
+
function readSmallFile(fs, filename, maxBytes = 4096) {
|
|
11
|
+
let fd;
|
|
12
|
+
try {
|
|
13
|
+
fd = fs.openSync(filename, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
14
|
+
const buffer = Buffer.alloc(maxBytes + 1);
|
|
15
|
+
const chunks = [];
|
|
16
|
+
let bytes = 0;
|
|
17
|
+
for (;;) {
|
|
18
|
+
const size = fs.readSync(fd, buffer, 0, buffer.length, null);
|
|
19
|
+
if (!size) break;
|
|
20
|
+
chunks.push(buffer.subarray(0, size));
|
|
21
|
+
bytes += size;
|
|
22
|
+
if (bytes > maxBytes) return { tooLarge: true };
|
|
23
|
+
}
|
|
24
|
+
return { text: Buffer.concat(chunks).toString('utf8') };
|
|
25
|
+
} catch (error) { return { error }; }
|
|
26
|
+
finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Parse the link fields of /proc/<pid>/stat: ppid, process group, session and
|
|
30
|
+
// start time. The comm field may contain spaces and parentheses, so field 3
|
|
31
|
+
// begins after its LAST ')'. Returns null when the text is malformed.
|
|
32
|
+
export function parseProcStatLink(text) {
|
|
33
|
+
if (typeof text !== 'string') return null;
|
|
34
|
+
const end = text.lastIndexOf(')');
|
|
35
|
+
if (end < 0) return null;
|
|
36
|
+
const fields = text.slice(end + 1).trim().split(/\s+/);
|
|
37
|
+
if (fields.length < 20) return null;
|
|
38
|
+
const [ppid, pgrp, session] = [fields[1], fields[2], fields[3]].map(Number);
|
|
39
|
+
const start = fields[19];
|
|
40
|
+
if (![ppid, pgrp, session].every(Number.isSafeInteger) || typeof start !== 'string' || !/^\d+$/.test(start)) return null;
|
|
41
|
+
return { ppid, pgrp, session, start };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Read /proc/<pid>/cgroup verbatim (cgroup membership stays readable for
|
|
45
|
+
// non-dumpable processes). Normalized like sampleProcessCgroup: the bare path
|
|
46
|
+
// for the unified hierarchy, raw content otherwise. Null when unreadable.
|
|
47
|
+
export function readProcCgroup(fs, path) {
|
|
48
|
+
const read = readSmallFile(fs, `${path}/cgroup`);
|
|
49
|
+
if (read.error || read.tooLarge) return null;
|
|
50
|
+
const raw = String(read.text ?? '').trim();
|
|
51
|
+
if (!raw) return null;
|
|
52
|
+
const unified = raw.split('\n').find(line => line.startsWith('0::'));
|
|
53
|
+
const group = unified ? unified.slice(3) : raw;
|
|
54
|
+
return group || null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const isTicks = value => typeof value === 'string' && /^\d+$/.test(value);
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Decide whether an unreadable same-uid process could be an escaped descendant
|
|
61
|
+
* of the recorded worker leader, using only evidence that stays readable for
|
|
62
|
+
* non-dumpable (hardened, e.g. Codex) processes: /proc/<pid>/stat (ppid,
|
|
63
|
+
* process group, session, start time) and /proc/<pid>/cgroup.
|
|
64
|
+
*
|
|
65
|
+
* Returns 'predates' (started strictly before the worker: cannot descend from
|
|
66
|
+
* it), 'unrelated' (its readable ancestry reaches another live process that
|
|
67
|
+
* is not ours, e.g. a different conductor's worker), 'vanished' (it
|
|
68
|
+
* disappeared mid-scan), or 'possible-descendant' (fail closed: ancestry
|
|
69
|
+
* reaches the worker leader or the worker's original session/process group,
|
|
70
|
+
* shares the worker's private cgroup, is reparented to init/a subreaper with
|
|
71
|
+
* no other link, or any link in the chain is unreadable).
|
|
72
|
+
*/
|
|
73
|
+
export function classifyUnreadableWorkerProcess({ fs, procRoot, pid, uid, workerPid,
|
|
74
|
+
workerPgid, workerSession, workerStart, workerCgroup, selfCgroup,
|
|
75
|
+
now = () => performance.now(), deadline = now() + 100, maxDepth = MAX_WORKER_ANCESTRY_DEPTH } = {}) {
|
|
76
|
+
const path = `${procRoot}/${pid}`;
|
|
77
|
+
const first = readSmallFile(fs, `${path}/stat`);
|
|
78
|
+
if (first.error) return vanished(first.error) ? 'vanished' : 'possible-descendant';
|
|
79
|
+
const link = !first.tooLarge && parseProcStatLink(first.text);
|
|
80
|
+
if (!link) return 'possible-descendant';
|
|
81
|
+
if (!isTicks(workerStart)) return 'possible-descendant';
|
|
82
|
+
if (BigInt(link.start) < BigInt(workerStart)) return 'predates';
|
|
83
|
+
// A process still in the worker's original session or process group is
|
|
84
|
+
// treated as a possible descendant even when its ppid chain is broken
|
|
85
|
+
// (an orphan that never called setsid keeps both).
|
|
86
|
+
if ((Number.isSafeInteger(workerPgid) && workerPgid > 0 && link.pgrp === workerPgid)
|
|
87
|
+
|| (Number.isSafeInteger(workerSession) && workerSession > 0 && link.session === workerSession)) {
|
|
88
|
+
return 'possible-descendant';
|
|
89
|
+
}
|
|
90
|
+
// A shared cgroup links the process to the worker only when the worker's
|
|
91
|
+
// own cgroup is private: a degraded worker shares its session cgroup with
|
|
92
|
+
// every sibling, so that match proves nothing and ancestry decides.
|
|
93
|
+
if (workerCgroup) {
|
|
94
|
+
const candidate = readProcCgroup(fs, path);
|
|
95
|
+
if (candidate !== null && candidate === workerCgroup && workerCgroup !== selfCgroup) return 'possible-descendant';
|
|
96
|
+
}
|
|
97
|
+
let current = link.ppid;
|
|
98
|
+
for (let depth = 0; ; depth++) {
|
|
99
|
+
if (now() >= deadline) return 'possible-descendant';
|
|
100
|
+
if (depth >= maxDepth) return 'possible-descendant';
|
|
101
|
+
if (current === workerPid) {
|
|
102
|
+
// A pid number alone is not proof: verify the generation, since the
|
|
103
|
+
// number may have been reused after the leader exited.
|
|
104
|
+
const leader = readSmallFile(fs, `${procRoot}/${current}/stat`);
|
|
105
|
+
if (leader.error || leader.tooLarge) return 'possible-descendant';
|
|
106
|
+
const leaderLink = parseProcStatLink(leader.text);
|
|
107
|
+
if (!leaderLink) return 'possible-descendant';
|
|
108
|
+
if (leaderLink.start === workerStart) return 'possible-descendant';
|
|
109
|
+
current = leaderLink.ppid;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
// Reparented to init (or the kernel): with no other link this is the
|
|
113
|
+
// honest unknown case, never proof of absence.
|
|
114
|
+
if (!Number.isSafeInteger(current) || current <= 1) return 'possible-descendant';
|
|
115
|
+
let owner;
|
|
116
|
+
try { owner = fs.statSync(`${procRoot}/${current}`).uid; }
|
|
117
|
+
catch { return 'possible-descendant'; }
|
|
118
|
+
// Another identity's tree (including container shims and managers owned
|
|
119
|
+
// by a different uid) can never be our worker's descendant. A setuid
|
|
120
|
+
// escape across uids is out of scope for this same-uid scan.
|
|
121
|
+
if (owner !== uid) return 'unrelated';
|
|
122
|
+
const next = readSmallFile(fs, `${procRoot}/${current}/stat`);
|
|
123
|
+
if (next.error || next.tooLarge) return 'possible-descendant';
|
|
124
|
+
const nextLink = parseProcStatLink(next.text);
|
|
125
|
+
if (!nextLink) return 'possible-descendant';
|
|
126
|
+
// This ancestor predates the worker, so it cannot descend from it — and
|
|
127
|
+
// neither can anything reached through it (a sibling subtree, another
|
|
128
|
+
// conductor's parallel worker, a session manager).
|
|
129
|
+
if (BigInt(nextLink.start) < BigInt(workerStart)) return 'unrelated';
|
|
130
|
+
current = nextLink.ppid;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Bind worker environments to an invocation and find surviving marked processes.
|
|
2
|
+
import { opendirSync, statSync, openSync, readSync, closeSync, constants } from 'node:fs';
|
|
3
|
+
import { StringDecoder } from 'node:string_decoder';
|
|
4
|
+
import { classifyUnreadableWorkerProcess, readProcCgroup } from './worker-invocation-ancestry.mjs';
|
|
5
|
+
|
|
6
|
+
export const WORKER_INVOCATION_MARKER = 'AIWG_WORKER_INVOCATION_ID';
|
|
7
|
+
|
|
8
|
+
function startedBeforeWorker(fs, path, startTicks) {
|
|
9
|
+
if (typeof startTicks !== 'string' || !/^\d+$/.test(startTicks)) return false;
|
|
10
|
+
let fd;
|
|
11
|
+
try {
|
|
12
|
+
fd = fs.openSync(`${path}/stat`, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
13
|
+
const buffer = Buffer.alloc(4096);
|
|
14
|
+
const size = fs.readSync(fd, buffer, 0, buffer.length, null);
|
|
15
|
+
if (size === buffer.length) return false;
|
|
16
|
+
const stat = buffer.toString('utf8', 0, size);
|
|
17
|
+
// comm may contain spaces and parentheses; field 3 begins after its LAST ')'.
|
|
18
|
+
const end = stat.lastIndexOf(')');
|
|
19
|
+
const ticks = end < 0 ? undefined : stat.slice(end + 1).trim().split(/\s+/)[19];
|
|
20
|
+
return /^\d+$/.test(ticks ?? '') && BigInt(ticks) < BigInt(startTicks);
|
|
21
|
+
} catch { return false; }
|
|
22
|
+
finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function buildWorkerInvocationEnvironment(envNames, invocationId, inherited = process.env) {
|
|
26
|
+
if (typeof invocationId !== 'string' || !invocationId || invocationId.includes('\0')) {
|
|
27
|
+
throw new Error('Worker environment requires a non-empty NUL-free invocation ID');
|
|
28
|
+
}
|
|
29
|
+
const names = [...new Set([...envNames, WORKER_INVOCATION_MARKER])];
|
|
30
|
+
const env = Object.fromEntries(envNames.filter(name => inherited[name] !== undefined)
|
|
31
|
+
.map(name => [name, inherited[name]]));
|
|
32
|
+
return { env: { ...env, [WORKER_INVOCATION_MARKER]: invocationId }, envNames: names };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function scanWorkerInvocationProcesses({ invocationId, startTicks, uid = process.getuid?.(),
|
|
36
|
+
platform = process.platform, procRoot = '/proc', now = () => performance.now(),
|
|
37
|
+
deadline = now() + 100, maxEntries = 4096, maxEnvironmentBytes = 1048576,
|
|
38
|
+
workerPid, workerPgid, workerSession, workerCgroup, selfCgroup, maxAncestryDepth,
|
|
39
|
+
fs = { opendirSync, statSync, openSync, readSync, closeSync } } = {}) {
|
|
40
|
+
const unknown = detail => ({ confirmed: false,
|
|
41
|
+
missing: `escaped-descendant evidence: ${detail}` });
|
|
42
|
+
if (platform !== 'linux') return unknown('same-uid invocation-marker scan requires Linux /proc');
|
|
43
|
+
if (!Number.isInteger(uid) || typeof invocationId !== 'string' || !invocationId || invocationId.includes('\0')) {
|
|
44
|
+
return unknown('launcher-bound invocation marker and uid');
|
|
45
|
+
}
|
|
46
|
+
// Ancestry discrimination needs the recorded worker leader. Without it the
|
|
47
|
+
// scan keeps the legacy fail-closed behaviour for every unreadable
|
|
48
|
+
// post-start process.
|
|
49
|
+
const workerKnown = Number.isSafeInteger(workerPid) && workerPid > 1
|
|
50
|
+
&& typeof startTicks === 'string' && /^\d+$/.test(startTicks);
|
|
51
|
+
let resolvedWorkerCgroup = workerCgroup;
|
|
52
|
+
if (resolvedWorkerCgroup === undefined && workerKnown) {
|
|
53
|
+
resolvedWorkerCgroup = readProcCgroup(fs, `${procRoot}/${workerPid}`);
|
|
54
|
+
}
|
|
55
|
+
let resolvedSelfCgroup = selfCgroup;
|
|
56
|
+
if (resolvedSelfCgroup === undefined) resolvedSelfCgroup = readProcCgroup(fs, `${procRoot}/self`);
|
|
57
|
+
const marker = `${WORKER_INVOCATION_MARKER}=${invocationId}`;
|
|
58
|
+
const present = () => ({ ...unknown('same-uid process still carries invocation marker (possible escaped descendant)'), markerPresent: true });
|
|
59
|
+
const gone = error => ['ENOENT', 'ESRCH'].includes(error.code);
|
|
60
|
+
let directory, uncertain = null;
|
|
61
|
+
try {
|
|
62
|
+
directory = fs.opendirSync(procRoot);
|
|
63
|
+
// Stream entries and cap per-process reads; neither a huge PID set nor
|
|
64
|
+
// a large environment can turn this proof into an unbounded allocation.
|
|
65
|
+
const buffer = Buffer.alloc(Math.min(65536, maxEnvironmentBytes + 1));
|
|
66
|
+
for (let count = 0; ; count++) {
|
|
67
|
+
if (now() >= deadline) return unknown('same-uid invocation-marker scan time bound reached');
|
|
68
|
+
const entry = directory.readSync();
|
|
69
|
+
if (!entry) return uncertain ?? { confirmed: true, missing: null };
|
|
70
|
+
if (count >= maxEntries) return unknown('same-uid invocation-marker scan count bound reached');
|
|
71
|
+
if (!/^\d+$/.test(entry.name)) continue;
|
|
72
|
+
const path = `${procRoot}/${entry.name}`;
|
|
73
|
+
let fd, sameUid = false;
|
|
74
|
+
try {
|
|
75
|
+
if (fs.statSync(path).uid !== uid) continue;
|
|
76
|
+
sameUid = true;
|
|
77
|
+
fd = fs.openSync(`${path}/environ`, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
78
|
+
let bytes = 0, tail = '';
|
|
79
|
+
const decoder = new StringDecoder('utf8');
|
|
80
|
+
for (;;) {
|
|
81
|
+
if (now() >= deadline) return unknown('same-uid invocation-marker scan time bound reached');
|
|
82
|
+
const size = fs.readSync(fd, buffer, 0, Math.min(buffer.length, maxEnvironmentBytes + 1 - bytes), null);
|
|
83
|
+
if (!size) break;
|
|
84
|
+
bytes += size;
|
|
85
|
+
if (bytes > maxEnvironmentBytes) {
|
|
86
|
+
uncertain = unknown('same-uid process environment exceeds scan byte bound'); break;
|
|
87
|
+
}
|
|
88
|
+
const fields = (tail + decoder.write(buffer.subarray(0, size))).split('\0');
|
|
89
|
+
tail = fields.pop();
|
|
90
|
+
if (fields.includes(marker)) return present();
|
|
91
|
+
}
|
|
92
|
+
if (tail + decoder.end() === marker) return present();
|
|
93
|
+
} catch (error) {
|
|
94
|
+
// An escaped descendant cannot predate its worker leader. Ownership
|
|
95
|
+
// failures and unreadable/missing start times still cannot prove absence.
|
|
96
|
+
if (gone(error)) continue;
|
|
97
|
+
if (!sameUid) {
|
|
98
|
+
uncertain = unknown('unreadable same-uid process environment or ownership');
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (!workerKnown) {
|
|
102
|
+
if (!startedBeforeWorker(fs, path, startTicks)) {
|
|
103
|
+
uncertain = unknown('unreadable same-uid process environment or ownership');
|
|
104
|
+
}
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
// Hardened (non-dumpable, e.g. Codex) processes hide their environment
|
|
108
|
+
// but not their stat/cgroup links: a parallel worker's tree reaches a
|
|
109
|
+
// live process that predates this worker and is provably not ours,
|
|
110
|
+
// while a genuine escapee still links to the leader, its session or
|
|
111
|
+
// group, or its private cgroup.
|
|
112
|
+
const verdict = classifyUnreadableWorkerProcess({ fs, procRoot, pid: Number(entry.name), uid,
|
|
113
|
+
workerPid, workerPgid, workerSession, workerStart: startTicks,
|
|
114
|
+
workerCgroup: resolvedWorkerCgroup, selfCgroup: resolvedSelfCgroup,
|
|
115
|
+
now, deadline, ...(maxAncestryDepth === undefined ? {} : { maxDepth: maxAncestryDepth }) });
|
|
116
|
+
if (verdict === 'possible-descendant') {
|
|
117
|
+
uncertain = unknown('unreadable same-uid process links to the worker (ancestry, session, group or cgroup)');
|
|
118
|
+
}
|
|
119
|
+
} finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
120
|
+
}
|
|
121
|
+
} catch { return unknown('readable Linux /proc process inventory'); }
|
|
122
|
+
finally { directory?.closeSync(); }
|
|
123
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Read retained unit exit state and its kernel-backed recursive task count.
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { buildSystemdClientEnv, resolveSystemdClient } from './systemd-client.mjs';
|
|
5
|
+
|
|
6
|
+
function emptyCgroup(fields, readCgroup) {
|
|
7
|
+
if (fields.TasksCurrent === '0') return true;
|
|
8
|
+
if (/^[1-9][0-9]*$/.test(fields.TasksCurrent ?? '')) return false;
|
|
9
|
+
if (!['inactive', 'failed'].includes(fields.ActiveState)) return false;
|
|
10
|
+
// A loaded inactive unit can have released its cgroup entirely.
|
|
11
|
+
if (fields.LoadState === 'loaded' && fields.ControlGroup === '') return true;
|
|
12
|
+
const group = fields.ControlGroup;
|
|
13
|
+
if (!group?.startsWith('/') || group === '/' || group.split('/').includes('..')) return false;
|
|
14
|
+
try { return /^populated 0$/m.test(readCgroup(`/sys/fs/cgroup${group}/cgroup.events`, 'utf8')); }
|
|
15
|
+
catch (error) { return error.code === 'ENOENT'; }
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Read retained service state after launch/exit without reaping or mutation.
|
|
20
|
+
* `memoryMax` is bytes. Query failures throw rather than reporting success.
|
|
21
|
+
*/
|
|
22
|
+
export function getWorkerIsolationStatus(unit, { spawn = spawnSync, timeoutMs = 10000, readCgroup = readFileSync } = {}) {
|
|
23
|
+
if (typeof unit !== 'string' || !/^[A-Za-z0-9:_.@-]+$/.test(unit)) {
|
|
24
|
+
throw new Error('Worker isolation requires a unit-safe unit name');
|
|
25
|
+
}
|
|
26
|
+
const shown = spawn(resolveSystemdClient('systemctl'), ['--user', 'show', unit, '-p', 'ActiveState',
|
|
27
|
+
'-p', 'Result', '-p', 'ExecMainStatus', '-p', 'ExecMainCode', '-p', 'MainPID', '-p', 'MemoryMax', '-p', 'TasksCurrent', '-p', 'ControlGroup', '-p', 'LoadState'], { encoding: 'utf8', timeout: timeoutMs, env: buildSystemdClientEnv({ envNames: [] }) });
|
|
28
|
+
if (shown.error || shown.status !== 0) {
|
|
29
|
+
throw new Error(`worker status unavailable: ${shown.error?.message ?? String(shown.stderr ?? '').trim()}`);
|
|
30
|
+
}
|
|
31
|
+
const fields = Object.fromEntries(String(shown.stdout ?? '').trim().split('\n').map(line => {
|
|
32
|
+
const index = line.indexOf('=');
|
|
33
|
+
return [line.slice(0, index), line.slice(index + 1)];
|
|
34
|
+
}));
|
|
35
|
+
if (!fields.ActiveState || !fields.Result || !fields.MemoryMax) {
|
|
36
|
+
throw new Error('worker status unavailable: missing systemd properties');
|
|
37
|
+
}
|
|
38
|
+
return {
|
|
39
|
+
active: ['inactive', 'failed'].includes(fields.ActiveState) ? false
|
|
40
|
+
: ['active', 'activating', 'deactivating', 'reloading'].includes(fields.ActiveState) ? true : null,
|
|
41
|
+
...(emptyCgroup(fields, readCgroup) ? { cgroupEmpty: true } : {}),
|
|
42
|
+
result: fields.Result,
|
|
43
|
+
...(fields.ExecMainStatus ? { execMainStatus: Number(fields.ExecMainStatus) } : {}),
|
|
44
|
+
memoryMax: fields.MemoryMax === 'infinity' ? Infinity : Number(fields.MemoryMax),
|
|
45
|
+
...(fields.MainPID ? { mainPid: Number(fields.MainPID) } : {}),
|
|
46
|
+
...(fields.ExecMainCode ? { execMainCode: Number(fields.ExecMainCode) } : {}),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
@@ -556,35 +556,7 @@ export function launchIsolatedWorker({
|
|
|
556
556
|
});
|
|
557
557
|
}
|
|
558
558
|
|
|
559
|
-
|
|
560
|
-
* Read retained service state after launch/exit without reaping or mutation.
|
|
561
|
-
* `memoryMax` is bytes. Query failures throw rather than reporting success.
|
|
562
|
-
*/
|
|
563
|
-
export function getWorkerIsolationStatus(unit, { spawn = spawnSync, timeoutMs = 10000 } = {}) {
|
|
564
|
-
if (typeof unit !== 'string' || !/^[A-Za-z0-9:_.@-]+$/.test(unit)) {
|
|
565
|
-
throw new Error('Worker isolation requires a unit-safe unit name');
|
|
566
|
-
}
|
|
567
|
-
const shown = spawn(resolveSystemdClient('systemctl'), ['--user', 'show', unit, '-p', 'ActiveState',
|
|
568
|
-
'-p', 'Result', '-p', 'ExecMainStatus', '-p', 'ExecMainCode', '-p', 'MainPID', '-p', 'MemoryMax'], { encoding: 'utf8', timeout: timeoutMs, env: buildSystemdClientEnv({ envNames: [] }) });
|
|
569
|
-
if (shown.error || shown.status !== 0) {
|
|
570
|
-
throw new Error(`worker status unavailable: ${shown.error?.message ?? String(shown.stderr ?? '').trim()}`);
|
|
571
|
-
}
|
|
572
|
-
const fields = Object.fromEntries(String(shown.stdout ?? '').trim().split('\n').map(line => {
|
|
573
|
-
const index = line.indexOf('=');
|
|
574
|
-
return [line.slice(0, index), line.slice(index + 1)];
|
|
575
|
-
}));
|
|
576
|
-
if (!fields.ActiveState || !fields.Result || !fields.ExecMainStatus || !fields.MemoryMax) {
|
|
577
|
-
throw new Error('worker status unavailable: missing systemd properties');
|
|
578
|
-
}
|
|
579
|
-
return {
|
|
580
|
-
active: ['active', 'activating', 'deactivating', 'reloading'].includes(fields.ActiveState),
|
|
581
|
-
result: fields.Result,
|
|
582
|
-
execMainStatus: Number(fields.ExecMainStatus),
|
|
583
|
-
memoryMax: fields.MemoryMax === 'infinity' ? Infinity : Number(fields.MemoryMax),
|
|
584
|
-
...(fields.MainPID ? { mainPid: Number(fields.MainPID) } : {}),
|
|
585
|
-
...(fields.ExecMainCode ? { execMainCode: Number(fields.ExecMainCode) } : {}),
|
|
586
|
-
};
|
|
587
|
-
}
|
|
559
|
+
export { getWorkerIsolationStatus } from './worker-isolation-status.mjs';
|
|
588
560
|
|
|
589
561
|
/**
|
|
590
562
|
* Request the unit's cgroup be torn down (`KillMode=control-group` is already
|
|
@@ -608,6 +580,22 @@ export function stopWorkerUnit(unit, { spawn = spawnSync, timeoutMs = 10000 } =
|
|
|
608
580
|
diagnostic: result.error ? result.error.message : String(result.stderr ?? '').trim() || null };
|
|
609
581
|
}
|
|
610
582
|
|
|
583
|
+
/**
|
|
584
|
+
* SIGKILL every process in the unit's cgroup and return without waiting.
|
|
585
|
+
* `stop` sends SIGTERM and then waits for the unit's stop timeout, which a
|
|
586
|
+
* worker that ignores SIGTERM outlasts (#3691).
|
|
587
|
+
*/
|
|
588
|
+
export function killWorkerUnit(unit, { spawn = spawnSync, timeoutMs = 10000, signal = 'SIGKILL' } = {}) {
|
|
589
|
+
if (typeof unit !== 'string' || !/^[A-Za-z0-9:_.@-]+$/.test(unit)) {
|
|
590
|
+
throw new Error('Worker isolation requires a unit-safe unit name');
|
|
591
|
+
}
|
|
592
|
+
if (!/^SIG[A-Z0-9]+$/.test(signal)) throw new Error('Worker isolation requires a named signal');
|
|
593
|
+
const result = spawn(resolveSystemdClient('systemctl'), ['--user', 'kill', '--kill-whom=all', `--signal=${signal}`, unit],
|
|
594
|
+
{ encoding: 'utf8', timeout: timeoutMs, killSignal: 'SIGKILL', env: buildSystemdClientEnv({ envNames: [] }) });
|
|
595
|
+
return { ok: !result.error && result.status === 0,
|
|
596
|
+
diagnostic: result.error ? result.error.message : String(result.stderr ?? '').trim() || null };
|
|
597
|
+
}
|
|
598
|
+
|
|
611
599
|
/**
|
|
612
600
|
* Read the kernel-reported `ControlGroup` systemd placed this unit's
|
|
613
601
|
* processes under. This is the containment authority for `systemd-unit`
|
|
@@ -12,6 +12,7 @@ export function parseWorkerLaunchArgs(args) {
|
|
|
12
12
|
if (seen.has(flag)) throw new Error(`Duplicate worker launch option: ${flag}`);
|
|
13
13
|
seen.add(flag);
|
|
14
14
|
if (flag === '--launch-worker') continue;
|
|
15
|
+
if (flag === '--check-descendants') { result.checkDescendants = true; continue; }
|
|
15
16
|
if (flag === '--allow-degraded-worker') { result.allowDegraded = true; continue; }
|
|
16
17
|
// A run directory declares a retry; its launcher ID is derived before validation.
|
|
17
18
|
const key = { '--invocation': 'invocationPath', '--run-dir': 'runDir',
|
|
@@ -24,11 +25,12 @@ export function parseWorkerLaunchArgs(args) {
|
|
|
24
25
|
else result[key] = value;
|
|
25
26
|
}
|
|
26
27
|
if (!seen.has('--launch-worker') || !result.invocationPath || !result.runDir) {
|
|
27
|
-
throw new Error('Usage: ralph-external --launch-worker --invocation FILE --run-dir DIR [--allow-degraded-worker] [--retry-of-run-dir DIR [--retry-of ID]]');
|
|
28
|
+
throw new Error('Usage: ralph-external --launch-worker --invocation FILE --run-dir DIR [--allow-degraded-worker] [--check-descendants] [--retry-of-run-dir DIR [--retry-of ID]]');
|
|
28
29
|
}
|
|
29
30
|
if (retryOfInvocationId !== undefined && retryOfRunDir === undefined) {
|
|
30
31
|
throw new Error('--retry-of and --retry-of-run-dir must be supplied together when an explicit ID is used');
|
|
31
32
|
}
|
|
33
|
+
if (result.checkDescendants && retryOfRunDir === undefined) throw new Error('--check-descendants requires --retry-of-run-dir');
|
|
32
34
|
if (retryOfRunDir !== undefined) result.retryOf = { runDir: retryOfRunDir, ...(retryOfInvocationId === undefined ? {} : { invocationId: retryOfInvocationId }) };
|
|
33
35
|
return result;
|
|
34
36
|
}
|
|
@@ -1,15 +1,30 @@
|
|
|
1
1
|
/** Stop authority and bounded termination proof for MCP stdout overflow. */
|
|
2
2
|
import { importImpl } from '../../_resolve-impl.mjs';
|
|
3
|
-
import { getWorkerIsolationStatus, stopWorkerUnit } from './worker-isolation.mjs';
|
|
3
|
+
import { getWorkerIsolationStatus, killWorkerUnit, stopWorkerUnit } from './worker-isolation.mjs';
|
|
4
4
|
import { signalOwnedWorker } from '../session-launcher.mjs';
|
|
5
5
|
import { sampleWorkerChildren, sampleWorkerIdentity, sameWorkerIdentity } from '../process-monitor.mjs';
|
|
6
6
|
const { signalChild } = await importImpl(import.meta.url, 'process/signal-child.mjs');
|
|
7
7
|
|
|
8
|
+
// Only identities sampled beneath a verified live supervisor enter this cache.
|
|
9
|
+
const ownedWorkers = new WeakMap();
|
|
10
|
+
function hasBirthIdentity(worker) {
|
|
11
|
+
return Number.isSafeInteger(worker?.pid) && worker.pid > 1 && worker.pid !== process.pid
|
|
12
|
+
&& Number.isSafeInteger(worker.pgid) && worker.pgid > 1
|
|
13
|
+
&& Number.isSafeInteger(worker.uid) && worker.uid === process.getuid?.()
|
|
14
|
+
&& Number.isSafeInteger(worker.sessionId) && worker.sessionId > 0
|
|
15
|
+
&& typeof worker.startTicks === 'string' && /^\d+$/.test(worker.startTicks)
|
|
16
|
+
&& ['bootId', 'host', 'executable'].every(key => typeof worker[key] === 'string' && worker[key].length > 0);
|
|
17
|
+
}
|
|
18
|
+
|
|
8
19
|
export async function stopOverflowWorker(launch, {
|
|
9
20
|
deadlineMs = 2000, pollIntervalMs = 25,
|
|
10
|
-
|
|
21
|
+
// Kill and stop are one unit-stop authority: a caller that substitutes the stop
|
|
22
|
+
// also controls the kill, so the real systemctl never bypasses it.
|
|
23
|
+
stopUnit = stopWorkerUnit, killUnit = stopUnit === stopWorkerUnit ? killWorkerUnit : () => ({ ok: false }),
|
|
24
|
+
unitStatus = getWorkerIsolationStatus,
|
|
11
25
|
sample = sampleWorkerIdentity, children = sampleWorkerChildren, platform = process.platform,
|
|
12
|
-
signal = signalOwnedWorker,
|
|
26
|
+
signal = signalOwnedWorker, killPid = (pid, sig) => process.kill(pid, sig),
|
|
27
|
+
clock = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
13
28
|
} = {}) {
|
|
14
29
|
let delivered = false;
|
|
15
30
|
const uncertain = diagnostic => ({ termination: 'uncertain', diagnostic, delivered });
|
|
@@ -18,8 +33,13 @@ export async function stopOverflowWorker(launch, {
|
|
|
18
33
|
// Subprocess timeouts cannot bound stalled kernel or /proc reads.
|
|
19
34
|
const remaining = () => Math.max(1, deadline - clock());
|
|
20
35
|
if (launch.mode === 'systemd-unit' && launch.unit) {
|
|
21
|
-
|
|
22
|
-
|
|
36
|
+
// SIGKILL the unit's whole cgroup first. A plain stop sends SIGTERM and then
|
|
37
|
+
// waits out the unit's stop timeout, which a worker ignoring SIGTERM outlasts
|
|
38
|
+
// (#3691). The stop that follows only cleans up the dead unit.
|
|
39
|
+
let killed = false;
|
|
40
|
+
try { killed = !!killUnit(launch.unit, { timeoutMs: remaining() }).ok; } catch { killed = false; }
|
|
41
|
+
try { delivered = !!stopUnit(launch.unit, { timeoutMs: remaining() }).ok || killed; } catch { delivered = killed; }
|
|
42
|
+
if (!delivered) diagnostic = 'OG_WORKER_MCP_STOP_FAILED';
|
|
23
43
|
} else {
|
|
24
44
|
const child = launch.child;
|
|
25
45
|
// The direct-child handle cannot prove descendant termination.
|
|
@@ -28,15 +48,30 @@ export async function stopOverflowWorker(launch, {
|
|
|
28
48
|
catch { return uncertain('OG_WORKER_MCP_STOP_FAILED'); }
|
|
29
49
|
return uncertain('OG_WORKER_MCP_STOP_UNCONFIRMED');
|
|
30
50
|
}
|
|
31
|
-
|
|
32
|
-
if (!
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
51
|
+
let worker = launch.transportSupervisor ? ownedWorkers.get(launch) : null;
|
|
52
|
+
if (!worker) {
|
|
53
|
+
const supervisor = sample(child?.pid);
|
|
54
|
+
if (!supervisor || supervisor.parentPid !== process.pid || supervisor.uid !== process.getuid?.()
|
|
55
|
+
|| child.exitCode !== null || child.signalCode !== null) return uncertain('OG_WORKER_MCP_STOP_UNCONFIRMED');
|
|
56
|
+
const candidates = launch.transportSupervisor ? children(supervisor.pid) : [supervisor];
|
|
57
|
+
worker = candidates?.length === 1 ? candidates[0] : null;
|
|
58
|
+
if (!worker || launch.transportSupervisor && (worker.parentPid !== supervisor.pid || !hasBirthIdentity(worker))
|
|
59
|
+
|| !sameWorkerIdentity(supervisor, sample(supervisor.pid))) return uncertain('OG_WORKER_MCP_STOP_UNCONFIRMED');
|
|
60
|
+
if (launch.transportSupervisor) ownedWorkers.set(launch, worker = Object.freeze({ ...worker }));
|
|
61
|
+
}
|
|
38
62
|
// Reuse the launcher's birth/session/UID checks; never signal a bare PID.
|
|
39
|
-
if (!(delivered = !!signal(worker, 'SIGKILL')))
|
|
63
|
+
if (!(delivered = !!signal(worker, 'SIGKILL'))) {
|
|
64
|
+
// #3691: reparenting changes ownership topology, not process birth.
|
|
65
|
+
// The cache pins birth/UID/session while the parent link is intact;
|
|
66
|
+
// compare every identity field except parentPid on a fresh kernel sample.
|
|
67
|
+
// This also works on retained-control retries after client.close() has
|
|
68
|
+
// killed the supervisor. Never refresh the pin from an orphan or disk.
|
|
69
|
+
const fresh = launch.transportSupervisor ? sample(worker.pid) : null;
|
|
70
|
+
if (fresh && sameWorkerIdentity(worker, { ...fresh, parentPid: worker.parentPid })) {
|
|
71
|
+
try { killPid(worker.pid, 'SIGKILL'); delivered = true; } catch { delivered = false; }
|
|
72
|
+
}
|
|
73
|
+
if (!delivered) return uncertain('OG_WORKER_MCP_STOP_FAILED');
|
|
74
|
+
}
|
|
40
75
|
// Descendants can escape the group, so its termination stays uncertain.
|
|
41
76
|
return uncertain('OG_WORKER_MCP_STOP_UNCONFIRMED');
|
|
42
77
|
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// Authenticate predecessor records and optionally recheck owned descendant absence.
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { importImpl } from '../../_resolve-impl.mjs';
|
|
4
|
+
import { lstat } from 'node:fs/promises';
|
|
5
|
+
import { boundedPrivateRead } from './worker-inputs.mjs';
|
|
6
|
+
import { realBeneath, stableTaskChainDigest, assertStopBeforeRetry, publishWorkerRecord, UNBOUND_RETRY_EVIDENCE_CODE } from '../session-launcher.mjs';
|
|
7
|
+
import { recheckWorkerTermination } from './worker-termination.mjs';
|
|
8
|
+
|
|
9
|
+
async function matchesLegacyTask(record, plan) {
|
|
10
|
+
// Non-recovery launches retained a digest of the request including output paths.
|
|
11
|
+
// Reconstruct only those paths; all task fields and the brief content stay bound.
|
|
12
|
+
if (![record.prompt?.path, record.cwd, record.paths?.final].every(value => typeof value === 'string')
|
|
13
|
+
|| record.prompt?.digest !== plan.invocation?.briefDigest) return false;
|
|
14
|
+
const { qualificationDigest } = await importImpl(import.meta.url, 'models/qualification-digest.mjs');
|
|
15
|
+
return record.taskDigest === qualificationDigest({ ...plan.request,
|
|
16
|
+
brief: record.prompt.path, workdir: record.cwd, final_output: record.paths.final });
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function verifyRetryPredecessor(retryOf, { newInvocationPath, inspect, workerWritableRoots, checkDescendants = false } = {}) {
|
|
20
|
+
const fail = (reason) => {
|
|
21
|
+
const error = new Error(`Retry predecessor evidence refused: ${reason}`);
|
|
22
|
+
error.code = UNBOUND_RETRY_EVIDENCE_CODE;
|
|
23
|
+
throw error;
|
|
24
|
+
};
|
|
25
|
+
if (!retryOf || typeof retryOf.invocationId !== 'string' || !retryOf.invocationId
|
|
26
|
+
|| typeof retryOf.runDir !== 'string' || !retryOf.runDir) {
|
|
27
|
+
fail('retryOf requires both a non-empty invocationId and runDir');
|
|
28
|
+
}
|
|
29
|
+
const { invocationId: prevId, runDir: prevRunDir } = retryOf;
|
|
30
|
+
// #2915 review 4 (finding 4): the minimal directory-authority repair —
|
|
31
|
+
// reject a predecessor location inside any worker-writable root, exactly
|
|
32
|
+
// like `launchOneWorker` already refuses for a NEW run directory. This
|
|
33
|
+
// closes the easiest forgery (planting a complete, well-formed fake
|
|
34
|
+
// predecessor inside the worktree the worker itself can write to) but is
|
|
35
|
+
// explicitly NOT a defense against an unconfined malicious same-UID
|
|
36
|
+
// worker in general: such a worker can still write anywhere this
|
|
37
|
+
// conductor process can (e.g. directly under a configured capture root,
|
|
38
|
+
// if that root is not itself outside every worker-writable path, or by
|
|
39
|
+
// racing a legitimate predecessor's own directory before this check
|
|
40
|
+
// runs). A real guarantee needs an enforced filesystem/process boundary
|
|
41
|
+
// protecting conductor-owned metadata, or a different execution
|
|
42
|
+
// identity — same-UID alone never provides it.
|
|
43
|
+
if (Array.isArray(workerWritableRoots)) {
|
|
44
|
+
for (const root of workerWritableRoots) {
|
|
45
|
+
if (await realBeneath(root, prevRunDir)) fail('predecessor location is inside a worker-writable root');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (!await inspect(prevRunDir, 'directory')) fail('predecessor run directory is missing');
|
|
49
|
+
const dirStat = await lstat(prevRunDir);
|
|
50
|
+
if (dirStat.uid !== process.getuid() || (dirStat.mode & 0o077)) fail('predecessor run directory is not privately owned');
|
|
51
|
+
let ownerBytes, invocationBytes, exitBytes;
|
|
52
|
+
try {
|
|
53
|
+
// #2915 review 4 (finding 4): `rejectInsecureMode: true` — an insecure
|
|
54
|
+
// mode on predecessor metadata REFUSES the retry instead of silently
|
|
55
|
+
// chmod-repairing it and proceeding (the former behaviour the review
|
|
56
|
+
// found: "the probe's 0666 files were accepted and changed to 0600").
|
|
57
|
+
ownerBytes = await boundedPrivateRead(join(prevRunDir, '.launch-owner'), 256, inspect, true, true);
|
|
58
|
+
invocationBytes = await boundedPrivateRead(join(prevRunDir, 'invocation.json'), 1048576, inspect, true, true);
|
|
59
|
+
exitBytes = await boundedPrivateRead(join(prevRunDir, 'exit.json'), 1048576, inspect, true, true);
|
|
60
|
+
} catch (error) { fail(`predecessor metadata is missing, insecurely-moded, or not a bounded, privately-owned, unaliased regular file (${error.message})`); }
|
|
61
|
+
if (ownerBytes.toString('utf8') !== prevId) fail('.launch-owner does not match retryOf.invocationId');
|
|
62
|
+
let invocationRecord, exitRecord;
|
|
63
|
+
try {
|
|
64
|
+
invocationRecord = JSON.parse(invocationBytes.toString('utf8'));
|
|
65
|
+
exitRecord = JSON.parse(exitBytes.toString('utf8'));
|
|
66
|
+
} catch { fail('predecessor metadata is not valid JSON'); }
|
|
67
|
+
if (invocationRecord?.invocationId !== prevId || invocationRecord?.origin !== 'launcher') {
|
|
68
|
+
fail('invocation.json identity does not match retryOf (or is not launcher-written)');
|
|
69
|
+
}
|
|
70
|
+
if (exitRecord?.invocationId !== prevId) fail('exit.json identity does not match retryOf');
|
|
71
|
+
if (newInvocationPath !== undefined) {
|
|
72
|
+
// #2915 review 4 (finding 4): the SAME stable task-chain identity
|
|
73
|
+
// `launchOneWorker` publishes — role/task/group plus the brief CONTENT
|
|
74
|
+
// digest, never `final_output`/`brief`/`workdir` PATHS, which legitimately
|
|
75
|
+
// differ between a confirmed predecessor and this fresh attempt.
|
|
76
|
+
let newTaskDigest, newPlan;
|
|
77
|
+
try {
|
|
78
|
+
const newPlanBytes = await boundedPrivateRead(newInvocationPath, 1048576, inspect);
|
|
79
|
+
newPlan = JSON.parse(newPlanBytes.toString('utf8'));
|
|
80
|
+
newTaskDigest = await stableTaskChainDigest(newPlan.request, newPlan.invocation?.briefDigest ?? null);
|
|
81
|
+
} catch { fail('new attempt plan is missing or invalid; cannot bind the retry to a task'); }
|
|
82
|
+
if (invocationRecord.taskDigest !== newTaskDigest && !await matchesLegacyTask(invocationRecord, newPlan)) fail('predecessor task does not match the new attempt (task-chain binding)');
|
|
83
|
+
}
|
|
84
|
+
if (exitRecord.termination !== 'confirmed') {
|
|
85
|
+
let evidence = null;
|
|
86
|
+
if (checkDescendants) {
|
|
87
|
+
try {
|
|
88
|
+
const processRecord = JSON.parse((await boundedPrivateRead(join(prevRunDir, 'process.json'), 65536, inspect, true, true)).toString('utf8'));
|
|
89
|
+
evidence = await recheckWorkerTermination({ processRecord, exitRecord });
|
|
90
|
+
} catch (error) { evidence = { confirmed: false, missing: `readable, private launcher process.json (${error.message})` }; }
|
|
91
|
+
if (evidence.confirmed) {
|
|
92
|
+
exitRecord = { ...exitRecord, termination: 'confirmed', descendants: 'none',
|
|
93
|
+
...(exitRecord.health ? { health: { ...exitRecord.health, termination: 'confirmed' } } : {}) };
|
|
94
|
+
await publishWorkerRecord(prevRunDir, 'exit.json', exitRecord, inspect);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (exitRecord.termination !== 'confirmed') {
|
|
98
|
+
assertStopBeforeRetry(exitRecord, { expectedInvocationId: prevId, runDir: prevRunDir, missing: evidence?.missing });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
assertStopBeforeRetry(exitRecord, { expectedInvocationId: prevId });
|
|
102
|
+
return exitRecord;
|
|
103
|
+
}
|