@aiwg/cli 2026.10.19 → 2026.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/agentic/code/providers/flue/README.md +2 -1
- package/dist/src/artifacts/cli.js +3 -2
- package/dist/src/artifacts/discover-facets.js +2 -0
- package/dist/src/artifacts/discover-session-facets.js +14 -0
- package/dist/src/artifacts/fortemi-core-query-adapter.js +4 -4
- package/dist/src/artifacts/framework-capabilities.js +7 -3
- package/dist/src/artifacts/index-builder.js +2 -3
- package/dist/src/artifacts/query-engine-discover-output.js +3 -3
- package/dist/src/artifacts/query-engine-discover.js +61 -14
- package/dist/src/artifacts/query-engine-entries.js +52 -9
- package/dist/src/artifacts/query-engine-installation.js +27 -0
- package/dist/src/artifacts/query-engine-rules.js +1 -2
- package/dist/src/artifacts/query-engine-scoring.js +33 -31
- package/dist/src/artifacts/query-engine-show-card.js +6 -2
- package/dist/src/artifacts/query-engine-show.js +2 -2
- package/dist/src/artifacts/relations.js +17 -10
- package/dist/src/cli/handlers/bundle-declaration.js +55 -0
- package/dist/src/cli/handlers/bundle-deployment-mode.js +71 -0
- package/dist/src/cli/handlers/bundle-hash-prune.js +37 -0
- package/dist/src/cli/handlers/bundle-reconciliation.js +140 -0
- package/dist/src/cli/handlers/checkpoint.js +143 -0
- package/dist/src/cli/handlers/help.js +1 -0
- package/dist/src/cli/handlers/index.js +4 -2
- package/dist/src/cli/handlers/models-resource-sample.js +7 -3
- package/dist/src/cli/handlers/ralph-launcher.js +1 -1
- package/dist/src/cli/handlers/refresh-bundles.js +35 -0
- package/dist/src/cli/handlers/refresh-rule-inventory.js +10 -0
- package/dist/src/cli/handlers/refresh.js +15 -52
- package/dist/src/cli/handlers/remove-bundled.js +7 -4
- package/dist/src/cli/handlers/remove-flue-modules.js +19 -0
- package/dist/src/cli/handlers/remove-historical-bundle.js +18 -0
- package/dist/src/cli/handlers/remove.js +8 -0
- package/dist/src/cli/handlers/script-runner.js +12 -1
- package/dist/src/cli/handlers/steward-signing.js +44 -0
- package/dist/src/cli/handlers/steward.js +5 -0
- package/dist/src/cli/handlers/subcommands-list-format.js +4 -4
- package/dist/src/cli/handlers/update.js +18 -8
- package/dist/src/cli/handlers/use-additive-bundles.js +87 -0
- package/dist/src/cli/handlers/use-bundle-counts.js +5 -1
- package/dist/src/cli/handlers/use-owned-counts.js +75 -0
- package/dist/src/cli/handlers/use-provider-results.js +17 -0
- package/dist/src/cli/handlers/use.js +53 -114
- package/dist/src/cli/router.js +12 -2
- package/dist/src/cli/services/deployment-verification.js +2 -1
- package/dist/src/cli/user-scope-mirror.js +3 -1
- package/dist/src/config/aiwg-config.js +15 -6
- package/dist/src/config/cli.js +1 -0
- package/dist/src/config/delivery-validation.js +21 -0
- package/dist/src/config/framework-aliases.js +14 -0
- package/dist/src/config/host-resource-report.d.mts +1 -1
- package/dist/src/config/host-resource-report.mjs +23 -4
- package/dist/src/config/host-resource-runtime.d.mts +8 -0
- package/dist/src/config/host-resource-runtime.mjs +67 -24
- package/dist/src/config/index-graph-overrides.js +4 -0
- package/dist/src/config/installed-selection.js +9 -1
- package/dist/src/config/installed-validation.js +35 -0
- package/dist/src/config/worker-attribution.d.mts +7 -0
- package/dist/src/config/worker-attribution.mjs +44 -0
- package/dist/src/delivery/git.js +62 -0
- package/dist/src/delivery/pr.js +149 -0
- package/dist/src/delivery/tracker.js +33 -0
- package/dist/src/discovery/interview.js +1 -1
- package/dist/src/extensions/claude-hooks-installer.js +1 -0
- package/dist/src/extensions/loader.js +2 -1
- package/dist/src/extensions/project-local-doctor.js +1 -1
- package/dist/src/extensions/project-local-remove.js +3 -3
- package/dist/src/issues/cli.js +12 -3
- package/dist/src/issues/external-cli.js +76 -0
- package/dist/src/issues/live.js +18 -1
- package/dist/src/issues/tracker-policy.js +17 -0
- package/dist/src/mcp/cli.mjs +12 -7
- package/dist/src/mcp/helpers.mjs +5 -1
- package/dist/src/mcp/http-options.mjs +101 -0
- package/dist/src/mcp/http-tools.mjs +51 -0
- package/dist/src/mcp/http-transport.mjs +201 -0
- package/dist/src/mcp/server.mjs +9 -7
- package/dist/src/mcp/tool-context.mjs +25 -0
- package/dist/src/mcp/tools/orchestration.mjs +2 -1
- package/dist/src/mcp/tools/worker-invoke.mjs +2 -1
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +12 -0
- package/dist/src/models/conductor-data-policy.d.mts +3 -2
- package/dist/src/models/conductor-data-policy.mjs +94 -5
- package/dist/src/models/worker-plan.js +3 -3
- package/dist/src/models/worker-selection-policy.js +2 -1
- package/dist/src/providers/deployment-lock.js +77 -0
- package/dist/src/providers/deployment-ownership.d.mts +12 -2
- package/dist/src/providers/deployment-ownership.mjs +127 -19
- package/dist/src/providers/deployment-preservation.d.mts +6 -0
- package/dist/src/providers/deployment-preservation.mjs +113 -0
- package/dist/src/providers/deployment-removal.js +4 -2
- package/dist/src/providers/flue-package-mode.js +19 -0
- package/dist/src/providers/provider-definitions.js +4 -0
- package/dist/src/providers/skill-ownership.mjs +1 -0
- package/dist/src/research/reference-match.js +4 -0
- package/dist/src/serve/worker-invocation-preparation.js +3 -1
- package/dist/src/serve/worker-invocation.js +2 -0
- package/dist/src/skills/run.js +5 -0
- package/dist/src/smiths/hook-bridge/codex-translator.js +3 -1
- package/dist/src/smiths/hook-bridge/hermes-translator.js +3 -1
- package/dist/src/smiths/hook-bridge/prompt-discovery-gaps.js +6 -0
- package/package.json +5 -2
- package/schemas/models/model-config-compatibility.v1.schema.json +4 -0
- package/tools/agents/deploy-agents.mjs +21 -1
- package/tools/agents/deploy-options.mjs +2 -0
- package/tools/agents/providers/base-contract.mjs +7 -1
- package/tools/agents/providers/base-inventory.mjs +41 -14
- package/tools/agents/providers/base-kernel-skills.mjs +9 -5
- package/tools/agents/providers/base-migrations.mjs +1 -0
- package/tools/agents/providers/base-skill-copy.mjs +9 -1
- package/tools/agents/providers/base.mjs +1 -1
- package/tools/agents/providers/claude.mjs +2 -1
- package/tools/agents/providers/codex.mjs +5 -1
- package/tools/agents/providers/conductor-data-guard.mjs +35 -11
- package/tools/agents/providers/flue-bundles.mjs +42 -0
- package/tools/agents/providers/flue-cloudflare.mjs +20 -0
- package/tools/agents/providers/flue-project.mjs +45 -6
- package/tools/agents/providers/flue-skill-copy.mjs +25 -0
- package/tools/agents/providers/flue-skills.mjs +45 -0
- package/tools/agents/providers/flue-specialists.mjs +36 -18
- package/tools/agents/providers/flue-validation.mjs +14 -0
- package/tools/agents/providers/flue.mjs +12 -8
- package/tools/agents/providers/hermes-skill-lifecycle.mjs +1 -0
- package/tools/agents/providers/muse-hooks.mjs +4 -0
- package/tools/agents/providers/muse-prompt-discovery.mjs +22 -0
- package/tools/agents/providers/provider-prepare-coordinator.mjs +15 -10
- package/tools/agents/providers/rule-delivery-surface.mjs +18 -1
- package/tools/agents/providers/rule-deployment-commit.mjs +24 -0
- package/tools/agents/providers/rule-deployment-prepare.mjs +4 -0
- package/tools/agents/providers/skill-copy-prune.mjs +49 -0
- package/tools/agents/providers/stale-artifact-prune.mjs +10 -8
- package/tools/ralph-external/lib/worker-capture.mjs +157 -0
- package/tools/ralph-external/lib/worker-health.mjs +3 -0
- package/tools/ralph-external/lib/worker-invocation-processes.mjs +9 -6
- package/tools/ralph-external/lib/worker-launch-cli.mjs +10 -1
- package/tools/ralph-external/lib/worker-process-identity.mjs +30 -0
- package/tools/ralph-external/lib/worker-retry-evidence.mjs +87 -6
- package/tools/ralph-external/lib/worker-termination.mjs +56 -6
- package/tools/ralph-external/process-monitor.mjs +12 -18
- package/tools/ralph-external/session-launcher.mjs +43 -29
- package/tools/skills/codex-skill-metadata.mjs +22 -0
- package/tools/skills/deploy-skills-codex.mjs +8 -7
- package/vscode-extension/schemas/aiwg.config.v1.json +59 -19
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// Preserve one-shot attempt captures as read-only snapshots bound to the terminal run record.
|
|
2
|
+
import { constants } from 'node:fs';
|
|
3
|
+
import { mkdir, open, readdir, lstat } from 'node:fs/promises';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
import { join, dirname, basename } from 'node:path';
|
|
6
|
+
import { withWorkerPath } from './worker-inputs.mjs';
|
|
7
|
+
import { importImpl } from '../../_resolve-impl.mjs';
|
|
8
|
+
|
|
9
|
+
const hash = bytes => `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
|
|
10
|
+
const reservationPath = filename => `${filename}.launch-owner`;
|
|
11
|
+
const directoryPath = runDir => join(dirname(runDir), `.captures-${basename(runDir)}`);
|
|
12
|
+
const fileFlags = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
|
13
|
+
const directoryFlags = constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW;
|
|
14
|
+
const sameFile = (a, b) => a.dev === b.dev && a.ino === b.ino;
|
|
15
|
+
|
|
16
|
+
export async function assertWorkerCapturePathsAvailable(paths, inspect, { runDir, writableRoots }) {
|
|
17
|
+
const { realBeneath } = await importImpl(import.meta.url, 'serve/worker-invocation-paths.js');
|
|
18
|
+
for (const root of writableRoots) {
|
|
19
|
+
if (await realBeneath(root, directoryPath(runDir))) throw new Error('Sealed captures must be outside worker-writable roots');
|
|
20
|
+
}
|
|
21
|
+
const filenames = Object.values(paths);
|
|
22
|
+
if (filenames.some(filename => filenames.includes(reservationPath(filename)))) {
|
|
23
|
+
throw new Error('Worker output paths collide with capture reservations');
|
|
24
|
+
}
|
|
25
|
+
for (const filename of filenames) {
|
|
26
|
+
// A removed live capture still belongs to the attempt whose snapshot survives beside its run directory.
|
|
27
|
+
if (await inspect(directoryPath(dirname(filename)), 'directory')) throw new Error('Worker output path belongs to an earlier attempt capture directory');
|
|
28
|
+
if (await inspect(reservationPath(filename), 'output')) throw new Error('Worker output path belongs to an earlier attempt');
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function ownedFile(filename, operation) {
|
|
33
|
+
return withWorkerPath(filename, async target => {
|
|
34
|
+
const handle = await open(target, fileFlags);
|
|
35
|
+
try {
|
|
36
|
+
const stat = await handle.stat({ bigint: true });
|
|
37
|
+
if (!stat.isFile() || stat.nlink !== 1n || stat.uid !== BigInt(process.getuid())) throw new Error('Capture must be an owned unaliased regular file');
|
|
38
|
+
const result = await operation(handle, stat);
|
|
39
|
+
const after = await handle.stat({ bigint: true });
|
|
40
|
+
if (stat.size !== after.size || stat.mtimeNs !== after.mtimeNs || stat.ctimeNs !== after.ctimeNs
|
|
41
|
+
|| !sameFile(after, await lstat(target, { bigint: true }))) {
|
|
42
|
+
throw new Error('Capture changed while sealing or verifying');
|
|
43
|
+
}
|
|
44
|
+
return result;
|
|
45
|
+
} finally { await handle.close(); }
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function digestFile(filename, expectedMode) {
|
|
50
|
+
return ownedFile(filename, async (handle, stat) => {
|
|
51
|
+
if (expectedMode !== undefined && Number(stat.mode & 0o777n) !== expectedMode) throw new Error('Sealed capture is not read-only');
|
|
52
|
+
const digest = createHash('sha256');
|
|
53
|
+
for await (const chunk of handle.createReadStream({ autoClose: false, start: 0 })) digest.update(chunk);
|
|
54
|
+
return { digest: `sha256:${digest.digest('hex')}`, byteLength: Number(stat.size) };
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export class WorkerCaptureOwner {
|
|
59
|
+
constructor({ runDir, paths, captures, inspect, invocationId }) {
|
|
60
|
+
Object.assign(this, { runDir, paths, captures, inspect, invocationId });
|
|
61
|
+
this.directory = directoryPath(runDir);
|
|
62
|
+
this.sources = { ...Object.fromEntries(Object.entries(paths).map(([key, value]) => [`output-${key}`, value])),
|
|
63
|
+
...Object.fromEntries(Object.entries(captures).map(([key, value]) => [`capture-${key}`, value])) };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
writer(writeRecord, publishSeal = writeRecord) {
|
|
67
|
+
this.publishSeal = publishSeal;
|
|
68
|
+
return async (runDir, name, value, inspect) => {
|
|
69
|
+
if (name === 'invocation.json') {
|
|
70
|
+
// Exclusive even if a conductor has deleted the old mutable run directory.
|
|
71
|
+
await withWorkerPath(this.directory, target => mkdir(target, { mode: 0o700 }));
|
|
72
|
+
this.started = true;
|
|
73
|
+
value.captureDirectory = this.directory;
|
|
74
|
+
this.invocationRecord = value;
|
|
75
|
+
}
|
|
76
|
+
if (name === 'exit.json') this.result = value;
|
|
77
|
+
return writeRecord(runDir, name, value, inspect);
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async reserveOutput(filename) {
|
|
82
|
+
if (Object.values(this.paths).includes(filename)) {
|
|
83
|
+
const marker = reservationPath(filename);
|
|
84
|
+
await this.inspect(marker, 'output');
|
|
85
|
+
const handle = await withWorkerPath(marker, target => open(target,
|
|
86
|
+
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600));
|
|
87
|
+
try { await handle.writeFile(JSON.stringify({ invocationId: this.invocationId, runDir: this.runDir })); await handle.sync(); }
|
|
88
|
+
finally { await handle.close(); }
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async finish() {
|
|
93
|
+
if (!this.started) return;
|
|
94
|
+
const files = [];
|
|
95
|
+
for (const [name, source] of Object.entries(this.sources)) {
|
|
96
|
+
const path = join(this.directory, name);
|
|
97
|
+
if (!await this.inspect(source, 'file')) { files.push({ source, path, missing: true }); continue; }
|
|
98
|
+
const recorded = await ownedFile(source, async (input, stat) => {
|
|
99
|
+
const output = await withWorkerPath(path, target => open(target,
|
|
100
|
+
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600));
|
|
101
|
+
const digest = createHash('sha256');
|
|
102
|
+
try {
|
|
103
|
+
for await (const chunk of input.createReadStream({ autoClose: false, start: 0 })) {
|
|
104
|
+
digest.update(chunk); await output.writeFile(chunk);
|
|
105
|
+
}
|
|
106
|
+
await output.chmod(0o400); await output.sync();
|
|
107
|
+
return { digest: `sha256:${digest.digest('hex')}`, byteLength: Number(stat.size) };
|
|
108
|
+
} finally { await output.close(); }
|
|
109
|
+
});
|
|
110
|
+
files.push({ source, path, ...recorded });
|
|
111
|
+
}
|
|
112
|
+
await withWorkerPath(this.directory, async target => {
|
|
113
|
+
const directory = await open(target, directoryFlags);
|
|
114
|
+
try { await directory.chmod(0o500); await directory.sync(); } finally { await directory.close(); }
|
|
115
|
+
});
|
|
116
|
+
// A capture-setup exception can precede exit.json; retain its seal in the invocation record.
|
|
117
|
+
const record = this.result ?? this.invocationRecord;
|
|
118
|
+
record.captureSeal = { directory: this.directory, sealedAt: new Date().toISOString(), files,
|
|
119
|
+
digest: hash(JSON.stringify(files)) };
|
|
120
|
+
await this.publishSeal(this.runDir, this.result ? 'exit.json' : 'invocation.json', record, this.inspect);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function verifyWorkerCaptureSeal(runDir, invocation, exit, inspect) {
|
|
125
|
+
// Pre-sealing launcher records remain readable; every new invocation declares its capture directory.
|
|
126
|
+
if (invocation.captureDirectory === undefined && exit.captureSeal === undefined) return;
|
|
127
|
+
const seal = exit.captureSeal;
|
|
128
|
+
if (invocation.captureDirectory !== directoryPath(runDir) || seal?.directory !== invocation.captureDirectory
|
|
129
|
+
|| !Array.isArray(seal.files) || seal.digest !== hash(JSON.stringify(seal.files))) throw new Error('Capture seal is missing or invalid');
|
|
130
|
+
const expected = { ...Object.fromEntries(Object.entries(invocation.paths).map(([key, value]) => [`output-${key}`, value])),
|
|
131
|
+
...Object.fromEntries(Object.entries(invocation.captures).map(([key, value]) => [`capture-${key}`, value])) };
|
|
132
|
+
if (seal.files.length !== Object.keys(expected).length) throw new Error('Capture seal omits attempt paths');
|
|
133
|
+
await withWorkerPath(seal.directory, async target => {
|
|
134
|
+
const directory = await open(target, directoryFlags);
|
|
135
|
+
try {
|
|
136
|
+
const stat = await directory.stat();
|
|
137
|
+
if (stat.uid !== process.getuid() || (stat.mode & 0o777) !== 0o500) throw new Error('Capture directory is not sealed read-only');
|
|
138
|
+
const names = (await readdir(target)).sort();
|
|
139
|
+
const present = seal.files.filter(file => !file.missing).map(file => Object.keys(expected).find(name => file.path === join(seal.directory, name))).sort();
|
|
140
|
+
if (JSON.stringify(names) !== JSON.stringify(present)) throw new Error('Sealed capture directory entries changed');
|
|
141
|
+
} finally { await directory.close(); }
|
|
142
|
+
});
|
|
143
|
+
const seen = new Set();
|
|
144
|
+
for (const file of seal.files) {
|
|
145
|
+
const name = Object.keys(expected).find(key => file.path === join(seal.directory, key));
|
|
146
|
+
if (!name || seen.has(name) || file.source !== expected[name]) throw new Error('Capture seal path binding mismatch');
|
|
147
|
+
seen.add(name);
|
|
148
|
+
if (file.missing) {
|
|
149
|
+
if (await inspect(file.source, 'file')) throw new Error('Previously missing capture has changed');
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
for (const [filename, mode] of [[file.path, 0o400], [file.source, undefined]]) {
|
|
153
|
+
const actual = await digestFile(filename, mode);
|
|
154
|
+
if (actual.digest !== file.digest || actual.byteLength !== file.byteLength) throw new Error(`Capture digest mismatch: ${filename}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
@@ -126,6 +126,7 @@ export function createWorkerHealthObserver(options = {}) {
|
|
|
126
126
|
let sequence = 0, generation = options.observerGeneration ?? 0, steps = null, total = 0, lastCheck = started, maxGap = 0;
|
|
127
127
|
let observed = null, activity = null, progress = null, heartbeat = null, handshakeAt = null, stalledAt = null;
|
|
128
128
|
let state = 'launching', handshake = 'unobserved', confidence = 'unobserved', reasonCode = 'AWAITING_HANDSHAKE';
|
|
129
|
+
let terminationMissing = null;
|
|
129
130
|
let termination = 'running', transitionCount = 0, live = null, disconnected = false, finalized = false;
|
|
130
131
|
let terminal = null, fatalFailure = false, cpuTimeMs = null, contentDigest = options.contentDigest ?? null;
|
|
131
132
|
const unavailable = new Set(profile.handshake === 'semantic' ? [] : ['semantic-start']);
|
|
@@ -350,6 +351,7 @@ export function createWorkerHealthObserver(options = {}) {
|
|
|
350
351
|
if (finalized) return snapshot();
|
|
351
352
|
for (const [id, cursor] of buffers) if (cursor.bytes.length || cursor.discarding) gap('MALFORMED_FRAME', { id });
|
|
352
353
|
finalized = true; observed = now(); termination = result.termination ?? 'uncertain';
|
|
354
|
+
terminationMissing = result.terminationMissing ?? null;
|
|
353
355
|
if (termination !== 'confirmed' || disconnected) {
|
|
354
356
|
termination = 'uncertain';
|
|
355
357
|
if (result.signal || result.exitCode != null && result.exitCode !== 0) transition(handshakeAt === null ? 'failed-to-start' : 'crashed', 'CRASH');
|
|
@@ -375,6 +377,7 @@ export function createWorkerHealthObserver(options = {}) {
|
|
|
375
377
|
return Math.max(1, Math.min(...pending) - time);
|
|
376
378
|
},
|
|
377
379
|
evidence() { return { invocationId: binding.invocationId, sequence, observerGeneration: generation,
|
|
380
|
+
...(terminationMissing ? { terminationMissing } : {}),
|
|
378
381
|
lastWorkerHeartbeatAt: utc(heartbeat), cpuTimeMs, steps: windows.stepEnforcement === 'unavailable' ? null : steps,
|
|
379
382
|
gaps: gaps.map(item => ({ ...item })), events: events.map(item => ({ ...item })) }; },
|
|
380
383
|
};
|
|
@@ -5,6 +5,7 @@ import { classifyUnreadableWorkerProcess, readProcCgroup } from './worker-invoca
|
|
|
5
5
|
import { importImpl } from '../../_resolve-impl.mjs';
|
|
6
6
|
const { WORKER_INVOCATION_MARKER } = await importImpl(import.meta.url, 'config/worker-invocation-marker.mjs');
|
|
7
7
|
export { WORKER_INVOCATION_MARKER };
|
|
8
|
+
const { workerAttributionEnvironment } = await importImpl(import.meta.url, 'config/worker-attribution.mjs');
|
|
8
9
|
|
|
9
10
|
function startedBeforeWorker(fs, path, startTicks) {
|
|
10
11
|
if (typeof startTicks !== 'string' || !/^\d+$/.test(startTicks)) return false;
|
|
@@ -23,14 +24,16 @@ function startedBeforeWorker(fs, path, startTicks) {
|
|
|
23
24
|
finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
export function buildWorkerInvocationEnvironment(envNames, invocationId, inherited = process.env) {
|
|
27
|
+
export function buildWorkerInvocationEnvironment(envNames, invocationId, inherited = process.env, attribution) {
|
|
27
28
|
if (typeof invocationId !== 'string' || !invocationId || invocationId.includes('\0')) {
|
|
28
29
|
throw new Error('Worker environment requires a non-empty NUL-free invocation ID');
|
|
29
30
|
}
|
|
30
|
-
const
|
|
31
|
+
const markers = attribution ? workerAttributionEnvironment(invocationId, attribution.cwd, attribution.selected.group)
|
|
32
|
+
: { [WORKER_INVOCATION_MARKER]: invocationId };
|
|
33
|
+
const names = [...new Set([...envNames, ...Object.keys(markers)])];
|
|
31
34
|
const env = Object.fromEntries(envNames.filter(name => inherited[name] !== undefined)
|
|
32
35
|
.map(name => [name, inherited[name]]));
|
|
33
|
-
return { env: { ...env,
|
|
36
|
+
return { env: { ...env, ...markers }, envNames: names };
|
|
34
37
|
}
|
|
35
38
|
|
|
36
39
|
export function scanWorkerInvocationProcesses({ invocationId, startTicks, uid = process.getuid?.(),
|
|
@@ -98,12 +101,12 @@ export function scanWorkerInvocationProcesses({ invocationId, startTicks, uid =
|
|
|
98
101
|
// failures and unreadable/missing start times still cannot prove absence.
|
|
99
102
|
if (gone(error)) continue;
|
|
100
103
|
if (!sameUid) {
|
|
101
|
-
uncertain = unknown('unreadable same-uid process environment or ownership');
|
|
104
|
+
uncertain = { ...unknown('unreadable same-uid process environment or ownership'), retryable: true };
|
|
102
105
|
continue;
|
|
103
106
|
}
|
|
104
107
|
if (!workerKnown) {
|
|
105
108
|
if (!startedBeforeWorker(fs, path, startTicks)) {
|
|
106
|
-
uncertain = unknown('unreadable same-uid process environment or ownership');
|
|
109
|
+
uncertain = { ...unknown('unreadable same-uid process environment or ownership'), retryable: true };
|
|
107
110
|
}
|
|
108
111
|
continue;
|
|
109
112
|
}
|
|
@@ -117,7 +120,7 @@ export function scanWorkerInvocationProcesses({ invocationId, startTicks, uid =
|
|
|
117
120
|
workerCgroup: resolvedWorkerCgroup, selfCgroup: resolvedSelfCgroup,
|
|
118
121
|
now, deadline, ...(maxAncestryDepth === undefined ? {} : { maxDepth: maxAncestryDepth }) });
|
|
119
122
|
if (verdict === 'possible-descendant') {
|
|
120
|
-
uncertain = unknown('unreadable same-uid process links to the worker (ancestry, session, group or cgroup)');
|
|
123
|
+
uncertain = { ...unknown('unreadable same-uid process links to the worker (ancestry, session, group or cgroup)'), retryable: true };
|
|
121
124
|
}
|
|
122
125
|
} finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
123
126
|
}
|
|
@@ -13,6 +13,7 @@ export function parseWorkerLaunchArgs(args) {
|
|
|
13
13
|
seen.add(flag);
|
|
14
14
|
if (flag === '--launch-worker') continue;
|
|
15
15
|
if (flag === '--check-descendants') { result.checkDescendants = true; continue; }
|
|
16
|
+
if (flag === '--retry-correction') continue;
|
|
16
17
|
if (flag === '--allow-degraded-worker') { result.allowDegraded = true; continue; }
|
|
17
18
|
// A run directory declares a retry; its launcher ID is derived before validation.
|
|
18
19
|
const key = { '--invocation': 'invocationPath', '--run-dir': 'runDir',
|
|
@@ -25,13 +26,21 @@ export function parseWorkerLaunchArgs(args) {
|
|
|
25
26
|
else result[key] = value;
|
|
26
27
|
}
|
|
27
28
|
if (!seen.has('--launch-worker') || !result.invocationPath || !result.runDir) {
|
|
28
|
-
throw new Error('Usage: ralph-external --launch-worker --invocation FILE --run-dir DIR [--allow-degraded-worker] [--check-descendants] [--retry-of-run-dir DIR [--retry-of ID]]');
|
|
29
|
+
throw new Error('Usage: ralph-external --launch-worker --invocation FILE --run-dir DIR [--allow-degraded-worker] [--check-descendants] [--retry-of-run-dir DIR [--retry-of ID] [--retry-correction]]');
|
|
30
|
+
}
|
|
31
|
+
// #3857: reject relative paths here, naming the flag, before any launch work.
|
|
32
|
+
for (const [flag, value] of [['--invocation', result.invocationPath], ['--run-dir', result.runDir], ['--retry-of-run-dir', retryOfRunDir]]) {
|
|
33
|
+
if (value !== undefined && !isAbsolute(value)) {
|
|
34
|
+
throw new Error(`${flag} must be an absolute path; relative paths are not resolved against cwd.`);
|
|
35
|
+
}
|
|
29
36
|
}
|
|
30
37
|
if (retryOfInvocationId !== undefined && retryOfRunDir === undefined) {
|
|
31
38
|
throw new Error('--retry-of and --retry-of-run-dir must be supplied together when an explicit ID is used');
|
|
32
39
|
}
|
|
33
40
|
if (result.checkDescendants && retryOfRunDir === undefined) throw new Error('--check-descendants requires --retry-of-run-dir');
|
|
41
|
+
if (seen.has('--retry-correction') && retryOfRunDir === undefined) throw new Error('--retry-correction requires --retry-of-run-dir');
|
|
34
42
|
if (retryOfRunDir !== undefined) result.retryOf = { runDir: retryOfRunDir, ...(retryOfInvocationId === undefined ? {} : { invocationId: retryOfInvocationId }) };
|
|
43
|
+
if (seen.has('--retry-correction')) result.retryOf.correction = true;
|
|
35
44
|
return result;
|
|
36
45
|
}
|
|
37
46
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Retain kernel birth identity when an exited child has already lost its executable link.
|
|
2
|
+
import { readFileSync, readlinkSync } from 'node:fs';
|
|
3
|
+
import { hostname } from 'node:os';
|
|
4
|
+
|
|
5
|
+
export function sampleWorkerProcIdentity(pid, { procRoot = '/proc', platform = process.platform,
|
|
6
|
+
host = hostname(), fs = { readFileSync, readlinkSync } } = {}) {
|
|
7
|
+
if (platform !== 'linux' || !Number.isSafeInteger(pid) || pid <= 0) return null;
|
|
8
|
+
try {
|
|
9
|
+
const parse = text => text.slice(text.lastIndexOf(')') + 2).trim().split(/\s+/);
|
|
10
|
+
const fields = parse(fs.readFileSync(`${procRoot}/${pid}/stat`, 'utf8'));
|
|
11
|
+
const uid = Number(/^Uid:\s+(\d+)/m.exec(fs.readFileSync(`${procRoot}/${pid}/status`, 'utf8'))?.[1]);
|
|
12
|
+
const bootId = fs.readFileSync(`${procRoot}/sys/kernel/random/boot_id`, 'utf8').trim();
|
|
13
|
+
if (!Number.isSafeInteger(uid) || !bootId || !/^\d+$/.test(fields[19] ?? '')) return null;
|
|
14
|
+
let executable;
|
|
15
|
+
try { executable = fs.readlinkSync(`${procRoot}/${pid}/exe`); }
|
|
16
|
+
catch (error) {
|
|
17
|
+
if (!['ENOENT', 'ESRCH'].includes(error.code)) return null;
|
|
18
|
+
// exit_mm removes /exe before wait() reaps the child. Its readable
|
|
19
|
+
// stat/status still prove birth, ownership and group, but grant no
|
|
20
|
+
// executable claim. Re-read stat to reject disappearance/PID reuse
|
|
21
|
+
// and any live process (including a transient exec failure).
|
|
22
|
+
const exited = parse(fs.readFileSync(`${procRoot}/${pid}/stat`, 'utf8'));
|
|
23
|
+
if (exited[0] !== 'Z' || [1, 2, 3, 19].some(index => exited[index] !== fields[index])) return null;
|
|
24
|
+
executable = null;
|
|
25
|
+
}
|
|
26
|
+
return { pid, pgid: Number(fields[2]), uid, parentPid: Number(fields[1]), sessionId: Number(fields[3]),
|
|
27
|
+
host, bootId, startTicks: fields[19], executable, state: executable === null ? 'Z' : fields[0],
|
|
28
|
+
cpuTicks: Number(fields[11]) + Number(fields[12]) + Number(fields[13]) + Number(fields[14]) };
|
|
29
|
+
} catch { return null; }
|
|
30
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
// Authenticate predecessor records and optionally recheck owned descendant absence.
|
|
2
|
-
import { join } from 'node:path';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
3
|
import { importImpl } from '../../_resolve-impl.mjs';
|
|
4
|
-
import { lstat } from 'node:fs/promises';
|
|
5
|
-
import { boundedPrivateRead } from './worker-inputs.mjs';
|
|
4
|
+
import { lstat, readdir, realpath } from 'node:fs/promises';
|
|
5
|
+
import { boundedPrivateRead, withWorkerPath } from './worker-inputs.mjs';
|
|
6
|
+
import { resolveRetryReference } from './worker-launch-cli.mjs';
|
|
7
|
+
import { verifyWorkerCaptureSeal } from './worker-capture.mjs';
|
|
6
8
|
import { realBeneath, stableTaskChainDigest, assertStopBeforeRetry, publishWorkerRecord, UNBOUND_RETRY_EVIDENCE_CODE } from '../session-launcher.mjs';
|
|
7
9
|
import { recheckWorkerTermination } from './worker-termination.mjs';
|
|
8
10
|
|
|
@@ -16,7 +18,15 @@ async function matchesLegacyTask(record, plan) {
|
|
|
16
18
|
brief: record.prompt.path, workdir: record.cwd, final_output: record.paths.final });
|
|
17
19
|
}
|
|
18
20
|
|
|
19
|
-
|
|
21
|
+
async function matchesTaskIdentity(record, plan) {
|
|
22
|
+
if (!record.prompt?.digest || !record.selected?.group || !await sameWorkspace(record.cwd, plan.request.workdir)) return false;
|
|
23
|
+
const oldPlan = { ...plan, request: { ...plan.request, group: record.selected.group },
|
|
24
|
+
invocation: { briefDigest: record.prompt.digest } };
|
|
25
|
+
const digest = await stableTaskChainDigest(oldPlan.request, oldPlan.invocation.briefDigest);
|
|
26
|
+
return (record.taskBindingDigest ?? record.taskDigest) === digest || await matchesLegacyTask(record, oldPlan);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function verifyRetryPredecessor(retryOf, { newInvocationPath, inspect, workerWritableRoots, checkDescendants = false, onVerified } = {}) {
|
|
20
30
|
const fail = (reason) => {
|
|
21
31
|
const error = new Error(`Retry predecessor evidence refused: ${reason}`);
|
|
22
32
|
error.code = UNBOUND_RETRY_EVIDENCE_CODE;
|
|
@@ -68,6 +78,9 @@ export async function verifyRetryPredecessor(retryOf, { newInvocationPath, inspe
|
|
|
68
78
|
fail('invocation.json identity does not match retryOf (or is not launcher-written)');
|
|
69
79
|
}
|
|
70
80
|
if (exitRecord?.invocationId !== prevId) fail('exit.json identity does not match retryOf');
|
|
81
|
+
let bindingDigest, correction;
|
|
82
|
+
try { await verifyWorkerCaptureSeal(prevRunDir, invocationRecord, exitRecord, inspect); }
|
|
83
|
+
catch (error) { fail(`capture seal verification failed (${error.message})`); }
|
|
71
84
|
if (newInvocationPath !== undefined) {
|
|
72
85
|
// #2915 review 4 (finding 4): the SAME stable task-chain identity
|
|
73
86
|
// `launchOneWorker` publishes — role/task/group plus the brief CONTENT
|
|
@@ -79,7 +92,18 @@ export async function verifyRetryPredecessor(retryOf, { newInvocationPath, inspe
|
|
|
79
92
|
newPlan = JSON.parse(newPlanBytes.toString('utf8'));
|
|
80
93
|
newTaskDigest = await stableTaskChainDigest(newPlan.request, newPlan.invocation?.briefDigest ?? null);
|
|
81
94
|
} catch { fail('new attempt plan is missing or invalid; cannot bind the retry to a task'); }
|
|
82
|
-
|
|
95
|
+
bindingDigest = newTaskDigest;
|
|
96
|
+
if ((invocationRecord.taskBindingDigest ?? invocationRecord.taskDigest) !== newTaskDigest && !await matchesLegacyTask(invocationRecord, newPlan)) {
|
|
97
|
+
if (retryOf.correction !== true) fail('predecessor task does not match the new attempt (task-chain binding); use --retry-correction for an explicit brief/group correction');
|
|
98
|
+
if (!await matchesTaskIdentity(invocationRecord, newPlan)) {
|
|
99
|
+
fail('correction must retain the predecessor role, task and workspace (task-chain binding)');
|
|
100
|
+
}
|
|
101
|
+
const { qualificationDigest } = await importImpl(import.meta.url, 'models/qualification-digest.mjs');
|
|
102
|
+
correction = {
|
|
103
|
+
oldBriefDigest: invocationRecord.prompt.digest, newBriefDigest: newPlan.invocation.briefDigest,
|
|
104
|
+
oldGroupDigest: qualificationDigest(invocationRecord.selected.group), newGroupDigest: qualificationDigest(newPlan.request.group),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
83
107
|
}
|
|
84
108
|
if (exitRecord.termination !== 'confirmed') {
|
|
85
109
|
let evidence = null;
|
|
@@ -88,16 +112,73 @@ export async function verifyRetryPredecessor(retryOf, { newInvocationPath, inspe
|
|
|
88
112
|
const processRecord = JSON.parse((await boundedPrivateRead(join(prevRunDir, 'process.json'), 65536, inspect, true, true)).toString('utf8'));
|
|
89
113
|
evidence = await recheckWorkerTermination({ processRecord, exitRecord });
|
|
90
114
|
} catch (error) { evidence = { confirmed: false, missing: `readable, private launcher process.json (${error.message})` }; }
|
|
115
|
+
exitRecord = { ...exitRecord, terminationMissing: evidence.missing ?? null,
|
|
116
|
+
...(exitRecord.healthEvidence ? { healthEvidence: { ...exitRecord.healthEvidence, terminationMissing: evidence.missing ?? null } } : {}) };
|
|
91
117
|
if (evidence.confirmed) {
|
|
92
118
|
exitRecord = { ...exitRecord, termination: 'confirmed', descendants: 'none',
|
|
93
119
|
...(exitRecord.health ? { health: { ...exitRecord.health, termination: 'confirmed' } } : {}) };
|
|
94
|
-
await publishWorkerRecord(prevRunDir, 'exit.json', exitRecord, inspect);
|
|
95
120
|
}
|
|
121
|
+
await publishWorkerRecord(prevRunDir, 'exit.json', exitRecord, inspect);
|
|
96
122
|
}
|
|
97
123
|
if (exitRecord.termination !== 'confirmed') {
|
|
98
124
|
assertStopBeforeRetry(exitRecord, { expectedInvocationId: prevId, runDir: prevRunDir, missing: evidence?.missing });
|
|
99
125
|
}
|
|
100
126
|
}
|
|
101
127
|
assertStopBeforeRetry(exitRecord, { expectedInvocationId: prevId });
|
|
128
|
+
onVerified?.({ previousAttemptId: prevId, taskDigest: invocationRecord.taskDigest,
|
|
129
|
+
taskBindingDigest: bindingDigest, ...(correction ? { retryCorrection: correction } : {}) });
|
|
102
130
|
return exitRecord;
|
|
103
131
|
}
|
|
132
|
+
|
|
133
|
+
async function sameWorkspace(left, right) {
|
|
134
|
+
if (typeof left !== 'string' || typeof right !== 'string') return false;
|
|
135
|
+
try { return await realpath(left) === await realpath(right); }
|
|
136
|
+
catch (error) { if (error.code === 'ENOENT') return false; throw error; }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Existing capture siblings are the durable inventory, including captures from
|
|
140
|
+
// older launchers. Brief and group changes cannot hide an open task predecessor.
|
|
141
|
+
async function assertNoOpenPredecessor({ runDir, captureRoot, saved, inspect, retryOf, workerWritableRoots }) {
|
|
142
|
+
const taskDigest = await stableTaskChainDigest(saved.request, saved.invocation.briefDigest);
|
|
143
|
+
const roots = new Set([dirname(runDir), ...(captureRoot ? [captureRoot] : []), ...(retryOf ? [dirname(retryOf.runDir)] : [])]);
|
|
144
|
+
for (const root of roots) {
|
|
145
|
+
if (!await inspect(root, 'directory')) continue;
|
|
146
|
+
const entries = await withWorkerPath(root, target => readdir(target, { withFileTypes: true }));
|
|
147
|
+
for (const entry of entries) {
|
|
148
|
+
if (!entry.isDirectory()) continue;
|
|
149
|
+
const previousDir = join(root, entry.name);
|
|
150
|
+
if (resolve(previousDir) === resolve(runDir)) continue;
|
|
151
|
+
const filename = join(previousDir, 'invocation.json');
|
|
152
|
+
if (!await inspect(filename, 'file')) continue;
|
|
153
|
+
const record = JSON.parse((await boundedPrivateRead(filename, 1048576, inspect, true, true)).toString('utf8'));
|
|
154
|
+
if (record.origin !== 'launcher' || ((record.taskBindingDigest ?? record.taskDigest) !== taskDigest
|
|
155
|
+
&& !await matchesTaskIdentity(record, saved))) continue;
|
|
156
|
+
try {
|
|
157
|
+
// Reuse the authenticated owned-stop loader, including missing exits.
|
|
158
|
+
await verifyRetryPredecessor({ invocationId: record.invocationId, runDir: previousDir }, { inspect, workerWritableRoots });
|
|
159
|
+
} catch (cause) {
|
|
160
|
+
const error = new Error(`Worker launch refused: open predecessor ${record.invocationId} at ${previousDir}; termination is not confirmed. Retry with --retry-of-run-dir and owned-stop proof.`, { cause });
|
|
161
|
+
error.code = 'OPEN_TASK_PREDECESSOR';
|
|
162
|
+
error.previousAttemptId = record.invocationId;
|
|
163
|
+
error.runDir = previousDir;
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function prepareWorkerRetry(options, { saved, inspect, workerWritableRoots }) {
|
|
171
|
+
let evidence = null;
|
|
172
|
+
const retryOf = options.retryOf === undefined ? undefined : await resolveRetryReference(options.retryOf, { inspect });
|
|
173
|
+
if (retryOf !== undefined) {
|
|
174
|
+
await verifyRetryPredecessor(retryOf, { newInvocationPath: options.invocationPath, inspect, workerWritableRoots,
|
|
175
|
+
checkDescendants: options.checkDescendants, onVerified: value => { evidence = value; } });
|
|
176
|
+
}
|
|
177
|
+
await assertNoOpenPredecessor({ ...options, saved, inspect, retryOf, workerWritableRoots });
|
|
178
|
+
return evidence;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export function workerRetryRecordFields(evidence) {
|
|
182
|
+
return { ...(evidence ? { taskBindingDigest: evidence.taskBindingDigest,
|
|
183
|
+
...(evidence.retryCorrection ? { retryCorrection: evidence.retryCorrection } : {}) } : {}) };
|
|
184
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// Confirm owned containment is empty independently of semantic observer coverage.
|
|
2
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
2
3
|
import { sampleWorkerIdentity, isSafeWorkerGroupTarget } from '../process-monitor.mjs';
|
|
3
4
|
import { getWorkerIsolationStatus } from './worker-isolation-status.mjs';
|
|
4
5
|
import { scanWorkerInvocationProcesses } from './worker-invocation-processes.mjs';
|
|
@@ -9,7 +10,7 @@ const TERMINATION_MAX_ATTEMPTS = 21;
|
|
|
9
10
|
|
|
10
11
|
export function observeWorkerTermination({ mode, native, workerExited, identity,
|
|
11
12
|
supervisor = sampleWorkerIdentity(process.pid), kill = process.kill, invocationId,
|
|
12
|
-
scan = scanWorkerInvocationProcesses, deadline, now = () => performance.now() } = {}) {
|
|
13
|
+
scan = scanWorkerInvocationProcesses, groupStates = sampleGroupStates, deadline, now = () => performance.now() } = {}) {
|
|
13
14
|
if (mode === 'systemd-unit') {
|
|
14
15
|
const exited = workerExited === true || [1, 2, 3].includes(native?.execMainCode);
|
|
15
16
|
return { confirmed: exited && native?.active === false && native?.cgroupEmpty === true,
|
|
@@ -22,22 +23,71 @@ export function observeWorkerTermination({ mode, native, workerExited, identity,
|
|
|
22
23
|
|| !isSafeWorkerGroupTarget(identity.pgid, supervisor)) {
|
|
23
24
|
return { confirmed: false, missing: 'launcher-owned process group identity on this host and boot' };
|
|
24
25
|
}
|
|
25
|
-
|
|
26
|
+
let groupEmpty = false;
|
|
27
|
+
try {
|
|
28
|
+
kill(-identity.pgid, 0);
|
|
29
|
+
// Signal 0 also reaches zombies. A reaper that never waits (a container's
|
|
30
|
+
// PID 1) can leave killed members as zombies indefinitely; a group whose
|
|
31
|
+
// remaining members are all zombies has no live process (#3853).
|
|
32
|
+
// Absence from /proc is never proof; only observed zombies explaining
|
|
33
|
+
// signal 0, with no live or unreadable member, count as an empty group.
|
|
34
|
+
const states = groupStates(identity.pgid);
|
|
35
|
+
groupEmpty = !!states && states.zombies > 0 && states.live === 0 && states.unreadable === 0;
|
|
36
|
+
}
|
|
26
37
|
catch (error) {
|
|
27
38
|
// Pass the recorded leader so the marker scan can tell a parallel
|
|
28
39
|
// hardened (non-dumpable) worker's tree apart from a genuine escapee via
|
|
29
40
|
// readable stat/cgroup links instead of failing closed on every
|
|
30
41
|
// unreadable post-start process.
|
|
31
42
|
if (error.code === 'ESRCH') {
|
|
32
|
-
//
|
|
33
|
-
// leaves the
|
|
43
|
+
// A standalone observation gets one scan, at most 2 s. An explicit
|
|
44
|
+
// deadline leaves the shared budget and retries to its caller.
|
|
34
45
|
return scan({ invocationId, startTicks: identity.startTicks, uid: identity.uid, now,
|
|
35
46
|
deadline: deadline ?? now() + TERMINATION_BUDGET_MS,
|
|
36
47
|
workerPid: identity.pid, workerPgid: identity.pgid, workerSession: identity.sessionId });
|
|
37
48
|
}
|
|
38
49
|
return { confirmed: false, missing: `empty owned process group (signal-0 returned ${error.code ?? 'unknown'})` };
|
|
39
50
|
}
|
|
40
|
-
|
|
51
|
+
if (groupEmpty) {
|
|
52
|
+
return scan({ invocationId, startTicks: identity.startTicks, uid: identity.uid, now,
|
|
53
|
+
deadline: deadline ?? now() + TERMINATION_BUDGET_MS,
|
|
54
|
+
workerPid: identity.pid, workerPgid: identity.pgid, workerSession: identity.sessionId });
|
|
55
|
+
}
|
|
56
|
+
return { confirmed: false, missing: 'empty owned process group (signal-0 still finds members)', retryable: true };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Count a process group's members by /proc state: zombies, live and unreadable. */
|
|
60
|
+
export function sampleGroupStates(pgid) {
|
|
61
|
+
if (!Number.isSafeInteger(pgid) || pgid <= 0 || process.platform !== 'linux') return null;
|
|
62
|
+
const states = { zombies: 0, live: 0, unreadable: 0 };
|
|
63
|
+
let entries;
|
|
64
|
+
try { entries = readdirSync('/proc'); } catch { return null; }
|
|
65
|
+
for (const entry of entries) {
|
|
66
|
+
if (!/^\d+$/.test(entry)) continue;
|
|
67
|
+
let stat;
|
|
68
|
+
try { stat = readFileSync(`/proc/${entry}/stat`, 'utf8'); }
|
|
69
|
+
catch (error) { if (error.code !== 'ENOENT' && error.code !== 'ESRCH') states.unreadable += 1; continue; }
|
|
70
|
+
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(/\s+/);
|
|
71
|
+
if (Number(fields[2]) !== pgid) continue;
|
|
72
|
+
if (fields[0] === 'Z') states.zombies += 1; else states.live += 1;
|
|
73
|
+
}
|
|
74
|
+
return states;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Share the exit-time budget across transient kernel observations; each scan
|
|
78
|
+
// gets all remaining time so a large inventory is never restarted at 100 ms.
|
|
79
|
+
export async function settleWorkerTermination({ now = () => performance.now(),
|
|
80
|
+
wait = ms => new Promise(resolve => setTimeout(resolve, ms)), ...options } = {}) {
|
|
81
|
+
const deadline = now() + TERMINATION_BUDGET_MS;
|
|
82
|
+
let evidence;
|
|
83
|
+
for (let attempt = 0; attempt < TERMINATION_MAX_ATTEMPTS; attempt++) {
|
|
84
|
+
if (evidence && now() >= deadline) return evidence;
|
|
85
|
+
evidence = observeWorkerTermination({ ...options, now, deadline });
|
|
86
|
+
if (evidence.confirmed || evidence.markerPresent || !evidence.retryable
|
|
87
|
+
|| now() >= deadline || attempt === TERMINATION_MAX_ATTEMPTS - 1) return evidence;
|
|
88
|
+
await wait(Math.min(TERMINATION_ATTEMPT_MS, Math.max(0, deadline - now())));
|
|
89
|
+
}
|
|
90
|
+
return evidence;
|
|
41
91
|
}
|
|
42
92
|
|
|
43
93
|
export function terminationRemedy(result, runDir) {
|
|
@@ -79,7 +129,7 @@ export async function recheckWorkerTermination({ processRecord, exitRecord,
|
|
|
79
129
|
catch { /* Unavailable is never empty. */ }
|
|
80
130
|
}
|
|
81
131
|
evidence = observe({ mode: processRecord.isolation, native, identity, invocationId: processRecord.invocationId,
|
|
82
|
-
deadline
|
|
132
|
+
deadline, now,
|
|
83
133
|
workerExited: Number.isInteger(exitRecord.exitCode) && exitRecord.exitCode >= 0 && exitRecord.exitCode <= 255
|
|
84
134
|
|| exitRecord.captureCoverage === 'complete' && exitRecord.workerTermination === 'confirmed' && /^SIG[A-Z0-9]+$/.test(exitRecord.signal ?? '') });
|
|
85
135
|
if (evidence.confirmed || evidence.markerPresent || now() >= deadline || attempt === TERMINATION_MAX_ATTEMPTS - 1) return evidence;
|
|
@@ -10,14 +10,14 @@
|
|
|
10
10
|
import { fileURLToPath } from 'node:url';
|
|
11
11
|
import { EventEmitter } from 'events';
|
|
12
12
|
import { execFileSync } from 'child_process';
|
|
13
|
-
import { existsSync, readFileSync, writeFileSync, mkdirSync,
|
|
14
|
-
import { hostname } from 'node:os';
|
|
13
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from 'fs';
|
|
15
14
|
import { join, dirname } from 'path';
|
|
16
15
|
import { constants } from 'node:fs';
|
|
17
16
|
import { open, opendir, rename, unlink } from 'node:fs/promises';
|
|
18
17
|
import { createHash, randomUUID } from 'node:crypto';
|
|
19
18
|
import path from 'node:path';
|
|
20
19
|
import { createWorkerHealthObserver } from './lib/worker-health.mjs';
|
|
20
|
+
import { sampleWorkerProcIdentity } from './lib/worker-process-identity.mjs';
|
|
21
21
|
|
|
22
22
|
// This module is loaded standalone in some contexts (e.g. staged/partial
|
|
23
23
|
// installs, or copied in isolation for a test) without its sibling
|
|
@@ -68,22 +68,16 @@ if (importImpl) {
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
/**
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
return { pid, pgid: Number(fields[2]), uid, parentPid: Number(fields[1]), sessionId: Number(fields[3]),
|
|
82
|
-
host: hostname(), bootId, startTicks: fields[19], executable, state: fields[0],
|
|
83
|
-
// /proc stat fields 14/15 are utime/stime; 16/17 are cutime/cstime
|
|
84
|
-
// accumulated for waited-for children, including child-heavy CLIs.
|
|
85
|
-
cpuTicks: Number(fields[11]) + Number(fields[12]) + Number(fields[13]) + Number(fields[14]) };
|
|
86
|
-
} catch { return null; }
|
|
71
|
+
/**
|
|
72
|
+
* Side-effect-free Linux sample. Unknown identity never grants signal authority.
|
|
73
|
+
* An exited, unreaped child (zombie without /proc/<pid>/exe) counts as gone, so
|
|
74
|
+
* liveness, group-member and ancestry checks never treat it as running. Only
|
|
75
|
+
* spawn-time identity capture passes `includeExited` to keep a fast-exiting
|
|
76
|
+
* worker's birth identity (#3835).
|
|
77
|
+
*/
|
|
78
|
+
export function sampleWorkerIdentity(pid, { includeExited = false } = {}) {
|
|
79
|
+
const identity = sampleWorkerProcIdentity(pid);
|
|
80
|
+
return identity && (includeExited || identity.executable !== null) ? identity : null;
|
|
87
81
|
}
|
|
88
82
|
|
|
89
83
|
export function sameWorkerIdentity(expected, actual) {
|