@smoothbricks/nx-plugin 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/dist/.build-inputs.json +12 -6
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +115 -62
- package/dist/executors/bounded-exec/host-memory.d.ts +54 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +115 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +137 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +56 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +147 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +22 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +22 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +79 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/verdict.d.ts +18 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +21 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +31 -2
- package/package.json +2 -2
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +121 -59
- package/src/executors/bounded-exec/host-memory.darwin.ts +147 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +147 -0
- package/src/executors/bounded-exec/host-memory.ts +77 -0
- package/src/executors/bounded-exec/leaf-peak.ts +152 -0
- package/src/executors/bounded-exec/mbx-reservations.ts +30 -0
- package/src/executors/bounded-exec/peak-waiter.ts +115 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/verdict.ts +30 -1
- package/test-timings.json +136 -132
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { darwinMemory } from './host-memory.darwin.js';
|
|
2
|
+
import { linuxMemory } from './host-memory.linux.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The memory facts a run's CPU pool charges by (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets
|
|
6
|
+
* itself"; 13_telemetry.md, "Job resource peaks"), read by each platform's own mechanism: `host-memory.linux.ts`,
|
|
7
|
+
* `host-memory.darwin.ts`. Callers read them here and never ask which platform they run on.
|
|
8
|
+
*/
|
|
9
|
+
export interface HostMemory {
|
|
10
|
+
/** The memory the run's pool charges its leaves' peaks against. */
|
|
11
|
+
readRunMemory(): Promise<RunMemoryReading>;
|
|
12
|
+
/**
|
|
13
|
+
* A leaf's peak in bytes, from the `maxRSS` that Bun's `Subprocess.resourceUsage()` on `bunVersion` reports for
|
|
14
|
+
* it: the exited leaf's `ru_maxrss`.
|
|
15
|
+
*/
|
|
16
|
+
peakBytes(maxRss: number, bunVersion: string): number;
|
|
17
|
+
/**
|
|
18
|
+
* What the process tree rooted at `root` holds resident at this instant: the sum over `root` and every descendant the
|
|
19
|
+
* kernel links to it, each process's footprint where the platform keeps one (macOS `ri_phys_footprint`, which counts
|
|
20
|
+
* its compressed pages) and its resident set where it does not (Linux `VmRSS`). A process that exits between the
|
|
21
|
+
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
22
|
+
*/
|
|
23
|
+
treeResidentBytes(root: number): Promise<number>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
27
|
+
export interface RunMemory {
|
|
28
|
+
readonly bytes: number;
|
|
29
|
+
readonly source: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Why the run's memory could not be read: the file or sysctl, and what reading it gave. */
|
|
33
|
+
export interface RunMemoryError {
|
|
34
|
+
readonly operation: 'read-run-memory';
|
|
35
|
+
/**
|
|
36
|
+
* `unreadable`: reading it failed; `malformed`: it held no byte count; `unmounted`: the run's cgroup v2 hierarchy is
|
|
37
|
+
* mounted nowhere this process can see; `unsupported-platform`: the platform has no reader.
|
|
38
|
+
*/
|
|
39
|
+
readonly code: 'unreadable' | 'malformed' | 'unmounted' | 'unsupported-platform';
|
|
40
|
+
readonly subject: string;
|
|
41
|
+
readonly cause: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type RunMemoryReading =
|
|
45
|
+
| { readonly ok: true; readonly memory: RunMemory }
|
|
46
|
+
| { readonly ok: false; readonly error: RunMemoryError };
|
|
47
|
+
|
|
48
|
+
/** A run whose memory cannot be read does not start: it is not shown to have room. */
|
|
49
|
+
export class RunMemoryUnreadable extends Error {
|
|
50
|
+
constructor(readonly error: RunMemoryError) {
|
|
51
|
+
super(
|
|
52
|
+
`bounded-exec: this run's memory cannot be read (${error.operation} ${error.code} ${error.subject}: ${error.cause}); a run whose memory cannot be read is not shown to have room, so it does not start`,
|
|
53
|
+
);
|
|
54
|
+
this.name = 'RunMemoryUnreadable';
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const PLATFORMS: Partial<Record<NodeJS.Platform, HostMemory>> = { darwin: darwinMemory, linux: linuxMemory };
|
|
59
|
+
|
|
60
|
+
/** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
|
|
61
|
+
export const HOST_MEMORY: HostMemory | null = PLATFORMS[process.platform] ?? null;
|
|
62
|
+
|
|
63
|
+
/** The run's memory on this platform, or why it cannot be read. */
|
|
64
|
+
export function readRunMemory(): Promise<RunMemoryReading> {
|
|
65
|
+
if (HOST_MEMORY === null) {
|
|
66
|
+
return Promise.resolve({
|
|
67
|
+
ok: false,
|
|
68
|
+
error: {
|
|
69
|
+
operation: 'read-run-memory',
|
|
70
|
+
code: 'unsupported-platform',
|
|
71
|
+
subject: process.platform,
|
|
72
|
+
cause: 'only Linux and macOS have a reader of the run memory',
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return HOST_MEMORY.readRunMemory();
|
|
77
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import type { IOType } from 'node:child_process';
|
|
2
|
+
import type { Readable } from 'node:stream';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import typia from 'typia';
|
|
5
|
+
import { HOST_MEMORY } from './host-memory.js';
|
|
6
|
+
import { resolveOnPath } from './runtime-pin.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A leaf's memory peak (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets itself"; 13_telemetry.md, "Job
|
|
10
|
+
* resource peaks"): the largest sum of what the exited command's process tree held side by side, as its waiter
|
|
11
|
+
* (`peak-waiter.ts`) sampled it, never less than the command's rusage `ru_maxrss`, which the waiter also reads. The
|
|
12
|
+
* leaf runs no cgroup of its own, even under a cowshed supervisor, whose subtree of the command is an observed fold
|
|
13
|
+
* inside the job's cgroup (07_api.md, `JobSubtreeHandle`), so the peak is read from the leaf's own tree on Linux and
|
|
14
|
+
* macOS alike. The executor writes it into the task's record, and the task's next run is charged it in its run's pool.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** The descriptor the waiter reports on: past the lifecycle channel (3), the subtree admission (4) and mbx's grant (5). */
|
|
18
|
+
export const PEAK_FD = 6;
|
|
19
|
+
|
|
20
|
+
/** The waiter's module, in the form this one was loaded in: compiled JavaScript under Nx, TypeScript under Bun. */
|
|
21
|
+
const WAITER = fileURLToPath(
|
|
22
|
+
new URL(import.meta.url.endsWith('.ts') ? './peak-waiter.ts' : './peak-waiter.js', import.meta.url),
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* One JSON line the waiter writes: the command's pid once it started it, then, once the command ended, its peak in
|
|
27
|
+
* bytes or why it has none.
|
|
28
|
+
*/
|
|
29
|
+
type WaiterLine =
|
|
30
|
+
| { readonly started: number & typia.tags.Type<'uint32'> }
|
|
31
|
+
| { readonly peakBytes: number & typia.tags.Type<'uint64'> }
|
|
32
|
+
| { readonly unmeasured: string };
|
|
33
|
+
const isWaiterLine = typia.createEquals<WaiterLine>();
|
|
34
|
+
|
|
35
|
+
/** A leaf's peak in bytes, or why it went unmeasured. */
|
|
36
|
+
export type LeafPeak =
|
|
37
|
+
| { readonly kind: 'measured'; readonly bytes: number }
|
|
38
|
+
| { readonly kind: 'unmeasured'; readonly reason: string };
|
|
39
|
+
|
|
40
|
+
/** The Bun a leaf's waiter runs on, or why its peak goes unmeasured. */
|
|
41
|
+
export type PeakWaiter =
|
|
42
|
+
| { readonly kind: 'waiter'; readonly runtime: string }
|
|
43
|
+
| { readonly kind: 'unmeasured'; readonly reason: string };
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The Bun that runs this executor, else the first `bun` on the executor's own PATH, never one a target's environment
|
|
47
|
+
* names: the waiter is the executor's, not the command's.
|
|
48
|
+
*/
|
|
49
|
+
export function peakWaiter(): PeakWaiter {
|
|
50
|
+
if (HOST_MEMORY === null) {
|
|
51
|
+
return { kind: 'unmeasured', reason: `${process.platform} has no reader of a leaf's peak` };
|
|
52
|
+
}
|
|
53
|
+
const runtime =
|
|
54
|
+
process.versions.bun === undefined ? resolveOnPath('bun', process.env.PATH, process.cwd()) : process.execPath;
|
|
55
|
+
return runtime === null
|
|
56
|
+
? { kind: 'unmeasured', reason: 'no bun is on PATH to wait on the command and read its rusage' }
|
|
57
|
+
: { kind: 'waiter', runtime };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The spawn of `file args` under the waiter on `runtime`. `descriptors` are the command's own past stderr, from fd 3:
|
|
62
|
+
* the waiter passes each one that is not ignored at its own number, and takes `PEAK_FD` for its report. The runtime's
|
|
63
|
+
* config is empty, so no preload a workspace's `bunfig.toml` names runs in the waiter.
|
|
64
|
+
*/
|
|
65
|
+
export function underWaiter(
|
|
66
|
+
runtime: string,
|
|
67
|
+
file: string,
|
|
68
|
+
args: readonly string[],
|
|
69
|
+
descriptors: readonly IOType[],
|
|
70
|
+
): { file: string; args: string[]; stdio: IOType[] } {
|
|
71
|
+
if (descriptors.length > PEAK_FD - 3) {
|
|
72
|
+
throw new RangeError(
|
|
73
|
+
`a leaf's own descriptors end before ${PEAK_FD}, the waiter's report; it names ${descriptors.length + 3}`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
const passed = descriptors.flatMap((io, at) => (io === 'ignore' ? [] : [at + 3]));
|
|
77
|
+
const padded = Array.from({ length: PEAK_FD - 3 }, (_, at): IOType => descriptors[at] ?? 'ignore');
|
|
78
|
+
return {
|
|
79
|
+
file: runtime,
|
|
80
|
+
args: ['--config=/dev/null', WAITER, String(PEAK_FD), passed.join(','), file, ...args],
|
|
81
|
+
stdio: ['pipe', 'pipe', 'pipe', ...padded, 'pipe'],
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** What a leaf's waiter reports on its descriptor. */
|
|
86
|
+
export interface WaiterReport {
|
|
87
|
+
/**
|
|
88
|
+
* The command's pid once the waiter started it: the leaf's runner, a member of the group the waiter leads. Null when
|
|
89
|
+
* the waiter ended without naming one.
|
|
90
|
+
*/
|
|
91
|
+
readonly started: Promise<number | null>;
|
|
92
|
+
/** Once the waiter ends: the exited command's peak, or why it reported none. */
|
|
93
|
+
readonly peak: Promise<LeafPeak>;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** What the waiter reports on `stream`: the command's pid as it starts it, and the command's peak once it ends. */
|
|
97
|
+
export function readWaiterReport(stream: Readable | null | undefined): WaiterReport {
|
|
98
|
+
if (stream === null || stream === undefined) {
|
|
99
|
+
return {
|
|
100
|
+
started: Promise.resolve(null),
|
|
101
|
+
peak: Promise.resolve({ kind: 'unmeasured', reason: 'the waiter was started without its report descriptor' }),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
// `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib es2022 from
|
|
105
|
+
// tsconfig.base.json. The report's close names null at the latest.
|
|
106
|
+
let name!: (pid: number | null) => void;
|
|
107
|
+
const started = new Promise<number | null>((named) => {
|
|
108
|
+
name = named;
|
|
109
|
+
});
|
|
110
|
+
// The waiter's last word: the line after its start, or what was wrong with a line.
|
|
111
|
+
let end: LeafPeak | null = null;
|
|
112
|
+
const line = (text: string): void => {
|
|
113
|
+
let parsed: unknown;
|
|
114
|
+
try {
|
|
115
|
+
parsed = JSON.parse(text);
|
|
116
|
+
} catch (error) {
|
|
117
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
118
|
+
}
|
|
119
|
+
if (!isWaiterLine(parsed)) {
|
|
120
|
+
end = { kind: 'unmeasured', reason: `the waiter reported ${JSON.stringify(text)}, which is no peak` };
|
|
121
|
+
} else if ('started' in parsed) {
|
|
122
|
+
name(parsed.started);
|
|
123
|
+
} else {
|
|
124
|
+
end =
|
|
125
|
+
'peakBytes' in parsed
|
|
126
|
+
? { kind: 'measured', bytes: parsed.peakBytes }
|
|
127
|
+
: { kind: 'unmeasured', reason: parsed.unmeasured };
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
let pending = '';
|
|
131
|
+
stream.setEncoding('utf8');
|
|
132
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: the report pipe is the waiter's alone (Bun closes every descriptor it does not pass), so it ends when the waiter does, and the executor's bounds end the waiter
|
|
133
|
+
const peak = new Promise<LeafPeak>((settle) => {
|
|
134
|
+
stream.on('data', (chunk: string) => {
|
|
135
|
+
pending += chunk;
|
|
136
|
+
for (let at = pending.indexOf('\n'); at !== -1; at = pending.indexOf('\n')) {
|
|
137
|
+
line(pending.slice(0, at));
|
|
138
|
+
pending = pending.slice(at + 1);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
stream.once('error', (error: Error) => {
|
|
142
|
+
name(null);
|
|
143
|
+
settle({ kind: 'unmeasured', reason: `its report descriptor: ${error.message}` });
|
|
144
|
+
});
|
|
145
|
+
stream.once('close', () => {
|
|
146
|
+
if (pending.trim() !== '') line(pending.trim());
|
|
147
|
+
name(null);
|
|
148
|
+
settle(end ?? { kind: 'unmeasured', reason: 'the waiter ended before it could report the command' });
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
return { started, peak };
|
|
152
|
+
}
|
|
@@ -21,6 +21,34 @@ import { mbxPool } from './mbx-pool.js';
|
|
|
21
21
|
*/
|
|
22
22
|
export const RESERVATIONS_ENV = 'MBX_RESERVATIONS_SOCKET';
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Why the run serves its tasks no endpoint, published beside {@link RESERVATIONS_ENV} when its endpoint did not start.
|
|
26
|
+
* Every task inherits it, so each reserved leaf's refusal names the cause: a gate's failures-only output shows the
|
|
27
|
+
* leaves' refusals, never the run's own warning.
|
|
28
|
+
*/
|
|
29
|
+
export const RESERVATIONS_UNSERVED_ENV = 'BOUNDED_EXEC_RESERVATIONS_UNSERVED';
|
|
30
|
+
|
|
31
|
+
/** What a task's environment says of its run's reservation endpoint. */
|
|
32
|
+
export type RunEndpoint =
|
|
33
|
+
| { readonly kind: 'served'; readonly socket: string }
|
|
34
|
+
| { readonly kind: 'unserved'; readonly reason: string }
|
|
35
|
+
| { readonly kind: 'unpublished' };
|
|
36
|
+
|
|
37
|
+
/** The endpoint `env` names, else why its run serves none, else that its run published neither. */
|
|
38
|
+
export function runEndpoint(env: Readonly<NodeJS.ProcessEnv>): RunEndpoint {
|
|
39
|
+
const socket = env[RESERVATIONS_ENV] ?? '';
|
|
40
|
+
if (socket !== '') return { kind: 'served', socket };
|
|
41
|
+
const reason = env[RESERVATIONS_UNSERVED_ENV] ?? '';
|
|
42
|
+
return reason === '' ? { kind: 'unpublished' } : { kind: 'unserved', reason };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Why a task's run serves it no endpoint, as its reserved leaf's refusal says it. */
|
|
46
|
+
export function describeUnserved(endpoint: Exclude<RunEndpoint, { kind: 'served' }>): string {
|
|
47
|
+
return endpoint.kind === 'unserved'
|
|
48
|
+
? `the run's endpoint did not start (${endpoint.reason})`
|
|
49
|
+
: `the run published neither an endpoint nor why it has none (${RESERVATIONS_UNSERVED_ENV} is unset too), so its preTasksExecution never served this run or its environment never reached this task`;
|
|
50
|
+
}
|
|
51
|
+
|
|
24
52
|
const ENDPOINT_DIRECTORY = 'mbx-r-';
|
|
25
53
|
const ENDPOINT_SOCKET = 'r.sock';
|
|
26
54
|
/** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
|
|
@@ -55,10 +83,12 @@ export async function serveRunReservations(
|
|
|
55
83
|
logger.warn(
|
|
56
84
|
`bounded-exec: this run's mbx reservation endpoint did not start (${started.error}); its reserved leaves refuse to run`,
|
|
57
85
|
);
|
|
86
|
+
env[RESERVATIONS_UNSERVED_ENV] = started.error;
|
|
58
87
|
return;
|
|
59
88
|
}
|
|
60
89
|
endpoints.set(context.id, started.endpoint);
|
|
61
90
|
env[RESERVATIONS_ENV] = started.socket;
|
|
91
|
+
delete env[RESERVATIONS_UNSERVED_ENV];
|
|
62
92
|
}
|
|
63
93
|
|
|
64
94
|
/** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { writeSync } from 'node:fs';
|
|
2
|
+
import { HOST_MEMORY } from './host-memory.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The waiter a leaf's command runs under (`leaf-peak.ts`; specs/cowshed/13_telemetry.md, "Job resource peaks"). Started
|
|
6
|
+
* on Bun as `bun --config=/dev/null peak-waiter.js <report fd> <fds> <file> [args…]`, it spawns `file args` with its
|
|
7
|
+
* own stdin, stdout and stderr and each descriptor `fds` names (comma-separated) at its own number, writes the
|
|
8
|
+
* command's pid to the report descriptor, waits for the command, writes its peak there, each as one JSON line, and
|
|
9
|
+
* ends as the command did: with its exit code, or with its signal raised on itself.
|
|
10
|
+
*
|
|
11
|
+
* The peak is what the command's tree held side by side. Every {@link SAMPLE_MS} while the command runs, the waiter
|
|
12
|
+
* sums what the command and each descendant the kernel links to it hold (`HostMemory.treeResidentBytes`) and keeps the
|
|
13
|
+
* largest sum. At the exit it takes the larger of that and the command's `ru_maxrss`, the largest resident set any one
|
|
14
|
+
* process of the tree it waited on reached, which a spike between two samples still reaches: Node, which runs the
|
|
15
|
+
* executor, reaps a child without its rusage; Bun's `Subprocess.resourceUsage()` reports it. A tree it could not read
|
|
16
|
+
* is said on the command's stderr, and its peak is then the larger of what was read and `ru_maxrss`.
|
|
17
|
+
*
|
|
18
|
+
* The waiter leads the leaf's process group, so the executor's group signals reach the command directly: it ignores
|
|
19
|
+
* SIGINT, SIGTERM and SIGHUP and outlives the command to report it. A SIGKILL of the group ends it unreported. The
|
|
20
|
+
* command whose pid it names is the leaf's runner, which a hang snapshot samples (specs/cowshed/08_testing.md).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
interface BunSubprocess {
|
|
24
|
+
readonly exited: Promise<number>;
|
|
25
|
+
readonly exitCode: number | null;
|
|
26
|
+
readonly signalCode: NodeJS.Signals | null;
|
|
27
|
+
readonly pid: number;
|
|
28
|
+
resourceUsage(): { readonly maxRSS: number } | undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The two Bun APIs the waiter uses. */
|
|
32
|
+
interface BunSpawner {
|
|
33
|
+
readonly version: string;
|
|
34
|
+
spawn(argv: string[], options: { stdio: (number | 'inherit' | 'ignore')[] }): BunSubprocess;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function isBunSpawner(runtime: unknown): runtime is BunSpawner {
|
|
38
|
+
return (
|
|
39
|
+
typeof runtime === 'object' &&
|
|
40
|
+
runtime !== null &&
|
|
41
|
+
'spawn' in runtime &&
|
|
42
|
+
typeof runtime.spawn === 'function' &&
|
|
43
|
+
'version' in runtime &&
|
|
44
|
+
typeof runtime.version === 'string'
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** How often, in ms, the waiter sums what its command's tree holds while the command runs. */
|
|
49
|
+
const SAMPLE_MS = 250;
|
|
50
|
+
const IGNORED = ['SIGINT', 'SIGTERM', 'SIGHUP'] as const;
|
|
51
|
+
const bun: unknown = Reflect.get(globalThis, 'Bun');
|
|
52
|
+
const [reportFd, list, file, ...args] = process.argv.slice(2);
|
|
53
|
+
if (!isBunSpawner(bun) || HOST_MEMORY === null || reportFd === undefined || list === undefined || file === undefined) {
|
|
54
|
+
throw new Error('peak-waiter runs on Bun, on Linux or macOS, as `peak-waiter.js <report fd> <fds> <file> [args…]`');
|
|
55
|
+
}
|
|
56
|
+
const memory = HOST_MEMORY;
|
|
57
|
+
const report = (line: { started: number } | { peakBytes: number } | { unmeasured: string }): void => {
|
|
58
|
+
try {
|
|
59
|
+
writeSync(Number(reportFd), `${JSON.stringify(line)}\n`);
|
|
60
|
+
} catch {
|
|
61
|
+
// The executor that would read the report is gone.
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const passed = list === '' ? [] : list.split(',').map(Number);
|
|
65
|
+
const stdio: (number | 'inherit' | 'ignore')[] = ['inherit', 'inherit', 'inherit'];
|
|
66
|
+
for (let fd = 3; fd <= Math.max(2, ...passed); fd += 1) stdio.push(passed.includes(fd) ? fd : 'ignore');
|
|
67
|
+
const ignore = (): void => {};
|
|
68
|
+
for (const signal of IGNORED) process.on(signal, ignore);
|
|
69
|
+
let child: BunSubprocess;
|
|
70
|
+
try {
|
|
71
|
+
child = bun.spawn([file, ...args], { stdio });
|
|
72
|
+
} catch (error) {
|
|
73
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
74
|
+
report({ unmeasured: `the command did not start: ${reason}` });
|
|
75
|
+
writeSync(2, `bounded-exec: could not start ${file}: ${reason}\n`);
|
|
76
|
+
process.exit(1);
|
|
77
|
+
}
|
|
78
|
+
report({ started: child.pid });
|
|
79
|
+
const root = child.pid;
|
|
80
|
+
let treePeak = 0;
|
|
81
|
+
let unreadTree: string | null = null;
|
|
82
|
+
let reading: Promise<void> | null = null;
|
|
83
|
+
const sample = async (): Promise<void> => {
|
|
84
|
+
try {
|
|
85
|
+
treePeak = Math.max(treePeak, await memory.treeResidentBytes(root));
|
|
86
|
+
} catch (error) {
|
|
87
|
+
unreadTree ??= error instanceof Error ? error.message : String(error);
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
// A read still running when the next period starts takes that period too: reads of one tree never overlap.
|
|
91
|
+
const sampler = setInterval(() => {
|
|
92
|
+
reading ??= sample().finally(() => {
|
|
93
|
+
reading = null;
|
|
94
|
+
});
|
|
95
|
+
}, SAMPLE_MS);
|
|
96
|
+
await child.exited;
|
|
97
|
+
clearInterval(sampler);
|
|
98
|
+
await reading;
|
|
99
|
+
if (unreadTree !== null) {
|
|
100
|
+
writeSync(
|
|
101
|
+
2,
|
|
102
|
+
`bounded-exec: the command tree's resident sum could not be read (${unreadTree}); its peak is the larger of what was read and its largest process's ru_maxrss\n`,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
const usage = child.resourceUsage();
|
|
106
|
+
report(
|
|
107
|
+
usage === undefined
|
|
108
|
+
? { unmeasured: 'Bun read no resource usage for the exited command' }
|
|
109
|
+
: { peakBytes: Math.max(treePeak, memory.peakBytes(usage.maxRSS, bun.version)) },
|
|
110
|
+
);
|
|
111
|
+
if (child.signalCode !== null) {
|
|
112
|
+
for (const signal of IGNORED) process.removeListener(signal, ignore);
|
|
113
|
+
process.kill(process.pid, child.signalCode);
|
|
114
|
+
}
|
|
115
|
+
process.exit(child.exitCode ?? 1);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The admission decision of a CPU pool (`cpu-pool.ts`; specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets
|
|
3
|
+
* itself"): which queued requests a pool grants, in FIFO order, from the tokens it has free and, for a run's own pool,
|
|
4
|
+
* the run's memory. Pure: the pool reads its jobserver fifo for the free count and keeps the holders' charges, and this
|
|
5
|
+
* decides.
|
|
6
|
+
*
|
|
7
|
+
* Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
|
|
8
|
+
* leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
|
|
9
|
+
* it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
|
|
10
|
+
* memory peak, and nothing while its task has none. A leaf whose own charge alone exceeds the run's memory could never
|
|
11
|
+
* fit, so it is granted once nothing else holds memory, and the grant says so.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** What a queued request asks: whole tokens, at least one, and its charge in bytes (0 for a task with no peak). */
|
|
15
|
+
export interface PoolAsk {
|
|
16
|
+
readonly want: number;
|
|
17
|
+
readonly memory: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The run's memory a pool charges against, and the bytes its holders are charged; null for a pool that charges none. */
|
|
21
|
+
export interface MemoryLedger {
|
|
22
|
+
readonly capacity: number;
|
|
23
|
+
readonly held: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** One request's grant: the request, its tokens, and whether its charge alone exceeds the run's memory. */
|
|
27
|
+
export interface PoolGrant<Ask extends PoolAsk> {
|
|
28
|
+
readonly request: Ask;
|
|
29
|
+
readonly tokens: number;
|
|
30
|
+
readonly oversized: boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
|
|
35
|
+
* tokens (`cpu`) or the run's memory (`memory`). `waiting` is null when every request was granted.
|
|
36
|
+
*/
|
|
37
|
+
export interface Admission<Ask extends PoolAsk> {
|
|
38
|
+
readonly grants: readonly PoolGrant<Ask>[];
|
|
39
|
+
readonly waiting: 'cpu' | 'memory' | null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function admit<Ask extends PoolAsk>(
|
|
43
|
+
queue: readonly Ask[],
|
|
44
|
+
available: number,
|
|
45
|
+
memory: MemoryLedger | null,
|
|
46
|
+
): Admission<Ask> {
|
|
47
|
+
const grants: PoolGrant<Ask>[] = [];
|
|
48
|
+
let free = available;
|
|
49
|
+
let held = memory?.held ?? 0;
|
|
50
|
+
for (const request of queue) {
|
|
51
|
+
const oversized = memory !== null && request.memory > memory.capacity;
|
|
52
|
+
if (memory !== null && held + request.memory > memory.capacity && !(oversized && held === 0)) {
|
|
53
|
+
return { grants, waiting: 'memory' };
|
|
54
|
+
}
|
|
55
|
+
const tokens = Math.min(request.want, free);
|
|
56
|
+
if (tokens === 0) {
|
|
57
|
+
return { grants, waiting: 'cpu' };
|
|
58
|
+
}
|
|
59
|
+
free -= tokens;
|
|
60
|
+
held += request.memory;
|
|
61
|
+
grants.push({ request, tokens, oversized });
|
|
62
|
+
}
|
|
63
|
+
return { grants, waiting: null };
|
|
64
|
+
}
|
|
@@ -5,6 +5,7 @@ import { logger } from 'nx/src/devkit-exports.js';
|
|
|
5
5
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
6
6
|
import { type CloseOutcome, type CpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
7
7
|
import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
|
|
8
|
+
import { type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
8
9
|
import { mbxPool } from './mbx-pool.js';
|
|
9
10
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
10
11
|
|
|
@@ -23,6 +24,10 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
23
24
|
* one token per core, or the inherited positive `BOUNDED_EXEC_CPU_TOKENS` when that is fewer: inside a cowshed job
|
|
24
25
|
* that count is the job's own, so the job's `--want` bounds the run.
|
|
25
26
|
*
|
|
27
|
+
* The pool also charges memory: each leaf is charged its task's last recorded peak, and granted only while the charges
|
|
28
|
+
* of the leaves holding grants and its own fit the run's memory (`host-memory.ts`: the cgroup's `memory.max` on Linux,
|
|
29
|
+
* else the host's). A run whose memory cannot be read does not start.
|
|
30
|
+
*
|
|
26
31
|
* The pool runs on a thread of its own (`cpu-pool-thread.ts`). The process that serves it, the Nx daemon when installed
|
|
27
32
|
* plugins load in it or the Nx CLI without one, hashes the run's tasks and builds its project graph in synchronous calls
|
|
28
33
|
* on its main thread. In a gate on 2026-10-28 one `hashPlans` ran for 14.2 s, and sixteen runners queued on a pool on
|
|
@@ -67,20 +72,26 @@ export function runPoolTokens(env: Readonly<NodeJS.ProcessEnv>, cores: number):
|
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
/**
|
|
70
|
-
* Serve `context`'s run a pool of `runPoolTokens
|
|
71
|
-
*
|
|
72
|
-
*
|
|
75
|
+
* Serve `context`'s run a pool of `runPoolTokens`, charged against the run's memory `readMemory` reads, and publish it
|
|
76
|
+
* in `env`, which every task inherits, unless the run is served none. A run whose memory cannot be read throws
|
|
77
|
+
* `RunMemoryUnreadable`, which fails the run before any task starts: it is not shown to have room. A pool that cannot
|
|
78
|
+
* be made is said, and the run's tasks are then sized only by the ceilings they inherit.
|
|
73
79
|
*/
|
|
74
80
|
export async function serveRunCpuPool(
|
|
75
81
|
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
76
82
|
env: NodeJS.ProcessEnv = process.env,
|
|
77
83
|
cores: number = availableParallelism(),
|
|
84
|
+
readMemory: () => Promise<RunMemoryReading> = readRunMemory,
|
|
78
85
|
): Promise<void> {
|
|
79
86
|
const tokens = runPoolTokens(env, cores);
|
|
80
87
|
if (tokens === null) {
|
|
81
88
|
return;
|
|
82
89
|
}
|
|
83
|
-
const
|
|
90
|
+
const memory = await readMemory();
|
|
91
|
+
if (!memory.ok) {
|
|
92
|
+
throw new RunMemoryUnreadable(memory.error);
|
|
93
|
+
}
|
|
94
|
+
const served = await servePoolThread(tokens, memory.memory.bytes);
|
|
84
95
|
if (!served.ok) {
|
|
85
96
|
logger.warn(
|
|
86
97
|
`bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`,
|
|
@@ -99,8 +110,11 @@ export async function serveRunCpuPool(
|
|
|
99
110
|
* has said how its pool closed fails whichever wait is open, a later close returns that failure, and a thread lost while
|
|
100
111
|
* its pool serves is said at once.
|
|
101
112
|
*/
|
|
102
|
-
function servePoolThread(tokens: number): Promise<ServedPool> {
|
|
103
|
-
const thread = new Worker(THREAD_ENTRY, {
|
|
113
|
+
function servePoolThread(tokens: number, memory: number): Promise<ServedPool> {
|
|
114
|
+
const thread = new Worker(THREAD_ENTRY, {
|
|
115
|
+
workerData: { tokens, memory } satisfies CpuPoolThreadData,
|
|
116
|
+
...THREAD_PRELOAD,
|
|
117
|
+
});
|
|
104
118
|
return new Promise<ServedPool>((served) => {
|
|
105
119
|
let state: ThreadState = { kind: 'starting', served };
|
|
106
120
|
let closing: Promise<CloseOutcome> | undefined;
|
|
@@ -101,6 +101,13 @@ export interface BoundedExecRecord {
|
|
|
101
101
|
* expired; written for the reader of a refused run, never read back.
|
|
102
102
|
*/
|
|
103
103
|
readonly cpuWait?: CpuWaitRow;
|
|
104
|
+
/**
|
|
105
|
+
* The command's memory peak in bytes when it ended (specs/cowshed/13_telemetry.md, "Job resource peaks"): its rusage
|
|
106
|
+
* `ru_maxrss`, the largest resident set any one process of its tree reached, as its waiter read it (`leaf-peak.ts`).
|
|
107
|
+
* Null when it was not measured: the command never ran, the waiter ended before it, or no Bun waited on it; never
|
|
108
|
+
* zero for that. The task's next run is charged it in its run's pool (`readLastPeak`).
|
|
109
|
+
*/
|
|
110
|
+
readonly peakBytes: number | null;
|
|
104
111
|
}
|
|
105
112
|
|
|
106
113
|
/** The Nx task a run executes as. Nx gives an executor no task id, only its parts. */
|
|
@@ -488,6 +495,23 @@ export async function readRecord(directory: string): Promise<BoundedExecRecord |
|
|
|
488
495
|
return parseRecord(JSON.parse(text));
|
|
489
496
|
}
|
|
490
497
|
|
|
498
|
+
/**
|
|
499
|
+
* The peak the task's last record in `directory` names: its charge in its run's pool. Null when the task has no record
|
|
500
|
+
* or its record no peak; a record that cannot be read is an error naming it, never a charge of nothing.
|
|
501
|
+
*/
|
|
502
|
+
export async function readLastPeak(
|
|
503
|
+
directory: string,
|
|
504
|
+
): Promise<{ ok: true; peakBytes: number | null } | { ok: false; error: string }> {
|
|
505
|
+
try {
|
|
506
|
+
return { ok: true, peakBytes: (await readRecord(directory))?.peakBytes ?? null };
|
|
507
|
+
} catch (error) {
|
|
508
|
+
return {
|
|
509
|
+
ok: false,
|
|
510
|
+
error: `the task's last record ${join(directory, VERDICT_FILE)} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
491
515
|
/** The JUnit report a command wrote into `directory`, or `null` when it wrote none. */
|
|
492
516
|
export async function readJunitReport(
|
|
493
517
|
directory: string,
|
|
@@ -511,7 +535,12 @@ function parseRecord(value: unknown): BoundedExecRecord {
|
|
|
511
535
|
if (!isObject(value) || typeof value.task !== 'string' || !(typeof value.hash === 'string' || value.hash === null)) {
|
|
512
536
|
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
513
537
|
}
|
|
514
|
-
|
|
538
|
+
// A record written before peaks were recorded carries none.
|
|
539
|
+
const peakBytes = value.peakBytes ?? null;
|
|
540
|
+
if (!(peakBytes === null || (typeof peakBytes === 'number' && Number.isSafeInteger(peakBytes) && peakBytes >= 0))) {
|
|
541
|
+
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
542
|
+
}
|
|
543
|
+
return { task: value.task, hash: value.hash, verdict: parseVerdict(value.verdict), peakBytes };
|
|
515
544
|
}
|
|
516
545
|
|
|
517
546
|
function parseVerdict(value: unknown): BoundedExecVerdict {
|