@smoothbricks/nx-plugin 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -30
- package/dist/.build-inputs.json +15 -8
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +192 -82
- package/dist/executors/bounded-exec/host-memory.d.ts +47 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +40 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +104 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +55 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +146 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +29 -11
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +53 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +6 -6
- package/dist/executors/bounded-exec/verdict.d.ts +21 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +27 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +202 -77
- package/src/executors/bounded-exec/host-memory.darwin.ts +44 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +115 -0
- package/src/executors/bounded-exec/host-memory.ts +70 -0
- package/src/executors/bounded-exec/leaf-peak.ts +151 -0
- package/src/executors/bounded-exec/mbx-pool.ts +33 -13
- package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
- package/src/executors/bounded-exec/peak-waiter.ts +82 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/runner-sizing.ts +6 -6
- package/src/executors/bounded-exec/verdict.ts +40 -6
- package/src/index.ts +4 -1
- package/test-timings.json +134 -131
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { darwinMemory } from './host-memory.darwin.js';
|
|
2
|
+
import { linuxMemory } from './host-memory.linux.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The memory facts a run's CPU pool charges by (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets
|
|
6
|
+
* itself"; 13_telemetry.md, "Job resource peaks"), read by each platform's own mechanism: `host-memory.linux.ts`,
|
|
7
|
+
* `host-memory.darwin.ts`. Callers read them here and never ask which platform they run on.
|
|
8
|
+
*/
|
|
9
|
+
export interface HostMemory {
|
|
10
|
+
/** The memory the run's pool charges its leaves' peaks against. */
|
|
11
|
+
readRunMemory(): Promise<RunMemoryReading>;
|
|
12
|
+
/**
|
|
13
|
+
* A leaf's peak in bytes, from the `maxRSS` that Bun's `Subprocess.resourceUsage()` on `bunVersion` reports for
|
|
14
|
+
* it: the exited leaf's `ru_maxrss`.
|
|
15
|
+
*/
|
|
16
|
+
peakBytes(maxRss: number, bunVersion: string): number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
20
|
+
export interface RunMemory {
|
|
21
|
+
readonly bytes: number;
|
|
22
|
+
readonly source: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Why the run's memory could not be read: the file or sysctl, and what reading it gave. */
|
|
26
|
+
export interface RunMemoryError {
|
|
27
|
+
readonly operation: 'read-run-memory';
|
|
28
|
+
/**
|
|
29
|
+
* `unreadable`: reading it failed; `malformed`: it held no byte count; `unmounted`: the run's cgroup v2 hierarchy is
|
|
30
|
+
* mounted nowhere this process can see; `unsupported-platform`: the platform has no reader.
|
|
31
|
+
*/
|
|
32
|
+
readonly code: 'unreadable' | 'malformed' | 'unmounted' | 'unsupported-platform';
|
|
33
|
+
readonly subject: string;
|
|
34
|
+
readonly cause: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type RunMemoryReading =
|
|
38
|
+
| { readonly ok: true; readonly memory: RunMemory }
|
|
39
|
+
| { readonly ok: false; readonly error: RunMemoryError };
|
|
40
|
+
|
|
41
|
+
/** A run whose memory cannot be read does not start: it is not shown to have room. */
|
|
42
|
+
export class RunMemoryUnreadable extends Error {
|
|
43
|
+
constructor(readonly error: RunMemoryError) {
|
|
44
|
+
super(
|
|
45
|
+
`bounded-exec: this run's memory cannot be read (${error.operation} ${error.code} ${error.subject}: ${error.cause}); a run whose memory cannot be read is not shown to have room, so it does not start`,
|
|
46
|
+
);
|
|
47
|
+
this.name = 'RunMemoryUnreadable';
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const PLATFORMS: Partial<Record<NodeJS.Platform, HostMemory>> = { darwin: darwinMemory, linux: linuxMemory };
|
|
52
|
+
|
|
53
|
+
/** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
|
|
54
|
+
export const HOST_MEMORY: HostMemory | null = PLATFORMS[process.platform] ?? null;
|
|
55
|
+
|
|
56
|
+
/** The run's memory on this platform, or why it cannot be read. */
|
|
57
|
+
export function readRunMemory(): Promise<RunMemoryReading> {
|
|
58
|
+
if (HOST_MEMORY === null) {
|
|
59
|
+
return Promise.resolve({
|
|
60
|
+
ok: false,
|
|
61
|
+
error: {
|
|
62
|
+
operation: 'read-run-memory',
|
|
63
|
+
code: 'unsupported-platform',
|
|
64
|
+
subject: process.platform,
|
|
65
|
+
cause: 'only Linux and macOS have a reader of the run memory',
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
return HOST_MEMORY.readRunMemory();
|
|
70
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import type { IOType } from 'node:child_process';
|
|
2
|
+
import type { Readable } from 'node:stream';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import typia from 'typia';
|
|
5
|
+
import { HOST_MEMORY } from './host-memory.js';
|
|
6
|
+
import { resolveOnPath } from './runtime-pin.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A leaf's memory peak (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets itself"; 13_telemetry.md, "Job
|
|
10
|
+
* resource peaks"): the exited command's rusage `ru_maxrss`, which its waiter (`peak-waiter.ts`) reads. The leaf runs
|
|
11
|
+
* no cgroup of its own, even under a cowshed supervisor, whose subtree of the command is an observed fold inside the
|
|
12
|
+
* job's cgroup (07_api.md, `JobSubtreeHandle`), so the peak is the rusage on Linux and macOS alike. The executor writes
|
|
13
|
+
* it into the task's record, and the task's next run is charged it in its run's pool.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** The descriptor the waiter reports on: past the lifecycle channel (3), the subtree admission (4) and mbx's grant (5). */
|
|
17
|
+
export const PEAK_FD = 6;
|
|
18
|
+
|
|
19
|
+
/** The waiter's module, in the form this one was loaded in: compiled JavaScript under Nx, TypeScript under Bun. */
|
|
20
|
+
const WAITER = fileURLToPath(
|
|
21
|
+
new URL(import.meta.url.endsWith('.ts') ? './peak-waiter.ts' : './peak-waiter.js', import.meta.url),
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* One JSON line the waiter writes: the command's pid once it started it, then, once the command ended, its peak in
|
|
26
|
+
* bytes or why it has none.
|
|
27
|
+
*/
|
|
28
|
+
type WaiterLine =
|
|
29
|
+
| { readonly started: number & typia.tags.Type<'uint32'> }
|
|
30
|
+
| { readonly peakBytes: number & typia.tags.Type<'uint64'> }
|
|
31
|
+
| { readonly unmeasured: string };
|
|
32
|
+
const isWaiterLine = typia.createEquals<WaiterLine>();
|
|
33
|
+
|
|
34
|
+
/** A leaf's peak in bytes, or why it went unmeasured. */
|
|
35
|
+
export type LeafPeak =
|
|
36
|
+
| { readonly kind: 'measured'; readonly bytes: number }
|
|
37
|
+
| { readonly kind: 'unmeasured'; readonly reason: string };
|
|
38
|
+
|
|
39
|
+
/** The Bun a leaf's waiter runs on, or why its peak goes unmeasured. */
|
|
40
|
+
export type PeakWaiter =
|
|
41
|
+
| { readonly kind: 'waiter'; readonly runtime: string }
|
|
42
|
+
| { readonly kind: 'unmeasured'; readonly reason: string };
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The Bun that runs this executor, else the first `bun` on the executor's own PATH, never one a target's environment
|
|
46
|
+
* names: the waiter is the executor's, not the command's.
|
|
47
|
+
*/
|
|
48
|
+
export function peakWaiter(): PeakWaiter {
|
|
49
|
+
if (HOST_MEMORY === null) {
|
|
50
|
+
return { kind: 'unmeasured', reason: `${process.platform} has no reader of a leaf's peak` };
|
|
51
|
+
}
|
|
52
|
+
const runtime =
|
|
53
|
+
process.versions.bun === undefined ? resolveOnPath('bun', process.env.PATH, process.cwd()) : process.execPath;
|
|
54
|
+
return runtime === null
|
|
55
|
+
? { kind: 'unmeasured', reason: 'no bun is on PATH to wait on the command and read its rusage' }
|
|
56
|
+
: { kind: 'waiter', runtime };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The spawn of `file args` under the waiter on `runtime`. `descriptors` are the command's own past stderr, from fd 3:
|
|
61
|
+
* the waiter passes each one that is not ignored at its own number, and takes `PEAK_FD` for its report. The runtime's
|
|
62
|
+
* config is empty, so no preload a workspace's `bunfig.toml` names runs in the waiter.
|
|
63
|
+
*/
|
|
64
|
+
export function underWaiter(
|
|
65
|
+
runtime: string,
|
|
66
|
+
file: string,
|
|
67
|
+
args: readonly string[],
|
|
68
|
+
descriptors: readonly IOType[],
|
|
69
|
+
): { file: string; args: string[]; stdio: IOType[] } {
|
|
70
|
+
if (descriptors.length > PEAK_FD - 3) {
|
|
71
|
+
throw new RangeError(
|
|
72
|
+
`a leaf's own descriptors end before ${PEAK_FD}, the waiter's report; it names ${descriptors.length + 3}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
const passed = descriptors.flatMap((io, at) => (io === 'ignore' ? [] : [at + 3]));
|
|
76
|
+
const padded = Array.from({ length: PEAK_FD - 3 }, (_, at): IOType => descriptors[at] ?? 'ignore');
|
|
77
|
+
return {
|
|
78
|
+
file: runtime,
|
|
79
|
+
args: ['--config=/dev/null', WAITER, String(PEAK_FD), passed.join(','), file, ...args],
|
|
80
|
+
stdio: ['pipe', 'pipe', 'pipe', ...padded, 'pipe'],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** What a leaf's waiter reports on its descriptor. */
|
|
85
|
+
export interface WaiterReport {
|
|
86
|
+
/**
|
|
87
|
+
* The command's pid once the waiter started it: the leaf's runner, a member of the group the waiter leads. Null when
|
|
88
|
+
* the waiter ended without naming one.
|
|
89
|
+
*/
|
|
90
|
+
readonly started: Promise<number | null>;
|
|
91
|
+
/** Once the waiter ends: the exited command's peak, or why it reported none. */
|
|
92
|
+
readonly peak: Promise<LeafPeak>;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** What the waiter reports on `stream`: the command's pid as it starts it, and the command's peak once it ends. */
|
|
96
|
+
export function readWaiterReport(stream: Readable | null | undefined): WaiterReport {
|
|
97
|
+
if (stream === null || stream === undefined) {
|
|
98
|
+
return {
|
|
99
|
+
started: Promise.resolve(null),
|
|
100
|
+
peak: Promise.resolve({ kind: 'unmeasured', reason: 'the waiter was started without its report descriptor' }),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
// `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib es2022 from
|
|
104
|
+
// tsconfig.base.json. The report's close names null at the latest.
|
|
105
|
+
let name!: (pid: number | null) => void;
|
|
106
|
+
const started = new Promise<number | null>((named) => {
|
|
107
|
+
name = named;
|
|
108
|
+
});
|
|
109
|
+
// The waiter's last word: the line after its start, or what was wrong with a line.
|
|
110
|
+
let end: LeafPeak | null = null;
|
|
111
|
+
const line = (text: string): void => {
|
|
112
|
+
let parsed: unknown;
|
|
113
|
+
try {
|
|
114
|
+
parsed = JSON.parse(text);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
if (!(error instanceof SyntaxError)) throw error;
|
|
117
|
+
}
|
|
118
|
+
if (!isWaiterLine(parsed)) {
|
|
119
|
+
end = { kind: 'unmeasured', reason: `the waiter reported ${JSON.stringify(text)}, which is no peak` };
|
|
120
|
+
} else if ('started' in parsed) {
|
|
121
|
+
name(parsed.started);
|
|
122
|
+
} else {
|
|
123
|
+
end =
|
|
124
|
+
'peakBytes' in parsed
|
|
125
|
+
? { kind: 'measured', bytes: parsed.peakBytes }
|
|
126
|
+
: { kind: 'unmeasured', reason: parsed.unmeasured };
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
let pending = '';
|
|
130
|
+
stream.setEncoding('utf8');
|
|
131
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: the report pipe is the waiter's alone (Bun closes every descriptor it does not pass), so it ends when the waiter does, and the executor's bounds end the waiter
|
|
132
|
+
const peak = new Promise<LeafPeak>((settle) => {
|
|
133
|
+
stream.on('data', (chunk: string) => {
|
|
134
|
+
pending += chunk;
|
|
135
|
+
for (let at = pending.indexOf('\n'); at !== -1; at = pending.indexOf('\n')) {
|
|
136
|
+
line(pending.slice(0, at));
|
|
137
|
+
pending = pending.slice(at + 1);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
stream.once('error', (error: Error) => {
|
|
141
|
+
name(null);
|
|
142
|
+
settle({ kind: 'unmeasured', reason: `its report descriptor: ${error.message}` });
|
|
143
|
+
});
|
|
144
|
+
stream.once('close', () => {
|
|
145
|
+
if (pending.trim() !== '') line(pending.trim());
|
|
146
|
+
name(null);
|
|
147
|
+
settle(end ?? { kind: 'unmeasured', reason: 'the waiter ended before it could report the command' });
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
return { started, peak };
|
|
151
|
+
}
|
|
@@ -2,6 +2,7 @@ import { shellTokens } from '../../runtime-input-policy.js';
|
|
|
2
2
|
import {
|
|
3
3
|
CPU_TOKENS_ENV,
|
|
4
4
|
type Demand,
|
|
5
|
+
GRANTED_CPUS_ENV,
|
|
5
6
|
grantedSizing,
|
|
6
7
|
invocationSpan,
|
|
7
8
|
type ReservedDemand,
|
|
@@ -21,10 +22,11 @@ export interface MbxPool {
|
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
24
|
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
24
|
-
* where this run is already inside an mbx reservation:
|
|
25
|
-
* non-Cargo work shares the enclosing grant
|
|
26
|
-
* inside a job"). A second reservation would
|
|
27
|
-
* nested leaf queued behind the very work it
|
|
25
|
+
* where this run is already inside an mbx reservation: whatever holds one hands its work `MBX_SCHED_DISABLE=1` (a
|
|
26
|
+
* reserved leaf's shell, `mbx reserve`, a held cowshed job), and nested non-Cargo work shares the enclosing grant
|
|
27
|
+
* instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A second reservation would
|
|
28
|
+
* wait in the pool while its parent holds a grant, so under a full pool the nested leaf queued behind the very work it
|
|
29
|
+
* belongs to.
|
|
28
30
|
*/
|
|
29
31
|
export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
|
|
30
32
|
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
|
|
@@ -35,9 +37,10 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
|
|
|
35
37
|
|
|
36
38
|
/**
|
|
37
39
|
* How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
|
|
38
|
-
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run
|
|
39
|
-
*
|
|
40
|
-
* the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
40
|
+
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
|
|
41
|
+
* run's mbx reservation endpoint (mbx-reservations.ts) for at most its runner's count: mbx grants as many of those
|
|
42
|
+
* cores as are free once one is, and the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
43
|
+
* nextest run of test binaries
|
|
41
44
|
* already built (the inferred extracted-archive run, or one from `--archive-file`): it compiles nothing, and beneath a
|
|
42
45
|
* reservation mbx schedules none of its test processes, so they run inside the one grant. Admitted one by one under
|
|
43
46
|
* `MBX_SCHEDULER_TESTS=1`, each queued for a ticket of its own: in a downstream gate on 2026-10-30, codec tests of at
|
|
@@ -156,8 +159,9 @@ export function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment:
|
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
/**
|
|
159
|
-
*
|
|
160
|
-
*
|
|
162
|
+
* The descriptor between a pooled leaf's shell and its executor. For a Cargo leaf mbx writes one byte on it per
|
|
163
|
+
* admission (g) and permit drop (r), every Cargo descendant inheriting it through MBX_GRANT_FD. A reserved leaf's shell
|
|
164
|
+
* writes the cores it asks for on it, one line, and reads back the count the pool granted (`reservation`).
|
|
161
165
|
*/
|
|
162
166
|
export const MBX_GRANT_FD = 5;
|
|
163
167
|
|
|
@@ -185,11 +189,27 @@ export function withCountedReservation(fixture: string, command: string, leaf: R
|
|
|
185
189
|
}
|
|
186
190
|
|
|
187
191
|
/**
|
|
188
|
-
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it
|
|
189
|
-
*
|
|
190
|
-
*
|
|
192
|
+
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it. The shell asks its
|
|
193
|
+
* executor on `MBX_GRANT_FD`, one line naming the count; the executor asks its run's mbx reservation endpoint and
|
|
194
|
+
* answers the cores granted, one line, or ends the descriptor unanswered when mbx refused, so the read fails and
|
|
195
|
+
* nothing runs (executor.ts). The shell then closes the descriptor, which the command never inherits, gives the command
|
|
196
|
+
* what `mbx reserve` gives its own (`GRANTED_CPUS_ENV`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`, so mbx work
|
|
197
|
+
* inside runs under the grant), sizes the runner to the grant (`grantedSizing`), says so on stderr and runs the command
|
|
198
|
+
* in the same shell: no process starts for the reservation.
|
|
191
199
|
*/
|
|
192
200
|
function reservation(command: string, leaf: ReservedLeaf): string {
|
|
193
201
|
const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
|
|
194
|
-
|
|
202
|
+
const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
|
|
203
|
+
const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
|
|
204
|
+
return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* The cores a reserved leaf's shell asked for on `MBX_GRANT_FD` (`reservation`): a whole count from one to `most`, the
|
|
209
|
+
* most its runner may ask for; anything else is `null`.
|
|
210
|
+
*/
|
|
211
|
+
export function askedCores(line: string, most: number): number | null {
|
|
212
|
+
if (!/^[1-9]\d{0,9}$/.test(line)) return null;
|
|
213
|
+
const cores = Number(line);
|
|
214
|
+
return cores <= most ? cores : null;
|
|
195
215
|
}
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import { type ChildProcessByStdio, spawn } from 'node:child_process';
|
|
2
|
+
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { createConnection, type Socket } from 'node:net';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import type { Readable, Writable } from 'node:stream';
|
|
6
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
7
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
8
|
+
import typia, { type tags } from 'typia';
|
|
9
|
+
import { ownedSocketDirectory } from '../../testing.js';
|
|
10
|
+
import { CONTROL_MS } from './lease-wait.js';
|
|
11
|
+
import { mbxPool } from './mbx-pool.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
|
|
15
|
+
* once ran under its own `mbx reserve`, two mbx processes per task whose dynamic loading alone kept the host's vnode
|
|
16
|
+
* table recycling. With mbx's pool, the run's `preTasksExecution` starts `mbx reservations <socket>` once and publishes
|
|
17
|
+
* the socket in `RESERVATIONS_ENV`, which every task inherits; each task's executor asks it for its leaf's cores over
|
|
18
|
+
* one connection per process (`reserveCores`), and the run's `postTasksExecution` closes the endpoint's standard input,
|
|
19
|
+
* which ends it. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
|
|
20
|
+
* until its connection closes, so an executor that dies gives back what it held.
|
|
21
|
+
*/
|
|
22
|
+
export const RESERVATIONS_ENV = 'MBX_RESERVATIONS_SOCKET';
|
|
23
|
+
|
|
24
|
+
const ENDPOINT_DIRECTORY = 'mbx-r-';
|
|
25
|
+
const ENDPOINT_SOCKET = 'r.sock';
|
|
26
|
+
/** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
|
|
27
|
+
const SAID_BYTES = 4096;
|
|
28
|
+
|
|
29
|
+
/** A run's endpoint: the `mbx reservations` process, the directory its socket is in, and how it ended once it has. */
|
|
30
|
+
interface Endpoint {
|
|
31
|
+
readonly server: ChildProcessByStdio<Writable, Readable, Readable>;
|
|
32
|
+
readonly directory: string;
|
|
33
|
+
readonly ended: Promise<string>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
type Started = { ok: true; endpoint: Endpoint; socket: string } | { ok: false; error: string };
|
|
37
|
+
|
|
38
|
+
/** The endpoints this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
39
|
+
const endpoints = new Map<string, Endpoint>();
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Serve `context`'s run an mbx reservation endpoint and publish it in `env`, which every task inherits, when the
|
|
43
|
+
* environment names mbx's pool (`mbxPool`). An endpoint that does not start is said, and each reserved leaf of the run
|
|
44
|
+
* then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before.
|
|
45
|
+
*/
|
|
46
|
+
export async function serveRunReservations(
|
|
47
|
+
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
48
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
49
|
+
): Promise<void> {
|
|
50
|
+
if (mbxPool(env) === null) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const started = await startEndpoint(env);
|
|
54
|
+
if (!started.ok) {
|
|
55
|
+
logger.warn(
|
|
56
|
+
`bounded-exec: this run's mbx reservation endpoint did not start (${started.error}); its reserved leaves refuse to run`,
|
|
57
|
+
);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
endpoints.set(context.id, started.endpoint);
|
|
61
|
+
env[RESERVATIONS_ENV] = started.socket;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
|
|
65
|
+
export async function closeRunReservations(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
66
|
+
const endpoint = endpoints.get(context.id);
|
|
67
|
+
if (endpoint === undefined) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
endpoints.delete(context.id);
|
|
71
|
+
endpoint.server.stdin.end();
|
|
72
|
+
const ended = await endedWithin(endpoint, CONTROL_MS);
|
|
73
|
+
await rm(endpoint.directory, { recursive: true, force: true });
|
|
74
|
+
if (ended !== 'mbx reservations ended with status 0') {
|
|
75
|
+
logger.warn(`bounded-exec: this run's mbx reservation endpoint did not close cleanly: ${ended}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Start `mbx reservations` by its PATH name in an owned socket directory, and wait until it says it listens. */
|
|
80
|
+
async function startEndpoint(env: NodeJS.ProcessEnv): Promise<Started> {
|
|
81
|
+
const directory = await ownedSocketDirectory(ENDPOINT_DIRECTORY, ENDPOINT_SOCKET);
|
|
82
|
+
const socket = join(directory, ENDPOINT_SOCKET);
|
|
83
|
+
const server = spawn('mbx', ['reservations', socket], { env, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
84
|
+
let said = '';
|
|
85
|
+
server.stderr.setEncoding('utf8');
|
|
86
|
+
server.stderr.on('data', (text: string) => {
|
|
87
|
+
said = (said + text).slice(-SAID_BYTES);
|
|
88
|
+
});
|
|
89
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a child that runs ends with its run's postTasksExecution, which ends its standard input and bounds its exit (`endedWithin`); one that cannot run says so at once
|
|
90
|
+
const ended = new Promise<string>((resolve) => {
|
|
91
|
+
server.once('error', (error) => resolve(`mbx reservations could not run: ${error.message}`));
|
|
92
|
+
server.once('close', (code, signal) => {
|
|
93
|
+
const last = said.trim().split('\n').at(-1) ?? '';
|
|
94
|
+
resolve(
|
|
95
|
+
`mbx reservations ended ${signal === null ? `with status ${code}` : `on ${signal}`}${last === '' ? '' : `: ${last}`}`,
|
|
96
|
+
);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
const endpoint: Endpoint = { server, directory, ended };
|
|
100
|
+
const listening = await saysListening(server.stdout, ended);
|
|
101
|
+
if (listening !== null) {
|
|
102
|
+
server.kill('SIGKILL');
|
|
103
|
+
await endedWithin(endpoint, CONTROL_MS);
|
|
104
|
+
await rm(directory, { recursive: true, force: true });
|
|
105
|
+
return { ok: false, error: listening };
|
|
106
|
+
}
|
|
107
|
+
return { ok: true, endpoint, socket };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Null once the endpoint's first line is `listening`; otherwise why not, within the control-plane bound. */
|
|
111
|
+
function saysListening(stdout: Readable, ended: Promise<string>): Promise<string | null> {
|
|
112
|
+
return new Promise<string | null>((resolve) => {
|
|
113
|
+
let text = '';
|
|
114
|
+
const timer = setTimeout(
|
|
115
|
+
() => resolve(`mbx reservations did not say it was listening within ${CONTROL_MS}ms`),
|
|
116
|
+
CONTROL_MS,
|
|
117
|
+
);
|
|
118
|
+
stdout.setEncoding('utf8');
|
|
119
|
+
stdout.on('data', (chunk: string) => {
|
|
120
|
+
text += chunk;
|
|
121
|
+
const end = text.indexOf('\n');
|
|
122
|
+
if (end === -1) return;
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
const line = text.slice(0, end);
|
|
125
|
+
resolve(line === 'listening' ? null : `mbx reservations said ${JSON.stringify(line)}, not that it was listening`);
|
|
126
|
+
});
|
|
127
|
+
void ended.then((why) => {
|
|
128
|
+
clearTimeout(timer);
|
|
129
|
+
resolve(why);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** How the endpoint ended, killing it once `ms` pass first. */
|
|
135
|
+
async function endedWithin(endpoint: Endpoint, ms: number): Promise<string> {
|
|
136
|
+
let timer: NodeJS.Timeout | undefined;
|
|
137
|
+
const late = new Promise<null>((resolve) => (timer = setTimeout(() => resolve(null), ms)));
|
|
138
|
+
const ended = await Promise.race([endpoint.ended, late]);
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
if (ended !== null) {
|
|
141
|
+
return ended;
|
|
142
|
+
}
|
|
143
|
+
endpoint.server.kill('SIGKILL');
|
|
144
|
+
return `${await endpoint.ended}, killed after it outlived its standard input by ${ms}ms`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The pool's answer to one reservation: the cores it holds and a promise of why it was lost, or why there are none. */
|
|
148
|
+
export type Granted =
|
|
149
|
+
| { readonly ok: true; readonly cpus: number; readonly lost: Promise<string> }
|
|
150
|
+
| { readonly ok: false; readonly error: string };
|
|
151
|
+
|
|
152
|
+
/** One request for cores on a run's endpoint. */
|
|
153
|
+
export interface Reservation {
|
|
154
|
+
/** Settles once the pool grants or refuses it, or its connection ends first. */
|
|
155
|
+
readonly granted: Promise<Granted>;
|
|
156
|
+
/** Gives the grant back, or withdraws the request while it waits. Later calls do nothing. */
|
|
157
|
+
release(): void;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
type Uint = number & tags.Type<'uint32'>;
|
|
161
|
+
|
|
162
|
+
/** The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). */
|
|
163
|
+
type ReservationRequest =
|
|
164
|
+
| { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: 'normal' | 'low' }
|
|
165
|
+
| { type: 'release'; id: Uint };
|
|
166
|
+
|
|
167
|
+
/** The lines the endpoint answers; an added field is mbx's to add. */
|
|
168
|
+
type ReservationReply =
|
|
169
|
+
| { type: 'granted'; id: Uint; cpus: Uint & tags.Minimum<1> }
|
|
170
|
+
| { type: 'refused'; id: Uint; error: string }
|
|
171
|
+
| { type: 'error'; error: string };
|
|
172
|
+
|
|
173
|
+
const stringifyRequest = typia.json.createStringify<ReservationRequest>();
|
|
174
|
+
const parseReply = typia.json.createValidateParse<ReservationReply>();
|
|
175
|
+
|
|
176
|
+
/** A request's place on its connection: waiting for its answer, or held until released or lost. */
|
|
177
|
+
type Entry =
|
|
178
|
+
| { readonly kind: 'waiting'; readonly answer: (granted: Granted) => void }
|
|
179
|
+
| { readonly kind: 'held'; readonly lose: (why: string) => void };
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* One connection to an endpoint, shared by every reservation this process makes there. It keeps the process alive only
|
|
183
|
+
* while one of them waits or holds.
|
|
184
|
+
*/
|
|
185
|
+
class Connection {
|
|
186
|
+
readonly #socket: Socket;
|
|
187
|
+
readonly #entries = new Map<number, Entry>();
|
|
188
|
+
#nextId = 1;
|
|
189
|
+
#unread = '';
|
|
190
|
+
#closed: string | null = null;
|
|
191
|
+
|
|
192
|
+
constructor(
|
|
193
|
+
private readonly endpoint: string,
|
|
194
|
+
private readonly forget: () => void,
|
|
195
|
+
) {
|
|
196
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a Unix-socket connect completes or fails at once, and a reservation's answer waits as long as the pool's admission holds it, as `mbx reserve`'s did; the connection's end answers every request still waiting
|
|
197
|
+
this.#socket = createConnection(endpoint);
|
|
198
|
+
this.#socket.setEncoding('utf8');
|
|
199
|
+
this.#socket.unref();
|
|
200
|
+
this.#socket.on('data', (text: string) => this.#read(text));
|
|
201
|
+
this.#socket.on('error', (error) => this.#close(`mbx reservation endpoint ${this.endpoint}: ${error.message}`));
|
|
202
|
+
this.#socket.on('close', () => this.#close(`mbx reservation endpoint ${this.endpoint} closed the connection`));
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
reserve(cpus: number, priority: 'normal' | 'low'): Reservation {
|
|
206
|
+
if (this.#closed !== null) {
|
|
207
|
+
return { granted: Promise.resolve({ ok: false, error: this.#closed }), release() {} };
|
|
208
|
+
}
|
|
209
|
+
const id = this.#nextId++;
|
|
210
|
+
let answer!: (granted: Granted) => void;
|
|
211
|
+
const granted = new Promise<Granted>((resolve) => (answer = resolve));
|
|
212
|
+
this.#entries.set(id, { kind: 'waiting', answer });
|
|
213
|
+
this.#socket.ref();
|
|
214
|
+
this.#socket.write(`${stringifyRequest({ type: 'reserve', id, cpus, priority })}\n`);
|
|
215
|
+
let released = false;
|
|
216
|
+
return {
|
|
217
|
+
granted,
|
|
218
|
+
release: () => {
|
|
219
|
+
if (released) return;
|
|
220
|
+
released = true;
|
|
221
|
+
const entry = this.#entries.get(id);
|
|
222
|
+
if (entry === undefined) return;
|
|
223
|
+
this.#entries.delete(id);
|
|
224
|
+
if (entry.kind === 'waiting') entry.answer({ ok: false, error: 'released while it waited' });
|
|
225
|
+
this.#socket.write(`${stringifyRequest({ type: 'release', id })}\n`);
|
|
226
|
+
if (this.#entries.size === 0) this.#socket.unref();
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
#read(text: string): void {
|
|
232
|
+
this.#unread += text;
|
|
233
|
+
for (let end = this.#unread.indexOf('\n'); end !== -1; end = this.#unread.indexOf('\n')) {
|
|
234
|
+
const line = this.#unread.slice(0, end);
|
|
235
|
+
this.#unread = this.#unread.slice(end + 1);
|
|
236
|
+
const parsed = parseReply(line);
|
|
237
|
+
if (!parsed.success) {
|
|
238
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} answered ${JSON.stringify(line)}, which is no reply`);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
this.#answer(parsed.data);
|
|
242
|
+
if (this.#closed !== null) return;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#answer(reply: ReservationReply): void {
|
|
247
|
+
if (reply.type === 'error') {
|
|
248
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} closed this connection: ${reply.error}`);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const entry = this.#entries.get(reply.id);
|
|
252
|
+
// Released before its answer arrived: the endpoint drops what it won.
|
|
253
|
+
if (entry === undefined) return;
|
|
254
|
+
if (entry.kind === 'held') {
|
|
255
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} answered reservation ${reply.id} twice`);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (reply.type === 'granted') {
|
|
259
|
+
let lose!: (why: string) => void;
|
|
260
|
+
const lost = new Promise<string>((resolve) => (lose = resolve));
|
|
261
|
+
this.#entries.set(reply.id, { kind: 'held', lose });
|
|
262
|
+
entry.answer({ ok: true, cpus: reply.cpus, lost });
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
this.#entries.delete(reply.id);
|
|
266
|
+
entry.answer({ ok: false, error: reply.error });
|
|
267
|
+
if (this.#entries.size === 0) this.#socket.unref();
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
#close(why: string): void {
|
|
271
|
+
if (this.#closed !== null) return;
|
|
272
|
+
this.#closed = why;
|
|
273
|
+
this.forget();
|
|
274
|
+
for (const entry of this.#entries.values()) {
|
|
275
|
+
if (entry.kind === 'waiting') entry.answer({ ok: false, error: why });
|
|
276
|
+
else entry.lose(why);
|
|
277
|
+
}
|
|
278
|
+
this.#entries.clear();
|
|
279
|
+
this.#socket.destroy();
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** This process's connection to each endpoint it has asked. */
|
|
284
|
+
const connections = new Map<string, Connection>();
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Ask the endpoint at `socket` for at most `cpus` cores at `priority`, over this process's one connection to it,
|
|
288
|
+
* opened by its first request and kept while another one may come.
|
|
289
|
+
*/
|
|
290
|
+
export function reserveCores(socket: string, cpus: number, priority: 'normal' | 'low'): Reservation {
|
|
291
|
+
let connection = connections.get(socket);
|
|
292
|
+
if (connection === undefined) {
|
|
293
|
+
const opened: Connection = new Connection(socket, () => {
|
|
294
|
+
if (connections.get(socket) === opened) connections.delete(socket);
|
|
295
|
+
});
|
|
296
|
+
connections.set(socket, opened);
|
|
297
|
+
connection = opened;
|
|
298
|
+
}
|
|
299
|
+
return connection.reserve(cpus, priority);
|
|
300
|
+
}
|