@smoothbricks/nx-plugin 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -27
- package/dist/.build-inputs.json +6 -5
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +95 -38
- package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +29 -11
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +6 -6
- package/dist/executors/bounded-exec/verdict.d.ts +3 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +6 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/executor.ts +100 -37
- package/src/executors/bounded-exec/mbx-pool.ts +33 -13
- package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
- package/src/executors/bounded-exec/runner-sizing.ts +6 -6
- package/src/executors/bounded-exec/verdict.ts +10 -5
- package/src/index.ts +4 -1
- package/test-timings.json +1 -0
|
@@ -38,8 +38,17 @@ import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
|
38
38
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
39
39
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
40
40
|
import type { LockWait } from './lock-wait.js';
|
|
41
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
askedCores,
|
|
43
|
+
MBX_GRANT_FD,
|
|
44
|
+
mbxIdleGated,
|
|
45
|
+
mbxLeaf,
|
|
46
|
+
mbxPool,
|
|
47
|
+
withCountedReservation,
|
|
48
|
+
withMbxLeaf,
|
|
49
|
+
} from './mbx-pool.js';
|
|
42
50
|
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
51
|
+
import { RESERVATIONS_ENV, type Reservation, reserveCores } from './mbx-reservations.js';
|
|
43
52
|
import { createNoSpaceWatch } from './no-space.js';
|
|
44
53
|
import {
|
|
45
54
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -568,8 +577,11 @@ export async function runBoundedExec(
|
|
|
568
577
|
sized = sizeRunner(demand, requested, demand.want);
|
|
569
578
|
}
|
|
570
579
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
571
|
-
// descendant still draws on tokens the grant has returned.
|
|
580
|
+
// descendant still draws on tokens the grant has returned. A reserved leaf's mbx grant covers the leaf's conclusion,
|
|
581
|
+
// its subtree's retirement included (11_shell.md), so it goes back after `releaseLease`.
|
|
572
582
|
let pool: CpuPool | null = null;
|
|
583
|
+
let reservation: Reservation | null = null;
|
|
584
|
+
const releaseReservation = (): void => reservation?.release();
|
|
573
585
|
let cpuReleased: Promise<void> | null = null;
|
|
574
586
|
const releaseCpu = (): Promise<void> => {
|
|
575
587
|
cpuReleased ??= (async () => {
|
|
@@ -596,8 +608,9 @@ export async function runBoundedExec(
|
|
|
596
608
|
let unconfirmedAdmission: string | null = null;
|
|
597
609
|
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
598
610
|
let confirmedAdmissionAt: number | null = null;
|
|
611
|
+
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
599
612
|
const queueWait = (): AdmissionQueueWait | null =>
|
|
600
|
-
mbxAdmitted || lastQueueRequests.length === 0
|
|
613
|
+
mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
601
614
|
? null
|
|
602
615
|
: {
|
|
603
616
|
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
@@ -679,6 +692,14 @@ export async function runBoundedExec(
|
|
|
679
692
|
if (parentRefusal !== null) {
|
|
680
693
|
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
681
694
|
}
|
|
695
|
+
// A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
|
|
696
|
+
// none said why in its own output; the leaf does not run beside the pool.
|
|
697
|
+
const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
|
|
698
|
+
if (reserved !== null && reserved.endpoint === '') {
|
|
699
|
+
return refuse(
|
|
700
|
+
`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`,
|
|
701
|
+
);
|
|
702
|
+
}
|
|
682
703
|
|
|
683
704
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
684
705
|
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
@@ -730,7 +751,8 @@ export async function runBoundedExec(
|
|
|
730
751
|
: withMbxLeaf(unadmitted, leaf);
|
|
731
752
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
732
753
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
733
|
-
// fd 4 is unused without a subtree admission, so
|
|
754
|
+
// fd 4 is unused without a subtree admission, so mbx's descriptor takes fd 5 either way: a Cargo leaf's admission
|
|
755
|
+
// stream, or a reserved leaf's ask for cores and their count (mbx-pool.ts, `MBX_GRANT_FD`).
|
|
734
756
|
const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
735
757
|
const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
|
|
736
758
|
const root = owner?.rootArgv(shellCommand);
|
|
@@ -740,7 +762,7 @@ export async function runBoundedExec(
|
|
|
740
762
|
resultWriter = results?.ok ? results.writer : null;
|
|
741
763
|
try {
|
|
742
764
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
743
|
-
// cowshed closes fd 4 before exec; fd 5
|
|
765
|
+
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant.
|
|
744
766
|
child =
|
|
745
767
|
root === undefined
|
|
746
768
|
? spawn(shellCommand, [], {
|
|
@@ -882,7 +904,7 @@ export async function runBoundedExec(
|
|
|
882
904
|
if (state.settled || state.expiry !== null) return;
|
|
883
905
|
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
884
906
|
if (scheduled?.kind !== kind || scheduled.at !== at) return;
|
|
885
|
-
if (kind === 'idle' && leaf
|
|
907
|
+
if (kind === 'idle' && leaf?.kind === 'cargo') {
|
|
886
908
|
if (checkingIdle) return;
|
|
887
909
|
checkingIdle = true;
|
|
888
910
|
armBoundTimer();
|
|
@@ -937,8 +959,6 @@ export async function runBoundedExec(
|
|
|
937
959
|
};
|
|
938
960
|
const refreshQueue = async (fresh: boolean): Promise<void> => {
|
|
939
961
|
if (fresh && queueSnapshot !== null) await queueSnapshot;
|
|
940
|
-
// A leaf that counts its units before reserving waits in no reservation queue while its fixture builds.
|
|
941
|
-
if (counted && !fixtureDone) return;
|
|
942
962
|
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
|
|
943
963
|
const pid = child.pid;
|
|
944
964
|
queueSnapshot ??= (async () => {
|
|
@@ -1079,13 +1099,14 @@ export async function runBoundedExec(
|
|
|
1079
1099
|
clearTimeout(boundTimer);
|
|
1080
1100
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
1081
1101
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
1082
|
-
signalCleanup = (leaf
|
|
1102
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
|
|
1083
1103
|
.finally(() => {
|
|
1084
1104
|
stopQueueObservation();
|
|
1085
1105
|
return killChildTree();
|
|
1086
1106
|
})
|
|
1087
1107
|
.finally(releaseCpu)
|
|
1088
1108
|
.finally(() => releaseLease(true))
|
|
1109
|
+
.finally(releaseReservation)
|
|
1089
1110
|
.finally(async () =>
|
|
1090
1111
|
recordRun(
|
|
1091
1112
|
signalToExitCode(signal),
|
|
@@ -1257,46 +1278,87 @@ export async function runBoundedExec(
|
|
|
1257
1278
|
fixtureFirst: counted && lifecycle !== null,
|
|
1258
1279
|
queued,
|
|
1259
1280
|
});
|
|
1260
|
-
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1261
1281
|
if (leaf !== null) {
|
|
1262
1282
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1263
|
-
if (!(channel instanceof
|
|
1264
|
-
let said = false;
|
|
1283
|
+
if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1265
1284
|
const failGrant = (detail: string): void => {
|
|
1266
1285
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
1267
1286
|
lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
|
|
1268
1287
|
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1269
1288
|
termination = terminate();
|
|
1270
1289
|
};
|
|
1271
|
-
channel.on('
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1290
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1291
|
+
if (reserved !== null) {
|
|
1292
|
+
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1293
|
+
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
1294
|
+
const { leaf: reservedLeaf, endpoint } = reserved;
|
|
1295
|
+
let asked = '';
|
|
1296
|
+
channel.setEncoding('utf8');
|
|
1297
|
+
channel.on('data', (text: string) => {
|
|
1298
|
+
if (reservation !== null) {
|
|
1299
|
+
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
asked += text;
|
|
1303
|
+
const end = asked.indexOf('\n');
|
|
1304
|
+
if (end === -1) return;
|
|
1305
|
+
const line = asked.slice(0, end);
|
|
1306
|
+
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1307
|
+
if (cpus === null) {
|
|
1308
|
+
failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
|
|
1309
|
+
channel.end();
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
|
|
1313
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1314
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1315
|
+
reservation = held;
|
|
1316
|
+
void held.granted.then((granted) => {
|
|
1317
|
+
if (!granted.ok) {
|
|
1318
|
+
failGrant(`mbx did not grant the reservation: ${granted.error}`);
|
|
1319
|
+
channel.end();
|
|
1320
|
+
return;
|
|
1282
1321
|
}
|
|
1322
|
+
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1323
|
+
if (state.settled || state.expiry !== null) return;
|
|
1324
|
+
mbxAdmitted = true;
|
|
1325
|
+
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1326
|
+
channel.end(`${granted.cpus}\n`);
|
|
1283
1327
|
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1328
|
+
void granted.lost.then((why) => {
|
|
1329
|
+
if (state.settled || cpuReleased !== null) return;
|
|
1330
|
+
appendStderr(
|
|
1331
|
+
`cowshed: mbx reservation lost while the command ran (${why}); it runs on unheld until it ends\n`,
|
|
1332
|
+
);
|
|
1333
|
+
});
|
|
1334
|
+
});
|
|
1335
|
+
});
|
|
1336
|
+
} else {
|
|
1337
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1338
|
+
let said = false;
|
|
1339
|
+
channel.on('data', (chunk: Buffer) => {
|
|
1340
|
+
for (const byte of chunk) {
|
|
1341
|
+
if (byte === 103) {
|
|
1342
|
+
if (!said) {
|
|
1343
|
+
mbxAdmitted = true;
|
|
1344
|
+
said = true;
|
|
1345
|
+
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1346
|
+
}
|
|
1347
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1348
|
+
} else if (byte === 114) {
|
|
1349
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1350
|
+
} else {
|
|
1351
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1352
|
+
return;
|
|
1353
|
+
}
|
|
1289
1354
|
}
|
|
1290
|
-
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
if (leaf !== null) {
|
|
1296
|
-
queueTimer = setInterval(() => {
|
|
1355
|
+
});
|
|
1356
|
+
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1357
|
+
queueTimer = setInterval(() => {
|
|
1358
|
+
void refreshQueue(false);
|
|
1359
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1297
1360
|
void refreshQueue(false);
|
|
1298
|
-
}
|
|
1299
|
-
void refreshQueue(false);
|
|
1361
|
+
}
|
|
1300
1362
|
}
|
|
1301
1363
|
|
|
1302
1364
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
@@ -1356,6 +1418,7 @@ export async function runBoundedExec(
|
|
|
1356
1418
|
}
|
|
1357
1419
|
}
|
|
1358
1420
|
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
1421
|
+
releaseReservation();
|
|
1359
1422
|
|
|
1360
1423
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1361
1424
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
@@ -2,6 +2,7 @@ import { shellTokens } from '../../runtime-input-policy.js';
|
|
|
2
2
|
import {
|
|
3
3
|
CPU_TOKENS_ENV,
|
|
4
4
|
type Demand,
|
|
5
|
+
GRANTED_CPUS_ENV,
|
|
5
6
|
grantedSizing,
|
|
6
7
|
invocationSpan,
|
|
7
8
|
type ReservedDemand,
|
|
@@ -21,10 +22,11 @@ export interface MbxPool {
|
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
24
|
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
24
|
-
* where this run is already inside an mbx reservation:
|
|
25
|
-
* non-Cargo work shares the enclosing grant
|
|
26
|
-
* inside a job"). A second reservation would
|
|
27
|
-
* nested leaf queued behind the very work it
|
|
25
|
+
* where this run is already inside an mbx reservation: whatever holds one hands its work `MBX_SCHED_DISABLE=1` (a
|
|
26
|
+
* reserved leaf's shell, `mbx reserve`, a held cowshed job), and nested non-Cargo work shares the enclosing grant
|
|
27
|
+
* instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A second reservation would
|
|
28
|
+
* wait in the pool while its parent holds a grant, so under a full pool the nested leaf queued behind the very work it
|
|
29
|
+
* belongs to.
|
|
28
30
|
*/
|
|
29
31
|
export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
|
|
30
32
|
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
|
|
@@ -35,9 +37,10 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
|
|
|
35
37
|
|
|
36
38
|
/**
|
|
37
39
|
* How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
|
|
38
|
-
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run
|
|
39
|
-
*
|
|
40
|
-
* the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
40
|
+
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
|
|
41
|
+
* run's mbx reservation endpoint (mbx-reservations.ts) for at most its runner's count: mbx grants as many of those
|
|
42
|
+
* cores as are free once one is, and the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
43
|
+
* nextest run of test binaries
|
|
41
44
|
* already built (the inferred extracted-archive run, or one from `--archive-file`): it compiles nothing, and beneath a
|
|
42
45
|
* reservation mbx schedules none of its test processes, so they run inside the one grant. Admitted one by one under
|
|
43
46
|
* `MBX_SCHEDULER_TESTS=1`, each queued for a ticket of its own: in a downstream gate on 2026-10-30, codec tests of at
|
|
@@ -156,8 +159,9 @@ export function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment:
|
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
/**
|
|
159
|
-
*
|
|
160
|
-
*
|
|
162
|
+
* The descriptor between a pooled leaf's shell and its executor. For a Cargo leaf mbx writes one byte on it per
|
|
163
|
+
* admission (g) and permit drop (r), every Cargo descendant inheriting it through MBX_GRANT_FD. A reserved leaf's shell
|
|
164
|
+
* writes the cores it asks for on it, one line, and reads back the count the pool granted (`reservation`).
|
|
161
165
|
*/
|
|
162
166
|
export const MBX_GRANT_FD = 5;
|
|
163
167
|
|
|
@@ -185,11 +189,27 @@ export function withCountedReservation(fixture: string, command: string, leaf: R
|
|
|
185
189
|
}
|
|
186
190
|
|
|
187
191
|
/**
|
|
188
|
-
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it
|
|
189
|
-
*
|
|
190
|
-
*
|
|
192
|
+
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it. The shell asks its
|
|
193
|
+
* executor on `MBX_GRANT_FD`, one line naming the count; the executor asks its run's mbx reservation endpoint and
|
|
194
|
+
* answers the cores granted, one line, or ends the descriptor unanswered when mbx refused, so the read fails and
|
|
195
|
+
* nothing runs (executor.ts). The shell then closes the descriptor, which the command never inherits, gives the command
|
|
196
|
+
* what `mbx reserve` gives its own (`GRANTED_CPUS_ENV`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`, so mbx work
|
|
197
|
+
* inside runs under the grant), sizes the runner to the grant (`grantedSizing`), says so on stderr and runs the command
|
|
198
|
+
* in the same shell: no process starts for the reservation.
|
|
191
199
|
*/
|
|
192
200
|
function reservation(command: string, leaf: ReservedLeaf): string {
|
|
193
201
|
const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
|
|
194
|
-
|
|
202
|
+
const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
|
|
203
|
+
const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
|
|
204
|
+
return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* The cores a reserved leaf's shell asked for on `MBX_GRANT_FD` (`reservation`): a whole count from one to `most`, the
|
|
209
|
+
* most its runner may ask for; anything else is `null`.
|
|
210
|
+
*/
|
|
211
|
+
export function askedCores(line: string, most: number): number | null {
|
|
212
|
+
if (!/^[1-9]\d{0,9}$/.test(line)) return null;
|
|
213
|
+
const cores = Number(line);
|
|
214
|
+
return cores <= most ? cores : null;
|
|
195
215
|
}
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import { type ChildProcessByStdio, spawn } from 'node:child_process';
|
|
2
|
+
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { createConnection, type Socket } from 'node:net';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import type { Readable, Writable } from 'node:stream';
|
|
6
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
7
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
8
|
+
import typia, { type tags } from 'typia';
|
|
9
|
+
import { ownedSocketDirectory } from '../../testing.js';
|
|
10
|
+
import { CONTROL_MS } from './lease-wait.js';
|
|
11
|
+
import { mbxPool } from './mbx-pool.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
|
|
15
|
+
* once ran under its own `mbx reserve`, two mbx processes per task whose dynamic loading alone kept the host's vnode
|
|
16
|
+
* table recycling. With mbx's pool, the run's `preTasksExecution` starts `mbx reservations <socket>` once and publishes
|
|
17
|
+
* the socket in `RESERVATIONS_ENV`, which every task inherits; each task's executor asks it for its leaf's cores over
|
|
18
|
+
* one connection per process (`reserveCores`), and the run's `postTasksExecution` closes the endpoint's standard input,
|
|
19
|
+
* which ends it. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
|
|
20
|
+
* until its connection closes, so an executor that dies gives back what it held.
|
|
21
|
+
*/
|
|
22
|
+
export const RESERVATIONS_ENV = 'MBX_RESERVATIONS_SOCKET';
|
|
23
|
+
|
|
24
|
+
const ENDPOINT_DIRECTORY = 'mbx-r-';
|
|
25
|
+
const ENDPOINT_SOCKET = 'r.sock';
|
|
26
|
+
/** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
|
|
27
|
+
const SAID_BYTES = 4096;
|
|
28
|
+
|
|
29
|
+
/** A run's endpoint: the `mbx reservations` process, the directory its socket is in, and how it ended once it has. */
|
|
30
|
+
interface Endpoint {
|
|
31
|
+
readonly server: ChildProcessByStdio<Writable, Readable, Readable>;
|
|
32
|
+
readonly directory: string;
|
|
33
|
+
readonly ended: Promise<string>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
type Started = { ok: true; endpoint: Endpoint; socket: string } | { ok: false; error: string };
|
|
37
|
+
|
|
38
|
+
/** The endpoints this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
39
|
+
const endpoints = new Map<string, Endpoint>();
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Serve `context`'s run an mbx reservation endpoint and publish it in `env`, which every task inherits, when the
|
|
43
|
+
* environment names mbx's pool (`mbxPool`). An endpoint that does not start is said, and each reserved leaf of the run
|
|
44
|
+
* then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before.
|
|
45
|
+
*/
|
|
46
|
+
export async function serveRunReservations(
|
|
47
|
+
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
48
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
49
|
+
): Promise<void> {
|
|
50
|
+
if (mbxPool(env) === null) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const started = await startEndpoint(env);
|
|
54
|
+
if (!started.ok) {
|
|
55
|
+
logger.warn(
|
|
56
|
+
`bounded-exec: this run's mbx reservation endpoint did not start (${started.error}); its reserved leaves refuse to run`,
|
|
57
|
+
);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
endpoints.set(context.id, started.endpoint);
|
|
61
|
+
env[RESERVATIONS_ENV] = started.socket;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
|
|
65
|
+
export async function closeRunReservations(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
66
|
+
const endpoint = endpoints.get(context.id);
|
|
67
|
+
if (endpoint === undefined) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
endpoints.delete(context.id);
|
|
71
|
+
endpoint.server.stdin.end();
|
|
72
|
+
const ended = await endedWithin(endpoint, CONTROL_MS);
|
|
73
|
+
await rm(endpoint.directory, { recursive: true, force: true });
|
|
74
|
+
if (ended !== 'mbx reservations ended with status 0') {
|
|
75
|
+
logger.warn(`bounded-exec: this run's mbx reservation endpoint did not close cleanly: ${ended}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Start `mbx reservations` by its PATH name in an owned socket directory, and wait until it says it listens. */
|
|
80
|
+
async function startEndpoint(env: NodeJS.ProcessEnv): Promise<Started> {
|
|
81
|
+
const directory = await ownedSocketDirectory(ENDPOINT_DIRECTORY, ENDPOINT_SOCKET);
|
|
82
|
+
const socket = join(directory, ENDPOINT_SOCKET);
|
|
83
|
+
const server = spawn('mbx', ['reservations', socket], { env, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
84
|
+
let said = '';
|
|
85
|
+
server.stderr.setEncoding('utf8');
|
|
86
|
+
server.stderr.on('data', (text: string) => {
|
|
87
|
+
said = (said + text).slice(-SAID_BYTES);
|
|
88
|
+
});
|
|
89
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a child that runs ends with its run's postTasksExecution, which ends its standard input and bounds its exit (`endedWithin`); one that cannot run says so at once
|
|
90
|
+
const ended = new Promise<string>((resolve) => {
|
|
91
|
+
server.once('error', (error) => resolve(`mbx reservations could not run: ${error.message}`));
|
|
92
|
+
server.once('close', (code, signal) => {
|
|
93
|
+
const last = said.trim().split('\n').at(-1) ?? '';
|
|
94
|
+
resolve(
|
|
95
|
+
`mbx reservations ended ${signal === null ? `with status ${code}` : `on ${signal}`}${last === '' ? '' : `: ${last}`}`,
|
|
96
|
+
);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
const endpoint: Endpoint = { server, directory, ended };
|
|
100
|
+
const listening = await saysListening(server.stdout, ended);
|
|
101
|
+
if (listening !== null) {
|
|
102
|
+
server.kill('SIGKILL');
|
|
103
|
+
await endedWithin(endpoint, CONTROL_MS);
|
|
104
|
+
await rm(directory, { recursive: true, force: true });
|
|
105
|
+
return { ok: false, error: listening };
|
|
106
|
+
}
|
|
107
|
+
return { ok: true, endpoint, socket };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Null once the endpoint's first line is `listening`; otherwise why not, within the control-plane bound. */
|
|
111
|
+
function saysListening(stdout: Readable, ended: Promise<string>): Promise<string | null> {
|
|
112
|
+
return new Promise<string | null>((resolve) => {
|
|
113
|
+
let text = '';
|
|
114
|
+
const timer = setTimeout(
|
|
115
|
+
() => resolve(`mbx reservations did not say it was listening within ${CONTROL_MS}ms`),
|
|
116
|
+
CONTROL_MS,
|
|
117
|
+
);
|
|
118
|
+
stdout.setEncoding('utf8');
|
|
119
|
+
stdout.on('data', (chunk: string) => {
|
|
120
|
+
text += chunk;
|
|
121
|
+
const end = text.indexOf('\n');
|
|
122
|
+
if (end === -1) return;
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
const line = text.slice(0, end);
|
|
125
|
+
resolve(line === 'listening' ? null : `mbx reservations said ${JSON.stringify(line)}, not that it was listening`);
|
|
126
|
+
});
|
|
127
|
+
void ended.then((why) => {
|
|
128
|
+
clearTimeout(timer);
|
|
129
|
+
resolve(why);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** How the endpoint ended, killing it once `ms` pass first. */
|
|
135
|
+
async function endedWithin(endpoint: Endpoint, ms: number): Promise<string> {
|
|
136
|
+
let timer: NodeJS.Timeout | undefined;
|
|
137
|
+
const late = new Promise<null>((resolve) => (timer = setTimeout(() => resolve(null), ms)));
|
|
138
|
+
const ended = await Promise.race([endpoint.ended, late]);
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
if (ended !== null) {
|
|
141
|
+
return ended;
|
|
142
|
+
}
|
|
143
|
+
endpoint.server.kill('SIGKILL');
|
|
144
|
+
return `${await endpoint.ended}, killed after it outlived its standard input by ${ms}ms`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The pool's answer to one reservation: the cores it holds and a promise of why it was lost, or why there are none. */
|
|
148
|
+
export type Granted =
|
|
149
|
+
| { readonly ok: true; readonly cpus: number; readonly lost: Promise<string> }
|
|
150
|
+
| { readonly ok: false; readonly error: string };
|
|
151
|
+
|
|
152
|
+
/** One request for cores on a run's endpoint. */
|
|
153
|
+
export interface Reservation {
|
|
154
|
+
/** Settles once the pool grants or refuses it, or its connection ends first. */
|
|
155
|
+
readonly granted: Promise<Granted>;
|
|
156
|
+
/** Gives the grant back, or withdraws the request while it waits. Later calls do nothing. */
|
|
157
|
+
release(): void;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
type Uint = number & tags.Type<'uint32'>;
|
|
161
|
+
|
|
162
|
+
/** The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). */
|
|
163
|
+
type ReservationRequest =
|
|
164
|
+
| { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: 'normal' | 'low' }
|
|
165
|
+
| { type: 'release'; id: Uint };
|
|
166
|
+
|
|
167
|
+
/** The lines the endpoint answers; an added field is mbx's to add. */
|
|
168
|
+
type ReservationReply =
|
|
169
|
+
| { type: 'granted'; id: Uint; cpus: Uint & tags.Minimum<1> }
|
|
170
|
+
| { type: 'refused'; id: Uint; error: string }
|
|
171
|
+
| { type: 'error'; error: string };
|
|
172
|
+
|
|
173
|
+
const stringifyRequest = typia.json.createStringify<ReservationRequest>();
|
|
174
|
+
const parseReply = typia.json.createValidateParse<ReservationReply>();
|
|
175
|
+
|
|
176
|
+
/** A request's place on its connection: waiting for its answer, or held until released or lost. */
|
|
177
|
+
type Entry =
|
|
178
|
+
| { readonly kind: 'waiting'; readonly answer: (granted: Granted) => void }
|
|
179
|
+
| { readonly kind: 'held'; readonly lose: (why: string) => void };
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* One connection to an endpoint, shared by every reservation this process makes there. It keeps the process alive only
|
|
183
|
+
* while one of them waits or holds.
|
|
184
|
+
*/
|
|
185
|
+
class Connection {
|
|
186
|
+
readonly #socket: Socket;
|
|
187
|
+
readonly #entries = new Map<number, Entry>();
|
|
188
|
+
#nextId = 1;
|
|
189
|
+
#unread = '';
|
|
190
|
+
#closed: string | null = null;
|
|
191
|
+
|
|
192
|
+
constructor(
|
|
193
|
+
private readonly endpoint: string,
|
|
194
|
+
private readonly forget: () => void,
|
|
195
|
+
) {
|
|
196
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a Unix-socket connect completes or fails at once, and a reservation's answer waits as long as the pool's admission holds it, as `mbx reserve`'s did; the connection's end answers every request still waiting
|
|
197
|
+
this.#socket = createConnection(endpoint);
|
|
198
|
+
this.#socket.setEncoding('utf8');
|
|
199
|
+
this.#socket.unref();
|
|
200
|
+
this.#socket.on('data', (text: string) => this.#read(text));
|
|
201
|
+
this.#socket.on('error', (error) => this.#close(`mbx reservation endpoint ${this.endpoint}: ${error.message}`));
|
|
202
|
+
this.#socket.on('close', () => this.#close(`mbx reservation endpoint ${this.endpoint} closed the connection`));
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
reserve(cpus: number, priority: 'normal' | 'low'): Reservation {
|
|
206
|
+
if (this.#closed !== null) {
|
|
207
|
+
return { granted: Promise.resolve({ ok: false, error: this.#closed }), release() {} };
|
|
208
|
+
}
|
|
209
|
+
const id = this.#nextId++;
|
|
210
|
+
let answer!: (granted: Granted) => void;
|
|
211
|
+
const granted = new Promise<Granted>((resolve) => (answer = resolve));
|
|
212
|
+
this.#entries.set(id, { kind: 'waiting', answer });
|
|
213
|
+
this.#socket.ref();
|
|
214
|
+
this.#socket.write(`${stringifyRequest({ type: 'reserve', id, cpus, priority })}\n`);
|
|
215
|
+
let released = false;
|
|
216
|
+
return {
|
|
217
|
+
granted,
|
|
218
|
+
release: () => {
|
|
219
|
+
if (released) return;
|
|
220
|
+
released = true;
|
|
221
|
+
const entry = this.#entries.get(id);
|
|
222
|
+
if (entry === undefined) return;
|
|
223
|
+
this.#entries.delete(id);
|
|
224
|
+
if (entry.kind === 'waiting') entry.answer({ ok: false, error: 'released while it waited' });
|
|
225
|
+
this.#socket.write(`${stringifyRequest({ type: 'release', id })}\n`);
|
|
226
|
+
if (this.#entries.size === 0) this.#socket.unref();
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
#read(text: string): void {
|
|
232
|
+
this.#unread += text;
|
|
233
|
+
for (let end = this.#unread.indexOf('\n'); end !== -1; end = this.#unread.indexOf('\n')) {
|
|
234
|
+
const line = this.#unread.slice(0, end);
|
|
235
|
+
this.#unread = this.#unread.slice(end + 1);
|
|
236
|
+
const parsed = parseReply(line);
|
|
237
|
+
if (!parsed.success) {
|
|
238
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} answered ${JSON.stringify(line)}, which is no reply`);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
this.#answer(parsed.data);
|
|
242
|
+
if (this.#closed !== null) return;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#answer(reply: ReservationReply): void {
|
|
247
|
+
if (reply.type === 'error') {
|
|
248
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} closed this connection: ${reply.error}`);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const entry = this.#entries.get(reply.id);
|
|
252
|
+
// Released before its answer arrived: the endpoint drops what it won.
|
|
253
|
+
if (entry === undefined) return;
|
|
254
|
+
if (entry.kind === 'held') {
|
|
255
|
+
this.#close(`mbx reservation endpoint ${this.endpoint} answered reservation ${reply.id} twice`);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (reply.type === 'granted') {
|
|
259
|
+
let lose!: (why: string) => void;
|
|
260
|
+
const lost = new Promise<string>((resolve) => (lose = resolve));
|
|
261
|
+
this.#entries.set(reply.id, { kind: 'held', lose });
|
|
262
|
+
entry.answer({ ok: true, cpus: reply.cpus, lost });
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
this.#entries.delete(reply.id);
|
|
266
|
+
entry.answer({ ok: false, error: reply.error });
|
|
267
|
+
if (this.#entries.size === 0) this.#socket.unref();
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
#close(why: string): void {
|
|
271
|
+
if (this.#closed !== null) return;
|
|
272
|
+
this.#closed = why;
|
|
273
|
+
this.forget();
|
|
274
|
+
for (const entry of this.#entries.values()) {
|
|
275
|
+
if (entry.kind === 'waiting') entry.answer({ ok: false, error: why });
|
|
276
|
+
else entry.lose(why);
|
|
277
|
+
}
|
|
278
|
+
this.#entries.clear();
|
|
279
|
+
this.#socket.destroy();
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** This process's connection to each endpoint it has asked. */
|
|
284
|
+
const connections = new Map<string, Connection>();
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Ask the endpoint at `socket` for at most `cpus` cores at `priority`, over this process's one connection to it,
|
|
288
|
+
* opened by its first request and kept while another one may come.
|
|
289
|
+
*/
|
|
290
|
+
export function reserveCores(socket: string, cpus: number, priority: 'normal' | 'low'): Reservation {
|
|
291
|
+
let connection = connections.get(socket);
|
|
292
|
+
if (connection === undefined) {
|
|
293
|
+
const opened: Connection = new Connection(socket, () => {
|
|
294
|
+
if (connections.get(socket) === opened) connections.delete(socket);
|
|
295
|
+
});
|
|
296
|
+
connections.set(socket, opened);
|
|
297
|
+
connection = opened;
|
|
298
|
+
}
|
|
299
|
+
return connection.reserve(cpus, priority);
|
|
300
|
+
}
|
|
@@ -56,8 +56,9 @@ export type ReservedDemand = Extract<Demand, { kind: 'nextest' | 'bun-parallel'
|
|
|
56
56
|
export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
57
57
|
|
|
58
58
|
/**
|
|
59
|
-
* The cores mbx granted the reservation a leaf runs under:
|
|
60
|
-
* as were free once one was
|
|
59
|
+
* The cores mbx granted the reservation a leaf runs under: its run's mbx reservation endpoint grants at most the N the
|
|
60
|
+
* leaf asks for, as many as were free once one was, and the leaf's shell reads that count into this variable
|
|
61
|
+
* (mbx-pool.ts, `reservation`), the one `mbx reserve --cpus N` names to its own command.
|
|
61
62
|
*/
|
|
62
63
|
export const GRANTED_CPUS_ENV = 'MBX_RESERVE_CPUS';
|
|
63
64
|
|
|
@@ -369,13 +370,12 @@ export function sizeRunner(
|
|
|
369
370
|
/**
|
|
370
371
|
* The shell statements that run first in a reserved leaf's reservation and size its runner to the grant, which only
|
|
371
372
|
* the command learns as it starts: `CPU_TOKENS_ENV` becomes the cores mbx granted (`GRANTED_CPUS_ENV`), never more
|
|
372
|
-
* than the leaf asked for (`WANT_ENV`, set before it reserved)
|
|
373
|
-
*
|
|
374
|
-
* ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`).
|
|
373
|
+
* than the leaf asked for (`WANT_ENV`, set before it reserved). The test-thread variables a sizing sets follow from it,
|
|
374
|
+
* under their inherited ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`).
|
|
375
375
|
*/
|
|
376
376
|
export function grantedSizing(demand: ReservedDemand): string {
|
|
377
377
|
const asked = `$${WANT_ENV}`;
|
|
378
|
-
const named =
|
|
378
|
+
const named = `$${GRANTED_CPUS_ENV}`;
|
|
379
379
|
const tokens = `$${CPU_TOKENS_ENV}`;
|
|
380
380
|
const atMost = (ceiling: number | null): string =>
|
|
381
381
|
ceiling === null ? `"${tokens}"` : `"$(( ${tokens} < ${ceiling} ? ${tokens} : ${ceiling} ))"`;
|