@smoothbricks/nx-plugin 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -30
- package/dist/.build-inputs.json +15 -8
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +192 -82
- package/dist/executors/bounded-exec/host-memory.d.ts +47 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +40 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +104 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +55 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +146 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +29 -11
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +53 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +6 -6
- package/dist/executors/bounded-exec/verdict.d.ts +21 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +27 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +202 -77
- package/src/executors/bounded-exec/host-memory.darwin.ts +44 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +115 -0
- package/src/executors/bounded-exec/host-memory.ts +70 -0
- package/src/executors/bounded-exec/leaf-peak.ts +151 -0
- package/src/executors/bounded-exec/mbx-pool.ts +33 -13
- package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
- package/src/executors/bounded-exec/peak-waiter.ts +82 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/runner-sizing.ts +6 -6
- package/src/executors/bounded-exec/verdict.ts +40 -6
- package/src/index.ts +4 -1
- package/test-timings.json +134 -131
|
@@ -36,10 +36,20 @@ import {
|
|
|
36
36
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
37
37
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
38
38
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
39
|
+
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
39
40
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
40
41
|
import type { LockWait } from './lock-wait.js';
|
|
41
|
-
import {
|
|
42
|
+
import {
|
|
43
|
+
askedCores,
|
|
44
|
+
MBX_GRANT_FD,
|
|
45
|
+
mbxIdleGated,
|
|
46
|
+
mbxLeaf,
|
|
47
|
+
mbxPool,
|
|
48
|
+
withCountedReservation,
|
|
49
|
+
withMbxLeaf,
|
|
50
|
+
} from './mbx-pool.js';
|
|
42
51
|
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
52
|
+
import { RESERVATIONS_ENV, type Reservation, reserveCores } from './mbx-reservations.js';
|
|
43
53
|
import { createNoSpaceWatch } from './no-space.js';
|
|
44
54
|
import {
|
|
45
55
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -83,6 +93,7 @@ import {
|
|
|
83
93
|
judgeRun,
|
|
84
94
|
type NoSpaceSighting,
|
|
85
95
|
readJunitReport,
|
|
96
|
+
readLastPeak,
|
|
86
97
|
taskDirectory,
|
|
87
98
|
withJunitReport,
|
|
88
99
|
writeRecord,
|
|
@@ -310,6 +321,8 @@ export async function runBoundedExec(
|
|
|
310
321
|
): Promise<BoundedExecResult> {
|
|
311
322
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
312
323
|
const directory = context.record?.directory ?? null;
|
|
324
|
+
// The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
|
|
325
|
+
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
313
326
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
314
327
|
if (directory !== null) {
|
|
315
328
|
await clearTaskDirectory(directory);
|
|
@@ -357,6 +370,20 @@ export async function runBoundedExec(
|
|
|
357
370
|
// Nobody is left to tell.
|
|
358
371
|
}
|
|
359
372
|
};
|
|
373
|
+
// A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
|
|
374
|
+
const refuseUnstarted = async (message: string): Promise<BoundedExecResult> => {
|
|
375
|
+
appendStderr(message);
|
|
376
|
+
if (context.record !== undefined) {
|
|
377
|
+
const { task, directory: recordDirectory } = context.record;
|
|
378
|
+
await writeRecord(recordDirectory, {
|
|
379
|
+
task: task.id,
|
|
380
|
+
hash: task.hash,
|
|
381
|
+
verdict: judgeOperationalFailure(1, null),
|
|
382
|
+
peakBytes: null,
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
386
|
+
};
|
|
360
387
|
|
|
361
388
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
362
389
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
@@ -365,14 +392,9 @@ export async function runBoundedExec(
|
|
|
365
392
|
(options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
|
|
366
393
|
(fixture !== null && !(fixture.timeoutMs > 0))
|
|
367
394
|
) {
|
|
368
|
-
|
|
395
|
+
return refuseUnstarted(
|
|
369
396
|
'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n',
|
|
370
397
|
);
|
|
371
|
-
if (context.record !== undefined) {
|
|
372
|
-
const { task, directory: recordDirectory } = context.record;
|
|
373
|
-
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
374
|
-
}
|
|
375
|
-
return { success: false, terminalOutput: outputChunks.join('') };
|
|
376
398
|
}
|
|
377
399
|
|
|
378
400
|
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
@@ -385,13 +407,15 @@ export async function runBoundedExec(
|
|
|
385
407
|
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
386
408
|
});
|
|
387
409
|
if (runtime !== null) {
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
410
|
+
return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
411
|
+
}
|
|
412
|
+
if (lastPeak !== null && !lastPeak.ok) {
|
|
413
|
+
return refuseUnstarted(
|
|
414
|
+
`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
|
|
415
|
+
);
|
|
394
416
|
}
|
|
417
|
+
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
418
|
+
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
395
419
|
|
|
396
420
|
let lease: RamTempLease | null = null;
|
|
397
421
|
let ordinaryTemp: string | null = null;
|
|
@@ -539,7 +563,7 @@ export async function runBoundedExec(
|
|
|
539
563
|
let parentRefusal: string | null = null;
|
|
540
564
|
let parentWait: CpuWaitRow | undefined;
|
|
541
565
|
if (inherited !== null) {
|
|
542
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
566
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
|
|
543
567
|
if (fromParent.granted) {
|
|
544
568
|
grant = fromParent;
|
|
545
569
|
} else if (fromParent.cause === 'absent') {
|
|
@@ -560,16 +584,24 @@ export async function runBoundedExec(
|
|
|
560
584
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
561
585
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
562
586
|
appendStderr(
|
|
563
|
-
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
587
|
+
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
|
|
564
588
|
);
|
|
589
|
+
if (grant.oversized !== null) {
|
|
590
|
+
appendStderr(
|
|
591
|
+
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
592
|
+
);
|
|
593
|
+
}
|
|
565
594
|
} else if (leaf?.kind === 'reserve') {
|
|
566
595
|
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
567
596
|
} else if (leaf !== null) {
|
|
568
597
|
sized = sizeRunner(demand, requested, demand.want);
|
|
569
598
|
}
|
|
570
599
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
571
|
-
// descendant still draws on tokens the grant has returned.
|
|
600
|
+
// descendant still draws on tokens the grant has returned. A reserved leaf's mbx grant covers the leaf's conclusion,
|
|
601
|
+
// its subtree's retirement included (11_shell.md), so it goes back after `releaseLease`.
|
|
572
602
|
let pool: CpuPool | null = null;
|
|
603
|
+
let reservation: Reservation | null = null;
|
|
604
|
+
const releaseReservation = (): void => reservation?.release();
|
|
573
605
|
let cpuReleased: Promise<void> | null = null;
|
|
574
606
|
const releaseCpu = (): Promise<void> => {
|
|
575
607
|
cpuReleased ??= (async () => {
|
|
@@ -596,8 +628,9 @@ export async function runBoundedExec(
|
|
|
596
628
|
let unconfirmedAdmission: string | null = null;
|
|
597
629
|
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
598
630
|
let confirmedAdmissionAt: number | null = null;
|
|
631
|
+
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
599
632
|
const queueWait = (): AdmissionQueueWait | null =>
|
|
600
|
-
mbxAdmitted || lastQueueRequests.length === 0
|
|
633
|
+
mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
601
634
|
? null
|
|
602
635
|
: {
|
|
603
636
|
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
@@ -620,6 +653,7 @@ export async function runBoundedExec(
|
|
|
620
653
|
operational: boolean,
|
|
621
654
|
noSpace: NoSpaceSighting | null,
|
|
622
655
|
lockWaits: readonly LockWait[],
|
|
656
|
+
peakBytes: number | null,
|
|
623
657
|
): Promise<void> => {
|
|
624
658
|
if (context.record === undefined) {
|
|
625
659
|
return;
|
|
@@ -665,6 +699,7 @@ export async function runBoundedExec(
|
|
|
665
699
|
verdict,
|
|
666
700
|
...(cpu ? { cpu } : {}),
|
|
667
701
|
...(parentWait ? { cpuWait: parentWait } : {}),
|
|
702
|
+
peakBytes,
|
|
668
703
|
});
|
|
669
704
|
};
|
|
670
705
|
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
@@ -673,17 +708,25 @@ export async function runBoundedExec(
|
|
|
673
708
|
await releaseCpu();
|
|
674
709
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
675
710
|
await releaseLease(false);
|
|
676
|
-
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
711
|
+
await recordRun(1, null, Date.now() - asked, false, null, [], null);
|
|
677
712
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
678
713
|
};
|
|
679
714
|
if (parentRefusal !== null) {
|
|
680
715
|
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
681
716
|
}
|
|
717
|
+
// A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
|
|
718
|
+
// none said why in its own output; the leaf does not run beside the pool.
|
|
719
|
+
const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
|
|
720
|
+
if (reserved !== null && reserved.endpoint === '') {
|
|
721
|
+
return refuse(
|
|
722
|
+
`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`,
|
|
723
|
+
);
|
|
724
|
+
}
|
|
682
725
|
|
|
683
726
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
684
727
|
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
685
728
|
if (grant !== null) {
|
|
686
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
729
|
+
const created = await createCpuPool(grant.tokens, grant.closed, null);
|
|
687
730
|
if (!created.ok) {
|
|
688
731
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
689
732
|
}
|
|
@@ -730,42 +773,72 @@ export async function runBoundedExec(
|
|
|
730
773
|
: withMbxLeaf(unadmitted, leaf);
|
|
731
774
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
732
775
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
733
|
-
// fd 4 is unused without a subtree admission, so
|
|
776
|
+
// fd 4 is unused without a subtree admission, so mbx's descriptor takes fd 5 either way: a Cargo leaf's admission
|
|
777
|
+
// stream, or a reserved leaf's ask for cores and their count (mbx-pool.ts, `MBX_GRANT_FD`).
|
|
734
778
|
const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
735
779
|
const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
|
|
736
780
|
const root = owner?.rootArgv(shellCommand);
|
|
781
|
+
// The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
|
|
782
|
+
const descriptors: IOType[] =
|
|
783
|
+
root === undefined
|
|
784
|
+
? hooked === null && leaf === null
|
|
785
|
+
? []
|
|
786
|
+
: [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
|
|
787
|
+
: [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
|
|
788
|
+
// A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
|
|
789
|
+
const waiter = context.record === undefined ? null : peakWaiter();
|
|
790
|
+
const measured =
|
|
791
|
+
waiter?.kind === 'waiter'
|
|
792
|
+
? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
|
|
793
|
+
: null;
|
|
737
794
|
let child: ChildProcess;
|
|
738
795
|
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
739
796
|
if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
|
|
740
797
|
resultWriter = results?.ok ? results.writer : null;
|
|
741
798
|
try {
|
|
742
799
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
743
|
-
// cowshed closes fd 4 before exec; fd 5
|
|
800
|
+
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
|
|
801
|
+
// the waiter, when there is one, passes them on and reports the command's peak on fd 6.
|
|
744
802
|
child =
|
|
745
|
-
|
|
746
|
-
? spawn(
|
|
803
|
+
measured !== null
|
|
804
|
+
? spawn(measured.file, measured.args, {
|
|
747
805
|
cwd,
|
|
748
806
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
749
|
-
|
|
750
|
-
stdio:
|
|
751
|
-
hooked === null && leaf === null
|
|
752
|
-
? 'pipe'
|
|
753
|
-
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
807
|
+
stdio: measured.stdio,
|
|
754
808
|
detached: process.platform !== 'win32',
|
|
755
809
|
windowsHide: true,
|
|
756
810
|
})
|
|
757
|
-
:
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
811
|
+
: root === undefined
|
|
812
|
+
? spawn(shellCommand, [], {
|
|
813
|
+
cwd,
|
|
814
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
815
|
+
shell: true,
|
|
816
|
+
stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
817
|
+
detached: process.platform !== 'win32',
|
|
818
|
+
windowsHide: true,
|
|
819
|
+
})
|
|
820
|
+
: spawn(root.file, root.args, {
|
|
821
|
+
cwd,
|
|
822
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
823
|
+
stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
824
|
+
detached: process.platform !== 'win32',
|
|
825
|
+
windowsHide: true,
|
|
826
|
+
});
|
|
764
827
|
} catch (error) {
|
|
765
828
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
766
829
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
767
830
|
}
|
|
768
831
|
subtreeRetired = false;
|
|
832
|
+
const reportChannel = child.stdio.at(PEAK_FD);
|
|
833
|
+
const waiterReport =
|
|
834
|
+
measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
|
|
835
|
+
// The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
|
|
836
|
+
// group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
|
|
837
|
+
const runnerPid: Promise<number | null> = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
|
|
838
|
+
let runner: number | null = null;
|
|
839
|
+
void runnerPid.then((pid) => {
|
|
840
|
+
runner = pid;
|
|
841
|
+
});
|
|
769
842
|
if (root !== undefined) {
|
|
770
843
|
const channel = child.stdio[HANDLE_FD];
|
|
771
844
|
const leased = lease;
|
|
@@ -825,22 +898,24 @@ export async function runBoundedExec(
|
|
|
825
898
|
captureTimer = setTimeout(
|
|
826
899
|
() => {
|
|
827
900
|
if (state.settled || state.expiry !== null || idleCapture !== capture) return;
|
|
828
|
-
const
|
|
829
|
-
if (
|
|
901
|
+
const group = child.pid;
|
|
902
|
+
if (group === undefined) {
|
|
830
903
|
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
831
904
|
return;
|
|
832
905
|
}
|
|
833
906
|
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
834
|
-
void
|
|
835
|
-
.then(() =>
|
|
836
|
-
|
|
907
|
+
void runnerPid
|
|
908
|
+
.then((pid) => {
|
|
909
|
+
if (pid === null)
|
|
910
|
+
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
911
|
+
return captureDiagnostics({
|
|
837
912
|
pid,
|
|
838
|
-
pgid: process.platform === 'win32' ? null :
|
|
913
|
+
pgid: process.platform === 'win32' ? null : group,
|
|
839
914
|
expiresAtMs: capture.expiresAtMs,
|
|
840
915
|
deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
|
|
841
916
|
signal: capture.controller.signal,
|
|
842
|
-
})
|
|
843
|
-
)
|
|
917
|
+
});
|
|
918
|
+
})
|
|
844
919
|
.then(
|
|
845
920
|
(snapshot) => {
|
|
846
921
|
if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
|
|
@@ -882,7 +957,7 @@ export async function runBoundedExec(
|
|
|
882
957
|
if (state.settled || state.expiry !== null) return;
|
|
883
958
|
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
884
959
|
if (scheduled?.kind !== kind || scheduled.at !== at) return;
|
|
885
|
-
if (kind === 'idle' && leaf
|
|
960
|
+
if (kind === 'idle' && leaf?.kind === 'cargo') {
|
|
886
961
|
if (checkingIdle) return;
|
|
887
962
|
checkingIdle = true;
|
|
888
963
|
armBoundTimer();
|
|
@@ -937,8 +1012,6 @@ export async function runBoundedExec(
|
|
|
937
1012
|
};
|
|
938
1013
|
const refreshQueue = async (fresh: boolean): Promise<void> => {
|
|
939
1014
|
if (fresh && queueSnapshot !== null) await queueSnapshot;
|
|
940
|
-
// A leaf that counts its units before reserving waits in no reservation queue while its fixture builds.
|
|
941
|
-
if (counted && !fixtureDone) return;
|
|
942
1015
|
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
|
|
943
1016
|
const pid = child.pid;
|
|
944
1017
|
queueSnapshot ??= (async () => {
|
|
@@ -1079,13 +1152,14 @@ export async function runBoundedExec(
|
|
|
1079
1152
|
clearTimeout(boundTimer);
|
|
1080
1153
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
1081
1154
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
1082
|
-
signalCleanup = (leaf
|
|
1155
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
|
|
1083
1156
|
.finally(() => {
|
|
1084
1157
|
stopQueueObservation();
|
|
1085
1158
|
return killChildTree();
|
|
1086
1159
|
})
|
|
1087
1160
|
.finally(releaseCpu)
|
|
1088
1161
|
.finally(() => releaseLease(true))
|
|
1162
|
+
.finally(releaseReservation)
|
|
1089
1163
|
.finally(async () =>
|
|
1090
1164
|
recordRun(
|
|
1091
1165
|
signalToExitCode(signal),
|
|
@@ -1094,6 +1168,7 @@ export async function runBoundedExec(
|
|
|
1094
1168
|
state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
|
|
1095
1169
|
state.noSpace === null ? null : await state.noSpace,
|
|
1096
1170
|
state.lockWaits,
|
|
1171
|
+
null,
|
|
1097
1172
|
),
|
|
1098
1173
|
)
|
|
1099
1174
|
.finally(() => process.kill(process.pid, signal));
|
|
@@ -1162,7 +1237,7 @@ export async function runBoundedExec(
|
|
|
1162
1237
|
const observed = idleCapture?.result;
|
|
1163
1238
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
1164
1239
|
appendStderr(
|
|
1165
|
-
`Hang diagnostics: runner pid=${
|
|
1240
|
+
`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
|
|
1166
1241
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
1167
1242
|
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
|
|
1168
1243
|
);
|
|
@@ -1257,46 +1332,87 @@ export async function runBoundedExec(
|
|
|
1257
1332
|
fixtureFirst: counted && lifecycle !== null,
|
|
1258
1333
|
queued,
|
|
1259
1334
|
});
|
|
1260
|
-
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1261
1335
|
if (leaf !== null) {
|
|
1262
1336
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1263
|
-
if (!(channel instanceof
|
|
1264
|
-
let said = false;
|
|
1337
|
+
if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1265
1338
|
const failGrant = (detail: string): void => {
|
|
1266
1339
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
1267
1340
|
lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
|
|
1268
1341
|
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1269
1342
|
termination = terminate();
|
|
1270
1343
|
};
|
|
1271
|
-
channel.on('
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1344
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1345
|
+
if (reserved !== null) {
|
|
1346
|
+
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1347
|
+
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
1348
|
+
const { leaf: reservedLeaf, endpoint } = reserved;
|
|
1349
|
+
let asked = '';
|
|
1350
|
+
channel.setEncoding('utf8');
|
|
1351
|
+
channel.on('data', (text: string) => {
|
|
1352
|
+
if (reservation !== null) {
|
|
1353
|
+
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
asked += text;
|
|
1357
|
+
const end = asked.indexOf('\n');
|
|
1358
|
+
if (end === -1) return;
|
|
1359
|
+
const line = asked.slice(0, end);
|
|
1360
|
+
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1361
|
+
if (cpus === null) {
|
|
1362
|
+
failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
|
|
1363
|
+
channel.end();
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
|
|
1367
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1368
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1369
|
+
reservation = held;
|
|
1370
|
+
void held.granted.then((granted) => {
|
|
1371
|
+
if (!granted.ok) {
|
|
1372
|
+
failGrant(`mbx did not grant the reservation: ${granted.error}`);
|
|
1373
|
+
channel.end();
|
|
1374
|
+
return;
|
|
1282
1375
|
}
|
|
1376
|
+
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1377
|
+
if (state.settled || state.expiry !== null) return;
|
|
1378
|
+
mbxAdmitted = true;
|
|
1379
|
+
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1380
|
+
channel.end(`${granted.cpus}\n`);
|
|
1283
1381
|
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1382
|
+
void granted.lost.then((why) => {
|
|
1383
|
+
if (state.settled || cpuReleased !== null) return;
|
|
1384
|
+
appendStderr(
|
|
1385
|
+
`cowshed: mbx reservation lost while the command ran (${why}); it runs on unheld until it ends\n`,
|
|
1386
|
+
);
|
|
1387
|
+
});
|
|
1388
|
+
});
|
|
1389
|
+
});
|
|
1390
|
+
} else {
|
|
1391
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1392
|
+
let said = false;
|
|
1393
|
+
channel.on('data', (chunk: Buffer) => {
|
|
1394
|
+
for (const byte of chunk) {
|
|
1395
|
+
if (byte === 103) {
|
|
1396
|
+
if (!said) {
|
|
1397
|
+
mbxAdmitted = true;
|
|
1398
|
+
said = true;
|
|
1399
|
+
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1400
|
+
}
|
|
1401
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1402
|
+
} else if (byte === 114) {
|
|
1403
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1404
|
+
} else {
|
|
1405
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1406
|
+
return;
|
|
1407
|
+
}
|
|
1289
1408
|
}
|
|
1290
|
-
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
if (leaf !== null) {
|
|
1296
|
-
queueTimer = setInterval(() => {
|
|
1409
|
+
});
|
|
1410
|
+
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1411
|
+
queueTimer = setInterval(() => {
|
|
1412
|
+
void refreshQueue(false);
|
|
1413
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1297
1414
|
void refreshQueue(false);
|
|
1298
|
-
}
|
|
1299
|
-
void refreshQueue(false);
|
|
1415
|
+
}
|
|
1300
1416
|
}
|
|
1301
1417
|
|
|
1302
1418
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
@@ -1356,11 +1472,20 @@ export async function runBoundedExec(
|
|
|
1356
1472
|
}
|
|
1357
1473
|
}
|
|
1358
1474
|
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
1475
|
+
releaseReservation();
|
|
1476
|
+
|
|
1477
|
+
// The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
|
|
1478
|
+
const reported: LeafPeak | null =
|
|
1479
|
+
waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
|
|
1480
|
+
if (reported?.kind === 'unmeasured') {
|
|
1481
|
+
appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
|
|
1482
|
+
}
|
|
1483
|
+
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1359
1484
|
|
|
1360
1485
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1361
1486
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1362
1487
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1363
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
1488
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1364
1489
|
return {
|
|
1365
1490
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1366
1491
|
terminalOutput: outputChunks.join(''),
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import type { HostMemory, RunMemoryReading } from './host-memory.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
|
|
7
|
+
* its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const run = promisify(execFile);
|
|
11
|
+
const HW_MEMSIZE = 'hw.memsize';
|
|
12
|
+
|
|
13
|
+
async function readRunMemory(): Promise<RunMemoryReading> {
|
|
14
|
+
let stdout: string;
|
|
15
|
+
try {
|
|
16
|
+
({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
|
|
17
|
+
} catch (error) {
|
|
18
|
+
return {
|
|
19
|
+
ok: false,
|
|
20
|
+
error: {
|
|
21
|
+
operation: 'read-run-memory',
|
|
22
|
+
code: 'unreadable',
|
|
23
|
+
subject: HW_MEMSIZE,
|
|
24
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const value = stdout.trim();
|
|
29
|
+
const bytes = Number(value);
|
|
30
|
+
if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
|
|
31
|
+
return {
|
|
32
|
+
ok: false,
|
|
33
|
+
error: {
|
|
34
|
+
operation: 'read-run-memory',
|
|
35
|
+
code: 'malformed',
|
|
36
|
+
subject: HW_MEMSIZE,
|
|
37
|
+
cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss };
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { posix } from 'node:path';
|
|
3
|
+
import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
|
|
7
|
+
* the cgroup v2 the run's process is in and of its ancestors, up to the root of the mounted hierarchy: the kernel holds
|
|
8
|
+
* a cgroup to its ancestors' limits too, so a job in cowshed's `background` class, whose own `memory.max` is `max`, has
|
|
9
|
+
* the class's three quarters of the host. A host with no cgroup v2 hierarchy, or a hierarchy that sets no limit, gives
|
|
10
|
+
* the run `MemTotal`.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const MEMINFO = '/proc/meminfo';
|
|
14
|
+
const CGROUP = '/proc/self/cgroup';
|
|
15
|
+
const MOUNTINFO = '/proc/self/mountinfo';
|
|
16
|
+
|
|
17
|
+
/** A file's text, or the errno code and message its read failed with. */
|
|
18
|
+
export type FileRead =
|
|
19
|
+
| { readonly ok: true; readonly text: string }
|
|
20
|
+
| { readonly ok: false; readonly code: string; readonly message: string };
|
|
21
|
+
|
|
22
|
+
const failed = (code: RunMemoryError['code'], subject: string, cause: string): RunMemoryReading => ({
|
|
23
|
+
ok: false,
|
|
24
|
+
error: { operation: 'read-run-memory', code, subject, cause },
|
|
25
|
+
});
|
|
26
|
+
const unreadable = (subject: string, read: Extract<FileRead, { ok: false }>): RunMemoryReading =>
|
|
27
|
+
failed('unreadable', subject, `${read.code}: ${read.message}`);
|
|
28
|
+
|
|
29
|
+
/** A mount point or root as mountinfo writes it, its space, tab, newline and backslash escaped in octal. */
|
|
30
|
+
const decodeOctal = (field: string): string =>
|
|
31
|
+
field.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(Number.parseInt(octal, 8)));
|
|
32
|
+
|
|
33
|
+
/** The cgroup v2 mounts mountinfo lists: the cgroup each shows (`root`) and where (`point`). */
|
|
34
|
+
function cgroup2Mounts(mountinfo: string): { root: string; point: string }[] {
|
|
35
|
+
const mounts: { root: string; point: string }[] = [];
|
|
36
|
+
for (const line of mountinfo.split('\n')) {
|
|
37
|
+
const fields = line.split(' ');
|
|
38
|
+
const separator = fields.indexOf('-');
|
|
39
|
+
const [root, point] = [fields[3], fields[4]];
|
|
40
|
+
if (separator !== -1 && fields[separator + 1] === 'cgroup2' && root !== undefined && point !== undefined) {
|
|
41
|
+
mounts.push({ root: decodeOctal(root), point: decodeOctal(point) });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return mounts;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The run's memory, reading every file through `read`. */
|
|
48
|
+
export async function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading> {
|
|
49
|
+
const meminfo = await read(MEMINFO);
|
|
50
|
+
if (!meminfo.ok) return unreadable(MEMINFO, meminfo);
|
|
51
|
+
const total = /^MemTotal:\s+(\d+) kB$/m.exec(meminfo.text)?.[1];
|
|
52
|
+
if (total === undefined) return failed('malformed', MEMINFO, 'it names no MemTotal in kB');
|
|
53
|
+
let least: RunMemory = { bytes: Number(total) * 1024, source: `${MEMINFO} MemTotal` };
|
|
54
|
+
const cgroups = await read(CGROUP);
|
|
55
|
+
if (!cgroups.ok) return unreadable(CGROUP, cgroups);
|
|
56
|
+
const path = /^0::(\/.*)$/m.exec(cgroups.text)?.[1];
|
|
57
|
+
if (path === undefined) {
|
|
58
|
+
// No cgroup v2 hierarchy holds the run: the host's memory is its memory.
|
|
59
|
+
return { ok: true, memory: least };
|
|
60
|
+
}
|
|
61
|
+
const mounts = await read(MOUNTINFO);
|
|
62
|
+
if (!mounts.ok) return unreadable(MOUNTINFO, mounts);
|
|
63
|
+
const mount = cgroup2Mounts(mounts.text).find(
|
|
64
|
+
({ root }) => root === '/' || path === root || path.startsWith(`${root}/`),
|
|
65
|
+
);
|
|
66
|
+
if (mount === undefined) {
|
|
67
|
+
return failed('unmounted', path, `${MOUNTINFO} lists no cgroup2 mount that shows the run's cgroup`);
|
|
68
|
+
}
|
|
69
|
+
const top = posix.resolve(mount.point);
|
|
70
|
+
const own = posix.resolve(top, `.${mount.root === '/' ? path : path.slice(mount.root.length)}`);
|
|
71
|
+
if (own !== top && !own.startsWith(`${top}/`)) {
|
|
72
|
+
return failed('unmounted', path, `the run's cgroup lies outside the cgroup2 mount at ${top}`);
|
|
73
|
+
}
|
|
74
|
+
for (let directory = own; ; ) {
|
|
75
|
+
const file = posix.join(directory, 'memory.max');
|
|
76
|
+
const limit = await read(file);
|
|
77
|
+
if (limit.ok) {
|
|
78
|
+
const value = limit.text.trim();
|
|
79
|
+
if (value !== 'max') {
|
|
80
|
+
if (!/^\d+$/.test(value)) {
|
|
81
|
+
return failed('malformed', file, `it holds ${JSON.stringify(value)}, not max or bytes`);
|
|
82
|
+
}
|
|
83
|
+
// A limit above the host's memory (a near-2^63 one included) is no constraint, and is never made a number.
|
|
84
|
+
if (BigInt(value) < BigInt(least.bytes)) least = { bytes: Number(value), source: file };
|
|
85
|
+
}
|
|
86
|
+
} else if (limit.code !== 'ENOENT') {
|
|
87
|
+
// ENOENT: the hierarchy's root, or a cgroup whose parent enables no memory controller, has no limit of its own.
|
|
88
|
+
return unreadable(file, limit);
|
|
89
|
+
}
|
|
90
|
+
if (directory === top) break;
|
|
91
|
+
directory = posix.dirname(directory);
|
|
92
|
+
}
|
|
93
|
+
return { ok: true, memory: least };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function readText(path: string): Promise<FileRead> {
|
|
97
|
+
try {
|
|
98
|
+
return { ok: true, text: await readFile(path, 'utf8') };
|
|
99
|
+
} catch (error) {
|
|
100
|
+
const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
|
|
101
|
+
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
|
|
106
|
+
function peakBytes(maxRss: number, bunVersion: string): number {
|
|
107
|
+
const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
|
|
108
|
+
if (version === null) {
|
|
109
|
+
throw new Error(`Bun names its version ${JSON.stringify(bunVersion)}, which is no semver`);
|
|
110
|
+
}
|
|
111
|
+
const [major, minor] = [Number(version[1]), Number(version[2])];
|
|
112
|
+
return major < 1 || (major === 1 && minor < 4) ? maxRss * 1024 : maxRss;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export const linuxMemory: HostMemory = { readRunMemory: () => linuxRunMemory(readText), peakBytes };
|