@smoothbricks/nx-plugin 0.7.12 → 0.7.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -27
- package/dist/.build-inputs.json +20 -12
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +1 -14
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +2 -49
- package/dist/executors/bounded-exec/executor.d.ts +5 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +51 -154
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +3 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +4 -25
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +97 -31
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool.ts +2 -33
- package/src/executors/bounded-exec/executor.ts +60 -201
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -1
- package/src/executors/bounded-exec/verdict.ts +7 -35
- package/src/index.ts +105 -31
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -139
|
@@ -13,31 +13,13 @@ import {
|
|
|
13
13
|
processGroupHasMembers,
|
|
14
14
|
signalProcessGroup,
|
|
15
15
|
} from '../../process-group.js';
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type BoundKind,
|
|
20
|
-
type Bounds,
|
|
21
|
-
earliestBound,
|
|
22
|
-
NO_ADMISSION_WAIT,
|
|
23
|
-
stepAdmissionWait,
|
|
24
|
-
stepBounds,
|
|
25
|
-
UNARMED,
|
|
26
|
-
} from './bounds.js';
|
|
27
|
-
import {
|
|
28
|
-
CPU_POOL_ENV,
|
|
29
|
-
type CpuPool,
|
|
30
|
-
createCpuPool,
|
|
31
|
-
type InheritedCpuPool,
|
|
32
|
-
readInheritedCpuPool,
|
|
33
|
-
type TokenGrant,
|
|
34
|
-
takeParentCpuTokens,
|
|
35
|
-
} from './cpu-pool.js';
|
|
16
|
+
import { AdmissionObservation, type QueueHost } from './admission-queue.js';
|
|
17
|
+
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
18
|
+
import { type CpuPool, createCpuPool } from './cpu-pool.js';
|
|
36
19
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
37
20
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
38
21
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
39
22
|
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
40
|
-
import type { CpuWaitRow } from './lease-wait.js';
|
|
41
23
|
import type { LockWait } from './lock-wait.js';
|
|
42
24
|
import {
|
|
43
25
|
askedCores,
|
|
@@ -48,8 +30,8 @@ import {
|
|
|
48
30
|
withCountedReservation,
|
|
49
31
|
withMbxLeaf,
|
|
50
32
|
} from './mbx-pool.js';
|
|
51
|
-
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
52
33
|
import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
34
|
+
import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
53
35
|
import { createNoSpaceWatch } from './no-space.js';
|
|
54
36
|
import {
|
|
55
37
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -60,6 +42,9 @@ import {
|
|
|
60
42
|
type SubtreeAdmission,
|
|
61
43
|
type SubtreeOwner,
|
|
62
44
|
} from './owned-subtree.js';
|
|
45
|
+
import { askParentPool, describeGrant } from './parent-grant.js';
|
|
46
|
+
import { type LastPeak, readLastPeak } from './peak-charge.js';
|
|
47
|
+
import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
|
|
63
48
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
64
49
|
import {
|
|
65
50
|
createHostCommands,
|
|
@@ -79,12 +64,10 @@ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
|
79
64
|
import type { BoundedExecOptions } from './schema.js';
|
|
80
65
|
import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
|
|
81
66
|
import {
|
|
82
|
-
type AdmissionQueueWait,
|
|
83
67
|
type BoundedExecRecord,
|
|
84
68
|
type BoundedExecTask,
|
|
85
69
|
BUN_TEST,
|
|
86
70
|
clearTaskDirectory,
|
|
87
|
-
describeAdmissionQueue,
|
|
88
71
|
describeNoSpace,
|
|
89
72
|
type ExpiredBound,
|
|
90
73
|
JUNIT_FILE,
|
|
@@ -93,7 +76,6 @@ import {
|
|
|
93
76
|
judgeRun,
|
|
94
77
|
type NoSpaceSighting,
|
|
95
78
|
readJunitReport,
|
|
96
|
-
readLastPeak,
|
|
97
79
|
taskDirectory,
|
|
98
80
|
withJunitReport,
|
|
99
81
|
writeRecord,
|
|
@@ -131,15 +113,7 @@ export interface NxTaskContext {
|
|
|
131
113
|
taskGraph?: { tasks: Record<string, { hash?: string }> };
|
|
132
114
|
}
|
|
133
115
|
|
|
134
|
-
export
|
|
135
|
-
success: boolean;
|
|
136
|
-
terminalOutput: string;
|
|
137
|
-
/**
|
|
138
|
-
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
139
|
-
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
140
|
-
*/
|
|
141
|
-
killedForMemory?: boolean;
|
|
142
|
-
}
|
|
116
|
+
export type BoundedExecResult = RunOutcome;
|
|
143
117
|
|
|
144
118
|
interface RunState {
|
|
145
119
|
settled: boolean;
|
|
@@ -157,8 +131,6 @@ interface RunState {
|
|
|
157
131
|
lockWaits: readonly LockWait[];
|
|
158
132
|
}
|
|
159
133
|
|
|
160
|
-
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
161
|
-
|
|
162
134
|
export interface ProcessTreeKiller {
|
|
163
135
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
164
136
|
}
|
|
@@ -318,39 +290,27 @@ export async function runBoundedExec(
|
|
|
318
290
|
inherited: InheritedCpuPool | null = null,
|
|
319
291
|
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
320
292
|
owner: SubtreeOwner | null = null,
|
|
321
|
-
queueHost?:
|
|
293
|
+
queueHost?: QueueHost,
|
|
322
294
|
): Promise<BoundedExecResult> {
|
|
323
295
|
const directory = context.record?.directory ?? null;
|
|
324
296
|
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
325
297
|
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
326
298
|
// killed run replaced, not by the peak it reached before it was killed.
|
|
327
299
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
// The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
|
|
338
|
-
// what it was killed beside is not there to be killed with it.
|
|
339
|
-
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
|
|
340
|
-
context.record === undefined ? '' : `: ${context.record.task.id}`
|
|
341
|
-
}); running it once more, alone in its run's pool\n`;
|
|
342
|
-
process.stderr.write(note);
|
|
343
|
-
const again = await run(true);
|
|
344
|
-
return {
|
|
345
|
-
success: again.success,
|
|
346
|
-
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
347
|
-
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
348
|
-
};
|
|
300
|
+
return runUntilNotKilledForMemory(
|
|
301
|
+
(alone) =>
|
|
302
|
+
runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
303
|
+
lastPeak,
|
|
304
|
+
alone,
|
|
305
|
+
}),
|
|
306
|
+
inherited !== null,
|
|
307
|
+
context.record?.task.id,
|
|
308
|
+
);
|
|
349
309
|
}
|
|
350
310
|
|
|
351
311
|
/** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
|
|
352
312
|
interface Attempt {
|
|
353
|
-
readonly lastPeak:
|
|
313
|
+
readonly lastPeak: LastPeak | null;
|
|
354
314
|
readonly alone: boolean;
|
|
355
315
|
}
|
|
356
316
|
|
|
@@ -363,7 +323,7 @@ async function runCommand(
|
|
|
363
323
|
inherited: InheritedCpuPool | null,
|
|
364
324
|
captureDiagnostics: typeof captureHangDiagnostics,
|
|
365
325
|
owner: SubtreeOwner | null,
|
|
366
|
-
queueHost:
|
|
326
|
+
queueHost: QueueHost | undefined,
|
|
367
327
|
attempt: Attempt,
|
|
368
328
|
): Promise<BoundedExecResult> {
|
|
369
329
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
@@ -608,24 +568,15 @@ async function runCommand(
|
|
|
608
568
|
const asked = Date.now();
|
|
609
569
|
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
610
570
|
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
if (fromParent.granted) {
|
|
621
|
-
grant = fromParent;
|
|
622
|
-
} else if (fromParent.cause === 'absent') {
|
|
623
|
-
appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
|
|
624
|
-
} else {
|
|
625
|
-
parentRefusal = fromParent.reason;
|
|
626
|
-
parentWait = fromParent.wait;
|
|
627
|
-
}
|
|
628
|
-
}
|
|
571
|
+
const parent = await askParentPool(inherited, {
|
|
572
|
+
want: demand.want,
|
|
573
|
+
charge,
|
|
574
|
+
parallelism: chargeParallelism,
|
|
575
|
+
alone: attempt.alone,
|
|
576
|
+
task: context.record?.task.id,
|
|
577
|
+
});
|
|
578
|
+
for (const said of parent.said) appendStderr(said);
|
|
579
|
+
const { grant, refusal: parentRefusal, wait: parentWait } = parent;
|
|
629
580
|
const waitedMs = Date.now() - asked;
|
|
630
581
|
let cpu: BoundedExecRecord['cpu'];
|
|
631
582
|
let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
|
|
@@ -636,13 +587,8 @@ async function runCommand(
|
|
|
636
587
|
if (grant !== null) {
|
|
637
588
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
638
589
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
);
|
|
642
|
-
if (grant.oversized !== null) {
|
|
643
|
-
appendStderr(
|
|
644
|
-
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
645
|
-
);
|
|
590
|
+
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
591
|
+
appendStderr(said);
|
|
646
592
|
}
|
|
647
593
|
} else if (leaf?.kind === 'reserve') {
|
|
648
594
|
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
@@ -670,33 +616,15 @@ async function runCommand(
|
|
|
670
616
|
})();
|
|
671
617
|
return cpuReleased;
|
|
672
618
|
};
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
683
|
-
let confirmedAdmissionAt: number | null = null;
|
|
684
|
-
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
685
|
-
const queueWait = (): AdmissionQueueWait | null =>
|
|
686
|
-
mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
687
|
-
? null
|
|
688
|
-
: {
|
|
689
|
-
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
690
|
-
requests: lastQueueRequests,
|
|
691
|
-
};
|
|
692
|
-
const queueAbort = new AbortController();
|
|
693
|
-
let queueTimer: NodeJS.Timeout | undefined;
|
|
694
|
-
let queueSnapshot: Promise<void> | null = null;
|
|
695
|
-
const stopQueueObservation = (): void => {
|
|
696
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
697
|
-
clearInterval(queueTimer);
|
|
698
|
-
queueAbort.abort();
|
|
699
|
-
};
|
|
619
|
+
const queue = new AdmissionObservation({
|
|
620
|
+
over: () => state.settled || state.expiry !== null,
|
|
621
|
+
pid: () => child.pid,
|
|
622
|
+
environment: () => mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
623
|
+
tell,
|
|
624
|
+
step: (event) => stepTo(event),
|
|
625
|
+
testing: () => isTesting(),
|
|
626
|
+
queueHost,
|
|
627
|
+
});
|
|
700
628
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
701
629
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
702
630
|
const recordRun = async (
|
|
@@ -730,7 +658,7 @@ async function runCommand(
|
|
|
730
658
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
731
659
|
}
|
|
732
660
|
});
|
|
733
|
-
const admissionQueue = queueWait();
|
|
661
|
+
const admissionQueue = queue.queueWait(reservation !== null);
|
|
734
662
|
const verdict =
|
|
735
663
|
operational || resultFailure !== null
|
|
736
664
|
? judgeOperationalFailure(exitCode, report)
|
|
@@ -1019,7 +947,7 @@ async function runCommand(
|
|
|
1019
947
|
try {
|
|
1020
948
|
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
1021
949
|
// ceilings remain armed while this observation runs.
|
|
1022
|
-
await
|
|
950
|
+
await queue.refresh(true);
|
|
1023
951
|
} finally {
|
|
1024
952
|
checkingIdle = false;
|
|
1025
953
|
}
|
|
@@ -1049,74 +977,6 @@ async function runCommand(
|
|
|
1049
977
|
armBoundTimer();
|
|
1050
978
|
};
|
|
1051
979
|
|
|
1052
|
-
let lastQueueDiagnostic = '';
|
|
1053
|
-
let lastQueueDiagnosticAt: number | null = null;
|
|
1054
|
-
let lastSnapshotDiagnostics = '';
|
|
1055
|
-
const unconfirmAdmission = (detail: string): void => {
|
|
1056
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
|
|
1057
|
-
if (unconfirmedAdmission !== detail) {
|
|
1058
|
-
tell(
|
|
1059
|
-
`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
|
|
1060
|
-
);
|
|
1061
|
-
}
|
|
1062
|
-
unconfirmedAdmission = detail;
|
|
1063
|
-
lastQueueRequests = [];
|
|
1064
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
1065
|
-
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
1066
|
-
stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
|
|
1067
|
-
};
|
|
1068
|
-
const refreshQueue = async (fresh: boolean): Promise<void> => {
|
|
1069
|
-
if (fresh && queueSnapshot !== null) await queueSnapshot;
|
|
1070
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
|
|
1071
|
-
const pid = child.pid;
|
|
1072
|
-
queueSnapshot ??= (async () => {
|
|
1073
|
-
const snapshot = await snapshotMbxQueue(
|
|
1074
|
-
pid,
|
|
1075
|
-
mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
1076
|
-
queueAbort.signal,
|
|
1077
|
-
queueHost,
|
|
1078
|
-
);
|
|
1079
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
|
|
1080
|
-
if (!snapshot.ok) {
|
|
1081
|
-
unconfirmAdmission(snapshot.detail);
|
|
1082
|
-
return;
|
|
1083
|
-
}
|
|
1084
|
-
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
1085
|
-
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
|
|
1086
|
-
lastSnapshotDiagnostics = diagnostics;
|
|
1087
|
-
const waiting = snapshot.requests.length > 0;
|
|
1088
|
-
const observedAt = Date.now();
|
|
1089
|
-
const observedWaitAt = performance.now();
|
|
1090
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
1091
|
-
unconfirmedAdmission = null;
|
|
1092
|
-
confirmedAdmissionAt = observedAt;
|
|
1093
|
-
if (waiting) {
|
|
1094
|
-
lastQueueRequests = snapshot.requests;
|
|
1095
|
-
const positions = JSON.stringify(snapshot.requests);
|
|
1096
|
-
if (
|
|
1097
|
-
positions !== lastQueueDiagnostic ||
|
|
1098
|
-
lastQueueDiagnosticAt === null ||
|
|
1099
|
-
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
|
|
1100
|
-
) {
|
|
1101
|
-
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
1102
|
-
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
1103
|
-
lastQueueDiagnostic = positions;
|
|
1104
|
-
lastQueueDiagnosticAt = Date.now();
|
|
1105
|
-
}
|
|
1106
|
-
} else {
|
|
1107
|
-
lastQueueDiagnostic = '';
|
|
1108
|
-
}
|
|
1109
|
-
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
1110
|
-
})()
|
|
1111
|
-
.catch((error: unknown) => {
|
|
1112
|
-
unconfirmAdmission(error instanceof Error ? error.message : String(error));
|
|
1113
|
-
})
|
|
1114
|
-
.finally(() => {
|
|
1115
|
-
queueSnapshot = null;
|
|
1116
|
-
});
|
|
1117
|
-
await queueSnapshot;
|
|
1118
|
-
};
|
|
1119
|
-
|
|
1120
980
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
1121
981
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
1122
982
|
// message extend the very bound that just fired.
|
|
@@ -1208,9 +1068,9 @@ async function runCommand(
|
|
|
1208
1068
|
clearTimeout(boundTimer);
|
|
1209
1069
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
1210
1070
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
1211
|
-
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() :
|
|
1071
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : queue.refresh(false))
|
|
1212
1072
|
.finally(() => {
|
|
1213
|
-
|
|
1073
|
+
queue.stop();
|
|
1214
1074
|
return killChildTree();
|
|
1215
1075
|
})
|
|
1216
1076
|
.finally(releaseCpu)
|
|
@@ -1252,12 +1112,12 @@ async function runCommand(
|
|
|
1252
1112
|
child.once('error', (error) => {
|
|
1253
1113
|
appendStderr(`${error.message}\n`);
|
|
1254
1114
|
state.settled = true;
|
|
1255
|
-
|
|
1115
|
+
queue.stop();
|
|
1256
1116
|
resolve({ code: 1, signal: null });
|
|
1257
1117
|
});
|
|
1258
1118
|
child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
|
|
1259
1119
|
state.settled = true;
|
|
1260
|
-
|
|
1120
|
+
queue.stop();
|
|
1261
1121
|
resolve({ code, signal });
|
|
1262
1122
|
});
|
|
1263
1123
|
});
|
|
@@ -1281,9 +1141,9 @@ async function runCommand(
|
|
|
1281
1141
|
? `still running: ${bound.tests.join(', ')}`
|
|
1282
1142
|
: 'the runner named no test still running'
|
|
1283
1143
|
} (cwd=${cwd}): ${command}\n${
|
|
1284
|
-
|
|
1144
|
+
queue.unconfirmed === null
|
|
1285
1145
|
? ''
|
|
1286
|
-
: `The last mbx admission snapshot failed (${
|
|
1146
|
+
: `The last mbx admission snapshot failed (${queue.unconfirmed}), so the wait it could not confirm earned no idle credit\n`
|
|
1287
1147
|
}`
|
|
1288
1148
|
: bound.kind === 'fixture'
|
|
1289
1149
|
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${segment ?? command}\n`
|
|
@@ -1420,7 +1280,7 @@ async function runCommand(
|
|
|
1420
1280
|
channel.end();
|
|
1421
1281
|
return;
|
|
1422
1282
|
}
|
|
1423
|
-
|
|
1283
|
+
queue.waiting();
|
|
1424
1284
|
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1425
1285
|
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1426
1286
|
reservation = held;
|
|
@@ -1432,7 +1292,7 @@ async function runCommand(
|
|
|
1432
1292
|
}
|
|
1433
1293
|
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1434
1294
|
if (state.settled || state.expiry !== null) return;
|
|
1435
|
-
|
|
1295
|
+
queue.admitted = true;
|
|
1436
1296
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1437
1297
|
channel.end(`${granted.cpus}\n`);
|
|
1438
1298
|
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
@@ -1451,7 +1311,7 @@ async function runCommand(
|
|
|
1451
1311
|
for (const byte of chunk) {
|
|
1452
1312
|
if (byte === 103) {
|
|
1453
1313
|
if (!said) {
|
|
1454
|
-
|
|
1314
|
+
queue.admitted = true;
|
|
1455
1315
|
said = true;
|
|
1456
1316
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1457
1317
|
}
|
|
@@ -1465,10 +1325,7 @@ async function runCommand(
|
|
|
1465
1325
|
}
|
|
1466
1326
|
});
|
|
1467
1327
|
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1468
|
-
|
|
1469
|
-
void refreshQueue(false);
|
|
1470
|
-
}, MBX_QUEUE_POLL_MS);
|
|
1471
|
-
void refreshQueue(false);
|
|
1328
|
+
queue.poll();
|
|
1472
1329
|
}
|
|
1473
1330
|
}
|
|
1474
1331
|
|
|
@@ -1493,8 +1350,8 @@ async function runCommand(
|
|
|
1493
1350
|
await signalCleanup;
|
|
1494
1351
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1495
1352
|
}
|
|
1496
|
-
|
|
1497
|
-
await
|
|
1353
|
+
queue.stop();
|
|
1354
|
+
await queue.idle();
|
|
1498
1355
|
await termination;
|
|
1499
1356
|
clearTimeout(boundTimer);
|
|
1500
1357
|
cancelCapture();
|
|
@@ -1549,10 +1406,12 @@ async function runCommand(
|
|
|
1549
1406
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1550
1407
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1551
1408
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1409
|
+
const killedForMemory = killedFromOutside({
|
|
1410
|
+
sentSignal: state.sentSignal,
|
|
1411
|
+
bounded: state.expiry !== null,
|
|
1412
|
+
exitSignal,
|
|
1413
|
+
code,
|
|
1414
|
+
});
|
|
1556
1415
|
return {
|
|
1557
1416
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1558
1417
|
terminalOutput: outputChunks.join(''),
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
|
|
3
|
+
* when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
|
|
4
|
+
* it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
|
|
5
|
+
* it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
|
|
6
|
+
* leaves it out (README, "Bounded exec key").
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** How one run of the command ended. */
|
|
10
|
+
export interface RunOutcome {
|
|
11
|
+
success: boolean;
|
|
12
|
+
terminalOutput: string;
|
|
13
|
+
/**
|
|
14
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
15
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
16
|
+
*/
|
|
17
|
+
killedForMemory?: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** What a run read of how its command ended and what this executor did to it. */
|
|
21
|
+
export interface Ending {
|
|
22
|
+
/** This executor sent the command's process group a signal. */
|
|
23
|
+
readonly sentSignal: boolean;
|
|
24
|
+
/** A bound of this executor ended the run. */
|
|
25
|
+
readonly bounded: boolean;
|
|
26
|
+
readonly exitSignal: NodeJS.Signals | null;
|
|
27
|
+
/** The exit status, or the status a signal folds to. */
|
|
28
|
+
readonly code: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
33
|
+
* run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
|
|
34
|
+
* it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
35
|
+
*/
|
|
36
|
+
export function killedFromOutside(ending: Ending): boolean {
|
|
37
|
+
return !ending.sentSignal && !ending.bounded && (ending.exitSignal === 'SIGKILL' || ending.code === 137);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
|
|
42
|
+
* two runs' output with a note naming the task.
|
|
43
|
+
*/
|
|
44
|
+
export async function runUntilNotKilledForMemory(
|
|
45
|
+
run: (alone: boolean) => Promise<RunOutcome>,
|
|
46
|
+
pooled: boolean,
|
|
47
|
+
task: string | undefined,
|
|
48
|
+
): Promise<RunOutcome> {
|
|
49
|
+
const first = await run(false);
|
|
50
|
+
if (first.killedForMemory !== true || !pooled) {
|
|
51
|
+
return first;
|
|
52
|
+
}
|
|
53
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
|
|
54
|
+
task === undefined ? '' : `: ${task}`
|
|
55
|
+
}); running it once more, alone in its run's pool\n`;
|
|
56
|
+
process.stderr.write(note);
|
|
57
|
+
const again = await run(true);
|
|
58
|
+
return {
|
|
59
|
+
success: again.success,
|
|
60
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
61
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { type TokenGrant, takeParentCpuTokens } from './cpu-pool.js';
|
|
2
|
+
import type { CpuWaitRow } from './lease-wait.js';
|
|
3
|
+
import type { InheritedCpuPool } from './pool-environment.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a run asks its parent's CPU pool and says of the answer. How many tokens the pool lends, how long the run waits
|
|
7
|
+
* and what it is charged are values that differ between runs of one task key; what a token count does to the command is
|
|
8
|
+
* `runner-sizing.ts`'s, keyed (README, "Bounded exec key").
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** Tokens a parent pool lent, held until released. */
|
|
12
|
+
export type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
13
|
+
|
|
14
|
+
/** The parent pool's answer to a run: a grant, a refusal that fails the run, or neither, and what to say of it. */
|
|
15
|
+
export interface ParentGrant {
|
|
16
|
+
readonly grant: GrantedTokens | null;
|
|
17
|
+
/** A pool that answered and refused is a live budget, and the run fails rather than run beside it. */
|
|
18
|
+
readonly refusal: string | null;
|
|
19
|
+
readonly wait: CpuWaitRow | undefined;
|
|
20
|
+
/** What the run says, in order. */
|
|
21
|
+
readonly said: readonly string[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** What a run asks for: its count, what its task's last recorded peak charges it, and whether it runs alone. */
|
|
25
|
+
export interface Ask {
|
|
26
|
+
readonly want: number;
|
|
27
|
+
readonly charge: number | null;
|
|
28
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
29
|
+
readonly parallelism: number | null;
|
|
30
|
+
readonly alone: boolean;
|
|
31
|
+
readonly task: string | undefined;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Ask the parent pool for the run's tokens. A run that inherits no pool runs as asked and says nothing; a pool whose
|
|
36
|
+
* endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
37
|
+
*/
|
|
38
|
+
export async function askParentPool(inherited: InheritedCpuPool | null, ask: Ask): Promise<ParentGrant> {
|
|
39
|
+
if (inherited === null) {
|
|
40
|
+
return { grant: null, refusal: null, wait: undefined, said: [] };
|
|
41
|
+
}
|
|
42
|
+
const answer = await takeParentCpuTokens(inherited, ask.want, ask.charge, {
|
|
43
|
+
parallelism: ask.parallelism ?? undefined,
|
|
44
|
+
alone: ask.alone ? true : undefined,
|
|
45
|
+
task: ask.task,
|
|
46
|
+
});
|
|
47
|
+
if (answer.granted) {
|
|
48
|
+
return { grant: answer, refusal: null, wait: undefined, said: [] };
|
|
49
|
+
}
|
|
50
|
+
if (answer.cause === 'absent') {
|
|
51
|
+
return {
|
|
52
|
+
grant: null,
|
|
53
|
+
refusal: null,
|
|
54
|
+
wait: undefined,
|
|
55
|
+
said: [`cowshed: cpu-pool parent pool absent: ${answer.reason}; running the command as asked\n`],
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
return { grant: null, refusal: answer.reason, wait: answer.wait, said: [] };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** What a run says once its grant is in hand. */
|
|
62
|
+
export function describeGrant(
|
|
63
|
+
grant: GrantedTokens,
|
|
64
|
+
run: { waitedMs: number; want: number; runner: string; charge: number | null },
|
|
65
|
+
): string[] {
|
|
66
|
+
const said = [
|
|
67
|
+
`cowshed: cpu-pool wait done elapsed=${run.waitedMs}ms status=ok tokens=${grant.tokens}/${run.want} runner=${run.runner} memory=${run.charge ?? 'none'}${grant.charged !== null && grant.charged !== run.charge ? ` charged=${grant.charged}` : ''}\n`,
|
|
68
|
+
];
|
|
69
|
+
if (grant.oversized !== null) {
|
|
70
|
+
said.push(
|
|
71
|
+
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${run.charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
return said;
|
|
75
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { readRecord, VERDICT_FILE } from './verdict.js';
|
|
3
|
+
|
|
4
|
+
/** What the task's last record charges its next run: the peak and the parallelism it was measured under. */
|
|
5
|
+
export type LastPeak =
|
|
6
|
+
| { ok: true; peakBytes: number | null; parallelism: number | null }
|
|
7
|
+
| { ok: false; error: string };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The peak the task's last record in `directory` names: its charge in its run's pool. Null when the task has no record
|
|
11
|
+
* or its record no peak; a record that cannot be read is an error naming it, never a charge of nothing. It decides what a
|
|
12
|
+
* pool charges a run, a value that differs between runs of one task key, so the key leaves it out (README, "Bounded
|
|
13
|
+
* exec key").
|
|
14
|
+
*/
|
|
15
|
+
export async function readLastPeak(directory: string): Promise<LastPeak> {
|
|
16
|
+
try {
|
|
17
|
+
const record = await readRecord(directory);
|
|
18
|
+
return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return {
|
|
21
|
+
ok: false,
|
|
22
|
+
error: `the task's last record ${join(directory, VERDICT_FILE)} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What a CPU pool puts in a command's environment (cpu-pool.ts serves the pool; this module decides what the command
|
|
5
|
+
* sees of it, so it is keyed: executors.json lists cpu-pool.js as output-neutral, which it may only be while it writes
|
|
6
|
+
* no variable the command reads).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
|
|
10
|
+
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
11
|
+
|
|
12
|
+
/** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
|
|
13
|
+
export interface InheritedCpuPool {
|
|
14
|
+
readonly socket: string;
|
|
15
|
+
/** The count the parent names beside its endpoint. */
|
|
16
|
+
readonly tokens: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
21
|
+
* or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
|
|
22
|
+
* descriptor still needs a live endpoint grant.
|
|
23
|
+
*/
|
|
24
|
+
export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null {
|
|
25
|
+
const socket = environment[CPU_POOL_ENV];
|
|
26
|
+
const tokens = parseInheritedTokens(environment[CPU_TOKENS_ENV]);
|
|
27
|
+
if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
return { socket, tokens };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The environment a child of a pool inherits to reach it: its endpoint, the count the pool holds, and its make
|
|
35
|
+
* jobserver, which Cargo and make read from the fifo; `-j` with a jobserver names no count of its own.
|
|
36
|
+
*/
|
|
37
|
+
export function poolEnvironment(socketPath: string, tokens: number, fifoPath: string): Record<string, string> {
|
|
38
|
+
const jobserverFlags = `-j --jobserver-auth=fifo:${fifoPath}`;
|
|
39
|
+
return {
|
|
40
|
+
[CPU_POOL_ENV]: socketPath,
|
|
41
|
+
[CPU_TOKENS_ENV]: String(tokens),
|
|
42
|
+
CARGO_MAKEFLAGS: jobserverFlags,
|
|
43
|
+
MAKEFLAGS: jobserverFlags,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -3,10 +3,11 @@ import { fileURLToPath } from 'node:url';
|
|
|
3
3
|
import { Worker } from 'node:worker_threads';
|
|
4
4
|
import { logger } from 'nx/src/devkit-exports.js';
|
|
5
5
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
6
|
-
import {
|
|
6
|
+
import type { CloseOutcome, CpuPool } from './cpu-pool.js';
|
|
7
7
|
import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
|
|
8
8
|
import { type RunMemory, type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
9
9
|
import { mbxPool } from './mbx-pool.js';
|
|
10
|
+
import { readInheritedCpuPool } from './pool-environment.js';
|
|
10
11
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
11
12
|
|
|
12
13
|
/**
|