@smoothbricks/nx-plugin 0.4.29 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -25
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +2 -8
- package/dist/cargo-closure-input.d.ts +11 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +16 -6
- package/dist/ensure-built-receipt.d.ts +8 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +45 -13
- package/dist/ensure-built-witness.d.ts +40 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +68 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +11 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts +7 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +9 -4
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +38 -14
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +3 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +15 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/graph-input-contract.d.ts +4 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +17 -4
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -3
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +15 -10
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +8 -1
- package/managed/raw/tooling/direnv/developer-links.ts +1 -8
- package/managed/raw/tooling/direnv/setup-environment.ts +13 -1
- package/package.json +15 -4
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/owned-installation.ts +4 -164
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.ts +2 -8
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +15 -112
- package/src/cargo-closure-input.ts +23 -6
- package/src/cargo-closure-key.test.ts +240 -0
- package/src/cargo-resolve-cache.test.ts +29 -7
- package/src/ensure-built-receipt.ts +56 -13
- package/src/ensure-built-witness.test.ts +113 -4
- package/src/ensure-built-witness.ts +75 -16
- package/src/ensure-built.test.ts +74 -2
- package/src/ensure-built.ts +13 -2
- package/src/executors/bounded-exec/cpu-pool.test.ts +18 -0
- package/src/executors/bounded-exec/cpu-pool.ts +10 -4
- package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +23 -11
- package/src/executors/bounded-exec/executor.test.ts +189 -63
- package/src/executors/bounded-exec/executor.ts +39 -13
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/ram-temp-host.test.ts +9 -8
- package/src/executors/bounded-exec/ram-temp.test.ts +32 -18
- package/src/executors/bounded-exec/ram-temp.ts +17 -4
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
- package/src/executors/bounded-exec/verdict.test.ts +125 -25
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/graph-input-contract.test.ts +8 -0
- package/src/graph-input-contract.ts +26 -4
- package/src/index.test.ts +12 -2
- package/src/index.ts +13 -3
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nx-daemon-claim.test.ts +5 -3
- package/src/nx-daemon-idle-policy.test.ts +22 -18
- package/src/nx-daemon-validity.test.ts +5 -13
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.ts +15 -10
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/testing.ts +8 -1
- package/test-timings.json +4 -0
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
} from './cpu-tokens.js';
|
|
24
24
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
25
25
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
26
|
+
import { createNoSpaceWatch } from './no-space.js';
|
|
26
27
|
import { createRunProgress, withProgressReport } from './progress.js';
|
|
27
28
|
import {
|
|
28
29
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -38,11 +39,13 @@ import {
|
|
|
38
39
|
type BoundedExecRecord,
|
|
39
40
|
type BoundedExecTask,
|
|
40
41
|
clearTaskDirectory,
|
|
42
|
+
describeNoSpace,
|
|
41
43
|
type ExpiredBound,
|
|
42
44
|
JUNIT_FILE,
|
|
43
45
|
JUNIT_REPORT_ENV,
|
|
44
46
|
judgeOperationalFailure,
|
|
45
47
|
judgeRun,
|
|
48
|
+
type NoSpaceSighting,
|
|
46
49
|
readJunitReport,
|
|
47
50
|
taskDirectory,
|
|
48
51
|
withJunitReport,
|
|
@@ -92,6 +95,8 @@ interface RunState {
|
|
|
92
95
|
forceKillNeeded: boolean;
|
|
93
96
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
94
97
|
poolCloseError: string | null;
|
|
98
|
+
/** The first output line that said ENOSPC, with the task's volume read when it arrived. */
|
|
99
|
+
noSpace: Promise<NoSpaceSighting> | null;
|
|
95
100
|
}
|
|
96
101
|
|
|
97
102
|
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
@@ -101,7 +106,7 @@ export interface ProcessTreeKiller {
|
|
|
101
106
|
}
|
|
102
107
|
|
|
103
108
|
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
104
|
-
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
|
|
109
|
+
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
|
|
105
110
|
|
|
106
111
|
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
107
112
|
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
@@ -248,6 +253,7 @@ export async function runBoundedExec(
|
|
|
248
253
|
expiry: null,
|
|
249
254
|
forceKillNeeded: false,
|
|
250
255
|
poolCloseError: null,
|
|
256
|
+
noSpace: null,
|
|
251
257
|
};
|
|
252
258
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
253
259
|
const text = chunk.toString();
|
|
@@ -308,14 +314,15 @@ export async function runBoundedExec(
|
|
|
308
314
|
}
|
|
309
315
|
};
|
|
310
316
|
|
|
311
|
-
// The wait is one span line, as cowshed's own spans read.
|
|
312
|
-
//
|
|
317
|
+
// The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
|
|
318
|
+
// pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
|
|
319
|
+
// asked and nothing is said.
|
|
313
320
|
const cores = availableParallelism();
|
|
314
321
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
315
322
|
const asked = Date.now();
|
|
316
|
-
// A run a parent's CPU pool
|
|
317
|
-
// A pool that answered and refused
|
|
318
|
-
//
|
|
323
|
+
// A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
|
|
324
|
+
// A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
|
|
325
|
+
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
319
326
|
let grant: GrantedTokens | null = null;
|
|
320
327
|
let parentRefusal: string | null = null;
|
|
321
328
|
if (inherited !== null) {
|
|
@@ -377,6 +384,7 @@ export async function runBoundedExec(
|
|
|
377
384
|
expiry: ExpiredBound | null,
|
|
378
385
|
elapsedMs: number,
|
|
379
386
|
operational: boolean,
|
|
387
|
+
noSpace: NoSpaceSighting | null,
|
|
380
388
|
): Promise<void> => {
|
|
381
389
|
if (context.record === undefined) {
|
|
382
390
|
return;
|
|
@@ -385,7 +393,7 @@ export async function runBoundedExec(
|
|
|
385
393
|
const report = await readJunitReport(recordDirectory);
|
|
386
394
|
const verdict = operational
|
|
387
395
|
? judgeOperationalFailure(exitCode, report)
|
|
388
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report });
|
|
396
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
389
397
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
390
398
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
391
399
|
}
|
|
@@ -396,7 +404,7 @@ export async function runBoundedExec(
|
|
|
396
404
|
appendStderr(message);
|
|
397
405
|
await releaseCpu();
|
|
398
406
|
await releaseLease();
|
|
399
|
-
await recordRun(1, null, Date.now() - asked, false);
|
|
407
|
+
await recordRun(1, null, Date.now() - asked, false, null);
|
|
400
408
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
401
409
|
};
|
|
402
410
|
if (parentRefusal !== null) {
|
|
@@ -461,6 +469,13 @@ export async function runBoundedExec(
|
|
|
461
469
|
|
|
462
470
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
463
471
|
const progress = createRunProgress();
|
|
472
|
+
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
473
|
+
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
474
|
+
const noSpace = createNoSpaceWatch();
|
|
475
|
+
const sight = async (line: string): Promise<NoSpaceSighting> => {
|
|
476
|
+
const held = lease;
|
|
477
|
+
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
478
|
+
};
|
|
464
479
|
let idleTimer: NodeJS.Timeout | undefined;
|
|
465
480
|
let captureTimer: NodeJS.Timeout | undefined;
|
|
466
481
|
let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
|
|
@@ -569,11 +584,15 @@ export async function runBoundedExec(
|
|
|
569
584
|
child.stdout?.on('data', (chunk: Buffer | string) => {
|
|
570
585
|
const text = chunk.toString();
|
|
571
586
|
if (progress.scan('stdout', text)) armIdleTimer();
|
|
587
|
+
const said = noSpace.scan('stdout', text);
|
|
588
|
+
if (said !== null) state.noSpace = sight(said);
|
|
572
589
|
appendStdout(text);
|
|
573
590
|
});
|
|
574
591
|
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
575
592
|
const text = chunk.toString();
|
|
576
593
|
if (progress.scan('stderr', text)) armIdleTimer();
|
|
594
|
+
const said = noSpace.scan('stderr', text);
|
|
595
|
+
if (said !== null) state.noSpace = sight(said);
|
|
577
596
|
appendStderr(text);
|
|
578
597
|
});
|
|
579
598
|
|
|
@@ -752,17 +771,24 @@ export async function runBoundedExec(
|
|
|
752
771
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
753
772
|
}
|
|
754
773
|
|
|
755
|
-
//
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
774
|
+
// ENOSPC said in the output names the writer's volume as it stood then. Without it, a volume still (nearly) full now
|
|
775
|
+
// is the likely cause, and the volume is named.
|
|
776
|
+
const seen = state.noSpace === null ? null : await state.noSpace;
|
|
777
|
+
if (seen !== null && (code !== 0 || state.expiry !== null)) {
|
|
778
|
+
const writer = context.record?.task.id;
|
|
779
|
+
appendStderr(`${writer === undefined ? '' : `${writer}: `}${describeNoSpace(seen)}\n`);
|
|
780
|
+
} else {
|
|
781
|
+
const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
|
|
782
|
+
if (full) {
|
|
783
|
+
appendStderr(`${describeRamTempError(full)}\n`);
|
|
784
|
+
}
|
|
759
785
|
}
|
|
760
786
|
await releaseLease();
|
|
761
787
|
|
|
762
788
|
// A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
|
|
763
789
|
// its verdict is an ordinary failure, never a bound.
|
|
764
790
|
const operationalFailure = state.poolCloseError !== null;
|
|
765
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
|
|
791
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
766
792
|
|
|
767
793
|
return {
|
|
768
794
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { describe, expect, it } from 'bun:test';
|
|
1
|
+
import { describe, expect, it, jest } from 'bun:test';
|
|
2
2
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
5
6
|
import {
|
|
6
7
|
captureHangDiagnostics,
|
|
7
8
|
type HangCaptureRequest,
|
|
@@ -23,8 +24,8 @@ const exited = (stdout: string, code = 0, stderr = ''): HangProbeResult => ({ st
|
|
|
23
24
|
const request = (): HangCaptureRequest => ({
|
|
24
25
|
pid: ROOT,
|
|
25
26
|
pgid: ROOT,
|
|
26
|
-
expiresAtMs: Date.now() +
|
|
27
|
-
deadlineMs:
|
|
27
|
+
expiresAtMs: Date.now() + BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
28
|
+
deadlineMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
28
29
|
});
|
|
29
30
|
|
|
30
31
|
function scriptedHost(results: readonly HangProbeResult[]) {
|
|
@@ -140,6 +141,7 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
140
141
|
return new Promise<HangProbeResult>(() => {});
|
|
141
142
|
},
|
|
142
143
|
};
|
|
144
|
+
// The probe is held off by construction, so the 1 ms deadline timer is the only event that can end this capture.
|
|
143
145
|
const capture = await captureHangDiagnostics({ ...request(), deadlineMs: 1 }, unresponsive);
|
|
144
146
|
expect(capture.outcome).toBe('incomplete');
|
|
145
147
|
expect(capture.text).toContain('1ms deadline reached during ps before');
|
|
@@ -228,12 +230,11 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
228
230
|
let descendant: number | undefined;
|
|
229
231
|
const failures: unknown[] = [];
|
|
230
232
|
try {
|
|
231
|
-
const reader = child.stdout
|
|
232
|
-
.pipeThrough(new TextDecoderStream(), { signal: AbortSignal.timeout(10_000) })
|
|
233
|
-
.getReader();
|
|
233
|
+
const reader = child.stdout.pipeThrough(new TextDecoderStream()).getReader();
|
|
234
234
|
let text = '';
|
|
235
235
|
try {
|
|
236
|
-
//
|
|
236
|
+
// The first complete line is the readiness event; a child that exits first ends the stream instead. No timer
|
|
237
|
+
// ends this loop.
|
|
237
238
|
for (;;) {
|
|
238
239
|
const chunk = await reader.read();
|
|
239
240
|
if (chunk.done) throw new Error('child stdout ended before a complete PID line');
|
|
@@ -252,7 +253,12 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
252
253
|
Number.isSafeInteger(descendant) && descendant > 0,
|
|
253
254
|
`expected a tagged PID line, observed ${JSON.stringify(line)}`,
|
|
254
255
|
).toBe(true);
|
|
255
|
-
|
|
256
|
+
// The capture budget (`HANG_CAPTURE_MS`) and probe limits are product constants this test cannot set, so the
|
|
257
|
+
// clock is stopped while the real `ps` and `lsof` run: each ends when it exits; no timer cuts it short.
|
|
258
|
+
jest.useFakeTimers();
|
|
259
|
+
const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null }).finally(() =>
|
|
260
|
+
jest.useRealTimers(),
|
|
261
|
+
);
|
|
256
262
|
expect(capture.outcome).toBe('holders');
|
|
257
263
|
expect(capture.text).toContain(String(child.pid));
|
|
258
264
|
expect(capture.text).toContain(String(descendant));
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { OutputStream } from './progress.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The operating system's own words for ENOSPC: the errno name Node and Bun print (`ENOSPC: no space left on device,
|
|
5
|
+
* write`), and `strerror(28)`, which Rust (`No space left on device (os error 28)`), Go and C print. Nothing else in a
|
|
6
|
+
* command's output says the disk was full, and a write that failed on it is otherwise reported as whatever its caller
|
|
7
|
+
* was doing: a failed test, an extraction error.
|
|
8
|
+
*/
|
|
9
|
+
const NO_SPACE = /\bENOSPC\b|No space left on device/;
|
|
10
|
+
const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
|
|
11
|
+
|
|
12
|
+
/** Watches a command's output for the first line that says a write found no space. */
|
|
13
|
+
export interface NoSpaceWatch {
|
|
14
|
+
/** One chunk of one output stream: the first matching line, once, as printed; `null` before and after it. */
|
|
15
|
+
scan(stream: OutputStream, chunk: string): string | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function createNoSpaceWatch(): NoSpaceWatch {
|
|
19
|
+
const partial: Record<OutputStream, string> = { stdout: '', stderr: '' };
|
|
20
|
+
let seen = false;
|
|
21
|
+
return {
|
|
22
|
+
scan(stream, chunk) {
|
|
23
|
+
if (seen) {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const lines = (partial[stream] + chunk).split('\n');
|
|
27
|
+
// The unfinished last line is kept for the next chunk, and read now too: a command that dies mid-line still
|
|
28
|
+
// printed the words.
|
|
29
|
+
partial[stream] = lines.at(-1) ?? '';
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const text = line.replace(ANSI, '').trim();
|
|
32
|
+
if (NO_SPACE.test(text)) {
|
|
33
|
+
seen = true;
|
|
34
|
+
return text;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -16,10 +16,11 @@ import {
|
|
|
16
16
|
} from 'node:fs';
|
|
17
17
|
import { tmpdir, userInfo } from 'node:os';
|
|
18
18
|
import { join } from 'node:path';
|
|
19
|
-
import { setTimeout as sleep } from 'node:timers/promises';
|
|
20
19
|
import { fileURLToPath } from 'node:url';
|
|
21
20
|
import type { OpContextBinding } from '@smoothbricks/lmao';
|
|
22
21
|
import { useTestSpan } from '@smoothbricks/lmao/testing/bun';
|
|
22
|
+
import { whenChanged } from '../../__tests__/directory-change.js';
|
|
23
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
23
24
|
import { processTable, terminate } from '../../testing.js';
|
|
24
25
|
import {
|
|
25
26
|
type CommandOutput,
|
|
@@ -386,9 +387,8 @@ async function nativeRamVolume(): Promise<void> {
|
|
|
386
387
|
mkdirSync(quiet);
|
|
387
388
|
writeFileSync(join(quiet, 'left'), 'by a task nothing of which still runs');
|
|
388
389
|
try {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
}
|
|
390
|
+
// Ends on the watch event that creates the writer's `w1` in the lease, not on a poll.
|
|
391
|
+
await whenChanged(leaseB.directory, () => existsSync(join(leaseB.directory, 'w1')));
|
|
392
392
|
const fourth = new RamTempVolume(paths, capacity, commands, 101, (pid) => alive.has(pid), processes);
|
|
393
393
|
const d = await fourth.acquire();
|
|
394
394
|
if (!d.ok || d.value.kind !== 'leased') {
|
|
@@ -476,11 +476,12 @@ async function aFailedStepLeavesNoNamedVolume(): Promise<void> {
|
|
|
476
476
|
(call) => call.file === '/usr/bin/hdiutil' && call.args[0] === 'detach' && call.output.status !== 0,
|
|
477
477
|
);
|
|
478
478
|
if (failedDetach !== undefined) {
|
|
479
|
-
|
|
479
|
+
// Diagnostic probes explain a failed detach: each waits for the host to answer, bounded by the leg backstop.
|
|
480
|
+
const mountTable = await host.run('/sbin/mount', [], BOUNDED_TEST_IDLE_TIMEOUT_MS);
|
|
480
481
|
const filesystemHolders = await host.run(
|
|
481
482
|
'/usr/bin/sudo',
|
|
482
483
|
['-n', '/usr/sbin/lsof', '-nP', '+f', '--', paths.staging],
|
|
483
|
-
|
|
484
|
+
BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
484
485
|
);
|
|
485
486
|
const added = calls.find(
|
|
486
487
|
(call) => call.file === '/usr/sbin/diskutil' && call.args[0] === 'apfs' && call.args[1] === 'addVolume',
|
|
@@ -494,7 +495,7 @@ async function aFailedStepLeavesNoNamedVolume(): Promise<void> {
|
|
|
494
495
|
const deviceHolders = await host.run(
|
|
495
496
|
'/usr/bin/sudo',
|
|
496
497
|
['-n', '/usr/sbin/lsof', '-nP', '--', ...devicePaths],
|
|
497
|
-
|
|
498
|
+
BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
498
499
|
);
|
|
499
500
|
const observations = JSON.stringify(
|
|
500
501
|
{
|
|
@@ -577,7 +578,7 @@ async function inheritedTempDirectory(): Promise<void> {
|
|
|
577
578
|
for (const directory of [inherited, supplied]) {
|
|
578
579
|
const options = {
|
|
579
580
|
command: `node ${JSON.stringify(probe)}`,
|
|
580
|
-
timeoutMs:
|
|
581
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
581
582
|
parallelism: 1,
|
|
582
583
|
env: {
|
|
583
584
|
// If the default runner incorrectly provisions an APFS volume here, it must fail,
|
|
@@ -7,7 +7,8 @@ import { createServer, type Socket } from 'node:net';
|
|
|
7
7
|
import { tmpdir } from 'node:os';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
9
|
import { createInterface } from 'node:readline';
|
|
10
|
-
import {
|
|
10
|
+
import { whenChanged } from '../../__tests__/directory-change.js';
|
|
11
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
11
12
|
import { awaitExit, ownedSocketDirectory, pidsWorkingIn, processTable, terminate } from '../../testing.js';
|
|
12
13
|
import {
|
|
13
14
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -180,15 +181,29 @@ async function ownedDiskFacadeFixture() {
|
|
|
180
181
|
|
|
181
182
|
describe('RAM temp volume on this host', () => {
|
|
182
183
|
it('kills a deadlined command together with everything it spawned', async () => {
|
|
183
|
-
// Ordinary helpers remain deadline-bound; disk-image tools are owned by cowshed instead.
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
184
|
+
// Ordinary helpers remain deadline-bound; disk-image tools are owned by cowshed instead. The command never answers:
|
|
185
|
+
// its shell opens `lifetime`, hands it to a background `tail -f /dev/null` that never ends on its own, closes its
|
|
186
|
+
// own copy and only then prints the tail's pid. So only the deadline ends the run, a printed pid means the FIFO
|
|
187
|
+
// has a writer, and EOF on it is that descendant's death. The bound returns once it has signalled the group, not
|
|
188
|
+
// once the kernel has torn it down, so a `ps` read straight after raced the kill (Linux CI saw it still running).
|
|
189
|
+
const directory = mkdtempSync(join(realpathSync(tmpdir()), 'deadline-group-'));
|
|
190
|
+
try {
|
|
191
|
+
const lifetime = join(directory, 'lifetime.fifo');
|
|
192
|
+
execFileSync('mkfifo', [lifetime]);
|
|
193
|
+
const ended = readFile(lifetime, 'utf8');
|
|
194
|
+
const result = await createHostCommands(500).run('/bin/sh', [
|
|
195
|
+
'-c',
|
|
196
|
+
'exec 3>"$1"; tail -f /dev/null & exec 3>&-; echo $!; wait',
|
|
197
|
+
'sh',
|
|
198
|
+
lifetime,
|
|
199
|
+
]);
|
|
200
|
+
expect(result.status).not.toBe(0);
|
|
201
|
+
expect(result.stderr).toMatch(/^no answer within 500 ms; killed process group \d+$/);
|
|
202
|
+
expect(result.stdout.trim()).toMatch(/^\d+$/);
|
|
203
|
+
expect(await ended).toBe('');
|
|
204
|
+
} finally {
|
|
205
|
+
rmSync(directory, { recursive: true, force: true });
|
|
206
|
+
}
|
|
192
207
|
});
|
|
193
208
|
|
|
194
209
|
it('routes disk tools and their lifetime together without shell interpolation', () => {
|
|
@@ -302,9 +317,9 @@ describe('RAM temp volume on this host', () => {
|
|
|
302
317
|
const directory = await ownedSocketDirectory('rt-', 'missing-cowshed');
|
|
303
318
|
try {
|
|
304
319
|
const executable = join(directory, 'missing-cowshed');
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
]);
|
|
320
|
+
// The bound is the leg backstop: a missing facade fails at spawn, so the run ends at once.
|
|
321
|
+
const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, undefined, undefined, executable);
|
|
322
|
+
const output = await host.run('/usr/sbin/diskutil', ['info']);
|
|
308
323
|
expect(output.status).toBe(1);
|
|
309
324
|
expect(output.stdout).toBe('');
|
|
310
325
|
expect(output.stderr).toContain(executable);
|
|
@@ -335,7 +350,7 @@ describe('RAM temp volume on this host', () => {
|
|
|
335
350
|
server.listen(socket, listened);
|
|
336
351
|
await listening;
|
|
337
352
|
server.off('error', failed);
|
|
338
|
-
const host = createHostCommands(
|
|
353
|
+
const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, socket);
|
|
339
354
|
// `umount` with no operand only prints its usage, and it is a disk tool that takes a lease.
|
|
340
355
|
expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
|
|
341
356
|
expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
|
|
@@ -369,7 +384,7 @@ describe('RAM temp volume on this host', () => {
|
|
|
369
384
|
server.listen(socket, listened);
|
|
370
385
|
await listening;
|
|
371
386
|
server.off('error', failed);
|
|
372
|
-
const host = createHostCommands(
|
|
387
|
+
const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, socket, 'alone');
|
|
373
388
|
// `umount` with no operand only prints its usage, and it is a disk tool that takes a lease.
|
|
374
389
|
expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
|
|
375
390
|
expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
|
|
@@ -684,9 +699,8 @@ describe('processes a task left working in its RAM lease', () => {
|
|
|
684
699
|
let child = 0;
|
|
685
700
|
try {
|
|
686
701
|
child = Number(String((await once(writer.stdout, 'data'))[0]).trim());
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
}
|
|
702
|
+
// Ends on the watch event that brings the lease to two entries: `nested`, and the writer's first file.
|
|
703
|
+
await whenChanged(lease, () => readdirSync(lease).length >= 2);
|
|
690
704
|
// None of them works in the lease by its working directory.
|
|
691
705
|
expect(await pidsWorkingIn(lease)).toEqual([]);
|
|
692
706
|
|
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
parseHeldVerdict,
|
|
40
40
|
validateHeldVerdict,
|
|
41
41
|
} from './ram-temp-held.js';
|
|
42
|
+
import type { TempVolumeSpace } from './verdict.js';
|
|
42
43
|
|
|
43
44
|
/**
|
|
44
45
|
* One RAM-backed APFS volume per host user, shared by every test task that runs
|
|
@@ -856,12 +857,24 @@ export class RamTempVolume {
|
|
|
856
857
|
|
|
857
858
|
/** After a failed task: whether the volume ran out of space, which is then the error to report. */
|
|
858
859
|
async fullness(lease: RamTempLease): Promise<RamTempError | null> {
|
|
860
|
+
const space = await this.space(lease);
|
|
861
|
+
if (space === null || space.freeBytes >= FULL_FREE_BYTES) {
|
|
862
|
+
return null;
|
|
863
|
+
}
|
|
864
|
+
const { mountpoint, capacityBytes, freeBytes } = space;
|
|
865
|
+
return { kind: 'volume-full', mountpoint, capacityBytes, freeBytes };
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/** The lease's volume and the space left on it now; `null` when the volume cannot be read. */
|
|
869
|
+
async space(lease: RamTempLease): Promise<TempVolumeSpace | null> {
|
|
859
870
|
try {
|
|
860
871
|
const fs = await statfs(lease.mountpoint);
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
:
|
|
872
|
+
return {
|
|
873
|
+
lease: lease.directory,
|
|
874
|
+
mountpoint: lease.mountpoint,
|
|
875
|
+
capacityBytes: lease.capacityBytes,
|
|
876
|
+
freeBytes: fs.bavail * fs.bsize,
|
|
877
|
+
};
|
|
865
878
|
} catch {
|
|
866
879
|
return null;
|
|
867
880
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { CPU_POOL_ENV, readInheritedCpuPool, takeParentCpuTokens } from './cpu-pool.js';
|
|
6
|
+
import { CPU_TOKENS_ENV } from './cpu-tokens.js';
|
|
7
|
+
import { holdLease } from './gateway-lease.js';
|
|
8
|
+
import { closeRunCpuPool, serveRunCpuPool } from './run-cpu-pool.js';
|
|
9
|
+
|
|
10
|
+
const directories: string[] = [];
|
|
11
|
+
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
for (const directory of directories.splice(0)) {
|
|
14
|
+
rmSync(directory, { recursive: true, force: true });
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
/** A gateway socket path in a fresh directory: absent unless the test makes it. */
|
|
19
|
+
function gatewayPath(): string {
|
|
20
|
+
const directory = mkdtempSync(join(tmpdir(), 'run-cpu-pool-'));
|
|
21
|
+
directories.push(directory);
|
|
22
|
+
return join(directory, 'gateway.sock');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe('a run without a gateway', () => {
|
|
26
|
+
it('serves its tasks one pool of one token per core, so a second runner waits for the first', async () => {
|
|
27
|
+
const env: NodeJS.ProcessEnv = {};
|
|
28
|
+
await serveRunCpuPool({ id: 'run-477' }, env, gatewayPath(), 4);
|
|
29
|
+
const pool = readInheritedCpuPool(env);
|
|
30
|
+
if (pool === null) {
|
|
31
|
+
throw new Error(`the run published no pool: ${JSON.stringify(env)}`);
|
|
32
|
+
}
|
|
33
|
+
expect(pool.tokens).toBe(4);
|
|
34
|
+
// Nightly 477's shape: two mutation sweeps, each asking for the whole host. The first holds the run's cores;
|
|
35
|
+
// the second queues for them instead of starting a second host's worth of workers beside it.
|
|
36
|
+
const first = await takeParentCpuTokens(pool, 31);
|
|
37
|
+
expect(first.granted && first.tokens).toBe(4);
|
|
38
|
+
const { promise: queued, resolve: queue } = Promise.withResolvers<void>();
|
|
39
|
+
const second = holdLease(
|
|
40
|
+
pool.socket,
|
|
41
|
+
{ op: 'cpu-pool', want: 31 },
|
|
42
|
+
'the run CPU pool',
|
|
43
|
+
{ ackMs: 2_000, afterCloseMs: 5_000, grantMs: null },
|
|
44
|
+
queue,
|
|
45
|
+
);
|
|
46
|
+
let secondSettled = false;
|
|
47
|
+
void second.then(() => {
|
|
48
|
+
secondSettled = true;
|
|
49
|
+
});
|
|
50
|
+
await queued;
|
|
51
|
+
expect(secondSettled).toBe(false);
|
|
52
|
+
if (first.granted) first.release();
|
|
53
|
+
const granted = await second;
|
|
54
|
+
expect(granted.granted && granted.answer.tokens).toBe(4);
|
|
55
|
+
if (granted.granted) granted.release();
|
|
56
|
+
|
|
57
|
+
await closeRunCpuPool({ id: 'run-477' });
|
|
58
|
+
// The run's end closes its pool: nothing is left to grant from it.
|
|
59
|
+
const late = await takeParentCpuTokens(pool, 1);
|
|
60
|
+
expect(late.granted === false && late.cause).toBe('absent');
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('leaves a run that inherits a pool to that pool', async () => {
|
|
64
|
+
const env: NodeJS.ProcessEnv = { [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' };
|
|
65
|
+
await serveRunCpuPool({ id: 'nested' }, env, gatewayPath(), 4);
|
|
66
|
+
expect(env).toEqual({ [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' });
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('leaves a run on a host whose gateway socket exists to the gateway, answering or not', async () => {
|
|
70
|
+
const gateway = gatewayPath();
|
|
71
|
+
writeFileSync(gateway, '');
|
|
72
|
+
const env: NodeJS.ProcessEnv = {};
|
|
73
|
+
await serveRunCpuPool({ id: 'gateway-host' }, env, gateway, 4);
|
|
74
|
+
expect(env).toEqual({});
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises';
|
|
2
|
+
import { availableParallelism } from 'node:os';
|
|
3
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
4
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
5
|
+
import { type CpuPool, createCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
6
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The CPU budget of an Nx run on a host without a gateway (specs/cowshed/05_gateway.md, "Host CPU budget", "A run
|
|
10
|
+
* without a gateway").
|
|
11
|
+
*
|
|
12
|
+
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
13
|
+
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
14
|
+
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
15
|
+
* task inherits it: `bounded-exec` and the `cargo` launcher take from it as from any parent pool, FIFO, the head
|
|
16
|
+
* taking `min(want, free)` once one token is free, so the runners of one run together stay within the cores.
|
|
17
|
+
*
|
|
18
|
+
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
19
|
+
* not exist. Only `ENOENT` proves the gateway absent, as for the launcher; a socket that exists but does not answer
|
|
20
|
+
* may be a live, saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
24
|
+
const pools = new Map<string, CpuPool>();
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
|
|
28
|
+
* inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
|
|
29
|
+
* unbudgeted, as a run without a gateway was before.
|
|
30
|
+
*/
|
|
31
|
+
export async function serveRunCpuPool(
|
|
32
|
+
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
33
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
34
|
+
gateway: string = GATEWAY_SOCKET,
|
|
35
|
+
cores: number = availableParallelism(),
|
|
36
|
+
): Promise<void> {
|
|
37
|
+
if (readInheritedCpuPool(env) !== null || !(await absent(gateway))) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
// The pool lives until the run's `postTasksExecution` closes it, or with this process.
|
|
41
|
+
const made = await createCpuPool(cores, new Promise<void>(() => {}), { begin() {}, end() {} });
|
|
42
|
+
if (!made.ok) {
|
|
43
|
+
logger.warn(
|
|
44
|
+
`bounded-exec: no gateway at ${gateway}, and this run's CPU pool failed (${made.error}); its tasks run unbudgeted`,
|
|
45
|
+
);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
pools.set(context.id, made.pool);
|
|
49
|
+
for (const [name, value] of Object.entries(made.pool.environment)) {
|
|
50
|
+
env[name] = value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Close the pool `context`'s run was served, if it was served one. */
|
|
55
|
+
export async function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
56
|
+
const pool = pools.get(context.id);
|
|
57
|
+
if (pool === undefined) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
pools.delete(context.id);
|
|
61
|
+
const closed = await pool.close();
|
|
62
|
+
if (!closed.ok) {
|
|
63
|
+
logger.warn(`bounded-exec: this run's CPU pool did not close cleanly: ${closed.error}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function absent(path: string): Promise<boolean> {
|
|
68
|
+
try {
|
|
69
|
+
await lstat(path);
|
|
70
|
+
return false;
|
|
71
|
+
} catch (error) {
|
|
72
|
+
const code = error instanceof Error && 'code' in error ? error.code : undefined;
|
|
73
|
+
return code === 'ENOENT' || code === 'ENOTDIR';
|
|
74
|
+
}
|
|
75
|
+
}
|