@smoothbricks/nx-plugin 0.4.28 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +277 -89
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +8 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +18 -30
- package/dist/cargo-closure-input.d.ts +23 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +41 -9
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +42 -19
- package/dist/ensure-built-receipt.d.ts +40 -6
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +297 -159
- package/dist/ensure-built-witness.d.ts +65 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +2237 -23
- package/dist/ensure-built.d.ts +99 -1
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +45 -56
- package/dist/executors/bounded-exec/cpu-pool.d.ts +22 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +51 -22
- package/dist/executors/bounded-exec/executor.d.ts +12 -3
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +174 -51
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +20 -11
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +42 -17
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/generators/bun-test-tracing/generator.js +2 -2
- package/dist/graph-input-contract.d.ts +102 -0
- package/dist/graph-input-contract.d.ts.map +1 -0
- package/dist/graph-input-contract.js +637 -0
- package/dist/index.d.ts +12 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +5 -2
- package/dist/managed-files/publish-workflow.d.ts +11 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts +19 -0
- package/dist/producer-receipts.d.ts.map +1 -0
- package/dist/producer-receipts.js +926 -0
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts +8 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +66 -1
- package/managed/raw/patches/nx@23.2.1.patch +328 -50
- package/managed/raw/tooling/direnv/developer-links.ts +148 -29
- package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
- package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
- package/managed/raw/tooling/direnv/repo-path +15 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +179 -115
- package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
- package/nextest.toml +1 -1
- package/package.json +25 -9
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/fixture-nx-env.ts +7 -0
- package/src/__tests__/owned-installation.ts +14 -0
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.test.ts +36 -22
- package/src/bounded-test-policy.ts +18 -32
- package/src/bun-bootstrap.test.ts +1 -1
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +73 -121
- package/src/cargo-closure-input.ts +47 -8
- package/src/cargo-closure-key.test.ts +239 -0
- package/src/cargo-resolve-cache.test.ts +168 -31
- package/src/cargo-source-hash.test.ts +78 -3
- package/src/cargo-source-hash.ts +48 -30
- package/src/cross-check-policy.test.ts +2 -2
- package/src/ensure-built-receipt.test.ts +111 -16
- package/src/ensure-built-receipt.ts +294 -149
- package/src/ensure-built-witness.test.ts +306 -13
- package/src/ensure-built-witness.ts +255 -24
- package/src/ensure-built.test.ts +228 -102
- package/src/ensure-built.ts +127 -66
- package/src/executors/bounded-exec/cpu-pool.test.ts +134 -32
- package/src/executors/bounded-exec/cpu-pool.ts +64 -20
- package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +74 -52
- package/src/executors/bounded-exec/executor.test.ts +343 -97
- package/src/executors/bounded-exec/executor.ts +192 -56
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/progress.test.ts +1 -1
- package/src/executors/bounded-exec/ram-temp-host.test.ts +51 -21
- package/src/executors/bounded-exec/ram-temp.test.ts +92 -16
- package/src/executors/bounded-exec/ram-temp.ts +49 -20
- package/src/executors/bounded-exec/verdict.test.ts +127 -22
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/generators/bounded-test-targets/generator.test.ts +1 -3
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/bun-test-tracing/generator.ts +2 -2
- package/src/graph-input-contract.test.ts +379 -0
- package/src/graph-input-contract.ts +627 -0
- package/src/index.test.ts +3 -3
- package/src/index.ts +24 -3
- package/src/linked-snapshots.nx.test.ts +112 -30
- package/src/managed-files/ci-workflow.ts +5 -2
- package/src/managed-files/publish-workflow.ts +12 -1
- package/src/managed-files/repo-path.test.ts +44 -1
- package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
- package/src/nextest-code-bound.test.ts +23 -17
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nextest-partitions.test.ts +7 -1
- package/src/nx-cache-restore-mtime.test.ts +2 -2
- package/src/nx-daemon-claim.test.ts +10 -8
- package/src/nx-daemon-idle-cpu.test.ts +0 -1
- package/src/nx-daemon-idle-policy.test.ts +61 -1
- package/src/nx-daemon-ignore-files.test.ts +2 -2
- package/src/nx-daemon-validity.test.ts +36 -22
- package/src/nx-dot-input-policy.test.ts +5 -5
- package/src/nx-files-archive-racy.test.ts +1 -1
- package/src/nx-incomplete-run-summary.test.ts +1 -1
- package/src/nx-plugin-load-errors.test.ts +2 -2
- package/src/nx-project-graph-errors.test.ts +70 -0
- package/src/nx-skipped-task-exit.test.ts +2 -2
- package/src/nx-store-resolution.test.ts +7 -25
- package/src/nx-task-output-lease.test.ts +3 -3
- package/src/nx-use-local.test.ts +7 -5
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.test.ts +164 -0
- package/src/producer-receipts.ts +636 -0
- package/src/runtime-input.test.ts +1 -1
- package/src/serial-group-wiring.test.ts +4 -4
- package/src/task-results.test.ts +18 -16
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/test-timings.test.ts +2 -2
- package/src/testing.test.ts +5 -5
- package/src/testing.ts +67 -1
- package/src/typecheck-test-policy.test.ts +1 -1
- package/src/typescript-resolution.test.ts +76 -0
- package/test-timings.json +8 -0
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { type ChildProcess, spawn } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants } from 'node:fs';
|
|
2
3
|
import { availableParallelism, userInfo } from 'node:os';
|
|
3
|
-
import { isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
5
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
5
6
|
|
|
6
7
|
import {
|
|
7
8
|
CPU_POOL_ENV,
|
|
8
9
|
type CpuPool,
|
|
9
10
|
createCpuPool,
|
|
11
|
+
type HostWaits,
|
|
10
12
|
type InheritedCpuPool,
|
|
11
13
|
readInheritedCpuPool,
|
|
12
14
|
takeParentCpuTokens,
|
|
@@ -21,6 +23,7 @@ import {
|
|
|
21
23
|
} from './cpu-tokens.js';
|
|
22
24
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
23
25
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
26
|
+
import { createNoSpaceWatch } from './no-space.js';
|
|
24
27
|
import { createRunProgress, withProgressReport } from './progress.js';
|
|
25
28
|
import {
|
|
26
29
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -36,11 +39,13 @@ import {
|
|
|
36
39
|
type BoundedExecRecord,
|
|
37
40
|
type BoundedExecTask,
|
|
38
41
|
clearTaskDirectory,
|
|
42
|
+
describeNoSpace,
|
|
39
43
|
type ExpiredBound,
|
|
40
44
|
JUNIT_FILE,
|
|
41
45
|
JUNIT_REPORT_ENV,
|
|
42
46
|
judgeOperationalFailure,
|
|
43
47
|
judgeRun,
|
|
48
|
+
type NoSpaceSighting,
|
|
44
49
|
readJunitReport,
|
|
45
50
|
taskDirectory,
|
|
46
51
|
withJunitReport,
|
|
@@ -88,10 +93,10 @@ interface RunState {
|
|
|
88
93
|
settled: boolean;
|
|
89
94
|
expiry: ExpiredBound | null;
|
|
90
95
|
forceKillNeeded: boolean;
|
|
91
|
-
/** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
|
|
92
|
-
grantLost: boolean;
|
|
93
96
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
94
97
|
poolCloseError: string | null;
|
|
98
|
+
/** The first output line that said ENOSPC, with the task's volume read when it arrived. */
|
|
99
|
+
noSpace: Promise<NoSpaceSighting> | null;
|
|
95
100
|
}
|
|
96
101
|
|
|
97
102
|
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
@@ -101,7 +106,53 @@ export interface ProcessTreeKiller {
|
|
|
101
106
|
}
|
|
102
107
|
|
|
103
108
|
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
104
|
-
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
|
|
109
|
+
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
|
|
110
|
+
|
|
111
|
+
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
112
|
+
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
113
|
+
'packages/cowshed/bin/cowshed',
|
|
114
|
+
'node_modules/@smoothbricks/cowshed/bin/cowshed',
|
|
115
|
+
] as const;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The cowshed a run's disk commands use, in precedence order: the target's named one, an
|
|
119
|
+
* explicit one inherited from the parent process, the installed one on PATH, and — only when
|
|
120
|
+
* PATH provides none — the launcher this workspace itself provides (its own cowshed package,
|
|
121
|
+
* or a linked one's bin). An environment with no cowshed anywhere keeps the bare name, so
|
|
122
|
+
* its spawn failure names the missing installed tool.
|
|
123
|
+
*/
|
|
124
|
+
export function resolveCowshedExecutable(
|
|
125
|
+
root: string,
|
|
126
|
+
named: string | undefined,
|
|
127
|
+
inherited: string | undefined,
|
|
128
|
+
pathValue: string | undefined,
|
|
129
|
+
cwd: string,
|
|
130
|
+
exists: (path: string) => boolean = (path) => {
|
|
131
|
+
try {
|
|
132
|
+
accessSync(path, constants.X_OK);
|
|
133
|
+
return true;
|
|
134
|
+
} catch {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
): string | undefined {
|
|
139
|
+
// A target-named executable is workspace-relative or absolute against the workspace root.
|
|
140
|
+
// An inherited explicit keeps the coordinate the parent gave it, exactly as the host
|
|
141
|
+
// commands always spawned it: an absolute one verbatim, a relative or bare one for
|
|
142
|
+
// spawn's own resolution — retargeting it to the root would silently change the tool.
|
|
143
|
+
if (named !== undefined) return resolvePath(root, named);
|
|
144
|
+
if (inherited !== undefined) return inherited;
|
|
145
|
+
// An empty PATH component names the spawning process's current directory, as POSIX
|
|
146
|
+
// spawn resolution does — the executor never chdirs, so that is the caller's cwd,
|
|
147
|
+
// not the workspace root.
|
|
148
|
+
const onPath = (pathValue ?? '').split(delimiter).some((dir) => exists(join(dir === '' ? cwd : dir, 'cowshed')));
|
|
149
|
+
if (onPath) return undefined;
|
|
150
|
+
for (const candidate of WORKSPACE_COWSHED_LAUNCHERS) {
|
|
151
|
+
const launcher = join(root, candidate);
|
|
152
|
+
if (exists(launcher)) return launcher;
|
|
153
|
+
}
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
105
156
|
|
|
106
157
|
export default function boundedExecExecutor(
|
|
107
158
|
options: BoundedExecOptions,
|
|
@@ -119,7 +170,13 @@ export function runDefaultBoundedExec(
|
|
|
119
170
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
120
171
|
// command, starting in another directory, could not find: it gets the absolute one.
|
|
121
172
|
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
122
|
-
const cowshed =
|
|
173
|
+
const cowshed = resolveCowshedExecutable(
|
|
174
|
+
context.root,
|
|
175
|
+
named,
|
|
176
|
+
process.env[COWSHED_EXECUTABLE_ENV],
|
|
177
|
+
process.env.PATH,
|
|
178
|
+
process.cwd(),
|
|
179
|
+
);
|
|
123
180
|
const run: BoundedExecOptions =
|
|
124
181
|
cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
125
182
|
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
@@ -167,8 +224,9 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
|
|
|
167
224
|
*
|
|
168
225
|
* `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
|
|
169
226
|
* and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
|
|
170
|
-
* descendants, closed before the grant returns. A
|
|
171
|
-
*
|
|
227
|
+
* descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
|
|
228
|
+
* restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
|
|
229
|
+
* its own exit, and the run says so.
|
|
172
230
|
*/
|
|
173
231
|
export async function runBoundedExec(
|
|
174
232
|
options: BoundedExecOptions,
|
|
@@ -194,8 +252,8 @@ export async function runBoundedExec(
|
|
|
194
252
|
settled: false,
|
|
195
253
|
expiry: null,
|
|
196
254
|
forceKillNeeded: false,
|
|
197
|
-
grantLost: false,
|
|
198
255
|
poolCloseError: null,
|
|
256
|
+
noSpace: null,
|
|
199
257
|
};
|
|
200
258
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
201
259
|
const text = chunk.toString();
|
|
@@ -261,13 +319,13 @@ export async function runBoundedExec(
|
|
|
261
319
|
const cores = availableParallelism();
|
|
262
320
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
263
321
|
const asked = Date.now();
|
|
264
|
-
// A run a parent's CPU pool
|
|
265
|
-
// A pool that answered and refused
|
|
266
|
-
//
|
|
322
|
+
// A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
|
|
323
|
+
// A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
|
|
324
|
+
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
267
325
|
let grant: GrantedTokens | null = null;
|
|
268
326
|
let parentRefusal: string | null = null;
|
|
269
327
|
if (inherited !== null) {
|
|
270
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want
|
|
328
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
271
329
|
if (fromParent.granted) {
|
|
272
330
|
grant = fromParent;
|
|
273
331
|
} else if (fromParent.cause === 'absent') {
|
|
@@ -318,13 +376,14 @@ export async function runBoundedExec(
|
|
|
318
376
|
return cpuReleased;
|
|
319
377
|
};
|
|
320
378
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
321
|
-
// An `operational` run failed on its CPU budget (a
|
|
322
|
-
//
|
|
379
|
+
// An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
|
|
380
|
+
// bound also fired, so it is never judged as a bound.
|
|
323
381
|
const recordRun = async (
|
|
324
382
|
exitCode: number,
|
|
325
383
|
expiry: ExpiredBound | null,
|
|
326
384
|
elapsedMs: number,
|
|
327
385
|
operational: boolean,
|
|
386
|
+
noSpace: NoSpaceSighting | null,
|
|
328
387
|
): Promise<void> => {
|
|
329
388
|
if (context.record === undefined) {
|
|
330
389
|
return;
|
|
@@ -333,7 +392,7 @@ export async function runBoundedExec(
|
|
|
333
392
|
const report = await readJunitReport(recordDirectory);
|
|
334
393
|
const verdict = operational
|
|
335
394
|
? judgeOperationalFailure(exitCode, report)
|
|
336
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report });
|
|
395
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
337
396
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
338
397
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
339
398
|
}
|
|
@@ -344,7 +403,7 @@ export async function runBoundedExec(
|
|
|
344
403
|
appendStderr(message);
|
|
345
404
|
await releaseCpu();
|
|
346
405
|
await releaseLease();
|
|
347
|
-
await recordRun(1, null, Date.now() - asked, false);
|
|
406
|
+
await recordRun(1, null, Date.now() - asked, false, null);
|
|
348
407
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
349
408
|
};
|
|
350
409
|
if (parentRefusal !== null) {
|
|
@@ -353,10 +412,26 @@ export async function runBoundedExec(
|
|
|
353
412
|
);
|
|
354
413
|
}
|
|
355
414
|
|
|
415
|
+
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
416
|
+
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
417
|
+
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
418
|
+
let heldWaits = 0;
|
|
419
|
+
let idleClock: { stop(): void; restart(): void } | null = null;
|
|
420
|
+
const hostWaits: HostWaits = {
|
|
421
|
+
begin: () => {
|
|
422
|
+
heldWaits += 1;
|
|
423
|
+
if (heldWaits === 1) idleClock?.stop();
|
|
424
|
+
},
|
|
425
|
+
end: () => {
|
|
426
|
+
heldWaits -= 1;
|
|
427
|
+
if (heldWaits === 0) idleClock?.restart();
|
|
428
|
+
},
|
|
429
|
+
};
|
|
430
|
+
|
|
356
431
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
357
432
|
// A pool that cannot be set up refuses the run: no command runs unbudgeted.
|
|
358
433
|
if (grant !== null) {
|
|
359
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
434
|
+
const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
|
|
360
435
|
if (!created.ok) {
|
|
361
436
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
362
437
|
}
|
|
@@ -393,6 +468,13 @@ export async function runBoundedExec(
|
|
|
393
468
|
|
|
394
469
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
395
470
|
const progress = createRunProgress();
|
|
471
|
+
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
472
|
+
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
473
|
+
const noSpace = createNoSpaceWatch();
|
|
474
|
+
const sight = async (line: string): Promise<NoSpaceSighting> => {
|
|
475
|
+
const held = lease;
|
|
476
|
+
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
477
|
+
};
|
|
396
478
|
let idleTimer: NodeJS.Timeout | undefined;
|
|
397
479
|
let captureTimer: NodeJS.Timeout | undefined;
|
|
398
480
|
let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
|
|
@@ -400,14 +482,17 @@ export async function runBoundedExec(
|
|
|
400
482
|
clearTimeout(captureTimer);
|
|
401
483
|
idleCapture?.controller.abort();
|
|
402
484
|
};
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
485
|
+
// What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
|
|
486
|
+
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
487
|
+
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
488
|
+
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
489
|
+
let queuedMs = 0;
|
|
490
|
+
let queuedSinceMs: number | null = null;
|
|
491
|
+
const scheduleIdle = (limitMs: number, leftMs: number): void => {
|
|
407
492
|
clearTimeout(idleTimer);
|
|
408
493
|
cancelCapture();
|
|
409
494
|
const capture: NonNullable<typeof idleCapture> = {
|
|
410
|
-
expiresAtMs: Date.now() +
|
|
495
|
+
expiresAtMs: Date.now() + leftMs,
|
|
411
496
|
controller: new AbortController(),
|
|
412
497
|
result: { kind: 'pending', startedAtMs: null },
|
|
413
498
|
};
|
|
@@ -445,12 +530,51 @@ export async function runBoundedExec(
|
|
|
445
530
|
},
|
|
446
531
|
);
|
|
447
532
|
},
|
|
448
|
-
Math.max(0,
|
|
449
|
-
);
|
|
450
|
-
idleTimer = setTimeout(
|
|
451
|
-
() => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
|
|
452
|
-
idleTimeoutMs,
|
|
533
|
+
Math.max(0, leftMs - HANG_CAPTURE_MS),
|
|
453
534
|
);
|
|
535
|
+
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
536
|
+
};
|
|
537
|
+
const armIdleTimer = (): void => {
|
|
538
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
idleLeftMs = idleTimeoutMs;
|
|
542
|
+
queuedMs = 0;
|
|
543
|
+
if (queuedSinceMs !== null) {
|
|
544
|
+
queuedSinceMs = Date.now();
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
547
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
548
|
+
};
|
|
549
|
+
idleClock = {
|
|
550
|
+
stop: () => {
|
|
551
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
const now = Date.now();
|
|
555
|
+
queuedSinceMs = now;
|
|
556
|
+
if (idleCapture !== undefined) {
|
|
557
|
+
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
558
|
+
}
|
|
559
|
+
clearTimeout(idleTimer);
|
|
560
|
+
cancelCapture();
|
|
561
|
+
idleCapture = undefined;
|
|
562
|
+
appendStderr(
|
|
563
|
+
`bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
564
|
+
);
|
|
565
|
+
},
|
|
566
|
+
restart: () => {
|
|
567
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
const waitedMs = Date.now() - queuedSinceMs;
|
|
571
|
+
queuedMs += waitedMs;
|
|
572
|
+
queuedSinceMs = null;
|
|
573
|
+
appendStderr(
|
|
574
|
+
`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
575
|
+
);
|
|
576
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
577
|
+
},
|
|
454
578
|
};
|
|
455
579
|
|
|
456
580
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
@@ -459,11 +583,15 @@ export async function runBoundedExec(
|
|
|
459
583
|
child.stdout?.on('data', (chunk: Buffer | string) => {
|
|
460
584
|
const text = chunk.toString();
|
|
461
585
|
if (progress.scan('stdout', text)) armIdleTimer();
|
|
586
|
+
const said = noSpace.scan('stdout', text);
|
|
587
|
+
if (said !== null) state.noSpace = sight(said);
|
|
462
588
|
appendStdout(text);
|
|
463
589
|
});
|
|
464
590
|
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
465
591
|
const text = chunk.toString();
|
|
466
592
|
if (progress.scan('stderr', text)) armIdleTimer();
|
|
593
|
+
const said = noSpace.scan('stderr', text);
|
|
594
|
+
if (said !== null) state.noSpace = sight(said);
|
|
467
595
|
appendStderr(text);
|
|
468
596
|
});
|
|
469
597
|
|
|
@@ -522,8 +650,9 @@ export async function runBoundedExec(
|
|
|
522
650
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
523
651
|
// two have opposite fixes.
|
|
524
652
|
let termination: Promise<void> = Promise.resolve();
|
|
525
|
-
|
|
526
|
-
|
|
653
|
+
// `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
|
|
654
|
+
const expire = (bound: ExpiredBound, queuedMs = 0): void => {
|
|
655
|
+
if (state.settled || state.expiry !== null) {
|
|
527
656
|
return;
|
|
528
657
|
}
|
|
529
658
|
state.expiry = bound;
|
|
@@ -532,7 +661,9 @@ export async function runBoundedExec(
|
|
|
532
661
|
const elapsedMs = Date.now() - startedAt;
|
|
533
662
|
appendStderr(
|
|
534
663
|
bound.kind === 'idle'
|
|
535
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})
|
|
664
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
|
|
665
|
+
queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
|
|
666
|
+
} after ${elapsedMs}ms of runtime; ${
|
|
536
667
|
bound.tests.length > 0
|
|
537
668
|
? `still running: ${bound.tests.join(', ')}`
|
|
538
669
|
: 'the runner named no test still running'
|
|
@@ -601,25 +732,23 @@ export async function runBoundedExec(
|
|
|
601
732
|
const totalTimer =
|
|
602
733
|
timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
|
|
603
734
|
armIdleTimer();
|
|
735
|
+
// A wait its pool heard before the command started stops the clock it was not yet there to stop.
|
|
736
|
+
if (heldWaits > 0) idleClock.stop();
|
|
604
737
|
|
|
605
|
-
// A
|
|
606
|
-
//
|
|
607
|
-
// is
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
if (state.expiry === null) termination = terminate();
|
|
620
|
-
};
|
|
621
|
-
if (grant?.inherited) {
|
|
622
|
-
void grant.closed.then(onGrantLost);
|
|
738
|
+
// A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
|
|
739
|
+
// restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
|
|
740
|
+
// command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
|
|
741
|
+
// the command settled is not a loss; `cpuReleased` marks it.
|
|
742
|
+
if (grant !== null) {
|
|
743
|
+
const held = grant;
|
|
744
|
+
void held.closed.then(() => {
|
|
745
|
+
if (state.settled || cpuReleased !== null) {
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
appendStderr(
|
|
749
|
+
`cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
|
|
750
|
+
);
|
|
751
|
+
});
|
|
623
752
|
}
|
|
624
753
|
|
|
625
754
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
@@ -641,17 +770,24 @@ export async function runBoundedExec(
|
|
|
641
770
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
642
771
|
}
|
|
643
772
|
|
|
644
|
-
//
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
773
|
+
// ENOSPC said in the output names the writer's volume as it stood then. Without it, a volume still (nearly) full now
|
|
774
|
+
// is the likely cause, and the volume is named.
|
|
775
|
+
const seen = state.noSpace === null ? null : await state.noSpace;
|
|
776
|
+
if (seen !== null && (code !== 0 || state.expiry !== null)) {
|
|
777
|
+
const writer = context.record?.task.id;
|
|
778
|
+
appendStderr(`${writer === undefined ? '' : `${writer}: `}${describeNoSpace(seen)}\n`);
|
|
779
|
+
} else {
|
|
780
|
+
const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
|
|
781
|
+
if (full) {
|
|
782
|
+
appendStderr(`${describeRamTempError(full)}\n`);
|
|
783
|
+
}
|
|
648
784
|
}
|
|
649
785
|
await releaseLease();
|
|
650
786
|
|
|
651
|
-
// A run whose
|
|
652
|
-
//
|
|
653
|
-
const operationalFailure = state.
|
|
654
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
|
|
787
|
+
// A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
|
|
788
|
+
// its verdict is an ordinary failure, never a bound.
|
|
789
|
+
const operationalFailure = state.poolCloseError !== null;
|
|
790
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
655
791
|
|
|
656
792
|
return {
|
|
657
793
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { describe, expect, it } from 'bun:test';
|
|
1
|
+
import { describe, expect, it, jest } from 'bun:test';
|
|
2
2
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
5
6
|
import {
|
|
6
7
|
captureHangDiagnostics,
|
|
7
8
|
type HangCaptureRequest,
|
|
@@ -23,8 +24,8 @@ const exited = (stdout: string, code = 0, stderr = ''): HangProbeResult => ({ st
|
|
|
23
24
|
const request = (): HangCaptureRequest => ({
|
|
24
25
|
pid: ROOT,
|
|
25
26
|
pgid: ROOT,
|
|
26
|
-
expiresAtMs: Date.now() +
|
|
27
|
-
deadlineMs:
|
|
27
|
+
expiresAtMs: Date.now() + BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
28
|
+
deadlineMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
28
29
|
});
|
|
29
30
|
|
|
30
31
|
function scriptedHost(results: readonly HangProbeResult[]) {
|
|
@@ -140,6 +141,7 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
140
141
|
return new Promise<HangProbeResult>(() => {});
|
|
141
142
|
},
|
|
142
143
|
};
|
|
144
|
+
// The probe is held off by construction, so the 1 ms deadline timer is the only event that can end this capture.
|
|
143
145
|
const capture = await captureHangDiagnostics({ ...request(), deadlineMs: 1 }, unresponsive);
|
|
144
146
|
expect(capture.outcome).toBe('incomplete');
|
|
145
147
|
expect(capture.text).toContain('1ms deadline reached during ps before');
|
|
@@ -228,12 +230,11 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
228
230
|
let descendant: number | undefined;
|
|
229
231
|
const failures: unknown[] = [];
|
|
230
232
|
try {
|
|
231
|
-
const reader = child.stdout
|
|
232
|
-
.pipeThrough(new TextDecoderStream(), { signal: AbortSignal.timeout(10_000) })
|
|
233
|
-
.getReader();
|
|
233
|
+
const reader = child.stdout.pipeThrough(new TextDecoderStream()).getReader();
|
|
234
234
|
let text = '';
|
|
235
235
|
try {
|
|
236
|
-
//
|
|
236
|
+
// The first complete line is the readiness event; a child that exits first ends the stream instead. No timer
|
|
237
|
+
// ends this loop.
|
|
237
238
|
for (;;) {
|
|
238
239
|
const chunk = await reader.read();
|
|
239
240
|
if (chunk.done) throw new Error('child stdout ended before a complete PID line');
|
|
@@ -252,7 +253,12 @@ describe('pre-expiry hang diagnostics', () => {
|
|
|
252
253
|
Number.isSafeInteger(descendant) && descendant > 0,
|
|
253
254
|
`expected a tagged PID line, observed ${JSON.stringify(line)}`,
|
|
254
255
|
).toBe(true);
|
|
255
|
-
|
|
256
|
+
// The capture budget (`HANG_CAPTURE_MS`) and probe limits are product constants this test cannot set, so the
|
|
257
|
+
// clock is stopped while the real `ps` and `lsof` run: each ends when it exits; no timer cuts it short.
|
|
258
|
+
jest.useFakeTimers();
|
|
259
|
+
const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null }).finally(() =>
|
|
260
|
+
jest.useRealTimers(),
|
|
261
|
+
);
|
|
256
262
|
expect(capture.outcome).toBe('holders');
|
|
257
263
|
expect(capture.text).toContain(String(child.pid));
|
|
258
264
|
expect(capture.text).toContain(String(descendant));
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { OutputStream } from './progress.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The operating system's own words for ENOSPC: the errno name Node and Bun print (`ENOSPC: no space left on device,
|
|
5
|
+
* write`), and `strerror(28)`, which Rust (`No space left on device (os error 28)`), Go and C print. Nothing else in a
|
|
6
|
+
* command's output says the disk was full, and a write that failed on it is otherwise reported as whatever its caller
|
|
7
|
+
* was doing: a failed test, an extraction error.
|
|
8
|
+
*/
|
|
9
|
+
const NO_SPACE = /\bENOSPC\b|No space left on device/;
|
|
10
|
+
const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
|
|
11
|
+
|
|
12
|
+
/** Watches a command's output for the first line that says a write found no space. */
|
|
13
|
+
export interface NoSpaceWatch {
|
|
14
|
+
/** One chunk of one output stream: the first matching line, once, as printed; `null` before and after it. */
|
|
15
|
+
scan(stream: OutputStream, chunk: string): string | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function createNoSpaceWatch(): NoSpaceWatch {
|
|
19
|
+
const partial: Record<OutputStream, string> = { stdout: '', stderr: '' };
|
|
20
|
+
let seen = false;
|
|
21
|
+
return {
|
|
22
|
+
scan(stream, chunk) {
|
|
23
|
+
if (seen) {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const lines = (partial[stream] + chunk).split('\n');
|
|
27
|
+
// The unfinished last line is kept for the next chunk, and read now too: a command that dies mid-line still
|
|
28
|
+
// printed the words.
|
|
29
|
+
partial[stream] = lines.at(-1) ?? '';
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const text = line.replace(ANSI, '').trim();
|
|
32
|
+
if (NO_SPACE.test(text)) {
|
|
33
|
+
seen = true;
|
|
34
|
+
return text;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -61,7 +61,7 @@ describe('run progress', () => {
|
|
|
61
61
|
|
|
62
62
|
describe('progress report', () => {
|
|
63
63
|
it('asks a lone bun test for dots, keeping the runtime the target named', () => {
|
|
64
|
-
expect(withProgressReport('bun test --timeout=
|
|
64
|
+
expect(withProgressReport('bun test --timeout=0 src')).toBe('bun test --dots --timeout=0 src');
|
|
65
65
|
expect(withProgressReport('../x/.runtime/bun test a')).toBe('../x/.runtime/bun test --dots a');
|
|
66
66
|
});
|
|
67
67
|
|