@smoothbricks/nx-plugin 0.4.28 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +277 -89
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +8 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +18 -30
- package/dist/cargo-closure-input.d.ts +23 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +41 -9
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +42 -19
- package/dist/ensure-built-receipt.d.ts +40 -6
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +297 -159
- package/dist/ensure-built-witness.d.ts +65 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +2237 -23
- package/dist/ensure-built.d.ts +99 -1
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +45 -56
- package/dist/executors/bounded-exec/cpu-pool.d.ts +22 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +51 -22
- package/dist/executors/bounded-exec/executor.d.ts +12 -3
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +174 -51
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +20 -11
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +42 -17
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/generators/bun-test-tracing/generator.js +2 -2
- package/dist/graph-input-contract.d.ts +102 -0
- package/dist/graph-input-contract.d.ts.map +1 -0
- package/dist/graph-input-contract.js +637 -0
- package/dist/index.d.ts +12 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +5 -2
- package/dist/managed-files/publish-workflow.d.ts +11 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts +19 -0
- package/dist/producer-receipts.d.ts.map +1 -0
- package/dist/producer-receipts.js +926 -0
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts +8 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +66 -1
- package/managed/raw/patches/nx@23.2.1.patch +328 -50
- package/managed/raw/tooling/direnv/developer-links.ts +148 -29
- package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
- package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
- package/managed/raw/tooling/direnv/repo-path +15 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +179 -115
- package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
- package/nextest.toml +1 -1
- package/package.json +25 -9
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/fixture-nx-env.ts +7 -0
- package/src/__tests__/owned-installation.ts +14 -0
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.test.ts +36 -22
- package/src/bounded-test-policy.ts +18 -32
- package/src/bun-bootstrap.test.ts +1 -1
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +73 -121
- package/src/cargo-closure-input.ts +47 -8
- package/src/cargo-closure-key.test.ts +239 -0
- package/src/cargo-resolve-cache.test.ts +168 -31
- package/src/cargo-source-hash.test.ts +78 -3
- package/src/cargo-source-hash.ts +48 -30
- package/src/cross-check-policy.test.ts +2 -2
- package/src/ensure-built-receipt.test.ts +111 -16
- package/src/ensure-built-receipt.ts +294 -149
- package/src/ensure-built-witness.test.ts +306 -13
- package/src/ensure-built-witness.ts +255 -24
- package/src/ensure-built.test.ts +228 -102
- package/src/ensure-built.ts +127 -66
- package/src/executors/bounded-exec/cpu-pool.test.ts +134 -32
- package/src/executors/bounded-exec/cpu-pool.ts +64 -20
- package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +74 -52
- package/src/executors/bounded-exec/executor.test.ts +343 -97
- package/src/executors/bounded-exec/executor.ts +192 -56
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/progress.test.ts +1 -1
- package/src/executors/bounded-exec/ram-temp-host.test.ts +51 -21
- package/src/executors/bounded-exec/ram-temp.test.ts +92 -16
- package/src/executors/bounded-exec/ram-temp.ts +49 -20
- package/src/executors/bounded-exec/verdict.test.ts +127 -22
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/generators/bounded-test-targets/generator.test.ts +1 -3
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/bun-test-tracing/generator.ts +2 -2
- package/src/graph-input-contract.test.ts +379 -0
- package/src/graph-input-contract.ts +627 -0
- package/src/index.test.ts +3 -3
- package/src/index.ts +24 -3
- package/src/linked-snapshots.nx.test.ts +112 -30
- package/src/managed-files/ci-workflow.ts +5 -2
- package/src/managed-files/publish-workflow.ts +12 -1
- package/src/managed-files/repo-path.test.ts +44 -1
- package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
- package/src/nextest-code-bound.test.ts +23 -17
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nextest-partitions.test.ts +7 -1
- package/src/nx-cache-restore-mtime.test.ts +2 -2
- package/src/nx-daemon-claim.test.ts +10 -8
- package/src/nx-daemon-idle-cpu.test.ts +0 -1
- package/src/nx-daemon-idle-policy.test.ts +61 -1
- package/src/nx-daemon-ignore-files.test.ts +2 -2
- package/src/nx-daemon-validity.test.ts +36 -22
- package/src/nx-dot-input-policy.test.ts +5 -5
- package/src/nx-files-archive-racy.test.ts +1 -1
- package/src/nx-incomplete-run-summary.test.ts +1 -1
- package/src/nx-plugin-load-errors.test.ts +2 -2
- package/src/nx-project-graph-errors.test.ts +70 -0
- package/src/nx-skipped-task-exit.test.ts +2 -2
- package/src/nx-store-resolution.test.ts +7 -25
- package/src/nx-task-output-lease.test.ts +3 -3
- package/src/nx-use-local.test.ts +7 -5
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.test.ts +164 -0
- package/src/producer-receipts.ts +636 -0
- package/src/runtime-input.test.ts +1 -1
- package/src/serial-group-wiring.test.ts +4 -4
- package/src/task-results.test.ts +18 -16
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/test-timings.test.ts +2 -2
- package/src/testing.test.ts +5 -5
- package/src/testing.ts +67 -1
- package/src/typecheck-test-policy.test.ts +1 -1
- package/src/typescript-resolution.test.ts +76 -0
- package/test-timings.json +8 -0
|
@@ -1,23 +1,27 @@
|
|
|
1
|
-
import { afterEach, describe, expect, it } from 'bun:test';
|
|
1
|
+
import { afterEach, describe, expect, it, jest } from 'bun:test';
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
3
|
import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { createServer } from 'node:net';
|
|
5
|
-
import {
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
6
|
import { dirname, join } from 'node:path';
|
|
7
7
|
import { onProgressClock } from '../../__tests__/progress-clock.js';
|
|
8
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
8
9
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
10
|
import { CPU_POOL_ENV, type CpuPool, createCpuPool, type InheritedCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
10
11
|
import { CPU_TOKENS_ENV, type CpuBudget, parseInheritedTokens, type TokenGrant } from './cpu-tokens.js';
|
|
11
12
|
import {
|
|
12
13
|
type BoundedExecContext,
|
|
14
|
+
type BoundedExecResult,
|
|
13
15
|
createProcessTreeKiller,
|
|
14
16
|
type ProcessTreeKiller,
|
|
17
|
+
resolveCowshedExecutable,
|
|
15
18
|
runBoundedExec,
|
|
16
19
|
type TempVolume,
|
|
17
20
|
} from './executor.js';
|
|
18
21
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
22
|
+
import { createNoSpaceWatch } from './no-space.js';
|
|
19
23
|
import type { RamTempAcquisition, RamTempError, RamTempLease, Result } from './ram-temp.js';
|
|
20
|
-
import { VERDICT_FILE } from './verdict.js';
|
|
24
|
+
import { readRecord, type TempVolumeSpace, VERDICT_FILE } from './verdict.js';
|
|
21
25
|
|
|
22
26
|
const workspaces: string[] = [];
|
|
23
27
|
const ownedProcessGroups = new Set<number>();
|
|
@@ -44,7 +48,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
44
48
|
try {
|
|
45
49
|
process.kill(-pid, 'SIGKILL');
|
|
46
50
|
} catch (cause) {
|
|
47
|
-
|
|
51
|
+
const code = cause instanceof Error && 'code' in cause ? cause.code : undefined;
|
|
52
|
+
// Darwin answers EPERM, not ESRCH, for a group whose only members are unreaped zombies (a killed
|
|
53
|
+
// descendant waiting for launchd to reap it): such a group has nothing left to kill.
|
|
54
|
+
if (code !== 'ESRCH' && !(code === 'EPERM' && onlyZombiesIn(pid))) {
|
|
48
55
|
throw new Error(`cleanup owned readiness process group pgid=${pid} failed`, { cause });
|
|
49
56
|
}
|
|
50
57
|
} finally {
|
|
@@ -61,12 +68,14 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
61
68
|
'console.log(process.cwd()); console.log(process.env.BOUNDED_EXEC_VALUE);\n',
|
|
62
69
|
);
|
|
63
70
|
|
|
71
|
+
// bounded-exec requires a ceiling; at the leg's backstop it cannot end a run whose `node` is slow to start
|
|
72
|
+
// under load.
|
|
64
73
|
const result = await runBoundedExec(
|
|
65
74
|
{
|
|
66
75
|
command: 'node print.js',
|
|
67
76
|
cwd: 'package',
|
|
68
77
|
env: { BOUNDED_EXEC_VALUE: 'from-env' },
|
|
69
|
-
timeoutMs:
|
|
78
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
70
79
|
},
|
|
71
80
|
workspace.context,
|
|
72
81
|
createProcessTreeKiller(),
|
|
@@ -83,7 +92,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
83
92
|
const workspace = await createWorkspace();
|
|
84
93
|
|
|
85
94
|
const result = await runBoundedExec(
|
|
86
|
-
{ command: 'node -e "process.exit(7)"', timeoutMs:
|
|
95
|
+
{ command: 'node -e "process.exit(7)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
87
96
|
workspace.context,
|
|
88
97
|
createProcessTreeKiller(),
|
|
89
98
|
null,
|
|
@@ -102,7 +111,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
102
111
|
process.stderr.write = captureWrite(stderr);
|
|
103
112
|
|
|
104
113
|
const result = await runBoundedExec(
|
|
105
|
-
{
|
|
114
|
+
{
|
|
115
|
+
command: "node -e \"console.log('out-value'); console.error('err-value')\"",
|
|
116
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
117
|
+
},
|
|
106
118
|
workspace.context,
|
|
107
119
|
createProcessTreeKiller(),
|
|
108
120
|
null,
|
|
@@ -119,8 +131,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
119
131
|
it('fails on timeout and reports bounded execution details', async () => {
|
|
120
132
|
const workspace = await createWorkspace();
|
|
121
133
|
|
|
134
|
+
// The command never exits, so only the ceiling under test can end it.
|
|
122
135
|
const result = await runBoundedExec(
|
|
123
|
-
{ command: 'node -e "
|
|
136
|
+
{ command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 0 },
|
|
124
137
|
workspace.context,
|
|
125
138
|
createProcessTreeKiller(),
|
|
126
139
|
null,
|
|
@@ -136,8 +149,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
136
149
|
it('kills a silent command on the progress bound, with no ceiling at all', async () => {
|
|
137
150
|
const workspace = await createWorkspace();
|
|
138
151
|
|
|
152
|
+
// The command never exits and never speaks, so only the progress bound under test can end it.
|
|
139
153
|
const result = await runBoundedExec(
|
|
140
|
-
{ command: 'node -e "
|
|
154
|
+
{ command: 'node -e "process.stdin.resume()"', idleTimeoutMs: 100, killAfterMs: 0 },
|
|
141
155
|
workspace.context,
|
|
142
156
|
createProcessTreeKiller(),
|
|
143
157
|
null,
|
|
@@ -186,6 +200,56 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
186
200
|
expect(result.terminalOutput).not.toContain('fake tests::quick (cwd=');
|
|
187
201
|
});
|
|
188
202
|
|
|
203
|
+
it("stops the idle clock while a descendant's work is queued for host CPU tokens, and runs it on after", async () => {
|
|
204
|
+
const workspace = await createWorkspace();
|
|
205
|
+
// The runner finishes a test, then says through its pool that work it started is queued for host tokens, ends that
|
|
206
|
+
// wait and goes silent. While the wait is held the clock moves well past the bound and nothing fires; once the wait
|
|
207
|
+
// ends the same move ends the run.
|
|
208
|
+
await workspace.write(
|
|
209
|
+
'queued-runner.js',
|
|
210
|
+
[
|
|
211
|
+
"const { connect } = require('node:net');",
|
|
212
|
+
"console.error(' PASS [ 0.010s] (1/2) fake tests::quick');",
|
|
213
|
+
`const socket = connect(process.env.${CPU_POOL_ENV});`,
|
|
214
|
+
`socket.on('connect', () => socket.write('{"op":"host-wait"}\\n'));`,
|
|
215
|
+
"socket.once('data', (reply) => { console.log('host wait ' + reply.toString().trim()); socket.end(); });",
|
|
216
|
+
"socket.on('close', () => setInterval(() => {}, 60_000));",
|
|
217
|
+
'',
|
|
218
|
+
].join('\n'),
|
|
219
|
+
);
|
|
220
|
+
const budget: CpuBudget = {
|
|
221
|
+
async take() {
|
|
222
|
+
return heldGrant(1, []);
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
const { result } = await onProgressClock(
|
|
227
|
+
(chunk) => chunk.includes('bounded-exec: idle clock'),
|
|
228
|
+
1_000,
|
|
229
|
+
() =>
|
|
230
|
+
runBoundedExec(
|
|
231
|
+
{ command: 'node queued-runner.js', idleTimeoutMs: 400, killAfterMs: 0 },
|
|
232
|
+
workspace.context,
|
|
233
|
+
createProcessTreeKiller(),
|
|
234
|
+
null,
|
|
235
|
+
budget,
|
|
236
|
+
),
|
|
237
|
+
);
|
|
238
|
+
|
|
239
|
+
const output = result.terminalOutput;
|
|
240
|
+
expect(result.success).toBe(false);
|
|
241
|
+
expect(output).toContain('host wait {"ok":true,"wait":"held"}');
|
|
242
|
+
const stopped = output.indexOf(
|
|
243
|
+
"bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens",
|
|
244
|
+
);
|
|
245
|
+
const restarted = output.indexOf('bounded-exec: idle clock restarted');
|
|
246
|
+
const expired = output.indexOf('Command made no progress: no test finished for 400ms (idleTimeoutMs=400)');
|
|
247
|
+
// The 1000 ms move while the wait was held fired nothing; the one after it ended the run.
|
|
248
|
+
expect(stopped).toBeGreaterThan(-1);
|
|
249
|
+
expect(restarted).toBeGreaterThan(stopped);
|
|
250
|
+
expect(expired).toBeGreaterThan(restarted);
|
|
251
|
+
});
|
|
252
|
+
|
|
189
253
|
it('emits a completed pre-expiry snapshot or an explicit incomplete capture without waiting for it', async () => {
|
|
190
254
|
const workspace = await createWorkspace();
|
|
191
255
|
// Child output drives the fake executor clock; the heartbeat period is not a timing oracle.
|
|
@@ -381,7 +445,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
381
445
|
await workspace.write('runaway.js', ["setInterval(() => console.log('still here'), 10);", ''].join('\n'));
|
|
382
446
|
|
|
383
447
|
const result = await runBoundedExec(
|
|
384
|
-
{ command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs:
|
|
448
|
+
{ command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, killAfterMs: 0 },
|
|
385
449
|
workspace.context,
|
|
386
450
|
createProcessTreeKiller(),
|
|
387
451
|
null,
|
|
@@ -396,17 +460,47 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
396
460
|
|
|
397
461
|
it('imposes no progress bound when idleTimeoutMs is omitted', async () => {
|
|
398
462
|
const workspace = await createWorkspace();
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
463
|
+
const go = join(workspace.root, 'go.fifo');
|
|
464
|
+
expect(spawnSync('mkfifo', [go]).status).toBe(0);
|
|
465
|
+
// Says `started`, then goes silent until the test writes `go.fifo`.
|
|
466
|
+
await workspace.write(
|
|
467
|
+
'silent.js',
|
|
468
|
+
[
|
|
469
|
+
"const fs = require('node:fs');",
|
|
470
|
+
"console.log('started');",
|
|
471
|
+
"fs.readFileSync('go.fifo');",
|
|
472
|
+
"console.log('late');",
|
|
473
|
+
'',
|
|
474
|
+
].join('\n'),
|
|
406
475
|
);
|
|
476
|
+
const { promise: started, resolve: sawStarted } = Promise.withResolvers<void>();
|
|
477
|
+
process.stdout.write = captureWrite([], (text) => {
|
|
478
|
+
if (text.includes('started')) sawStarted();
|
|
479
|
+
});
|
|
480
|
+
// Every bound is armed as the command spawns, before it can say `started`. The clock then moves to just short of
|
|
481
|
+
// the ceiling while the command is silent: any progress bound would have fired there, and the ceiling has not.
|
|
482
|
+
// Only then is the command released, so the run ends by its own exit.
|
|
483
|
+
jest.useFakeTimers();
|
|
484
|
+
let result: BoundedExecResult;
|
|
485
|
+
try {
|
|
486
|
+
const run = runBoundedExec(
|
|
487
|
+
{ command: 'node silent.js', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
488
|
+
workspace.context,
|
|
489
|
+
createProcessTreeKiller(),
|
|
490
|
+
null,
|
|
491
|
+
null,
|
|
492
|
+
);
|
|
493
|
+
await started;
|
|
494
|
+
jest.advanceTimersByTime(BOUNDED_TEST_IDLE_TIMEOUT_MS - 1);
|
|
495
|
+
await writeFile(go, 'go\n');
|
|
496
|
+
result = await run;
|
|
497
|
+
} finally {
|
|
498
|
+
jest.useRealTimers();
|
|
499
|
+
}
|
|
407
500
|
|
|
408
501
|
expect(result.success).toBe(true);
|
|
409
502
|
expect(result.terminalOutput).toContain('late');
|
|
503
|
+
expect(result.terminalOutput).not.toContain('made no progress');
|
|
410
504
|
});
|
|
411
505
|
|
|
412
506
|
it('uses graceful timeout termination before force-killing on POSIX', async () => {
|
|
@@ -427,7 +521,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
427
521
|
' process.exit(0);',
|
|
428
522
|
'});',
|
|
429
523
|
"fs.writeFileSync('cooperative-ready.fifo', 'ready pid=' + process.pid + '\\n');",
|
|
430
|
-
'
|
|
524
|
+
'process.stdin.resume();',
|
|
431
525
|
'',
|
|
432
526
|
].join('\n'),
|
|
433
527
|
);
|
|
@@ -437,8 +531,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
437
531
|
expect(handlerPid, `handler readiness must name the command leader pid=${pid}`).toBe(pid);
|
|
438
532
|
});
|
|
439
533
|
|
|
534
|
+
// The ceiling is the subject; the grace after the graceful signal is the leg's backstop, so the command ends by its
|
|
535
|
+
// handler alone however slowly a loaded host runs it.
|
|
440
536
|
const result = await runBoundedExec(
|
|
441
|
-
{ command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs:
|
|
537
|
+
{ command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
442
538
|
workspace.context,
|
|
443
539
|
termination.killer,
|
|
444
540
|
null,
|
|
@@ -498,15 +594,18 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
498
594
|
leaked = pid;
|
|
499
595
|
},
|
|
500
596
|
};
|
|
501
|
-
const started = Date.now();
|
|
502
597
|
const result = await runBoundedExec(
|
|
503
|
-
{ command: 'node -e "
|
|
598
|
+
{ command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 10 },
|
|
504
599
|
workspace.context,
|
|
505
600
|
killer,
|
|
506
601
|
null,
|
|
507
602
|
null,
|
|
508
603
|
);
|
|
509
|
-
|
|
604
|
+
// The command never exits and the killer delivers nothing, so the run returned by giving up on the tree after its
|
|
605
|
+
// force-kill, not by the command ending: the command is still alive here.
|
|
606
|
+
if (leaked === undefined) throw new Error('the killer was never asked to kill the command');
|
|
607
|
+
const pid = leaked;
|
|
608
|
+
expect(() => process.kill(pid, 0)).not.toThrow();
|
|
510
609
|
expect(result.success).toBe(false);
|
|
511
610
|
expect(result.terminalOutput).toContain('Force-killing timed out command');
|
|
512
611
|
if (leaked !== undefined) {
|
|
@@ -531,7 +630,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
531
630
|
command: 'node -e "console.log(process.argv.slice(1).join(\',\'))"',
|
|
532
631
|
args: ['first'],
|
|
533
632
|
__unparsed__: ['second'],
|
|
534
|
-
timeoutMs:
|
|
633
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
535
634
|
},
|
|
536
635
|
workspace.context,
|
|
537
636
|
createProcessTreeKiller(),
|
|
@@ -544,7 +643,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
544
643
|
args: ['first'],
|
|
545
644
|
__unparsed__: ['second'],
|
|
546
645
|
forwardAllArgs: false,
|
|
547
|
-
timeoutMs:
|
|
646
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
548
647
|
},
|
|
549
648
|
workspace.context,
|
|
550
649
|
createProcessTreeKiller(),
|
|
@@ -565,7 +664,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
565
664
|
}
|
|
566
665
|
|
|
567
666
|
const workspace = await createWorkspace();
|
|
568
|
-
const marker = join(workspace.root, 'marker.txt');
|
|
569
667
|
const ready = join(workspace.root, 'descendant-ready.fifo');
|
|
570
668
|
const lifetime = join(workspace.root, 'descendant-lifetime.fifo');
|
|
571
669
|
expect(spawnSync('mkfifo', [ready, lifetime]).status).toBe(0);
|
|
@@ -575,7 +673,8 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
575
673
|
"const fs = require('node:fs');",
|
|
576
674
|
"fs.openSync('descendant-lifetime.fifo', 'w');",
|
|
577
675
|
"fs.writeFileSync('descendant-ready.fifo', 'ready pid=' + process.pid + '\\n');",
|
|
578
|
-
|
|
676
|
+
// Its stdin is /dev/null, so it holds itself: a wait no event ever ends, until a signal does.
|
|
677
|
+
'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);',
|
|
579
678
|
'',
|
|
580
679
|
].join('\n'),
|
|
581
680
|
);
|
|
@@ -583,12 +682,12 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
583
682
|
'spawn-child.js',
|
|
584
683
|
[
|
|
585
684
|
"import { spawn } from 'node:child_process';",
|
|
586
|
-
"spawn(process.execPath, ['group-child.js'
|
|
587
|
-
'
|
|
685
|
+
"spawn(process.execPath, ['group-child.js'], { stdio: 'ignore' });",
|
|
686
|
+
'process.stdin.resume();',
|
|
588
687
|
'',
|
|
589
688
|
].join('\n'),
|
|
590
689
|
);
|
|
591
|
-
// Only the descendant owns the writer
|
|
690
|
+
// Only the descendant owns the writer and it never exits on its own: EOF observes its kill, not a time horizon.
|
|
592
691
|
const closed = readToEnd(lifetime);
|
|
593
692
|
let descendantPid: number | undefined;
|
|
594
693
|
const termination = armedKiller('process-group descendant', async (pid) => {
|
|
@@ -597,7 +696,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
597
696
|
});
|
|
598
697
|
|
|
599
698
|
const result = await runBoundedExec(
|
|
600
|
-
{ command:
|
|
699
|
+
{ command: 'exec node spawn-child.js', timeoutMs: 50, killAfterMs: 50 },
|
|
601
700
|
workspace.context,
|
|
602
701
|
termination.killer,
|
|
603
702
|
null,
|
|
@@ -607,7 +706,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
607
706
|
await termination.armed();
|
|
608
707
|
expect(await closed, `ready descendant pid=${descendantPid} must close its lifetime FIFO on exit`).toBe('');
|
|
609
708
|
expect(result.success).toBe(false);
|
|
610
|
-
expect(await exists(marker)).toBe(false);
|
|
611
709
|
});
|
|
612
710
|
|
|
613
711
|
it('force-kills what outlives a timed-out command in its process group', async () => {
|
|
@@ -620,27 +718,26 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
620
718
|
const ready = join(workspace.root, 'straggler-ready.fifo');
|
|
621
719
|
expect(spawnSync('mkfifo', [fifo, ready]).status).toBe(0);
|
|
622
720
|
// The leader exits on SIGTERM. The subshell beside it ignores SIGTERM, a disposition its
|
|
623
|
-
// `
|
|
721
|
+
// `tail` inherits, and announces itself only once it holds the FIFO: from then on the
|
|
624
722
|
// graceful signal cannot end it. A process forked while that signal goes out is the same
|
|
625
723
|
// case, without the trap.
|
|
626
724
|
await workspace.write(
|
|
627
725
|
'straggle.sh',
|
|
628
726
|
[
|
|
629
|
-
'( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2";
|
|
630
|
-
' read -r line; echo "$line"; exec
|
|
727
|
+
'( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; exec tail -f /dev/null ) | {',
|
|
728
|
+
' read -r line; echo "$line"; exec tail -f /dev/null',
|
|
631
729
|
'}',
|
|
632
730
|
'',
|
|
633
731
|
].join('\n'),
|
|
634
732
|
);
|
|
635
|
-
// The straggler is the FIFO's only writer, so the read ends when it is
|
|
636
|
-
// written if it was killed, with "survived" if it lived to finish its sleep.
|
|
733
|
+
// The straggler is the FIFO's only writer and never ends on its own, so the read ends only when it is killed.
|
|
637
734
|
const straggler = readToEnd(fifo);
|
|
638
735
|
const termination = armedKiller('TERM-ignoring FIFO straggler', async (pid) => {
|
|
639
736
|
expect(await readToEnd(ready), `straggler readiness pgid=${pid} must follow trap and FIFO open`).toBe('armed\n');
|
|
640
737
|
});
|
|
641
738
|
|
|
642
739
|
const result = await runBoundedExec(
|
|
643
|
-
{ command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs:
|
|
740
|
+
{ command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
644
741
|
workspace.context,
|
|
645
742
|
termination.killer,
|
|
646
743
|
null,
|
|
@@ -652,7 +749,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
652
749
|
expect(await straggler).toBe('');
|
|
653
750
|
expect(result.success).toBe(false);
|
|
654
751
|
expect(result.terminalOutput).toContain('Force-killing what the timed-out command left in its process group');
|
|
655
|
-
}
|
|
752
|
+
});
|
|
656
753
|
|
|
657
754
|
it('runs the command with its RAM lease as TMPDIR, names held dead leases, and ends the lease after it exits', async () => {
|
|
658
755
|
const workspace = await createWorkspace();
|
|
@@ -665,7 +762,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
665
762
|
});
|
|
666
763
|
|
|
667
764
|
const result = await runBoundedExec(
|
|
668
|
-
{ command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs:
|
|
765
|
+
{ command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
669
766
|
workspace.context,
|
|
670
767
|
createProcessTreeKiller(),
|
|
671
768
|
volume,
|
|
@@ -692,7 +789,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
692
789
|
);
|
|
693
790
|
|
|
694
791
|
const result = await runBoundedExec(
|
|
695
|
-
{ command: 'node -e "0"', timeoutMs:
|
|
792
|
+
{ command: 'node -e "0"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
696
793
|
workspace.context,
|
|
697
794
|
createProcessTreeKiller(),
|
|
698
795
|
volume,
|
|
@@ -714,7 +811,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
714
811
|
const volume = scriptedVolume({ ok: true, value: { kind: 'sandboxed', detail: 'lock: EPERM' } });
|
|
715
812
|
|
|
716
813
|
const result = await runBoundedExec(
|
|
717
|
-
{
|
|
814
|
+
{
|
|
815
|
+
command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`,
|
|
816
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
817
|
+
env: { TMPDIR: '/shed/tmp' },
|
|
818
|
+
},
|
|
718
819
|
workspace.context,
|
|
719
820
|
createProcessTreeKiller(),
|
|
720
821
|
volume,
|
|
@@ -736,7 +837,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
736
837
|
});
|
|
737
838
|
|
|
738
839
|
const result = await runBoundedExec(
|
|
739
|
-
{ command: `touch ${marker}`, timeoutMs:
|
|
840
|
+
{ command: `touch ${marker}`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
740
841
|
workspace.context,
|
|
741
842
|
createProcessTreeKiller(),
|
|
742
843
|
volume,
|
|
@@ -758,7 +859,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
758
859
|
);
|
|
759
860
|
|
|
760
861
|
const result = await runBoundedExec(
|
|
761
|
-
{ command: 'node -e "process.exit(1)"', timeoutMs:
|
|
862
|
+
{ command: 'node -e "process.exit(1)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
762
863
|
workspace.context,
|
|
763
864
|
createProcessTreeKiller(),
|
|
764
865
|
volume,
|
|
@@ -770,6 +871,48 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
770
871
|
expect(volume.released).toEqual([directory]);
|
|
771
872
|
});
|
|
772
873
|
|
|
874
|
+
it('names ENOSPC, its writer and the volume as it stood when the output said so, not after the command exited', async () => {
|
|
875
|
+
// From a downstream land gate: a compiler test shard printed ENOSPC for a 1 KB write into its lease and failed two.
|
|
876
|
+
// By its exit the volume had space again (`fullness` answers null here), so a probe after exit named nothing and
|
|
877
|
+
// the run read as two failed tests.
|
|
878
|
+
const workspace = await createWorkspace();
|
|
879
|
+
const directory = join(workspace.root, 'lease');
|
|
880
|
+
await mkdir(directory);
|
|
881
|
+
const lease = leaseAt(directory);
|
|
882
|
+
const space = { lease: directory, mountpoint: workspace.root, capacityBytes: lease.capacityBytes, freeBytes: 4096 };
|
|
883
|
+
const volume = scriptedVolume(
|
|
884
|
+
{ ok: true, value: { kind: 'leased', lease, held: [], reaped: [] } },
|
|
885
|
+
null,
|
|
886
|
+
[],
|
|
887
|
+
space,
|
|
888
|
+
);
|
|
889
|
+
const record = { task: { id: 'compiler:test-10', hash: 'h' }, directory: join(workspace.root, 'record') };
|
|
890
|
+
const line = 'ENOSPC: no space left on device, write';
|
|
891
|
+
|
|
892
|
+
const result = await runBoundedExec(
|
|
893
|
+
{ command: `node -e "console.error('${line}'); process.exit(1)"`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
894
|
+
{ ...workspace.context, record },
|
|
895
|
+
createProcessTreeKiller(),
|
|
896
|
+
volume,
|
|
897
|
+
null,
|
|
898
|
+
);
|
|
899
|
+
|
|
900
|
+
expect(result.success).toBe(false);
|
|
901
|
+
expect(result.terminalOutput).toContain(
|
|
902
|
+
`compiler:test-10: ENOSPC (no space left on device): the command printed "${line}"; its TMPDIR ` +
|
|
903
|
+
`${directory} is on the RAM temp volume ${workspace.root}, which had 0.0 MiB free of 1024.0 MiB when that ` +
|
|
904
|
+
'line arrived\n',
|
|
905
|
+
);
|
|
906
|
+
expect(result.terminalOutput).not.toContain('is full:');
|
|
907
|
+
expect(await readRecord(record.directory)).toEqual({
|
|
908
|
+
task: record.task.id,
|
|
909
|
+
hash: 'h',
|
|
910
|
+
verdict: { outcome: 'no-space', exitCode: 1, tests: [], line, volume: space },
|
|
911
|
+
});
|
|
912
|
+
// Read while the lease was still the task's, before it was reaped and released.
|
|
913
|
+
expect(volume.order).toEqual(['space', `reap ${directory}`, `release ${directory}`]);
|
|
914
|
+
});
|
|
915
|
+
|
|
773
916
|
it('keeps inherited nextest and Cargo ceilings under oversized grants', async () => {
|
|
774
917
|
const previousNextest = process.env.NEXTEST_TEST_THREADS;
|
|
775
918
|
const previousCargo = process.env.CARGO_BUILD_JOBS;
|
|
@@ -796,7 +939,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
796
939
|
const result = await runBoundedExec(
|
|
797
940
|
{
|
|
798
941
|
command,
|
|
799
|
-
timeoutMs:
|
|
942
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
800
943
|
parallelism: 18,
|
|
801
944
|
env: { NEXTEST_TEST_THREADS: '64', CARGO_BUILD_JOBS: '64', RUST_TEST_THREADS: '64' },
|
|
802
945
|
},
|
|
@@ -836,7 +979,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
836
979
|
"node -e \"console.log('threads=' + process.env.NEXTEST_TEST_THREADS + ' tokens=' + process.env.BOUNDED_EXEC_CPU_TOKENS)\" # nextest run";
|
|
837
980
|
|
|
838
981
|
const result = await runBoundedExec(
|
|
839
|
-
{ command, timeoutMs:
|
|
982
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
|
|
840
983
|
workspace.context,
|
|
841
984
|
createProcessTreeKiller(),
|
|
842
985
|
null,
|
|
@@ -867,7 +1010,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
867
1010
|
const result = await runBoundedExec(
|
|
868
1011
|
{
|
|
869
1012
|
command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
|
|
870
|
-
timeoutMs:
|
|
1013
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
871
1014
|
env: { NEXTEST_TEST_THREADS: '64' },
|
|
872
1015
|
},
|
|
873
1016
|
workspace.context,
|
|
@@ -892,7 +1035,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
892
1035
|
};
|
|
893
1036
|
|
|
894
1037
|
const result = await runBoundedExec(
|
|
895
|
-
{ command: 'node -e "console.log(\'ran\')"', timeoutMs:
|
|
1038
|
+
{ command: 'node -e "console.log(\'ran\')"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
896
1039
|
workspace.context,
|
|
897
1040
|
createProcessTreeKiller(),
|
|
898
1041
|
null,
|
|
@@ -903,6 +1046,52 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
903
1046
|
expect(result.terminalOutput).toBe('ran\n');
|
|
904
1047
|
});
|
|
905
1048
|
|
|
1049
|
+
it('resolves the cowshed a run uses, in full precedence order', () => {
|
|
1050
|
+
const root = '/workspace';
|
|
1051
|
+
// The executor never chdirs, so an empty PATH component names its actual spawn cwd,
|
|
1052
|
+
// deliberately not the workspace root: the two directories differ on purpose here.
|
|
1053
|
+
const cwd = '/elsewhere';
|
|
1054
|
+
// Every level present at once, removed one at a time from the top: each step must fall
|
|
1055
|
+
// to exactly the next level, so the order itself is the assertion, not each level alone.
|
|
1056
|
+
const everywhere = new Set([
|
|
1057
|
+
'/opt/bin/cowshed',
|
|
1058
|
+
'/elsewhere/cowshed',
|
|
1059
|
+
'/workspace/packages/cowshed/bin/cowshed',
|
|
1060
|
+
'/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed',
|
|
1061
|
+
]);
|
|
1062
|
+
const present = (path: string) => everywhere.has(path);
|
|
1063
|
+
const select = (named?: string, inherited?: string, pathValue?: string) =>
|
|
1064
|
+
resolveCowshedExecutable(root, named, inherited, pathValue, cwd, present);
|
|
1065
|
+
|
|
1066
|
+
// 1. A target-named executable wins, made absolute against the workspace root.
|
|
1067
|
+
expect(select('bin/cowshed', '/valid/bin/cowshed', '/opt/bin')).toBe('/workspace/bin/cowshed');
|
|
1068
|
+
// 2. An explicit inherited executable outranks PATH and the workspace launcher — the
|
|
1069
|
+
// precedence createHostCommands always had — and keeps the coordinate the parent
|
|
1070
|
+
// gave it: absolute verbatim, relative untouched for spawn's own resolution,
|
|
1071
|
+
// never retargeted to the workspace root.
|
|
1072
|
+
expect(select(undefined, '/valid/bin/cowshed', '/opt/bin')).toBe('/valid/bin/cowshed');
|
|
1073
|
+
expect(select(undefined, 'bin/cowshed', '/opt/bin')).toBe('bin/cowshed');
|
|
1074
|
+
// 3. An installed cowshed on PATH wins (undefined: the bare name spawn already finds it).
|
|
1075
|
+
expect(select(undefined, undefined, '/opt/bin')).toBeUndefined();
|
|
1076
|
+
// 3b. An empty PATH component (the leading ':' below) names the current directory, as
|
|
1077
|
+
// POSIX spawn resolution does — here the executor's cwd, not the workspace root:
|
|
1078
|
+
// a cowshed in the cwd is installed too, and one merely in the root is not.
|
|
1079
|
+
expect(select(undefined, undefined, ':/usr/bin')).toBeUndefined();
|
|
1080
|
+
everywhere.delete('/elsewhere/cowshed');
|
|
1081
|
+
everywhere.add('/workspace/cowshed');
|
|
1082
|
+
expect(select(undefined, undefined, ':/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
|
|
1083
|
+
everywhere.delete('/workspace/cowshed');
|
|
1084
|
+
// 4. Only when PATH provides none does the workspace's own launcher answer.
|
|
1085
|
+
expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
|
|
1086
|
+
// 5. A linked workspace's launcher answers when the workspace has none of its own.
|
|
1087
|
+
everywhere.delete('/workspace/packages/cowshed/bin/cowshed');
|
|
1088
|
+
expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed');
|
|
1089
|
+
// 6. No cowshed anywhere keeps the bare name, so the spawn failure names the missing tool.
|
|
1090
|
+
everywhere.clear();
|
|
1091
|
+
expect(select(undefined, undefined, '/usr/bin')).toBeUndefined();
|
|
1092
|
+
expect(select(undefined, undefined, undefined)).toBeUndefined();
|
|
1093
|
+
});
|
|
1094
|
+
|
|
906
1095
|
it('sizes a runner to the grant its parent pool lent, takes nothing from the host, and returns nothing to it', async () => {
|
|
907
1096
|
const workspace = await createWorkspace();
|
|
908
1097
|
const asked: number[] = [];
|
|
@@ -919,7 +1108,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
919
1108
|
runBoundedExec(
|
|
920
1109
|
{
|
|
921
1110
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
|
|
922
|
-
timeoutMs:
|
|
1111
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
923
1112
|
parallelism: 8,
|
|
924
1113
|
},
|
|
925
1114
|
workspace.context,
|
|
@@ -945,7 +1134,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
945
1134
|
runBoundedExec(
|
|
946
1135
|
{
|
|
947
1136
|
command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
|
|
948
|
-
timeoutMs:
|
|
1137
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
949
1138
|
parallelism: 2,
|
|
950
1139
|
},
|
|
951
1140
|
workspace.context,
|
|
@@ -973,7 +1162,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
973
1162
|
};
|
|
974
1163
|
|
|
975
1164
|
const result = await runBoundedExec(
|
|
976
|
-
{ command: 'node -e "console.log(1)" # nextest run', timeoutMs:
|
|
1165
|
+
{ command: 'node -e "console.log(1)" # nextest run', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
977
1166
|
workspace.context,
|
|
978
1167
|
createProcessTreeKiller(),
|
|
979
1168
|
null,
|
|
@@ -1008,7 +1197,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1008
1197
|
|
|
1009
1198
|
const borrowed = await withPool(3, (pool) =>
|
|
1010
1199
|
runBoundedExec(
|
|
1011
|
-
{ command, timeoutMs:
|
|
1200
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
1012
1201
|
{ ...workspace.context, record: { task, directory: borrowedDir } },
|
|
1013
1202
|
createProcessTreeKiller(),
|
|
1014
1203
|
null,
|
|
@@ -1017,7 +1206,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1017
1206
|
),
|
|
1018
1207
|
);
|
|
1019
1208
|
const owned = await runBoundedExec(
|
|
1020
|
-
{ command, timeoutMs:
|
|
1209
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
1021
1210
|
{ ...workspace.context, record: { task, directory: ownedDir } },
|
|
1022
1211
|
createProcessTreeKiller(),
|
|
1023
1212
|
null,
|
|
@@ -1039,41 +1228,45 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1039
1228
|
expect(ownedRecord).not.toHaveProperty('cpu.inherited');
|
|
1040
1229
|
});
|
|
1041
1230
|
|
|
1042
|
-
it('
|
|
1231
|
+
it('keeps a command running when its borrowed grant closes, and judges it by its own exit', async () => {
|
|
1043
1232
|
if (process.platform === 'win32') return;
|
|
1044
1233
|
const workspace = await createWorkspace();
|
|
1045
|
-
const directory = join(workspace.root, '
|
|
1046
|
-
const
|
|
1047
|
-
const
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1234
|
+
const directory = join(workspace.root, 'outlives-its-grant');
|
|
1235
|
+
const started = join(workspace.root, 'started.fifo');
|
|
1236
|
+
const go = join(workspace.root, 'go.fifo');
|
|
1237
|
+
expect(spawnSync('mkfifo', [started, go]).status).toBe(0);
|
|
1238
|
+
// The run says the grant ended on its stderr; that line is the signal the close was heard while the command runs.
|
|
1239
|
+
const { promise: heard, resolve: hear } = Promise.withResolvers<void>();
|
|
1240
|
+
process.stderr.write = captureWrite([], (text) => {
|
|
1241
|
+
if (text.includes('cpu-tokens grant ended')) hear();
|
|
1242
|
+
});
|
|
1243
|
+
const result = await withPool(1, async (pool) => {
|
|
1244
|
+
const running = runBoundedExec(
|
|
1056
1245
|
{
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
await ignoring.armed();
|
|
1060
|
-
const closed = await pool.close();
|
|
1061
|
-
if (!closed.ok) throw new Error(closed.error);
|
|
1062
|
-
}
|
|
1063
|
-
calls.push(signal);
|
|
1064
|
-
await createProcessTreeKiller().kill(pid, signal);
|
|
1065
|
-
},
|
|
1246
|
+
command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
|
|
1247
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1066
1248
|
},
|
|
1249
|
+
{ ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
|
|
1250
|
+
createProcessTreeKiller(),
|
|
1067
1251
|
null,
|
|
1068
1252
|
null,
|
|
1069
1253
|
inheritedFrom(pool),
|
|
1070
|
-
)
|
|
1254
|
+
);
|
|
1255
|
+
// The command has started and waits on `go` while its borrowed grant closes under it.
|
|
1256
|
+
expect(await readToEnd(started)).toBe('started\n');
|
|
1257
|
+
const closed = await pool.close();
|
|
1258
|
+
if (!closed.ok) throw new Error(closed.error);
|
|
1259
|
+
await heard;
|
|
1260
|
+
await writeFile(go, 'go\n');
|
|
1261
|
+
return running;
|
|
1262
|
+
});
|
|
1263
|
+
expect(result.terminalOutput).toContain(
|
|
1264
|
+
'cowshed: cpu-tokens grant ended: the parent pool closed its 1-token grant while the command ran; it runs on over budget until it ends',
|
|
1071
1265
|
);
|
|
1072
|
-
expect(result.
|
|
1073
|
-
expect(
|
|
1074
|
-
expect(result.terminalOutput).toContain('cpu-tokens grant lost');
|
|
1266
|
+
expect(result.terminalOutput).not.toContain('Force-killing');
|
|
1267
|
+
expect(result.success).toBe(true);
|
|
1075
1268
|
const record: unknown = JSON.parse(await readFile(join(directory, VERDICT_FILE), 'utf8'));
|
|
1076
|
-
expect(record).toHaveProperty('verdict.outcome', '
|
|
1269
|
+
expect(record).toHaveProperty('verdict.outcome', 'passed');
|
|
1077
1270
|
});
|
|
1078
1271
|
|
|
1079
1272
|
it('asks the host budget when a parent pool refuses its descriptor, and says why', async () => {
|
|
@@ -1089,7 +1282,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1089
1282
|
const gone = { socket: join(workspace.root, 'gone.sock'), tokens: 3 };
|
|
1090
1283
|
|
|
1091
1284
|
const result = await runBoundedExec(
|
|
1092
|
-
{
|
|
1285
|
+
{
|
|
1286
|
+
command: 'node -e "console.log(\'ran\')" # nextest run',
|
|
1287
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1288
|
+
parallelism: 8,
|
|
1289
|
+
},
|
|
1093
1290
|
workspace.context,
|
|
1094
1291
|
createProcessTreeKiller(),
|
|
1095
1292
|
null,
|
|
@@ -1111,7 +1308,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1111
1308
|
const result = await runBoundedExec(
|
|
1112
1309
|
{
|
|
1113
1310
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)"',
|
|
1114
|
-
timeoutMs:
|
|
1311
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1115
1312
|
env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
|
|
1116
1313
|
},
|
|
1117
1314
|
workspace.context,
|
|
@@ -1139,7 +1336,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1139
1336
|
const result = await runBoundedExec(
|
|
1140
1337
|
{
|
|
1141
1338
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
|
|
1142
|
-
timeoutMs:
|
|
1339
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1143
1340
|
parallelism: 8,
|
|
1144
1341
|
env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
|
|
1145
1342
|
},
|
|
@@ -1176,7 +1373,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1176
1373
|
};
|
|
1177
1374
|
try {
|
|
1178
1375
|
const result = await runBoundedExec(
|
|
1179
|
-
{ command: 'touch ran.marker', timeoutMs:
|
|
1376
|
+
{ command: 'touch ran.marker', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
1180
1377
|
{
|
|
1181
1378
|
...workspace.context,
|
|
1182
1379
|
record: { task: { id: 'refused', hash: null }, directory: join(workspace.root, 'refused') },
|
|
@@ -1214,7 +1411,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1214
1411
|
};
|
|
1215
1412
|
const unbudgeted = await withPool(3, (pool) =>
|
|
1216
1413
|
runBoundedExec(
|
|
1217
|
-
{
|
|
1414
|
+
{
|
|
1415
|
+
command: 'node probe.js',
|
|
1416
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1417
|
+
env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
|
|
1418
|
+
},
|
|
1218
1419
|
workspace.context,
|
|
1219
1420
|
createProcessTreeKiller(),
|
|
1220
1421
|
null,
|
|
@@ -1228,7 +1429,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1228
1429
|
const granted = await withPool(3, (pool) => {
|
|
1229
1430
|
injected = inheritedFrom(pool).socket;
|
|
1230
1431
|
return runBoundedExec(
|
|
1231
|
-
{
|
|
1432
|
+
{
|
|
1433
|
+
command: 'node probe.js',
|
|
1434
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1435
|
+
env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
|
|
1436
|
+
},
|
|
1232
1437
|
workspace.context,
|
|
1233
1438
|
createProcessTreeKiller(),
|
|
1234
1439
|
null,
|
|
@@ -1260,6 +1465,25 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1260
1465
|
});
|
|
1261
1466
|
});
|
|
1262
1467
|
|
|
1468
|
+
describe('createNoSpaceWatch', () => {
|
|
1469
|
+
it("finds nextest's ENOSPC split across chunks, past colour codes, and reports it once", () => {
|
|
1470
|
+
// nextest's own report of a failed archive extraction into a full RAM lease, from a downstream land gate.
|
|
1471
|
+
const watch = createNoSpaceWatch();
|
|
1472
|
+
expect(watch.scan('stderr', 'Caused by:\n No space left')).toBeNull();
|
|
1473
|
+
expect(watch.scan('stdout', ' on device (os error 28)\n')).toBeNull();
|
|
1474
|
+
expect(watch.scan('stderr', ' on device \u001b[1m(os error 28)\u001b[0m\nCommand exited')).toBe(
|
|
1475
|
+
'No space left on device (os error 28)',
|
|
1476
|
+
);
|
|
1477
|
+
expect(watch.scan('stderr', 'ENOSPC: no space left on device, write\n')).toBeNull();
|
|
1478
|
+
});
|
|
1479
|
+
|
|
1480
|
+
it("finds Bun's errno name, and not a word that merely contains it", () => {
|
|
1481
|
+
const watch = createNoSpaceWatch();
|
|
1482
|
+
expect(watch.scan('stdout', 'XENOSPCY is not an errno\n')).toBeNull();
|
|
1483
|
+
expect(watch.scan('stdout', ' code: "ENOSPC"\n')).toBe('code: "ENOSPC"');
|
|
1484
|
+
});
|
|
1485
|
+
});
|
|
1486
|
+
|
|
1263
1487
|
interface WorkspaceFixture {
|
|
1264
1488
|
root: string;
|
|
1265
1489
|
context: BoundedExecContext;
|
|
@@ -1320,17 +1544,18 @@ async function armedPid(path: string, pgid: number, subject: string): Promise<nu
|
|
|
1320
1544
|
}
|
|
1321
1545
|
|
|
1322
1546
|
/**
|
|
1323
|
-
* A command that ignores SIGTERM, a disposition its `
|
|
1324
|
-
*
|
|
1325
|
-
*
|
|
1326
|
-
*
|
|
1327
|
-
* the announcement can neither precede the trap nor
|
|
1547
|
+
* A command that ignores SIGTERM, a disposition its `tail` inherits, and never exits on its own, so only the escalation
|
|
1548
|
+
* can end it; and `armed`, which resolves once it ignores SIGTERM. A killer that awaits `armed` before the graceful
|
|
1549
|
+
* signal sends it to a command that cannot die of it: a bound that fires before the command's shell has even run
|
|
1550
|
+
* `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the default disposition, and the escalation under
|
|
1551
|
+
* test never runs. The command's FIFO write waits for the reader, so the announcement can neither precede the trap nor
|
|
1552
|
+
* be missed.
|
|
1328
1553
|
*/
|
|
1329
1554
|
function ignoresTerm(root: string): { command: string; armed: () => Promise<void> } {
|
|
1330
1555
|
const fifo = join(root, 'armed.fifo');
|
|
1331
1556
|
expect(spawnSync('mkfifo', [fifo]).status).toBe(0);
|
|
1332
1557
|
return {
|
|
1333
|
-
command: `sh -c 'trap "" TERM; echo armed > "$0"; exec
|
|
1558
|
+
command: `sh -c 'trap "" TERM; echo armed > "$0"; exec tail -f /dev/null' '${fifo}'`,
|
|
1334
1559
|
armed: async () => {
|
|
1335
1560
|
expect(await readToEnd(fifo)).toBe('armed\n');
|
|
1336
1561
|
},
|
|
@@ -1355,11 +1580,12 @@ function leaseAt(directory: string): RamTempLease {
|
|
|
1355
1580
|
return { directory, mountpoint: dirname(directory), capacityBytes: 1024 * 1024 * 1024 };
|
|
1356
1581
|
}
|
|
1357
1582
|
|
|
1358
|
-
/** A temp volume whose answers are fixed, recording which leases were reaped and released,
|
|
1583
|
+
/** A temp volume whose answers are fixed, recording which leases were reaped and released, and when space was read. */
|
|
1359
1584
|
function scriptedVolume(
|
|
1360
1585
|
acquisition: Result<RamTempAcquisition, RamTempError>,
|
|
1361
1586
|
full: RamTempError | null = null,
|
|
1362
1587
|
leftovers: string[] = [],
|
|
1588
|
+
space: TempVolumeSpace | null = null,
|
|
1363
1589
|
): TempVolume & { released: string[]; order: string[] } {
|
|
1364
1590
|
const released: string[] = [];
|
|
1365
1591
|
const order: string[] = [];
|
|
@@ -1377,16 +1603,22 @@ function scriptedVolume(
|
|
|
1377
1603
|
return Promise.resolve({ ok: true, value: undefined });
|
|
1378
1604
|
},
|
|
1379
1605
|
fullness: () => Promise.resolve(full),
|
|
1606
|
+
space() {
|
|
1607
|
+
order.push('space');
|
|
1608
|
+
return Promise.resolve(space);
|
|
1609
|
+
},
|
|
1380
1610
|
};
|
|
1381
1611
|
}
|
|
1382
1612
|
|
|
1383
|
-
function captureWrite(chunks: string[]): typeof process.stdout.write {
|
|
1613
|
+
function captureWrite(chunks: string[], onChunk?: (text: string) => void): typeof process.stdout.write {
|
|
1384
1614
|
return ((
|
|
1385
1615
|
chunk: string | Uint8Array,
|
|
1386
1616
|
encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),
|
|
1387
1617
|
callback?: (error?: Error | null) => void,
|
|
1388
1618
|
) => {
|
|
1389
|
-
|
|
1619
|
+
const text = chunk.toString();
|
|
1620
|
+
chunks.push(text);
|
|
1621
|
+
onChunk?.(text);
|
|
1390
1622
|
const cb = typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
|
|
1391
1623
|
cb?.();
|
|
1392
1624
|
return true;
|
|
@@ -1419,9 +1651,10 @@ function heldGrant(tokens: number, released: number[]): TokenGrant {
|
|
|
1419
1651
|
};
|
|
1420
1652
|
}
|
|
1421
1653
|
|
|
1422
|
-
/** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own
|
|
1654
|
+
/** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own, and no run owns the
|
|
1655
|
+
* host waits its children report. */
|
|
1423
1656
|
async function withPool<T>(tokens: number, body: (pool: CpuPool) => Promise<T>): Promise<T> {
|
|
1424
|
-
const created = await createCpuPool(tokens, new Promise<void>(() => {}));
|
|
1657
|
+
const created = await createCpuPool(tokens, new Promise<void>(() => {}), { begin() {}, end() {} });
|
|
1425
1658
|
if (!created.ok) {
|
|
1426
1659
|
throw new Error(`cpu pool: ${created.error}`);
|
|
1427
1660
|
}
|
|
@@ -1449,7 +1682,20 @@ function inheritedFrom(pool: CpuPool): InheritedCpuPool {
|
|
|
1449
1682
|
return parent;
|
|
1450
1683
|
}
|
|
1451
1684
|
|
|
1452
|
-
/** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens
|
|
1685
|
+
/** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens). */
|
|
1453
1686
|
function lentCount(want: number, tokens: number): number {
|
|
1454
|
-
return Math.min(want, tokens
|
|
1687
|
+
return Math.min(want, tokens);
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
/** Every process left in group `pgid` is a zombie: `ps` lists no member in any other state. */
|
|
1691
|
+
function onlyZombiesIn(pgid: number): boolean {
|
|
1692
|
+
const listed = spawnSync('ps', ['-A', '-o', 'pgid=,stat='], { encoding: 'utf8' });
|
|
1693
|
+
if (listed.status !== 0) {
|
|
1694
|
+
return false;
|
|
1695
|
+
}
|
|
1696
|
+
return listed.stdout
|
|
1697
|
+
.split('\n')
|
|
1698
|
+
.map((line) => line.trim().split(/\s+/))
|
|
1699
|
+
.filter(([group]) => Number(group) === pgid)
|
|
1700
|
+
.every(([, state]) => state?.startsWith('Z') === true);
|
|
1455
1701
|
}
|