@smoothbricks/nx-plugin 0.4.29 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -15
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +2 -8
- package/dist/cargo-closure-input.d.ts +11 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +16 -6
- package/dist/ensure-built-receipt.d.ts +8 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +45 -13
- package/dist/ensure-built-witness.d.ts +40 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +68 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +11 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts +7 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +9 -4
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +35 -12
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +3 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +15 -4
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/graph-input-contract.d.ts +4 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +17 -4
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +15 -10
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +8 -1
- package/managed/raw/tooling/direnv/developer-links.ts +1 -8
- package/managed/raw/tooling/direnv/setup-environment.ts +0 -1
- package/package.json +15 -4
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/owned-installation.ts +4 -164
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.ts +2 -8
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +15 -112
- package/src/cargo-closure-input.ts +23 -6
- package/src/cargo-closure-key.test.ts +239 -0
- package/src/cargo-resolve-cache.test.ts +29 -7
- package/src/ensure-built-receipt.ts +56 -13
- package/src/ensure-built-witness.test.ts +113 -4
- package/src/ensure-built-witness.ts +75 -16
- package/src/ensure-built.test.ts +74 -2
- package/src/ensure-built.ts +13 -2
- package/src/executors/bounded-exec/cpu-pool.test.ts +18 -0
- package/src/executors/bounded-exec/cpu-pool.ts +10 -4
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +23 -11
- package/src/executors/bounded-exec/executor.test.ts +189 -63
- package/src/executors/bounded-exec/executor.ts +36 -11
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/ram-temp-host.test.ts +9 -8
- package/src/executors/bounded-exec/ram-temp.test.ts +32 -18
- package/src/executors/bounded-exec/ram-temp.ts +17 -4
- package/src/executors/bounded-exec/verdict.test.ts +125 -25
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/graph-input-contract.test.ts +8 -0
- package/src/graph-input-contract.ts +26 -4
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nx-daemon-claim.test.ts +5 -3
- package/src/nx-daemon-idle-policy.test.ts +22 -18
- package/src/nx-daemon-validity.test.ts +5 -13
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.ts +15 -10
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/testing.ts +8 -1
- package/test-timings.json +3 -0
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
import { afterEach, describe, expect, it } from 'bun:test';
|
|
1
|
+
import { afterEach, describe, expect, it, jest } from 'bun:test';
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
3
|
import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { createServer } from 'node:net';
|
|
5
5
|
import { tmpdir } from 'node:os';
|
|
6
6
|
import { dirname, join } from 'node:path';
|
|
7
7
|
import { onProgressClock } from '../../__tests__/progress-clock.js';
|
|
8
|
+
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
8
9
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
10
|
import { CPU_POOL_ENV, type CpuPool, createCpuPool, type InheritedCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
10
11
|
import { CPU_TOKENS_ENV, type CpuBudget, parseInheritedTokens, type TokenGrant } from './cpu-tokens.js';
|
|
11
12
|
import {
|
|
12
13
|
type BoundedExecContext,
|
|
14
|
+
type BoundedExecResult,
|
|
13
15
|
createProcessTreeKiller,
|
|
14
16
|
type ProcessTreeKiller,
|
|
15
17
|
resolveCowshedExecutable,
|
|
@@ -17,8 +19,9 @@ import {
|
|
|
17
19
|
type TempVolume,
|
|
18
20
|
} from './executor.js';
|
|
19
21
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
22
|
+
import { createNoSpaceWatch } from './no-space.js';
|
|
20
23
|
import type { RamTempAcquisition, RamTempError, RamTempLease, Result } from './ram-temp.js';
|
|
21
|
-
import { VERDICT_FILE } from './verdict.js';
|
|
24
|
+
import { readRecord, type TempVolumeSpace, VERDICT_FILE } from './verdict.js';
|
|
22
25
|
|
|
23
26
|
const workspaces: string[] = [];
|
|
24
27
|
const ownedProcessGroups = new Set<number>();
|
|
@@ -65,12 +68,14 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
65
68
|
'console.log(process.cwd()); console.log(process.env.BOUNDED_EXEC_VALUE);\n',
|
|
66
69
|
);
|
|
67
70
|
|
|
71
|
+
// bounded-exec requires a ceiling; at the leg's backstop it cannot end a run whose `node` is slow to start
|
|
72
|
+
// under load.
|
|
68
73
|
const result = await runBoundedExec(
|
|
69
74
|
{
|
|
70
75
|
command: 'node print.js',
|
|
71
76
|
cwd: 'package',
|
|
72
77
|
env: { BOUNDED_EXEC_VALUE: 'from-env' },
|
|
73
|
-
timeoutMs:
|
|
78
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
74
79
|
},
|
|
75
80
|
workspace.context,
|
|
76
81
|
createProcessTreeKiller(),
|
|
@@ -87,7 +92,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
87
92
|
const workspace = await createWorkspace();
|
|
88
93
|
|
|
89
94
|
const result = await runBoundedExec(
|
|
90
|
-
{ command: 'node -e "process.exit(7)"', timeoutMs:
|
|
95
|
+
{ command: 'node -e "process.exit(7)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
91
96
|
workspace.context,
|
|
92
97
|
createProcessTreeKiller(),
|
|
93
98
|
null,
|
|
@@ -106,7 +111,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
106
111
|
process.stderr.write = captureWrite(stderr);
|
|
107
112
|
|
|
108
113
|
const result = await runBoundedExec(
|
|
109
|
-
{
|
|
114
|
+
{
|
|
115
|
+
command: "node -e \"console.log('out-value'); console.error('err-value')\"",
|
|
116
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
117
|
+
},
|
|
110
118
|
workspace.context,
|
|
111
119
|
createProcessTreeKiller(),
|
|
112
120
|
null,
|
|
@@ -123,8 +131,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
123
131
|
it('fails on timeout and reports bounded execution details', async () => {
|
|
124
132
|
const workspace = await createWorkspace();
|
|
125
133
|
|
|
134
|
+
// The command never exits, so only the ceiling under test can end it.
|
|
126
135
|
const result = await runBoundedExec(
|
|
127
|
-
{ command: 'node -e "
|
|
136
|
+
{ command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 0 },
|
|
128
137
|
workspace.context,
|
|
129
138
|
createProcessTreeKiller(),
|
|
130
139
|
null,
|
|
@@ -140,8 +149,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
140
149
|
it('kills a silent command on the progress bound, with no ceiling at all', async () => {
|
|
141
150
|
const workspace = await createWorkspace();
|
|
142
151
|
|
|
152
|
+
// The command never exits and never speaks, so only the progress bound under test can end it.
|
|
143
153
|
const result = await runBoundedExec(
|
|
144
|
-
{ command: 'node -e "
|
|
154
|
+
{ command: 'node -e "process.stdin.resume()"', idleTimeoutMs: 100, killAfterMs: 0 },
|
|
145
155
|
workspace.context,
|
|
146
156
|
createProcessTreeKiller(),
|
|
147
157
|
null,
|
|
@@ -435,7 +445,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
435
445
|
await workspace.write('runaway.js', ["setInterval(() => console.log('still here'), 10);", ''].join('\n'));
|
|
436
446
|
|
|
437
447
|
const result = await runBoundedExec(
|
|
438
|
-
{ command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs:
|
|
448
|
+
{ command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, killAfterMs: 0 },
|
|
439
449
|
workspace.context,
|
|
440
450
|
createProcessTreeKiller(),
|
|
441
451
|
null,
|
|
@@ -450,17 +460,47 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
450
460
|
|
|
451
461
|
it('imposes no progress bound when idleTimeoutMs is omitted', async () => {
|
|
452
462
|
const workspace = await createWorkspace();
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
463
|
+
const go = join(workspace.root, 'go.fifo');
|
|
464
|
+
expect(spawnSync('mkfifo', [go]).status).toBe(0);
|
|
465
|
+
// Says `started`, then goes silent until the test writes `go.fifo`.
|
|
466
|
+
await workspace.write(
|
|
467
|
+
'silent.js',
|
|
468
|
+
[
|
|
469
|
+
"const fs = require('node:fs');",
|
|
470
|
+
"console.log('started');",
|
|
471
|
+
"fs.readFileSync('go.fifo');",
|
|
472
|
+
"console.log('late');",
|
|
473
|
+
'',
|
|
474
|
+
].join('\n'),
|
|
460
475
|
);
|
|
476
|
+
const { promise: started, resolve: sawStarted } = Promise.withResolvers<void>();
|
|
477
|
+
process.stdout.write = captureWrite([], (text) => {
|
|
478
|
+
if (text.includes('started')) sawStarted();
|
|
479
|
+
});
|
|
480
|
+
// Every bound is armed as the command spawns, before it can say `started`. The clock then moves to just short of
|
|
481
|
+
// the ceiling while the command is silent: any progress bound would have fired there, and the ceiling has not.
|
|
482
|
+
// Only then is the command released, so the run ends by its own exit.
|
|
483
|
+
jest.useFakeTimers();
|
|
484
|
+
let result: BoundedExecResult;
|
|
485
|
+
try {
|
|
486
|
+
const run = runBoundedExec(
|
|
487
|
+
{ command: 'node silent.js', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
488
|
+
workspace.context,
|
|
489
|
+
createProcessTreeKiller(),
|
|
490
|
+
null,
|
|
491
|
+
null,
|
|
492
|
+
);
|
|
493
|
+
await started;
|
|
494
|
+
jest.advanceTimersByTime(BOUNDED_TEST_IDLE_TIMEOUT_MS - 1);
|
|
495
|
+
await writeFile(go, 'go\n');
|
|
496
|
+
result = await run;
|
|
497
|
+
} finally {
|
|
498
|
+
jest.useRealTimers();
|
|
499
|
+
}
|
|
461
500
|
|
|
462
501
|
expect(result.success).toBe(true);
|
|
463
502
|
expect(result.terminalOutput).toContain('late');
|
|
503
|
+
expect(result.terminalOutput).not.toContain('made no progress');
|
|
464
504
|
});
|
|
465
505
|
|
|
466
506
|
it('uses graceful timeout termination before force-killing on POSIX', async () => {
|
|
@@ -481,7 +521,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
481
521
|
' process.exit(0);',
|
|
482
522
|
'});',
|
|
483
523
|
"fs.writeFileSync('cooperative-ready.fifo', 'ready pid=' + process.pid + '\\n');",
|
|
484
|
-
'
|
|
524
|
+
'process.stdin.resume();',
|
|
485
525
|
'',
|
|
486
526
|
].join('\n'),
|
|
487
527
|
);
|
|
@@ -491,8 +531,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
491
531
|
expect(handlerPid, `handler readiness must name the command leader pid=${pid}`).toBe(pid);
|
|
492
532
|
});
|
|
493
533
|
|
|
534
|
+
// The ceiling is the subject; the grace after the graceful signal is the leg's backstop, so the command ends by its
|
|
535
|
+
// handler alone however slowly a loaded host runs it.
|
|
494
536
|
const result = await runBoundedExec(
|
|
495
|
-
{ command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs:
|
|
537
|
+
{ command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
496
538
|
workspace.context,
|
|
497
539
|
termination.killer,
|
|
498
540
|
null,
|
|
@@ -552,15 +594,18 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
552
594
|
leaked = pid;
|
|
553
595
|
},
|
|
554
596
|
};
|
|
555
|
-
const started = Date.now();
|
|
556
597
|
const result = await runBoundedExec(
|
|
557
|
-
{ command: 'node -e "
|
|
598
|
+
{ command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 10 },
|
|
558
599
|
workspace.context,
|
|
559
600
|
killer,
|
|
560
601
|
null,
|
|
561
602
|
null,
|
|
562
603
|
);
|
|
563
|
-
|
|
604
|
+
// The command never exits and the killer delivers nothing, so the run returned by giving up on the tree after its
|
|
605
|
+
// force-kill, not by the command ending: the command is still alive here.
|
|
606
|
+
if (leaked === undefined) throw new Error('the killer was never asked to kill the command');
|
|
607
|
+
const pid = leaked;
|
|
608
|
+
expect(() => process.kill(pid, 0)).not.toThrow();
|
|
564
609
|
expect(result.success).toBe(false);
|
|
565
610
|
expect(result.terminalOutput).toContain('Force-killing timed out command');
|
|
566
611
|
if (leaked !== undefined) {
|
|
@@ -585,7 +630,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
585
630
|
command: 'node -e "console.log(process.argv.slice(1).join(\',\'))"',
|
|
586
631
|
args: ['first'],
|
|
587
632
|
__unparsed__: ['second'],
|
|
588
|
-
timeoutMs:
|
|
633
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
589
634
|
},
|
|
590
635
|
workspace.context,
|
|
591
636
|
createProcessTreeKiller(),
|
|
@@ -598,7 +643,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
598
643
|
args: ['first'],
|
|
599
644
|
__unparsed__: ['second'],
|
|
600
645
|
forwardAllArgs: false,
|
|
601
|
-
timeoutMs:
|
|
646
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
602
647
|
},
|
|
603
648
|
workspace.context,
|
|
604
649
|
createProcessTreeKiller(),
|
|
@@ -619,7 +664,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
619
664
|
}
|
|
620
665
|
|
|
621
666
|
const workspace = await createWorkspace();
|
|
622
|
-
const marker = join(workspace.root, 'marker.txt');
|
|
623
667
|
const ready = join(workspace.root, 'descendant-ready.fifo');
|
|
624
668
|
const lifetime = join(workspace.root, 'descendant-lifetime.fifo');
|
|
625
669
|
expect(spawnSync('mkfifo', [ready, lifetime]).status).toBe(0);
|
|
@@ -629,7 +673,8 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
629
673
|
"const fs = require('node:fs');",
|
|
630
674
|
"fs.openSync('descendant-lifetime.fifo', 'w');",
|
|
631
675
|
"fs.writeFileSync('descendant-ready.fifo', 'ready pid=' + process.pid + '\\n');",
|
|
632
|
-
|
|
676
|
+
// Its stdin is /dev/null, so it holds itself: a wait no event ever ends, until a signal does.
|
|
677
|
+
'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);',
|
|
633
678
|
'',
|
|
634
679
|
].join('\n'),
|
|
635
680
|
);
|
|
@@ -637,12 +682,12 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
637
682
|
'spawn-child.js',
|
|
638
683
|
[
|
|
639
684
|
"import { spawn } from 'node:child_process';",
|
|
640
|
-
"spawn(process.execPath, ['group-child.js'
|
|
641
|
-
'
|
|
685
|
+
"spawn(process.execPath, ['group-child.js'], { stdio: 'ignore' });",
|
|
686
|
+
'process.stdin.resume();',
|
|
642
687
|
'',
|
|
643
688
|
].join('\n'),
|
|
644
689
|
);
|
|
645
|
-
// Only the descendant owns the writer
|
|
690
|
+
// Only the descendant owns the writer and it never exits on its own: EOF observes its kill, not a time horizon.
|
|
646
691
|
const closed = readToEnd(lifetime);
|
|
647
692
|
let descendantPid: number | undefined;
|
|
648
693
|
const termination = armedKiller('process-group descendant', async (pid) => {
|
|
@@ -651,7 +696,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
651
696
|
});
|
|
652
697
|
|
|
653
698
|
const result = await runBoundedExec(
|
|
654
|
-
{ command:
|
|
699
|
+
{ command: 'exec node spawn-child.js', timeoutMs: 50, killAfterMs: 50 },
|
|
655
700
|
workspace.context,
|
|
656
701
|
termination.killer,
|
|
657
702
|
null,
|
|
@@ -661,7 +706,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
661
706
|
await termination.armed();
|
|
662
707
|
expect(await closed, `ready descendant pid=${descendantPid} must close its lifetime FIFO on exit`).toBe('');
|
|
663
708
|
expect(result.success).toBe(false);
|
|
664
|
-
expect(await exists(marker)).toBe(false);
|
|
665
709
|
});
|
|
666
710
|
|
|
667
711
|
it('force-kills what outlives a timed-out command in its process group', async () => {
|
|
@@ -674,27 +718,26 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
674
718
|
const ready = join(workspace.root, 'straggler-ready.fifo');
|
|
675
719
|
expect(spawnSync('mkfifo', [fifo, ready]).status).toBe(0);
|
|
676
720
|
// The leader exits on SIGTERM. The subshell beside it ignores SIGTERM, a disposition its
|
|
677
|
-
// `
|
|
721
|
+
// `tail` inherits, and announces itself only once it holds the FIFO: from then on the
|
|
678
722
|
// graceful signal cannot end it. A process forked while that signal goes out is the same
|
|
679
723
|
// case, without the trap.
|
|
680
724
|
await workspace.write(
|
|
681
725
|
'straggle.sh',
|
|
682
726
|
[
|
|
683
|
-
'( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2";
|
|
684
|
-
' read -r line; echo "$line"; exec
|
|
727
|
+
'( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; exec tail -f /dev/null ) | {',
|
|
728
|
+
' read -r line; echo "$line"; exec tail -f /dev/null',
|
|
685
729
|
'}',
|
|
686
730
|
'',
|
|
687
731
|
].join('\n'),
|
|
688
732
|
);
|
|
689
|
-
// The straggler is the FIFO's only writer, so the read ends when it is
|
|
690
|
-
// written if it was killed, with "survived" if it lived to finish its sleep.
|
|
733
|
+
// The straggler is the FIFO's only writer and never ends on its own, so the read ends only when it is killed.
|
|
691
734
|
const straggler = readToEnd(fifo);
|
|
692
735
|
const termination = armedKiller('TERM-ignoring FIFO straggler', async (pid) => {
|
|
693
736
|
expect(await readToEnd(ready), `straggler readiness pgid=${pid} must follow trap and FIFO open`).toBe('armed\n');
|
|
694
737
|
});
|
|
695
738
|
|
|
696
739
|
const result = await runBoundedExec(
|
|
697
|
-
{ command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs:
|
|
740
|
+
{ command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
698
741
|
workspace.context,
|
|
699
742
|
termination.killer,
|
|
700
743
|
null,
|
|
@@ -719,7 +762,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
719
762
|
});
|
|
720
763
|
|
|
721
764
|
const result = await runBoundedExec(
|
|
722
|
-
{ command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs:
|
|
765
|
+
{ command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
723
766
|
workspace.context,
|
|
724
767
|
createProcessTreeKiller(),
|
|
725
768
|
volume,
|
|
@@ -746,7 +789,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
746
789
|
);
|
|
747
790
|
|
|
748
791
|
const result = await runBoundedExec(
|
|
749
|
-
{ command: 'node -e "0"', timeoutMs:
|
|
792
|
+
{ command: 'node -e "0"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
750
793
|
workspace.context,
|
|
751
794
|
createProcessTreeKiller(),
|
|
752
795
|
volume,
|
|
@@ -768,7 +811,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
768
811
|
const volume = scriptedVolume({ ok: true, value: { kind: 'sandboxed', detail: 'lock: EPERM' } });
|
|
769
812
|
|
|
770
813
|
const result = await runBoundedExec(
|
|
771
|
-
{
|
|
814
|
+
{
|
|
815
|
+
command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`,
|
|
816
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
817
|
+
env: { TMPDIR: '/shed/tmp' },
|
|
818
|
+
},
|
|
772
819
|
workspace.context,
|
|
773
820
|
createProcessTreeKiller(),
|
|
774
821
|
volume,
|
|
@@ -790,7 +837,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
790
837
|
});
|
|
791
838
|
|
|
792
839
|
const result = await runBoundedExec(
|
|
793
|
-
{ command: `touch ${marker}`, timeoutMs:
|
|
840
|
+
{ command: `touch ${marker}`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
794
841
|
workspace.context,
|
|
795
842
|
createProcessTreeKiller(),
|
|
796
843
|
volume,
|
|
@@ -812,7 +859,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
812
859
|
);
|
|
813
860
|
|
|
814
861
|
const result = await runBoundedExec(
|
|
815
|
-
{ command: 'node -e "process.exit(1)"', timeoutMs:
|
|
862
|
+
{ command: 'node -e "process.exit(1)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
816
863
|
workspace.context,
|
|
817
864
|
createProcessTreeKiller(),
|
|
818
865
|
volume,
|
|
@@ -824,6 +871,48 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
824
871
|
expect(volume.released).toEqual([directory]);
|
|
825
872
|
});
|
|
826
873
|
|
|
874
|
+
it('names ENOSPC, its writer and the volume as it stood when the output said so, not after the command exited', async () => {
|
|
875
|
+
// From a downstream land gate: a compiler test shard printed ENOSPC for a 1 KB write into its lease and failed two.
|
|
876
|
+
// By its exit the volume had space again (`fullness` answers null here), so a probe after exit named nothing and
|
|
877
|
+
// the run read as two failed tests.
|
|
878
|
+
const workspace = await createWorkspace();
|
|
879
|
+
const directory = join(workspace.root, 'lease');
|
|
880
|
+
await mkdir(directory);
|
|
881
|
+
const lease = leaseAt(directory);
|
|
882
|
+
const space = { lease: directory, mountpoint: workspace.root, capacityBytes: lease.capacityBytes, freeBytes: 4096 };
|
|
883
|
+
const volume = scriptedVolume(
|
|
884
|
+
{ ok: true, value: { kind: 'leased', lease, held: [], reaped: [] } },
|
|
885
|
+
null,
|
|
886
|
+
[],
|
|
887
|
+
space,
|
|
888
|
+
);
|
|
889
|
+
const record = { task: { id: 'compiler:test-10', hash: 'h' }, directory: join(workspace.root, 'record') };
|
|
890
|
+
const line = 'ENOSPC: no space left on device, write';
|
|
891
|
+
|
|
892
|
+
const result = await runBoundedExec(
|
|
893
|
+
{ command: `node -e "console.error('${line}'); process.exit(1)"`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
894
|
+
{ ...workspace.context, record },
|
|
895
|
+
createProcessTreeKiller(),
|
|
896
|
+
volume,
|
|
897
|
+
null,
|
|
898
|
+
);
|
|
899
|
+
|
|
900
|
+
expect(result.success).toBe(false);
|
|
901
|
+
expect(result.terminalOutput).toContain(
|
|
902
|
+
`compiler:test-10: ENOSPC (no space left on device): the command printed "${line}"; its TMPDIR ` +
|
|
903
|
+
`${directory} is on the RAM temp volume ${workspace.root}, which had 0.0 MiB free of 1024.0 MiB when that ` +
|
|
904
|
+
'line arrived\n',
|
|
905
|
+
);
|
|
906
|
+
expect(result.terminalOutput).not.toContain('is full:');
|
|
907
|
+
expect(await readRecord(record.directory)).toEqual({
|
|
908
|
+
task: record.task.id,
|
|
909
|
+
hash: 'h',
|
|
910
|
+
verdict: { outcome: 'no-space', exitCode: 1, tests: [], line, volume: space },
|
|
911
|
+
});
|
|
912
|
+
// Read while the lease was still the task's, before it was reaped and released.
|
|
913
|
+
expect(volume.order).toEqual(['space', `reap ${directory}`, `release ${directory}`]);
|
|
914
|
+
});
|
|
915
|
+
|
|
827
916
|
it('keeps inherited nextest and Cargo ceilings under oversized grants', async () => {
|
|
828
917
|
const previousNextest = process.env.NEXTEST_TEST_THREADS;
|
|
829
918
|
const previousCargo = process.env.CARGO_BUILD_JOBS;
|
|
@@ -850,7 +939,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
850
939
|
const result = await runBoundedExec(
|
|
851
940
|
{
|
|
852
941
|
command,
|
|
853
|
-
timeoutMs:
|
|
942
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
854
943
|
parallelism: 18,
|
|
855
944
|
env: { NEXTEST_TEST_THREADS: '64', CARGO_BUILD_JOBS: '64', RUST_TEST_THREADS: '64' },
|
|
856
945
|
},
|
|
@@ -890,7 +979,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
890
979
|
"node -e \"console.log('threads=' + process.env.NEXTEST_TEST_THREADS + ' tokens=' + process.env.BOUNDED_EXEC_CPU_TOKENS)\" # nextest run";
|
|
891
980
|
|
|
892
981
|
const result = await runBoundedExec(
|
|
893
|
-
{ command, timeoutMs:
|
|
982
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
|
|
894
983
|
workspace.context,
|
|
895
984
|
createProcessTreeKiller(),
|
|
896
985
|
null,
|
|
@@ -921,7 +1010,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
921
1010
|
const result = await runBoundedExec(
|
|
922
1011
|
{
|
|
923
1012
|
command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
|
|
924
|
-
timeoutMs:
|
|
1013
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
925
1014
|
env: { NEXTEST_TEST_THREADS: '64' },
|
|
926
1015
|
},
|
|
927
1016
|
workspace.context,
|
|
@@ -946,7 +1035,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
946
1035
|
};
|
|
947
1036
|
|
|
948
1037
|
const result = await runBoundedExec(
|
|
949
|
-
{ command: 'node -e "console.log(\'ran\')"', timeoutMs:
|
|
1038
|
+
{ command: 'node -e "console.log(\'ran\')"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
950
1039
|
workspace.context,
|
|
951
1040
|
createProcessTreeKiller(),
|
|
952
1041
|
null,
|
|
@@ -1019,7 +1108,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1019
1108
|
runBoundedExec(
|
|
1020
1109
|
{
|
|
1021
1110
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
|
|
1022
|
-
timeoutMs:
|
|
1111
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1023
1112
|
parallelism: 8,
|
|
1024
1113
|
},
|
|
1025
1114
|
workspace.context,
|
|
@@ -1045,7 +1134,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1045
1134
|
runBoundedExec(
|
|
1046
1135
|
{
|
|
1047
1136
|
command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
|
|
1048
|
-
timeoutMs:
|
|
1137
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1049
1138
|
parallelism: 2,
|
|
1050
1139
|
},
|
|
1051
1140
|
workspace.context,
|
|
@@ -1073,7 +1162,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1073
1162
|
};
|
|
1074
1163
|
|
|
1075
1164
|
const result = await runBoundedExec(
|
|
1076
|
-
{ command: 'node -e "console.log(1)" # nextest run', timeoutMs:
|
|
1165
|
+
{ command: 'node -e "console.log(1)" # nextest run', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
1077
1166
|
workspace.context,
|
|
1078
1167
|
createProcessTreeKiller(),
|
|
1079
1168
|
null,
|
|
@@ -1108,7 +1197,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1108
1197
|
|
|
1109
1198
|
const borrowed = await withPool(3, (pool) =>
|
|
1110
1199
|
runBoundedExec(
|
|
1111
|
-
{ command, timeoutMs:
|
|
1200
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
1112
1201
|
{ ...workspace.context, record: { task, directory: borrowedDir } },
|
|
1113
1202
|
createProcessTreeKiller(),
|
|
1114
1203
|
null,
|
|
@@ -1117,7 +1206,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1117
1206
|
),
|
|
1118
1207
|
);
|
|
1119
1208
|
const owned = await runBoundedExec(
|
|
1120
|
-
{ command, timeoutMs:
|
|
1209
|
+
{ command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
|
|
1121
1210
|
{ ...workspace.context, record: { task, directory: ownedDir } },
|
|
1122
1211
|
createProcessTreeKiller(),
|
|
1123
1212
|
null,
|
|
@@ -1155,7 +1244,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1155
1244
|
const running = runBoundedExec(
|
|
1156
1245
|
{
|
|
1157
1246
|
command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
|
|
1158
|
-
timeoutMs:
|
|
1247
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1159
1248
|
},
|
|
1160
1249
|
{ ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
|
|
1161
1250
|
createProcessTreeKiller(),
|
|
@@ -1193,7 +1282,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1193
1282
|
const gone = { socket: join(workspace.root, 'gone.sock'), tokens: 3 };
|
|
1194
1283
|
|
|
1195
1284
|
const result = await runBoundedExec(
|
|
1196
|
-
{
|
|
1285
|
+
{
|
|
1286
|
+
command: 'node -e "console.log(\'ran\')" # nextest run',
|
|
1287
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1288
|
+
parallelism: 8,
|
|
1289
|
+
},
|
|
1197
1290
|
workspace.context,
|
|
1198
1291
|
createProcessTreeKiller(),
|
|
1199
1292
|
null,
|
|
@@ -1215,7 +1308,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1215
1308
|
const result = await runBoundedExec(
|
|
1216
1309
|
{
|
|
1217
1310
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)"',
|
|
1218
|
-
timeoutMs:
|
|
1311
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1219
1312
|
env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
|
|
1220
1313
|
},
|
|
1221
1314
|
workspace.context,
|
|
@@ -1243,7 +1336,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1243
1336
|
const result = await runBoundedExec(
|
|
1244
1337
|
{
|
|
1245
1338
|
command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
|
|
1246
|
-
timeoutMs:
|
|
1339
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1247
1340
|
parallelism: 8,
|
|
1248
1341
|
env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
|
|
1249
1342
|
},
|
|
@@ -1280,7 +1373,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1280
1373
|
};
|
|
1281
1374
|
try {
|
|
1282
1375
|
const result = await runBoundedExec(
|
|
1283
|
-
{ command: 'touch ran.marker', timeoutMs:
|
|
1376
|
+
{ command: 'touch ran.marker', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
|
|
1284
1377
|
{
|
|
1285
1378
|
...workspace.context,
|
|
1286
1379
|
record: { task: { id: 'refused', hash: null }, directory: join(workspace.root, 'refused') },
|
|
@@ -1318,7 +1411,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1318
1411
|
};
|
|
1319
1412
|
const unbudgeted = await withPool(3, (pool) =>
|
|
1320
1413
|
runBoundedExec(
|
|
1321
|
-
{
|
|
1414
|
+
{
|
|
1415
|
+
command: 'node probe.js',
|
|
1416
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1417
|
+
env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
|
|
1418
|
+
},
|
|
1322
1419
|
workspace.context,
|
|
1323
1420
|
createProcessTreeKiller(),
|
|
1324
1421
|
null,
|
|
@@ -1332,7 +1429,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1332
1429
|
const granted = await withPool(3, (pool) => {
|
|
1333
1430
|
injected = inheritedFrom(pool).socket;
|
|
1334
1431
|
return runBoundedExec(
|
|
1335
|
-
{
|
|
1432
|
+
{
|
|
1433
|
+
command: 'node probe.js',
|
|
1434
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1435
|
+
env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
|
|
1436
|
+
},
|
|
1336
1437
|
workspace.context,
|
|
1337
1438
|
createProcessTreeKiller(),
|
|
1338
1439
|
null,
|
|
@@ -1364,6 +1465,25 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1364
1465
|
});
|
|
1365
1466
|
});
|
|
1366
1467
|
|
|
1468
|
+
describe('createNoSpaceWatch', () => {
|
|
1469
|
+
it("finds nextest's ENOSPC split across chunks, past colour codes, and reports it once", () => {
|
|
1470
|
+
// nextest's own report of a failed archive extraction into a full RAM lease, from a downstream land gate.
|
|
1471
|
+
const watch = createNoSpaceWatch();
|
|
1472
|
+
expect(watch.scan('stderr', 'Caused by:\n No space left')).toBeNull();
|
|
1473
|
+
expect(watch.scan('stdout', ' on device (os error 28)\n')).toBeNull();
|
|
1474
|
+
expect(watch.scan('stderr', ' on device \u001b[1m(os error 28)\u001b[0m\nCommand exited')).toBe(
|
|
1475
|
+
'No space left on device (os error 28)',
|
|
1476
|
+
);
|
|
1477
|
+
expect(watch.scan('stderr', 'ENOSPC: no space left on device, write\n')).toBeNull();
|
|
1478
|
+
});
|
|
1479
|
+
|
|
1480
|
+
it("finds Bun's errno name, and not a word that merely contains it", () => {
|
|
1481
|
+
const watch = createNoSpaceWatch();
|
|
1482
|
+
expect(watch.scan('stdout', 'XENOSPCY is not an errno\n')).toBeNull();
|
|
1483
|
+
expect(watch.scan('stdout', ' code: "ENOSPC"\n')).toBe('code: "ENOSPC"');
|
|
1484
|
+
});
|
|
1485
|
+
});
|
|
1486
|
+
|
|
1367
1487
|
interface WorkspaceFixture {
|
|
1368
1488
|
root: string;
|
|
1369
1489
|
context: BoundedExecContext;
|
|
@@ -1424,17 +1544,18 @@ async function armedPid(path: string, pgid: number, subject: string): Promise<nu
|
|
|
1424
1544
|
}
|
|
1425
1545
|
|
|
1426
1546
|
/**
|
|
1427
|
-
* A command that ignores SIGTERM, a disposition its `
|
|
1428
|
-
*
|
|
1429
|
-
*
|
|
1430
|
-
*
|
|
1431
|
-
* the announcement can neither precede the trap nor
|
|
1547
|
+
* A command that ignores SIGTERM, a disposition its `tail` inherits, and never exits on its own, so only the escalation
|
|
1548
|
+
* can end it; and `armed`, which resolves once it ignores SIGTERM. A killer that awaits `armed` before the graceful
|
|
1549
|
+
* signal sends it to a command that cannot die of it: a bound that fires before the command's shell has even run
|
|
1550
|
+
* `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the default disposition, and the escalation under
|
|
1551
|
+
* test never runs. The command's FIFO write waits for the reader, so the announcement can neither precede the trap nor
|
|
1552
|
+
* be missed.
|
|
1432
1553
|
*/
|
|
1433
1554
|
function ignoresTerm(root: string): { command: string; armed: () => Promise<void> } {
|
|
1434
1555
|
const fifo = join(root, 'armed.fifo');
|
|
1435
1556
|
expect(spawnSync('mkfifo', [fifo]).status).toBe(0);
|
|
1436
1557
|
return {
|
|
1437
|
-
command: `sh -c 'trap "" TERM; echo armed > "$0"; exec
|
|
1558
|
+
command: `sh -c 'trap "" TERM; echo armed > "$0"; exec tail -f /dev/null' '${fifo}'`,
|
|
1438
1559
|
armed: async () => {
|
|
1439
1560
|
expect(await readToEnd(fifo)).toBe('armed\n');
|
|
1440
1561
|
},
|
|
@@ -1459,11 +1580,12 @@ function leaseAt(directory: string): RamTempLease {
|
|
|
1459
1580
|
return { directory, mountpoint: dirname(directory), capacityBytes: 1024 * 1024 * 1024 };
|
|
1460
1581
|
}
|
|
1461
1582
|
|
|
1462
|
-
/** A temp volume whose answers are fixed, recording which leases were reaped and released,
|
|
1583
|
+
/** A temp volume whose answers are fixed, recording which leases were reaped and released, and when space was read. */
|
|
1463
1584
|
function scriptedVolume(
|
|
1464
1585
|
acquisition: Result<RamTempAcquisition, RamTempError>,
|
|
1465
1586
|
full: RamTempError | null = null,
|
|
1466
1587
|
leftovers: string[] = [],
|
|
1588
|
+
space: TempVolumeSpace | null = null,
|
|
1467
1589
|
): TempVolume & { released: string[]; order: string[] } {
|
|
1468
1590
|
const released: string[] = [];
|
|
1469
1591
|
const order: string[] = [];
|
|
@@ -1481,6 +1603,10 @@ function scriptedVolume(
|
|
|
1481
1603
|
return Promise.resolve({ ok: true, value: undefined });
|
|
1482
1604
|
},
|
|
1483
1605
|
fullness: () => Promise.resolve(full),
|
|
1606
|
+
space() {
|
|
1607
|
+
order.push('space');
|
|
1608
|
+
return Promise.resolve(space);
|
|
1609
|
+
},
|
|
1484
1610
|
};
|
|
1485
1611
|
}
|
|
1486
1612
|
|