@smoothbricks/nx-plugin 0.4.28 → 0.4.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +203 -83
- package/dist/bounded-test-policy.d.ts +10 -3
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +19 -25
- package/dist/cargo-closure-input.d.ts +12 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +29 -7
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +42 -19
- package/dist/ensure-built-receipt.d.ts +34 -6
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +265 -159
- package/dist/ensure-built-witness.d.ts +27 -2
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +2176 -15
- package/dist/ensure-built.d.ts +99 -1
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +34 -53
- package/dist/executors/bounded-exec/cpu-pool.d.ts +16 -5
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +43 -19
- package/dist/executors/bounded-exec/executor.d.ts +11 -2
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +139 -39
- package/dist/executors/bounded-exec/ram-temp.d.ts +17 -11
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +27 -13
- package/dist/generators/bun-test-tracing/generator.js +2 -2
- package/dist/graph-input-contract.d.ts +98 -0
- package/dist/graph-input-contract.d.ts.map +1 -0
- package/dist/graph-input-contract.js +624 -0
- package/dist/index.d.ts +12 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +5 -2
- package/dist/managed-files/publish-workflow.d.ts +11 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/producer-receipts.d.ts +19 -0
- package/dist/producer-receipts.d.ts.map +1 -0
- package/dist/producer-receipts.js +921 -0
- package/dist/testing.d.ts +8 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +58 -0
- package/managed/raw/patches/nx@23.2.1.patch +328 -50
- package/managed/raw/tooling/direnv/developer-links.ts +147 -21
- package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
- package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
- package/managed/raw/tooling/direnv/repo-path +15 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +179 -114
- package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
- package/nextest.toml +1 -1
- package/package.json +11 -6
- package/src/__tests__/fixture-nx-env.ts +7 -0
- package/src/__tests__/owned-installation.ts +174 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
- package/src/bounded-test-policy.test.ts +36 -22
- package/src/bounded-test-policy.ts +19 -27
- package/src/bun-bootstrap.test.ts +1 -1
- package/src/cargo-closure-input.test.ts +60 -11
- package/src/cargo-closure-input.ts +28 -6
- package/src/cargo-resolve-cache.test.ts +140 -25
- package/src/cargo-source-hash.test.ts +78 -3
- package/src/cargo-source-hash.ts +48 -30
- package/src/cross-check-policy.test.ts +2 -2
- package/src/ensure-built-receipt.test.ts +111 -16
- package/src/ensure-built-receipt.ts +251 -149
- package/src/ensure-built-witness.test.ts +197 -13
- package/src/ensure-built-witness.ts +187 -15
- package/src/ensure-built.test.ts +156 -102
- package/src/ensure-built.ts +114 -64
- package/src/executors/bounded-exec/cpu-pool.test.ts +116 -32
- package/src/executors/bounded-exec/cpu-pool.ts +55 -17
- package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
- package/src/executors/bounded-exec/executor-host.test.ts +51 -41
- package/src/executors/bounded-exec/executor.test.ts +155 -35
- package/src/executors/bounded-exec/executor.ts +156 -45
- package/src/executors/bounded-exec/progress.test.ts +1 -1
- package/src/executors/bounded-exec/ram-temp-host.test.ts +46 -17
- package/src/executors/bounded-exec/ram-temp.test.ts +65 -3
- package/src/executors/bounded-exec/ram-temp.ts +32 -16
- package/src/executors/bounded-exec/verdict.test.ts +17 -12
- package/src/generators/bounded-test-targets/generator.test.ts +1 -3
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/bun-test-tracing/generator.ts +2 -2
- package/src/graph-input-contract.test.ts +371 -0
- package/src/graph-input-contract.ts +605 -0
- package/src/index.test.ts +3 -3
- package/src/index.ts +24 -3
- package/src/linked-snapshots.nx.test.ts +112 -30
- package/src/managed-files/ci-workflow.ts +5 -2
- package/src/managed-files/publish-workflow.ts +12 -1
- package/src/managed-files/repo-path.test.ts +44 -1
- package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
- package/src/nextest-code-bound.test.ts +23 -17
- package/src/nextest-partitions.test.ts +7 -1
- package/src/nx-cache-restore-mtime.test.ts +2 -2
- package/src/nx-daemon-claim.test.ts +5 -5
- package/src/nx-daemon-idle-cpu.test.ts +0 -1
- package/src/nx-daemon-idle-policy.test.ts +56 -0
- package/src/nx-daemon-ignore-files.test.ts +2 -2
- package/src/nx-daemon-validity.test.ts +31 -9
- package/src/nx-dot-input-policy.test.ts +5 -5
- package/src/nx-files-archive-racy.test.ts +1 -1
- package/src/nx-incomplete-run-summary.test.ts +1 -1
- package/src/nx-plugin-load-errors.test.ts +2 -2
- package/src/nx-project-graph-errors.test.ts +70 -0
- package/src/nx-skipped-task-exit.test.ts +2 -2
- package/src/nx-store-resolution.test.ts +7 -25
- package/src/nx-task-output-lease.test.ts +3 -3
- package/src/nx-use-local.test.ts +7 -5
- package/src/producer-receipts.test.ts +164 -0
- package/src/producer-receipts.ts +631 -0
- package/src/runtime-input.test.ts +1 -1
- package/src/serial-group-wiring.test.ts +4 -4
- package/src/task-results.test.ts +18 -16
- package/src/test-timings.test.ts +2 -2
- package/src/testing.test.ts +5 -5
- package/src/testing.ts +59 -0
- package/src/typecheck-test-policy.test.ts +1 -1
- package/src/typescript-resolution.test.ts +76 -0
- package/test-timings.json +5 -0
|
@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it } from 'bun:test';
|
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
3
|
import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { createServer } from 'node:net';
|
|
5
|
-
import {
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
6
|
import { dirname, join } from 'node:path';
|
|
7
7
|
import { onProgressClock } from '../../__tests__/progress-clock.js';
|
|
8
8
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
type BoundedExecContext,
|
|
13
13
|
createProcessTreeKiller,
|
|
14
14
|
type ProcessTreeKiller,
|
|
15
|
+
resolveCowshedExecutable,
|
|
15
16
|
runBoundedExec,
|
|
16
17
|
type TempVolume,
|
|
17
18
|
} from './executor.js';
|
|
@@ -44,7 +45,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
44
45
|
try {
|
|
45
46
|
process.kill(-pid, 'SIGKILL');
|
|
46
47
|
} catch (cause) {
|
|
47
|
-
|
|
48
|
+
const code = cause instanceof Error && 'code' in cause ? cause.code : undefined;
|
|
49
|
+
// Darwin answers EPERM, not ESRCH, for a group whose only members are unreaped zombies (a killed
|
|
50
|
+
// descendant waiting for launchd to reap it): such a group has nothing left to kill.
|
|
51
|
+
if (code !== 'ESRCH' && !(code === 'EPERM' && onlyZombiesIn(pid))) {
|
|
48
52
|
throw new Error(`cleanup owned readiness process group pgid=${pid} failed`, { cause });
|
|
49
53
|
}
|
|
50
54
|
} finally {
|
|
@@ -186,6 +190,56 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
186
190
|
expect(result.terminalOutput).not.toContain('fake tests::quick (cwd=');
|
|
187
191
|
});
|
|
188
192
|
|
|
193
|
+
it("stops the idle clock while a descendant's work is queued for host CPU tokens, and runs it on after", async () => {
|
|
194
|
+
const workspace = await createWorkspace();
|
|
195
|
+
// The runner finishes a test, then says through its pool that work it started is queued for host tokens, ends that
|
|
196
|
+
// wait and goes silent. While the wait is held the clock moves well past the bound and nothing fires; once the wait
|
|
197
|
+
// ends the same move ends the run.
|
|
198
|
+
await workspace.write(
|
|
199
|
+
'queued-runner.js',
|
|
200
|
+
[
|
|
201
|
+
"const { connect } = require('node:net');",
|
|
202
|
+
"console.error(' PASS [ 0.010s] (1/2) fake tests::quick');",
|
|
203
|
+
`const socket = connect(process.env.${CPU_POOL_ENV});`,
|
|
204
|
+
`socket.on('connect', () => socket.write('{"op":"host-wait"}\\n'));`,
|
|
205
|
+
"socket.once('data', (reply) => { console.log('host wait ' + reply.toString().trim()); socket.end(); });",
|
|
206
|
+
"socket.on('close', () => setInterval(() => {}, 60_000));",
|
|
207
|
+
'',
|
|
208
|
+
].join('\n'),
|
|
209
|
+
);
|
|
210
|
+
const budget: CpuBudget = {
|
|
211
|
+
async take() {
|
|
212
|
+
return heldGrant(1, []);
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
const { result } = await onProgressClock(
|
|
217
|
+
(chunk) => chunk.includes('bounded-exec: idle clock'),
|
|
218
|
+
1_000,
|
|
219
|
+
() =>
|
|
220
|
+
runBoundedExec(
|
|
221
|
+
{ command: 'node queued-runner.js', idleTimeoutMs: 400, killAfterMs: 0 },
|
|
222
|
+
workspace.context,
|
|
223
|
+
createProcessTreeKiller(),
|
|
224
|
+
null,
|
|
225
|
+
budget,
|
|
226
|
+
),
|
|
227
|
+
);
|
|
228
|
+
|
|
229
|
+
const output = result.terminalOutput;
|
|
230
|
+
expect(result.success).toBe(false);
|
|
231
|
+
expect(output).toContain('host wait {"ok":true,"wait":"held"}');
|
|
232
|
+
const stopped = output.indexOf(
|
|
233
|
+
"bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens",
|
|
234
|
+
);
|
|
235
|
+
const restarted = output.indexOf('bounded-exec: idle clock restarted');
|
|
236
|
+
const expired = output.indexOf('Command made no progress: no test finished for 400ms (idleTimeoutMs=400)');
|
|
237
|
+
// The 1000 ms move while the wait was held fired nothing; the one after it ended the run.
|
|
238
|
+
expect(stopped).toBeGreaterThan(-1);
|
|
239
|
+
expect(restarted).toBeGreaterThan(stopped);
|
|
240
|
+
expect(expired).toBeGreaterThan(restarted);
|
|
241
|
+
});
|
|
242
|
+
|
|
189
243
|
it('emits a completed pre-expiry snapshot or an explicit incomplete capture without waiting for it', async () => {
|
|
190
244
|
const workspace = await createWorkspace();
|
|
191
245
|
// Child output drives the fake executor clock; the heartbeat period is not a timing oracle.
|
|
@@ -652,7 +706,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
652
706
|
expect(await straggler).toBe('');
|
|
653
707
|
expect(result.success).toBe(false);
|
|
654
708
|
expect(result.terminalOutput).toContain('Force-killing what the timed-out command left in its process group');
|
|
655
|
-
}
|
|
709
|
+
});
|
|
656
710
|
|
|
657
711
|
it('runs the command with its RAM lease as TMPDIR, names held dead leases, and ends the lease after it exits', async () => {
|
|
658
712
|
const workspace = await createWorkspace();
|
|
@@ -903,6 +957,52 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
903
957
|
expect(result.terminalOutput).toBe('ran\n');
|
|
904
958
|
});
|
|
905
959
|
|
|
960
|
+
it('resolves the cowshed a run uses, in full precedence order', () => {
|
|
961
|
+
const root = '/workspace';
|
|
962
|
+
// The executor never chdirs, so an empty PATH component names its actual spawn cwd,
|
|
963
|
+
// deliberately not the workspace root: the two directories differ on purpose here.
|
|
964
|
+
const cwd = '/elsewhere';
|
|
965
|
+
// Every level present at once, removed one at a time from the top: each step must fall
|
|
966
|
+
// to exactly the next level, so the order itself is the assertion, not each level alone.
|
|
967
|
+
const everywhere = new Set([
|
|
968
|
+
'/opt/bin/cowshed',
|
|
969
|
+
'/elsewhere/cowshed',
|
|
970
|
+
'/workspace/packages/cowshed/bin/cowshed',
|
|
971
|
+
'/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed',
|
|
972
|
+
]);
|
|
973
|
+
const present = (path: string) => everywhere.has(path);
|
|
974
|
+
const select = (named?: string, inherited?: string, pathValue?: string) =>
|
|
975
|
+
resolveCowshedExecutable(root, named, inherited, pathValue, cwd, present);
|
|
976
|
+
|
|
977
|
+
// 1. A target-named executable wins, made absolute against the workspace root.
|
|
978
|
+
expect(select('bin/cowshed', '/valid/bin/cowshed', '/opt/bin')).toBe('/workspace/bin/cowshed');
|
|
979
|
+
// 2. An explicit inherited executable outranks PATH and the workspace launcher — the
|
|
980
|
+
// precedence createHostCommands always had — and keeps the coordinate the parent
|
|
981
|
+
// gave it: absolute verbatim, relative untouched for spawn's own resolution,
|
|
982
|
+
// never retargeted to the workspace root.
|
|
983
|
+
expect(select(undefined, '/valid/bin/cowshed', '/opt/bin')).toBe('/valid/bin/cowshed');
|
|
984
|
+
expect(select(undefined, 'bin/cowshed', '/opt/bin')).toBe('bin/cowshed');
|
|
985
|
+
// 3. An installed cowshed on PATH wins (undefined: the bare name spawn already finds it).
|
|
986
|
+
expect(select(undefined, undefined, '/opt/bin')).toBeUndefined();
|
|
987
|
+
// 3b. An empty PATH component (the leading ':' below) names the current directory, as
|
|
988
|
+
// POSIX spawn resolution does — here the executor's cwd, not the workspace root:
|
|
989
|
+
// a cowshed in the cwd is installed too, and one merely in the root is not.
|
|
990
|
+
expect(select(undefined, undefined, ':/usr/bin')).toBeUndefined();
|
|
991
|
+
everywhere.delete('/elsewhere/cowshed');
|
|
992
|
+
everywhere.add('/workspace/cowshed');
|
|
993
|
+
expect(select(undefined, undefined, ':/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
|
|
994
|
+
everywhere.delete('/workspace/cowshed');
|
|
995
|
+
// 4. Only when PATH provides none does the workspace's own launcher answer.
|
|
996
|
+
expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
|
|
997
|
+
// 5. A linked workspace's launcher answers when the workspace has none of its own.
|
|
998
|
+
everywhere.delete('/workspace/packages/cowshed/bin/cowshed');
|
|
999
|
+
expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed');
|
|
1000
|
+
// 6. No cowshed anywhere keeps the bare name, so the spawn failure names the missing tool.
|
|
1001
|
+
everywhere.clear();
|
|
1002
|
+
expect(select(undefined, undefined, '/usr/bin')).toBeUndefined();
|
|
1003
|
+
expect(select(undefined, undefined, undefined)).toBeUndefined();
|
|
1004
|
+
});
|
|
1005
|
+
|
|
906
1006
|
it('sizes a runner to the grant its parent pool lent, takes nothing from the host, and returns nothing to it', async () => {
|
|
907
1007
|
const workspace = await createWorkspace();
|
|
908
1008
|
const asked: number[] = [];
|
|
@@ -1039,41 +1139,45 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
1039
1139
|
expect(ownedRecord).not.toHaveProperty('cpu.inherited');
|
|
1040
1140
|
});
|
|
1041
1141
|
|
|
1042
|
-
it('
|
|
1142
|
+
it('keeps a command running when its borrowed grant closes, and judges it by its own exit', async () => {
|
|
1043
1143
|
if (process.platform === 'win32') return;
|
|
1044
1144
|
const workspace = await createWorkspace();
|
|
1045
|
-
const directory = join(workspace.root, '
|
|
1046
|
-
const
|
|
1047
|
-
const
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1145
|
+
const directory = join(workspace.root, 'outlives-its-grant');
|
|
1146
|
+
const started = join(workspace.root, 'started.fifo');
|
|
1147
|
+
const go = join(workspace.root, 'go.fifo');
|
|
1148
|
+
expect(spawnSync('mkfifo', [started, go]).status).toBe(0);
|
|
1149
|
+
// The run says the grant ended on its stderr; that line is the signal the close was heard while the command runs.
|
|
1150
|
+
const { promise: heard, resolve: hear } = Promise.withResolvers<void>();
|
|
1151
|
+
process.stderr.write = captureWrite([], (text) => {
|
|
1152
|
+
if (text.includes('cpu-tokens grant ended')) hear();
|
|
1153
|
+
});
|
|
1154
|
+
const result = await withPool(1, async (pool) => {
|
|
1155
|
+
const running = runBoundedExec(
|
|
1056
1156
|
{
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
await ignoring.armed();
|
|
1060
|
-
const closed = await pool.close();
|
|
1061
|
-
if (!closed.ok) throw new Error(closed.error);
|
|
1062
|
-
}
|
|
1063
|
-
calls.push(signal);
|
|
1064
|
-
await createProcessTreeKiller().kill(pid, signal);
|
|
1065
|
-
},
|
|
1157
|
+
command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
|
|
1158
|
+
timeoutMs: 20_000,
|
|
1066
1159
|
},
|
|
1160
|
+
{ ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
|
|
1161
|
+
createProcessTreeKiller(),
|
|
1067
1162
|
null,
|
|
1068
1163
|
null,
|
|
1069
1164
|
inheritedFrom(pool),
|
|
1070
|
-
)
|
|
1165
|
+
);
|
|
1166
|
+
// The command has started and waits on `go` while its borrowed grant closes under it.
|
|
1167
|
+
expect(await readToEnd(started)).toBe('started\n');
|
|
1168
|
+
const closed = await pool.close();
|
|
1169
|
+
if (!closed.ok) throw new Error(closed.error);
|
|
1170
|
+
await heard;
|
|
1171
|
+
await writeFile(go, 'go\n');
|
|
1172
|
+
return running;
|
|
1173
|
+
});
|
|
1174
|
+
expect(result.terminalOutput).toContain(
|
|
1175
|
+
'cowshed: cpu-tokens grant ended: the parent pool closed its 1-token grant while the command ran; it runs on over budget until it ends',
|
|
1071
1176
|
);
|
|
1072
|
-
expect(result.
|
|
1073
|
-
expect(
|
|
1074
|
-
expect(result.terminalOutput).toContain('cpu-tokens grant lost');
|
|
1177
|
+
expect(result.terminalOutput).not.toContain('Force-killing');
|
|
1178
|
+
expect(result.success).toBe(true);
|
|
1075
1179
|
const record: unknown = JSON.parse(await readFile(join(directory, VERDICT_FILE), 'utf8'));
|
|
1076
|
-
expect(record).toHaveProperty('verdict.outcome', '
|
|
1180
|
+
expect(record).toHaveProperty('verdict.outcome', 'passed');
|
|
1077
1181
|
});
|
|
1078
1182
|
|
|
1079
1183
|
it('asks the host budget when a parent pool refuses its descriptor, and says why', async () => {
|
|
@@ -1380,13 +1484,15 @@ function scriptedVolume(
|
|
|
1380
1484
|
};
|
|
1381
1485
|
}
|
|
1382
1486
|
|
|
1383
|
-
function captureWrite(chunks: string[]): typeof process.stdout.write {
|
|
1487
|
+
function captureWrite(chunks: string[], onChunk?: (text: string) => void): typeof process.stdout.write {
|
|
1384
1488
|
return ((
|
|
1385
1489
|
chunk: string | Uint8Array,
|
|
1386
1490
|
encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),
|
|
1387
1491
|
callback?: (error?: Error | null) => void,
|
|
1388
1492
|
) => {
|
|
1389
|
-
|
|
1493
|
+
const text = chunk.toString();
|
|
1494
|
+
chunks.push(text);
|
|
1495
|
+
onChunk?.(text);
|
|
1390
1496
|
const cb = typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
|
|
1391
1497
|
cb?.();
|
|
1392
1498
|
return true;
|
|
@@ -1419,9 +1525,10 @@ function heldGrant(tokens: number, released: number[]): TokenGrant {
|
|
|
1419
1525
|
};
|
|
1420
1526
|
}
|
|
1421
1527
|
|
|
1422
|
-
/** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own
|
|
1528
|
+
/** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own, and no run owns the
|
|
1529
|
+
* host waits its children report. */
|
|
1423
1530
|
async function withPool<T>(tokens: number, body: (pool: CpuPool) => Promise<T>): Promise<T> {
|
|
1424
|
-
const created = await createCpuPool(tokens, new Promise<void>(() => {}));
|
|
1531
|
+
const created = await createCpuPool(tokens, new Promise<void>(() => {}), { begin() {}, end() {} });
|
|
1425
1532
|
if (!created.ok) {
|
|
1426
1533
|
throw new Error(`cpu pool: ${created.error}`);
|
|
1427
1534
|
}
|
|
@@ -1449,7 +1556,20 @@ function inheritedFrom(pool: CpuPool): InheritedCpuPool {
|
|
|
1449
1556
|
return parent;
|
|
1450
1557
|
}
|
|
1451
1558
|
|
|
1452
|
-
/** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens
|
|
1559
|
+
/** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens). */
|
|
1453
1560
|
function lentCount(want: number, tokens: number): number {
|
|
1454
|
-
return Math.min(want, tokens
|
|
1561
|
+
return Math.min(want, tokens);
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/** Every process left in group `pgid` is a zombie: `ps` lists no member in any other state. */
|
|
1565
|
+
function onlyZombiesIn(pgid: number): boolean {
|
|
1566
|
+
const listed = spawnSync('ps', ['-A', '-o', 'pgid=,stat='], { encoding: 'utf8' });
|
|
1567
|
+
if (listed.status !== 0) {
|
|
1568
|
+
return false;
|
|
1569
|
+
}
|
|
1570
|
+
return listed.stdout
|
|
1571
|
+
.split('\n')
|
|
1572
|
+
.map((line) => line.trim().split(/\s+/))
|
|
1573
|
+
.filter(([group]) => Number(group) === pgid)
|
|
1574
|
+
.every(([, state]) => state?.startsWith('Z') === true);
|
|
1455
1575
|
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { type ChildProcess, spawn } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants } from 'node:fs';
|
|
2
3
|
import { availableParallelism, userInfo } from 'node:os';
|
|
3
|
-
import { isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
5
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
5
6
|
|
|
6
7
|
import {
|
|
7
8
|
CPU_POOL_ENV,
|
|
8
9
|
type CpuPool,
|
|
9
10
|
createCpuPool,
|
|
11
|
+
type HostWaits,
|
|
10
12
|
type InheritedCpuPool,
|
|
11
13
|
readInheritedCpuPool,
|
|
12
14
|
takeParentCpuTokens,
|
|
@@ -88,8 +90,6 @@ interface RunState {
|
|
|
88
90
|
settled: boolean;
|
|
89
91
|
expiry: ExpiredBound | null;
|
|
90
92
|
forceKillNeeded: boolean;
|
|
91
|
-
/** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
|
|
92
|
-
grantLost: boolean;
|
|
93
93
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
94
94
|
poolCloseError: string | null;
|
|
95
95
|
}
|
|
@@ -103,6 +103,52 @@ export interface ProcessTreeKiller {
|
|
|
103
103
|
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
104
104
|
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
|
|
105
105
|
|
|
106
|
+
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
107
|
+
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
108
|
+
'packages/cowshed/bin/cowshed',
|
|
109
|
+
'node_modules/@smoothbricks/cowshed/bin/cowshed',
|
|
110
|
+
] as const;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The cowshed a run's disk commands use, in precedence order: the target's named one, an
|
|
114
|
+
* explicit one inherited from the parent process, the installed one on PATH, and — only when
|
|
115
|
+
* PATH provides none — the launcher this workspace itself provides (its own cowshed package,
|
|
116
|
+
* or a linked one's bin). An environment with no cowshed anywhere keeps the bare name, so
|
|
117
|
+
* its spawn failure names the missing installed tool.
|
|
118
|
+
*/
|
|
119
|
+
export function resolveCowshedExecutable(
|
|
120
|
+
root: string,
|
|
121
|
+
named: string | undefined,
|
|
122
|
+
inherited: string | undefined,
|
|
123
|
+
pathValue: string | undefined,
|
|
124
|
+
cwd: string,
|
|
125
|
+
exists: (path: string) => boolean = (path) => {
|
|
126
|
+
try {
|
|
127
|
+
accessSync(path, constants.X_OK);
|
|
128
|
+
return true;
|
|
129
|
+
} catch {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
): string | undefined {
|
|
134
|
+
// A target-named executable is workspace-relative or absolute against the workspace root.
|
|
135
|
+
// An inherited explicit keeps the coordinate the parent gave it, exactly as the host
|
|
136
|
+
// commands always spawned it: an absolute one verbatim, a relative or bare one for
|
|
137
|
+
// spawn's own resolution — retargeting it to the root would silently change the tool.
|
|
138
|
+
if (named !== undefined) return resolvePath(root, named);
|
|
139
|
+
if (inherited !== undefined) return inherited;
|
|
140
|
+
// An empty PATH component names the spawning process's current directory, as POSIX
|
|
141
|
+
// spawn resolution does — the executor never chdirs, so that is the caller's cwd,
|
|
142
|
+
// not the workspace root.
|
|
143
|
+
const onPath = (pathValue ?? '').split(delimiter).some((dir) => exists(join(dir === '' ? cwd : dir, 'cowshed')));
|
|
144
|
+
if (onPath) return undefined;
|
|
145
|
+
for (const candidate of WORKSPACE_COWSHED_LAUNCHERS) {
|
|
146
|
+
const launcher = join(root, candidate);
|
|
147
|
+
if (exists(launcher)) return launcher;
|
|
148
|
+
}
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
|
|
106
152
|
export default function boundedExecExecutor(
|
|
107
153
|
options: BoundedExecOptions,
|
|
108
154
|
context: NxTaskContext,
|
|
@@ -119,7 +165,13 @@ export function runDefaultBoundedExec(
|
|
|
119
165
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
120
166
|
// command, starting in another directory, could not find: it gets the absolute one.
|
|
121
167
|
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
122
|
-
const cowshed =
|
|
168
|
+
const cowshed = resolveCowshedExecutable(
|
|
169
|
+
context.root,
|
|
170
|
+
named,
|
|
171
|
+
process.env[COWSHED_EXECUTABLE_ENV],
|
|
172
|
+
process.env.PATH,
|
|
173
|
+
process.cwd(),
|
|
174
|
+
);
|
|
123
175
|
const run: BoundedExecOptions =
|
|
124
176
|
cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
125
177
|
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
@@ -167,8 +219,9 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
|
|
|
167
219
|
*
|
|
168
220
|
* `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
|
|
169
221
|
* and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
|
|
170
|
-
* descendants, closed before the grant returns. A
|
|
171
|
-
*
|
|
222
|
+
* descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
|
|
223
|
+
* restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
|
|
224
|
+
* its own exit, and the run says so.
|
|
172
225
|
*/
|
|
173
226
|
export async function runBoundedExec(
|
|
174
227
|
options: BoundedExecOptions,
|
|
@@ -194,7 +247,6 @@ export async function runBoundedExec(
|
|
|
194
247
|
settled: false,
|
|
195
248
|
expiry: null,
|
|
196
249
|
forceKillNeeded: false,
|
|
197
|
-
grantLost: false,
|
|
198
250
|
poolCloseError: null,
|
|
199
251
|
};
|
|
200
252
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
@@ -267,7 +319,7 @@ export async function runBoundedExec(
|
|
|
267
319
|
let grant: GrantedTokens | null = null;
|
|
268
320
|
let parentRefusal: string | null = null;
|
|
269
321
|
if (inherited !== null) {
|
|
270
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want
|
|
322
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
271
323
|
if (fromParent.granted) {
|
|
272
324
|
grant = fromParent;
|
|
273
325
|
} else if (fromParent.cause === 'absent') {
|
|
@@ -318,8 +370,8 @@ export async function runBoundedExec(
|
|
|
318
370
|
return cpuReleased;
|
|
319
371
|
};
|
|
320
372
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
321
|
-
// An `operational` run failed on its CPU budget (a
|
|
322
|
-
//
|
|
373
|
+
// An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
|
|
374
|
+
// bound also fired, so it is never judged as a bound.
|
|
323
375
|
const recordRun = async (
|
|
324
376
|
exitCode: number,
|
|
325
377
|
expiry: ExpiredBound | null,
|
|
@@ -353,10 +405,26 @@ export async function runBoundedExec(
|
|
|
353
405
|
);
|
|
354
406
|
}
|
|
355
407
|
|
|
408
|
+
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
409
|
+
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
410
|
+
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
411
|
+
let heldWaits = 0;
|
|
412
|
+
let idleClock: { stop(): void; restart(): void } | null = null;
|
|
413
|
+
const hostWaits: HostWaits = {
|
|
414
|
+
begin: () => {
|
|
415
|
+
heldWaits += 1;
|
|
416
|
+
if (heldWaits === 1) idleClock?.stop();
|
|
417
|
+
},
|
|
418
|
+
end: () => {
|
|
419
|
+
heldWaits -= 1;
|
|
420
|
+
if (heldWaits === 0) idleClock?.restart();
|
|
421
|
+
},
|
|
422
|
+
};
|
|
423
|
+
|
|
356
424
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
357
425
|
// A pool that cannot be set up refuses the run: no command runs unbudgeted.
|
|
358
426
|
if (grant !== null) {
|
|
359
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
427
|
+
const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
|
|
360
428
|
if (!created.ok) {
|
|
361
429
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
362
430
|
}
|
|
@@ -400,14 +468,17 @@ export async function runBoundedExec(
|
|
|
400
468
|
clearTimeout(captureTimer);
|
|
401
469
|
idleCapture?.controller.abort();
|
|
402
470
|
};
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
471
|
+
// What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
|
|
472
|
+
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
473
|
+
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
474
|
+
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
475
|
+
let queuedMs = 0;
|
|
476
|
+
let queuedSinceMs: number | null = null;
|
|
477
|
+
const scheduleIdle = (limitMs: number, leftMs: number): void => {
|
|
407
478
|
clearTimeout(idleTimer);
|
|
408
479
|
cancelCapture();
|
|
409
480
|
const capture: NonNullable<typeof idleCapture> = {
|
|
410
|
-
expiresAtMs: Date.now() +
|
|
481
|
+
expiresAtMs: Date.now() + leftMs,
|
|
411
482
|
controller: new AbortController(),
|
|
412
483
|
result: { kind: 'pending', startedAtMs: null },
|
|
413
484
|
};
|
|
@@ -445,12 +516,51 @@ export async function runBoundedExec(
|
|
|
445
516
|
},
|
|
446
517
|
);
|
|
447
518
|
},
|
|
448
|
-
Math.max(0,
|
|
449
|
-
);
|
|
450
|
-
idleTimer = setTimeout(
|
|
451
|
-
() => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
|
|
452
|
-
idleTimeoutMs,
|
|
519
|
+
Math.max(0, leftMs - HANG_CAPTURE_MS),
|
|
453
520
|
);
|
|
521
|
+
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
522
|
+
};
|
|
523
|
+
const armIdleTimer = (): void => {
|
|
524
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
idleLeftMs = idleTimeoutMs;
|
|
528
|
+
queuedMs = 0;
|
|
529
|
+
if (queuedSinceMs !== null) {
|
|
530
|
+
queuedSinceMs = Date.now();
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
534
|
+
};
|
|
535
|
+
idleClock = {
|
|
536
|
+
stop: () => {
|
|
537
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
const now = Date.now();
|
|
541
|
+
queuedSinceMs = now;
|
|
542
|
+
if (idleCapture !== undefined) {
|
|
543
|
+
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
544
|
+
}
|
|
545
|
+
clearTimeout(idleTimer);
|
|
546
|
+
cancelCapture();
|
|
547
|
+
idleCapture = undefined;
|
|
548
|
+
appendStderr(
|
|
549
|
+
`bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
550
|
+
);
|
|
551
|
+
},
|
|
552
|
+
restart: () => {
|
|
553
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
const waitedMs = Date.now() - queuedSinceMs;
|
|
557
|
+
queuedMs += waitedMs;
|
|
558
|
+
queuedSinceMs = null;
|
|
559
|
+
appendStderr(
|
|
560
|
+
`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
561
|
+
);
|
|
562
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
563
|
+
},
|
|
454
564
|
};
|
|
455
565
|
|
|
456
566
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
@@ -522,8 +632,9 @@ export async function runBoundedExec(
|
|
|
522
632
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
523
633
|
// two have opposite fixes.
|
|
524
634
|
let termination: Promise<void> = Promise.resolve();
|
|
525
|
-
|
|
526
|
-
|
|
635
|
+
// `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
|
|
636
|
+
const expire = (bound: ExpiredBound, queuedMs = 0): void => {
|
|
637
|
+
if (state.settled || state.expiry !== null) {
|
|
527
638
|
return;
|
|
528
639
|
}
|
|
529
640
|
state.expiry = bound;
|
|
@@ -532,7 +643,9 @@ export async function runBoundedExec(
|
|
|
532
643
|
const elapsedMs = Date.now() - startedAt;
|
|
533
644
|
appendStderr(
|
|
534
645
|
bound.kind === 'idle'
|
|
535
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})
|
|
646
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
|
|
647
|
+
queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
|
|
648
|
+
} after ${elapsedMs}ms of runtime; ${
|
|
536
649
|
bound.tests.length > 0
|
|
537
650
|
? `still running: ${bound.tests.join(', ')}`
|
|
538
651
|
: 'the runner named no test still running'
|
|
@@ -601,25 +714,23 @@ export async function runBoundedExec(
|
|
|
601
714
|
const totalTimer =
|
|
602
715
|
timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
|
|
603
716
|
armIdleTimer();
|
|
717
|
+
// A wait its pool heard before the command started stops the clock it was not yet there to stop.
|
|
718
|
+
if (heldWaits > 0) idleClock.stop();
|
|
604
719
|
|
|
605
|
-
// A
|
|
606
|
-
//
|
|
607
|
-
// is
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
if (state.expiry === null) termination = terminate();
|
|
620
|
-
};
|
|
621
|
-
if (grant?.inherited) {
|
|
622
|
-
void grant.closed.then(onGrantLost);
|
|
720
|
+
// A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
|
|
721
|
+
// restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
|
|
722
|
+
// command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
|
|
723
|
+
// the command settled is not a loss; `cpuReleased` marks it.
|
|
724
|
+
if (grant !== null) {
|
|
725
|
+
const held = grant;
|
|
726
|
+
void held.closed.then(() => {
|
|
727
|
+
if (state.settled || cpuReleased !== null) {
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
appendStderr(
|
|
731
|
+
`cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
|
|
732
|
+
);
|
|
733
|
+
});
|
|
623
734
|
}
|
|
624
735
|
|
|
625
736
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
@@ -648,9 +759,9 @@ export async function runBoundedExec(
|
|
|
648
759
|
}
|
|
649
760
|
await releaseLease();
|
|
650
761
|
|
|
651
|
-
// A run whose
|
|
652
|
-
//
|
|
653
|
-
const operationalFailure = state.
|
|
762
|
+
// A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
|
|
763
|
+
// its verdict is an ordinary failure, never a bound.
|
|
764
|
+
const operationalFailure = state.poolCloseError !== null;
|
|
654
765
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
|
|
655
766
|
|
|
656
767
|
return {
|
|
@@ -61,7 +61,7 @@ describe('run progress', () => {
|
|
|
61
61
|
|
|
62
62
|
describe('progress report', () => {
|
|
63
63
|
it('asks a lone bun test for dots, keeping the runtime the target named', () => {
|
|
64
|
-
expect(withProgressReport('bun test --timeout=
|
|
64
|
+
expect(withProgressReport('bun test --timeout=0 src')).toBe('bun test --dots --timeout=0 src');
|
|
65
65
|
expect(withProgressReport('../x/.runtime/bun test a')).toBe('../x/.runtime/bun test --dots a');
|
|
66
66
|
});
|
|
67
67
|
|