@smoothbricks/nx-plugin 0.4.29 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -25
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +2 -8
- package/dist/cargo-closure-input.d.ts +11 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +16 -6
- package/dist/ensure-built-receipt.d.ts +8 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +45 -13
- package/dist/ensure-built-witness.d.ts +40 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +68 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +11 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts +7 -3
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +9 -4
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +38 -14
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +3 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +15 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/graph-input-contract.d.ts +4 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +17 -4
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -3
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +15 -10
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +8 -1
- package/managed/raw/tooling/direnv/developer-links.ts +1 -8
- package/managed/raw/tooling/direnv/setup-environment.ts +13 -1
- package/package.json +15 -4
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/owned-installation.ts +4 -164
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.ts +2 -8
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +15 -112
- package/src/cargo-closure-input.ts +23 -6
- package/src/cargo-closure-key.test.ts +240 -0
- package/src/cargo-resolve-cache.test.ts +29 -7
- package/src/ensure-built-receipt.ts +56 -13
- package/src/ensure-built-witness.test.ts +113 -4
- package/src/ensure-built-witness.ts +75 -16
- package/src/ensure-built.test.ts +74 -2
- package/src/ensure-built.ts +13 -2
- package/src/executors/bounded-exec/cpu-pool.test.ts +18 -0
- package/src/executors/bounded-exec/cpu-pool.ts +10 -4
- package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +23 -11
- package/src/executors/bounded-exec/executor.test.ts +189 -63
- package/src/executors/bounded-exec/executor.ts +39 -13
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/ram-temp-host.test.ts +9 -8
- package/src/executors/bounded-exec/ram-temp.test.ts +32 -18
- package/src/executors/bounded-exec/ram-temp.ts +17 -4
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
- package/src/executors/bounded-exec/verdict.test.ts +125 -25
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/graph-input-contract.test.ts +8 -0
- package/src/graph-input-contract.ts +26 -4
- package/src/index.test.ts +12 -2
- package/src/index.ts +13 -3
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nx-daemon-claim.test.ts +5 -3
- package/src/nx-daemon-idle-policy.test.ts +22 -18
- package/src/nx-daemon-validity.test.ts +5 -13
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.ts +15 -10
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/testing.ts +8 -1
- package/test-timings.json +4 -0
|
@@ -25,6 +25,7 @@ export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
|
25
25
|
/** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
|
|
26
26
|
export interface InheritedCpuPool {
|
|
27
27
|
readonly socket: string;
|
|
28
|
+
/** The count the parent names beside its endpoint: 0 for a resident's relay pool, which holds no token. */
|
|
28
29
|
readonly tokens: number;
|
|
29
30
|
}
|
|
30
31
|
|
|
@@ -120,11 +121,16 @@ const frame = (reply: CpuPoolReply): string => `${stringifyReply(reply)}\n`;
|
|
|
120
121
|
|
|
121
122
|
/**
|
|
122
123
|
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
123
|
-
* or NUL-bearing endpoint, proves no allocation. A
|
|
124
|
+
* or NUL-bearing endpoint, proves no allocation. A resident's relay pool names a count of 0 beside its
|
|
125
|
+
* endpoint: it holds no token for its lifetime and asks the host for each child that asks it, so zero is
|
|
126
|
+
* valid only with the endpoint, and every other count is a positive whole number. A valid descriptor still
|
|
127
|
+
* needs a live endpoint grant (specs/cowshed/05_gateway.md, "Host CPU budget", "A resident holds no token").
|
|
124
128
|
*/
|
|
125
129
|
export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null {
|
|
126
130
|
const socket = environment[CPU_POOL_ENV];
|
|
127
|
-
const
|
|
131
|
+
const count = environment[CPU_TOKENS_ENV];
|
|
132
|
+
// Zero is valid only as the literal 0 a relay pool names; any other count must be a positive safe integer.
|
|
133
|
+
const tokens = count === '0' ? 0 : parseInheritedTokens(count);
|
|
128
134
|
if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
|
|
129
135
|
return null;
|
|
130
136
|
}
|
|
@@ -133,8 +139,8 @@ export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>):
|
|
|
133
139
|
|
|
134
140
|
/**
|
|
135
141
|
* Ask the immediate parent pool for `want` tokens. Neither the inherited count nor the host's core count bounds the
|
|
136
|
-
* ask: a lent pool grants at most what it holds, and a gate's relay pool asks the host for each
|
|
137
|
-
* lease is the child's local allocation: its connection is the grant, and closing it returns only
|
|
142
|
+
* ask: a lent pool grants at most what it holds, and a gate's or a resident's relay pool asks the host for each
|
|
143
|
+
* child. A granted lease is the child's local allocation: its connection is the grant, and closing it returns only
|
|
138
144
|
* that allocation. An absent endpoint allows normal host acquisition; a connected endpoint's refusal
|
|
139
145
|
* or protocol failure must fail admission rather than taking a second grant above its parent.
|
|
140
146
|
*/
|
|
@@ -54,6 +54,24 @@ describe('cowshed CPU budget client', () => {
|
|
|
54
54
|
want: 12,
|
|
55
55
|
});
|
|
56
56
|
expect(runnerDemand('cargo nextest run --test-threads=4', undefined, 18, {})).toEqual({ kind: 'nextest', want: 4 });
|
|
57
|
+
// `--no-capture` runs tests serially and ignores `--test-threads`: a host smoke of one test asks for one
|
|
58
|
+
// token, not one per core it would hold idle while its own scratch jobs queue for the host's tokens.
|
|
59
|
+
const smoke =
|
|
60
|
+
"cargo --frozen nextest run --archive-file a.tar.zst -E 'test(smoke)' --run-ignored only --no-capture";
|
|
61
|
+
expect(runnerDemand(smoke, undefined, 18, {})).toEqual({ kind: 'nextest', want: 1 });
|
|
62
|
+
expect(runnerDemand(`${smoke} --test-threads 4`, undefined, 18, {})).toEqual({ kind: 'nextest', want: 1 });
|
|
63
|
+
expect(runnerDemand('cargo nextest run -E all() -- --nocapture', undefined, 18, {})).toEqual({
|
|
64
|
+
kind: 'nextest',
|
|
65
|
+
want: 1,
|
|
66
|
+
});
|
|
67
|
+
// A flag that only shares the prefix is not it.
|
|
68
|
+
expect(runnerDemand('cargo nextest run --no-capture-x', undefined, 18, {})).toEqual({ kind: 'nextest', want: 18 });
|
|
69
|
+
// The same words in a cargo build are not nextest's.
|
|
70
|
+
expect(runnerDemand('cargo test --workspace -- --nocapture', undefined, 18, {})).toEqual({
|
|
71
|
+
kind: 'cargo',
|
|
72
|
+
want: 18,
|
|
73
|
+
testThreads: null,
|
|
74
|
+
});
|
|
57
75
|
expect(runnerDemand('bun test --timeout=0 --parallel=3', undefined, 18, {})).toEqual({
|
|
58
76
|
kind: 'bun-parallel',
|
|
59
77
|
want: 3,
|
|
@@ -88,6 +106,193 @@ describe('cowshed CPU budget client', () => {
|
|
|
88
106
|
expect(runnerDemand('cargo nextest run --test-threads=4', 6, 18, {})).toEqual({ kind: 'nextest', want: 6 });
|
|
89
107
|
});
|
|
90
108
|
|
|
109
|
+
it("asks for Stryker's --concurrency, else its own default of one worker per core less one past four cores", () => {
|
|
110
|
+
expect(runnerDemand('bun stryker run --concurrency 4', undefined, 18, {})).toEqual({ kind: 'stryker', want: 4 });
|
|
111
|
+
expect(runnerDemand('bun stryker run -c 6', undefined, 18, {})).toEqual({ kind: 'stryker', want: 6 });
|
|
112
|
+
expect(runnerDemand('npx stryker run --concurrency=3', undefined, 18, {})).toEqual({ kind: 'stryker', want: 3 });
|
|
113
|
+
expect(runnerDemand('../../node_modules/.bin/stryker run -c 2', undefined, 18, {})).toEqual({
|
|
114
|
+
kind: 'stryker',
|
|
115
|
+
want: 2,
|
|
116
|
+
});
|
|
117
|
+
// Stryker takes the last count it is given.
|
|
118
|
+
expect(runnerDemand('bun stryker run -c 8 --concurrency 5', undefined, 18, {})).toEqual({
|
|
119
|
+
kind: 'stryker',
|
|
120
|
+
want: 5,
|
|
121
|
+
});
|
|
122
|
+
// A share is of the cores, and never less than one worker.
|
|
123
|
+
expect(runnerDemand('bun stryker run --concurrency=50%', undefined, 18, {})).toEqual({ kind: 'stryker', want: 9 });
|
|
124
|
+
expect(runnerDemand('bun stryker run --concurrency 0%', undefined, 18, {})).toEqual({ kind: 'stryker', want: 1 });
|
|
125
|
+
// Stryker's default, which sized nightly 477's sweeps to the whole host: n - 1 past four cores, else n.
|
|
126
|
+
expect(runnerDemand('bun stryker run', undefined, 32, {})).toEqual({ kind: 'stryker', want: 31 });
|
|
127
|
+
expect(runnerDemand('bun stryker run', undefined, 18, {})).toEqual({ kind: 'stryker', want: 17 });
|
|
128
|
+
expect(runnerDemand('bun stryker run', undefined, 4, {})).toEqual({ kind: 'stryker', want: 4 });
|
|
129
|
+
// Another command's `-c` in the same line is not Stryker's.
|
|
130
|
+
expect(runnerDemand('ulimit -c 0 && bun stryker run', undefined, 18, {})).toEqual({ kind: 'stryker', want: 17 });
|
|
131
|
+
expect(runnerDemand('bun stryker run && nice -c 3 true', undefined, 18, {})).toEqual({ kind: 'stryker', want: 17 });
|
|
132
|
+
// A declared parallelism outranks the flag.
|
|
133
|
+
expect(runnerDemand('bun stryker run --concurrency 8', 2, 18, {})).toEqual({ kind: 'stryker', want: 2 });
|
|
134
|
+
// A test file named for Stryker is one bun process.
|
|
135
|
+
expect(runnerDemand('bun test stryker.test.ts', undefined, 18, {})).toEqual({ kind: 'process', want: 1 });
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it("asks for Vitest's --maxWorkers and go test's -p, else each runner's own per-core default", () => {
|
|
139
|
+
// Vitest outside watch mode runs one worker per core less one.
|
|
140
|
+
expect(runnerDemand('bun vitest run', undefined, 18, {})).toEqual({ kind: 'vitest', want: 17 });
|
|
141
|
+
expect(runnerDemand('bun vitest run --config vitest.config.workers.ts', undefined, 18, {})).toEqual({
|
|
142
|
+
kind: 'vitest',
|
|
143
|
+
want: 17,
|
|
144
|
+
});
|
|
145
|
+
expect(runnerDemand('bun vitest run --maxWorkers=4', undefined, 18, {})).toEqual({ kind: 'vitest', want: 4 });
|
|
146
|
+
expect(runnerDemand('vitest run --max-workers 50%', undefined, 18, {})).toEqual({ kind: 'vitest', want: 9 });
|
|
147
|
+
expect(runnerDemand('bun vitest run --no-file-parallelism', undefined, 18, {})).toEqual({
|
|
148
|
+
kind: 'vitest',
|
|
149
|
+
want: 1,
|
|
150
|
+
});
|
|
151
|
+
// `go test -p` defaults to GOMAXPROCS, normally one per core.
|
|
152
|
+
expect(runnerDemand('go test -count=1 ./hts/... ./host/...', undefined, 18, {})).toEqual({
|
|
153
|
+
kind: 'go-test',
|
|
154
|
+
want: 18,
|
|
155
|
+
});
|
|
156
|
+
expect(runnerDemand('cd go/ttscwire && go test ./...', undefined, 18, { GOMAXPROCS: '4' })).toEqual({
|
|
157
|
+
kind: 'go-test',
|
|
158
|
+
want: 4,
|
|
159
|
+
});
|
|
160
|
+
expect(runnerDemand('go test -p 3 -parallel 8 ./...', undefined, 18, {})).toEqual({ kind: 'go-test', want: 3 });
|
|
161
|
+
// A shell word that is no count leaves the runner's default; it is still sized by the grant.
|
|
162
|
+
expect(runnerDemand('bun ../scripts/gen-go-work.mjs && go test ./... -p="$(nproc)"', undefined, 18, {})).toEqual({
|
|
163
|
+
kind: 'go-test',
|
|
164
|
+
want: 18,
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("asks for cargo-mutants' own jobserver, one task per core unless --jobserver-tasks names fewer", () => {
|
|
169
|
+
// The plugin's inferred `mutation` command, with no flag: `--jobs` defaults to one cargo process, but its
|
|
170
|
+
// jobserver defaults to one task per core (`n_tasks=<cores>`), shared by every cargo that process starts.
|
|
171
|
+
expect(runnerDemand('cargo --frozen mutants --workspace', undefined, 18, {})).toEqual({
|
|
172
|
+
kind: 'cargo-mutants',
|
|
173
|
+
want: 18,
|
|
174
|
+
jobs: 1,
|
|
175
|
+
});
|
|
176
|
+
// `--jobs` counts cargo processes, which still share that per-core jobserver.
|
|
177
|
+
expect(runnerDemand('cargo mutants --jobs 4', undefined, 18, {})).toEqual({
|
|
178
|
+
kind: 'cargo-mutants',
|
|
179
|
+
want: 18,
|
|
180
|
+
jobs: 4,
|
|
181
|
+
});
|
|
182
|
+
expect(runnerDemand('cargo mutants --jobserver-tasks 4', undefined, 18, {})).toEqual({
|
|
183
|
+
kind: 'cargo-mutants',
|
|
184
|
+
want: 4,
|
|
185
|
+
jobs: 1,
|
|
186
|
+
});
|
|
187
|
+
expect(runnerDemand('cargo mutants -j2 --jobserver-tasks=4', undefined, 18, {})).toEqual({
|
|
188
|
+
kind: 'cargo-mutants',
|
|
189
|
+
want: 4,
|
|
190
|
+
jobs: 2,
|
|
191
|
+
});
|
|
192
|
+
// Each cargo process runs at least one thread, whatever the jobserver allows.
|
|
193
|
+
expect(runnerDemand('cargo mutants --jobs=6 --jobserver-tasks 4', undefined, 18, {})).toEqual({
|
|
194
|
+
kind: 'cargo-mutants',
|
|
195
|
+
want: 6,
|
|
196
|
+
jobs: 6,
|
|
197
|
+
});
|
|
198
|
+
// `CARGO_MUTANTS_JOBS` as the command will see it: the target's own environment over the inherited one.
|
|
199
|
+
expect(runnerDemand('cargo mutants --jobserver-tasks 4', undefined, 18, { CARGO_MUTANTS_JOBS: '3' })).toEqual({
|
|
200
|
+
kind: 'cargo-mutants',
|
|
201
|
+
want: 4,
|
|
202
|
+
jobs: 3,
|
|
203
|
+
});
|
|
204
|
+
expect(
|
|
205
|
+
runnerDemand(
|
|
206
|
+
'cargo mutants --jobserver-tasks 4',
|
|
207
|
+
undefined,
|
|
208
|
+
18,
|
|
209
|
+
{ CARGO_MUTANTS_JOBS: '3' },
|
|
210
|
+
{ CARGO_MUTANTS_JOBS: '2' },
|
|
211
|
+
),
|
|
212
|
+
).toEqual({ kind: 'cargo-mutants', want: 4, jobs: 2 });
|
|
213
|
+
expect(runnerDemand('cargo-mutants mutants --in-diff pr.diff', undefined, 12, {})).toEqual({
|
|
214
|
+
kind: 'cargo-mutants',
|
|
215
|
+
want: 12,
|
|
216
|
+
jobs: 1,
|
|
217
|
+
});
|
|
218
|
+
// Another command's `-j` in the same line is not cargo-mutants'.
|
|
219
|
+
expect(runnerDemand('cargo mutants --jobserver-tasks 4 && make -j 9', undefined, 18, {})).toEqual({
|
|
220
|
+
kind: 'cargo-mutants',
|
|
221
|
+
want: 4,
|
|
222
|
+
jobs: 1,
|
|
223
|
+
});
|
|
224
|
+
expect(runnerDemand('cargo mutants', 3, 18, {})).toEqual({ kind: 'cargo-mutants', want: 3, jobs: 1 });
|
|
225
|
+
// A package named `mutants` is an ordinary cargo build.
|
|
226
|
+
expect(runnerDemand('cargo test -p mutants', undefined, 18, {})).toEqual({
|
|
227
|
+
kind: 'cargo',
|
|
228
|
+
want: 18,
|
|
229
|
+
testThreads: null,
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it('sizes the worker-flag runners and cargo-mutants to their grant', () => {
|
|
234
|
+
expect(sizeRunner({ kind: 'stryker', want: 17 }, 'bun stryker run', 4)).toEqual({
|
|
235
|
+
command: 'bun stryker run --concurrency=4',
|
|
236
|
+
env: { [CPU_TOKENS_ENV]: '4' },
|
|
237
|
+
});
|
|
238
|
+
expect(sizeRunner({ kind: 'stryker', want: 8 }, 'bun stryker run -c 8 --fileLogLevel off', 3).command).toBe(
|
|
239
|
+
'bun stryker run --concurrency=3 --fileLogLevel off',
|
|
240
|
+
);
|
|
241
|
+
expect(
|
|
242
|
+
sizeRunner({ kind: 'stryker', want: 8 }, 'ulimit -c 0 && bun stryker run --concurrency 8 && echo done', 2)
|
|
243
|
+
.command,
|
|
244
|
+
).toBe('ulimit -c 0 && bun stryker run --concurrency=2 && echo done');
|
|
245
|
+
// A grant larger than the ask sizes to the ask.
|
|
246
|
+
expect(sizeRunner({ kind: 'stryker', want: 4 }, 'bun stryker run --concurrency=4', 9).command).toBe(
|
|
247
|
+
'bun stryker run --concurrency=4',
|
|
248
|
+
);
|
|
249
|
+
expect(sizeRunner({ kind: 'cargo-mutants', want: 18, jobs: 1 }, 'cargo --frozen mutants --workspace', 5)).toEqual({
|
|
250
|
+
command: 'cargo --frozen mutants --jobserver-tasks=5 --workspace',
|
|
251
|
+
env: { [CPU_TOKENS_ENV]: '5', CARGO_BUILD_JOBS: '5', RUST_TEST_THREADS: '5', NEXTEST_TEST_THREADS: '5' },
|
|
252
|
+
});
|
|
253
|
+
// Its cargo processes share the grant: each tests on its share, and there are never more of them than tokens.
|
|
254
|
+
expect(
|
|
255
|
+
sizeRunner({ kind: 'cargo-mutants', want: 18, jobs: 4 }, 'cargo mutants --jobs 4 --jobserver-tasks 18', 8),
|
|
256
|
+
).toEqual({
|
|
257
|
+
command: 'cargo mutants --jobs=4 --jobserver-tasks=8',
|
|
258
|
+
env: {
|
|
259
|
+
[CPU_TOKENS_ENV]: '8',
|
|
260
|
+
CARGO_MUTANTS_JOBS: '4',
|
|
261
|
+
CARGO_BUILD_JOBS: '2',
|
|
262
|
+
RUST_TEST_THREADS: '2',
|
|
263
|
+
NEXTEST_TEST_THREADS: '2',
|
|
264
|
+
},
|
|
265
|
+
});
|
|
266
|
+
expect(sizeRunner({ kind: 'cargo-mutants', want: 18, jobs: 4 }, 'cargo mutants -j4', 2)).toEqual({
|
|
267
|
+
command: 'cargo mutants --jobserver-tasks=2 --jobs=2',
|
|
268
|
+
env: {
|
|
269
|
+
[CPU_TOKENS_ENV]: '2',
|
|
270
|
+
CARGO_MUTANTS_JOBS: '2',
|
|
271
|
+
CARGO_BUILD_JOBS: '1',
|
|
272
|
+
RUST_TEST_THREADS: '1',
|
|
273
|
+
NEXTEST_TEST_THREADS: '1',
|
|
274
|
+
},
|
|
275
|
+
});
|
|
276
|
+
// `--jobs` is never added: cargo-mutants refuses it beside `--in-place`.
|
|
277
|
+
expect(sizeRunner({ kind: 'cargo-mutants', want: 18, jobs: 1 }, 'cargo mutants --in-place', 3).command).toBe(
|
|
278
|
+
'cargo mutants --jobserver-tasks=3 --in-place',
|
|
279
|
+
);
|
|
280
|
+
expect(sizeRunner({ kind: 'vitest', want: 17 }, 'bun vitest run --config vitest.config.workers.ts', 3)).toEqual({
|
|
281
|
+
command: 'bun vitest run --maxWorkers=3 --config vitest.config.workers.ts',
|
|
282
|
+
env: { [CPU_TOKENS_ENV]: '3' },
|
|
283
|
+
});
|
|
284
|
+
expect(sizeRunner({ kind: 'vitest', want: 4 }, 'bun vitest run --maxWorkers 4', 2).command).toBe(
|
|
285
|
+
'bun vitest run --maxWorkers=2',
|
|
286
|
+
);
|
|
287
|
+
expect(
|
|
288
|
+
sizeRunner({ kind: 'go-test', want: 18 }, 'bun ../scripts/gen-go-work.mjs && go test ./... -p="$(nproc)"', 4)
|
|
289
|
+
.command,
|
|
290
|
+
).toBe('bun ../scripts/gen-go-work.mjs && go test ./... -p=4');
|
|
291
|
+
expect(sizeRunner({ kind: 'go-test', want: 18 }, 'cd go/ttscwire && go test ./...', 2).command).toBe(
|
|
292
|
+
'cd go/ttscwire && go test -p=2 ./...',
|
|
293
|
+
);
|
|
294
|
+
});
|
|
295
|
+
|
|
91
296
|
it('sizes each runner to its grant', () => {
|
|
92
297
|
expect(sizeRunner({ kind: 'nextest', want: 18 }, NEXTEST_PIECE, 5)).toEqual({
|
|
93
298
|
command: NEXTEST_PIECE,
|
|
@@ -19,15 +19,23 @@ export type RunnerKind =
|
|
|
19
19
|
| 'nextest'
|
|
20
20
|
/** `bun test --parallel`: its worker processes. */
|
|
21
21
|
| 'bun-parallel'
|
|
22
|
+
/** A runner whose worker count is one flag of its own (`WORKER_FLAGS`). */
|
|
23
|
+
| WorkerFlagKind
|
|
24
|
+
/** `cargo mutants`: its own jobserver's tasks, `--jobserver-tasks`, shared by the `--jobs` cargo processes it starts. */
|
|
25
|
+
| 'cargo-mutants'
|
|
22
26
|
/** A cargo build (`cargo build`/`test`/`clippy`/…, `nextest archive`, `napi build`): its jobs. */
|
|
23
27
|
| 'cargo'
|
|
24
28
|
/** Anything else, one `bun test` process included: one process unless it declares more. */
|
|
25
29
|
| 'process';
|
|
26
30
|
|
|
27
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* What a runner asks the budget for. Cargo also carries its independent test-thread ceiling; cargo-mutants carries
|
|
33
|
+
* how many cargo processes it runs at once, which share its grant.
|
|
34
|
+
*/
|
|
28
35
|
export type Demand =
|
|
29
36
|
| { kind: 'cargo'; want: number; testThreads: number | null }
|
|
30
|
-
| { kind:
|
|
37
|
+
| { kind: 'cargo-mutants'; want: number; jobs: number }
|
|
38
|
+
| { kind: Exclude<RunnerKind, 'cargo' | 'cargo-mutants'>; want: number };
|
|
31
39
|
|
|
32
40
|
/** Every budgeted command learns its grant here, so a script that starts a runner can size it. */
|
|
33
41
|
export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
@@ -35,14 +43,75 @@ export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
|
35
43
|
const CARGO_BUILD =
|
|
36
44
|
/(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*\s+(?:build|test|check|clippy|rustc|doc|run|nextest\s+archive)|napi\s+build)(?=\s|$)/;
|
|
37
45
|
const TEST_THREADS = /--test-threads(?:=|\s+)(\d+)/g;
|
|
46
|
+
/** nextest's `--no-capture` (`--nocapture` past `--`): it runs tests serially and ignores `--test-threads`. */
|
|
47
|
+
const NO_CAPTURE = /(?<![\w-])--no-?capture(?![\w-])/;
|
|
38
48
|
const BUN_PARALLEL = /--parallel(?:=(\d+))?(?=\s|$)/g;
|
|
49
|
+
/** `cargo mutants`, cargo's global options allowed between, or the `cargo-mutants` binary itself. */
|
|
50
|
+
const CARGO_MUTANTS = /(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*|cargo-mutants)\s+mutants(?=\s|$)/;
|
|
51
|
+
/** cargo-mutants' own jobserver, one per run, whose tasks every cargo it starts shares: `--jobserver-tasks N`. */
|
|
52
|
+
const JOBSERVER_TASKS = /(?<![\w-])--jobserver-tasks(?:=|\s+)(\d+)(?![\w.])/g;
|
|
53
|
+
/** cargo-mutants' cargo build/test processes at once: `--jobs N` or `-j N`. */
|
|
54
|
+
const MUTANTS_JOBS = /(?<![\w-])(?:--jobs(?:=|\s+)|-j(?:=|\s*))(\d+)(?![\w.])/g;
|
|
55
|
+
/** A count, or a share of the cores (`N%`), optionally quoted. Any other word (`"$(nproc)"`) names no count. */
|
|
56
|
+
const WORKER_COUNT = /^(["']?)(\d+)(%?)\1$/;
|
|
39
57
|
const INHERITED_COUNT = /^[1-9]\d*$/;
|
|
40
58
|
|
|
59
|
+
type WorkerFlagKind = 'stryker' | 'vitest' | 'go-test';
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* A runner that starts its own workers, one per core or close to it unless one flag of its own names a count or a
|
|
63
|
+
* share of the cores. The grant always sizes it through that flag, which outranks the runner's config file.
|
|
64
|
+
*/
|
|
65
|
+
interface WorkerFlag {
|
|
66
|
+
/** How the runner is started; the flag counts only in this invocation's own arguments. */
|
|
67
|
+
readonly invocation: RegExp;
|
|
68
|
+
/** The flag with its value, one shell word, captured. */
|
|
69
|
+
readonly flag: RegExp;
|
|
70
|
+
/** The flag naming `count`. */
|
|
71
|
+
readonly sized: (count: number) => string;
|
|
72
|
+
/** The runner's own count when the flag names none, from the cores and the environment the command will see. */
|
|
73
|
+
readonly ownDefault: (cores: number, env: Readonly<NodeJS.ProcessEnv>) => number;
|
|
74
|
+
/** A flag with which the runner runs one worker whatever count it names. */
|
|
75
|
+
readonly serial?: RegExp;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** One shell word: quoted, a `$(…)` substitution, or bare. */
|
|
79
|
+
const WORD = String.raw`("[^"]*"|'[^']*'|\$\([^)]*\)|[^\s;&|()'"]+)`;
|
|
80
|
+
|
|
81
|
+
const WORKER_FLAGS: Readonly<Record<WorkerFlagKind, WorkerFlag>> = {
|
|
82
|
+
/** StrykerJS's `concurrency`: n - 1 workers past four cores, else n. */
|
|
83
|
+
stryker: {
|
|
84
|
+
invocation: /(?<=^|[\s;&|()/])stryker\s+run(?=\s|$)/,
|
|
85
|
+
flag: new RegExp(String.raw`(?<![\w-])(?:--concurrency|-c)(?:=|\s+)${WORD}`, 'g'),
|
|
86
|
+
sized: (count) => `--concurrency=${count}`,
|
|
87
|
+
ownDefault: (cores) => (cores > 4 ? cores - 1 : cores),
|
|
88
|
+
},
|
|
89
|
+
/** Vitest's `maxWorkers`: n - 1 workers outside watch mode, one with file parallelism off. */
|
|
90
|
+
vitest: {
|
|
91
|
+
invocation: /(?<=^|[\s;&|()/])vitest(?:\s+run)?(?=\s|$)/,
|
|
92
|
+
flag: new RegExp(String.raw`(?<![\w-])--max(?:W|-w)orkers(?:=|\s+)${WORD}`, 'g'),
|
|
93
|
+
sized: (count) => `--maxWorkers=${count}`,
|
|
94
|
+
ownDefault: (cores) => Math.max(cores - 1, 1),
|
|
95
|
+
serial: /(?<![\w-])(?:--no-file-parallelism|--file(?:P|-p)arallelism=false)(?![\w-])/,
|
|
96
|
+
},
|
|
97
|
+
/** `go test -p`: packages built and tested at once, `GOMAXPROCS` by default, normally one per core. */
|
|
98
|
+
'go-test': {
|
|
99
|
+
invocation: /(?<=^|[\s;&|()])go\s+test(?=\s|$)/,
|
|
100
|
+
flag: new RegExp(String.raw`(?<![\w-])-p(?:=|\s+)${WORD}`, 'g'),
|
|
101
|
+
sized: (count) => `-p=${count}`,
|
|
102
|
+
ownDefault: (cores, env) => parseInheritedTokens(env.GOMAXPROCS) ?? cores,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
|
|
41
106
|
/**
|
|
42
107
|
* What `command` asks for: its kind by invocation, and `declared` (the target's `parallelism`)
|
|
43
|
-
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N
|
|
44
|
-
* runner's own default — one thread per core for
|
|
45
|
-
*
|
|
108
|
+
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N`, a worker
|
|
109
|
+
* flag's N or N%, `--jobserver-tasks=N`), else the runner's own default — one thread per core for
|
|
110
|
+
* nextest, a bare `--parallel` and cargo; a worker-flag runner's own (`WORKER_FLAGS`); one per core
|
|
111
|
+
* for cargo-mutants, whose own jobserver defaults to one task per core whatever `--jobs` says; one
|
|
112
|
+
* process for anything else. A nextest run with `--no-capture` runs one test at a time whatever
|
|
113
|
+
* thread count it names, so it asks for one. Positive inherited runner counts are ceilings;
|
|
114
|
+
* configured counts may only lower them.
|
|
46
115
|
*/
|
|
47
116
|
export function runnerDemand(
|
|
48
117
|
command: string,
|
|
@@ -51,20 +120,15 @@ export function runnerDemand(
|
|
|
51
120
|
inherited: Readonly<NodeJS.ProcessEnv>,
|
|
52
121
|
configured?: Readonly<NodeJS.ProcessEnv>,
|
|
53
122
|
): Demand {
|
|
54
|
-
const kind
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
const named =
|
|
64
|
-
counted === null
|
|
65
|
-
? []
|
|
66
|
-
: [...command.matchAll(counted)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
67
|
-
const requested = declared ?? (named.length > 0 ? Math.max(...named) : kind === 'process' ? 1 : cores);
|
|
123
|
+
const kind = runnerKind(command);
|
|
124
|
+
// cargo-mutants' cargo processes at once: `--jobs`, else `CARGO_MUTANTS_JOBS` as the command will see it, else one.
|
|
125
|
+
const jobs =
|
|
126
|
+
kind === 'cargo-mutants'
|
|
127
|
+
? (namedCounts(invocationArgs(command, CARGO_MUTANTS), MUTANTS_JOBS).at(-1) ??
|
|
128
|
+
parseInheritedTokens(configured?.CARGO_MUTANTS_JOBS ?? inherited.CARGO_MUTANTS_JOBS) ??
|
|
129
|
+
1)
|
|
130
|
+
: 1;
|
|
131
|
+
const requested = declared ?? ownParallelism(kind, command, cores, jobs, { ...inherited, ...configured });
|
|
68
132
|
const ceiling =
|
|
69
133
|
kind === 'nextest'
|
|
70
134
|
? inheritedCeiling(inherited.NEXTEST_TEST_THREADS, configured?.NEXTEST_TEST_THREADS)
|
|
@@ -72,9 +136,126 @@ export function runnerDemand(
|
|
|
72
136
|
? inheritedCeiling(inherited.CARGO_BUILD_JOBS, configured?.CARGO_BUILD_JOBS)
|
|
73
137
|
: null;
|
|
74
138
|
const want = Math.max(1, Math.min(requested, ceiling ?? requested));
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
139
|
+
switch (kind) {
|
|
140
|
+
case 'cargo':
|
|
141
|
+
return { kind, want, testThreads: inheritedCeiling(inherited.RUST_TEST_THREADS, configured?.RUST_TEST_THREADS) };
|
|
142
|
+
case 'cargo-mutants':
|
|
143
|
+
return { kind, want, jobs };
|
|
144
|
+
default:
|
|
145
|
+
return { kind, want };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function runnerKind(command: string): RunnerKind {
|
|
150
|
+
if (command.search(NEXTEST_RUN) !== -1) {
|
|
151
|
+
return 'nextest';
|
|
152
|
+
}
|
|
153
|
+
// Stryker before Vitest: its vitest runner is Stryker's workers, not a `vitest` invocation of the command.
|
|
154
|
+
for (const kind of ['stryker', 'vitest', 'go-test'] as const) {
|
|
155
|
+
if (WORKER_FLAGS[kind].invocation.test(command)) {
|
|
156
|
+
return kind;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (CARGO_MUTANTS.test(command)) {
|
|
160
|
+
return 'cargo-mutants';
|
|
161
|
+
}
|
|
162
|
+
if (command.search(BUN_TEST) !== -1 && command.search(BUN_PARALLEL) !== -1) {
|
|
163
|
+
return 'bun-parallel';
|
|
164
|
+
}
|
|
165
|
+
return CARGO_BUILD.test(command) ? 'cargo' : 'process';
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** What the runner runs at once when the target declares nothing: the count it names, else its own default. */
|
|
169
|
+
function ownParallelism(
|
|
170
|
+
kind: RunnerKind,
|
|
171
|
+
command: string,
|
|
172
|
+
cores: number,
|
|
173
|
+
jobs: number,
|
|
174
|
+
env: Readonly<NodeJS.ProcessEnv>,
|
|
175
|
+
): number {
|
|
176
|
+
switch (kind) {
|
|
177
|
+
case 'nextest': {
|
|
178
|
+
const named = namedCounts(command, TEST_THREADS);
|
|
179
|
+
return NO_CAPTURE.test(command) ? 1 : named.length > 0 ? Math.max(...named) : cores;
|
|
180
|
+
}
|
|
181
|
+
case 'bun-parallel': {
|
|
182
|
+
const named = namedCounts(command, BUN_PARALLEL);
|
|
183
|
+
return named.length > 0 ? Math.max(...named) : cores;
|
|
184
|
+
}
|
|
185
|
+
case 'stryker':
|
|
186
|
+
case 'vitest':
|
|
187
|
+
case 'go-test': {
|
|
188
|
+
const runner = WORKER_FLAGS[kind];
|
|
189
|
+
const args = invocationArgs(command, runner.invocation);
|
|
190
|
+
if (runner.serial?.test(args)) {
|
|
191
|
+
return 1;
|
|
192
|
+
}
|
|
193
|
+
// The runner takes the last count it is given; a share is of the cores, between one worker and all of them.
|
|
194
|
+
const named = [...args.matchAll(runner.flag)].flatMap((match) => {
|
|
195
|
+
const count = WORKER_COUNT.exec(match[1] ?? '');
|
|
196
|
+
if (count === null) {
|
|
197
|
+
return [];
|
|
198
|
+
}
|
|
199
|
+
const value = Number(count[2]);
|
|
200
|
+
return [count[3] === '%' ? Math.max(1, Math.min(cores, Math.round((cores * value) / 100))) : value];
|
|
201
|
+
});
|
|
202
|
+
return named.at(-1) ?? runner.ownDefault(cores, env);
|
|
203
|
+
}
|
|
204
|
+
case 'cargo-mutants':
|
|
205
|
+
// Its compiles share its own jobserver (one task per core unless named); its `jobs` test runs need a thread each.
|
|
206
|
+
return Math.max(namedCounts(invocationArgs(command, CARGO_MUTANTS), JOBSERVER_TASKS).at(-1) ?? cores, jobs);
|
|
207
|
+
case 'cargo':
|
|
208
|
+
return cores;
|
|
209
|
+
case 'process':
|
|
210
|
+
return 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function namedCounts(text: string, flag: RegExp): number[] {
|
|
215
|
+
return [...text.matchAll(flag)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Where `invocation` ends in `command`, and where its own arguments end: the next `;`, `&`, `|`, `(` or `)` outside
|
|
220
|
+
* quotes and `$(…)`.
|
|
221
|
+
*/
|
|
222
|
+
function invocationSpan(command: string, invocation: RegExp): { end: number; argsEnd: number } {
|
|
223
|
+
const found = invocation.exec(command);
|
|
224
|
+
const end = found === null ? command.length : found.index + found[0].length;
|
|
225
|
+
let quote: string | null = null;
|
|
226
|
+
let substitutions = 0;
|
|
227
|
+
for (let at = end; at < command.length; at += 1) {
|
|
228
|
+
const char = command[at];
|
|
229
|
+
if (quote !== null) {
|
|
230
|
+
quote = char === quote ? null : quote;
|
|
231
|
+
} else if (char === '"' || char === "'") {
|
|
232
|
+
quote = char;
|
|
233
|
+
} else if (char === '$' && command[at + 1] === '(') {
|
|
234
|
+
substitutions += 1;
|
|
235
|
+
at += 1;
|
|
236
|
+
} else if (char === ')' && substitutions > 0) {
|
|
237
|
+
substitutions -= 1;
|
|
238
|
+
} else if (substitutions === 0 && (char === ';' || char === '&' || char === '|' || char === '(' || char === ')')) {
|
|
239
|
+
return { end, argsEnd: at };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return { end, argsEnd: command.length };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function invocationArgs(command: string, invocation: RegExp): string {
|
|
246
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
247
|
+
return command.slice(end, argsEnd);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* `command` with every `flag` in `invocation`'s own arguments replaced by `sized`; when it names none, `sized` is
|
|
252
|
+
* added right after the invocation if `add` says so.
|
|
253
|
+
*/
|
|
254
|
+
function withSizedFlag(command: string, invocation: RegExp, flag: RegExp, sized: string, add: boolean): string {
|
|
255
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
256
|
+
const args = command.slice(end, argsEnd);
|
|
257
|
+
const sizedArgs = args.search(flag) === -1 && add ? ` ${sized}${args}` : args.replace(flag, sized);
|
|
258
|
+
return `${command.slice(0, end)}${sizedArgs}${command.slice(argsEnd)}`;
|
|
78
259
|
}
|
|
79
260
|
|
|
80
261
|
/**
|
|
@@ -106,6 +287,32 @@ export function sizeRunner(
|
|
|
106
287
|
RUST_TEST_THREADS: String(Math.min(runnerCount, demand.testThreads ?? runnerCount)),
|
|
107
288
|
},
|
|
108
289
|
};
|
|
290
|
+
case 'stryker':
|
|
291
|
+
case 'vitest':
|
|
292
|
+
case 'go-test': {
|
|
293
|
+
const runner = WORKER_FLAGS[demand.kind];
|
|
294
|
+
return {
|
|
295
|
+
command: withSizedFlag(command, runner.invocation, runner.flag, runner.sized(runnerCount), true),
|
|
296
|
+
env: { [CPU_TOKENS_ENV]: held },
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
case 'cargo-mutants': {
|
|
300
|
+
// Its own jobserver bounds every compile it starts; each of its `jobs` cargo processes runs tests on its share.
|
|
301
|
+
// `--jobs` is lowered only where it was set: cargo-mutants refuses it beside `--in-place`.
|
|
302
|
+
const jobs = Math.min(demand.jobs, runnerCount);
|
|
303
|
+
const share = String(Math.max(1, Math.floor(runnerCount / jobs)));
|
|
304
|
+
const tasks = withSizedFlag(command, CARGO_MUTANTS, JOBSERVER_TASKS, `--jobserver-tasks=${count}`, true);
|
|
305
|
+
return {
|
|
306
|
+
command: withSizedFlag(tasks, CARGO_MUTANTS, MUTANTS_JOBS, `--jobs=${jobs}`, false),
|
|
307
|
+
env: {
|
|
308
|
+
[CPU_TOKENS_ENV]: held,
|
|
309
|
+
...(demand.jobs > 1 ? { CARGO_MUTANTS_JOBS: String(jobs) } : {}),
|
|
310
|
+
CARGO_BUILD_JOBS: share,
|
|
311
|
+
RUST_TEST_THREADS: share,
|
|
312
|
+
NEXTEST_TEST_THREADS: share,
|
|
313
|
+
},
|
|
314
|
+
};
|
|
315
|
+
}
|
|
109
316
|
case 'process':
|
|
110
317
|
return { command, env: { [CPU_TOKENS_ENV]: held } };
|
|
111
318
|
}
|