@smoothbricks/nx-plugin 0.4.30 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -21
- package/dist/cargo-toolchain-policy.d.ts +6 -6
- package/dist/cargo-toolchain-policy.js +6 -6
- package/dist/cross-check-policy.d.ts +31 -14
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +43 -15
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +3 -2
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts +60 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -0
- package/dist/executors/cargo-zigbuild/executor.js +266 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts +15 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts.map +1 -0
- package/dist/index.d.ts +3 -16
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +69 -92
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3 -2
- package/executors.json +4 -4
- package/managed/raw/patches/nx@23.2.1.patch +177 -9
- package/managed/raw/tooling/direnv/devenv.smoo.nix +46 -46
- package/managed/raw/tooling/direnv/setup-environment.ts +13 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +41 -23
- package/package.json +1 -1
- package/src/cargo-closure-key.test.ts +3 -2
- package/src/cargo-toolchain-policy.ts +6 -6
- package/src/cross-check-policy.test.ts +90 -11
- package/src/cross-check-policy.ts +47 -17
- package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
- package/src/executors/bounded-exec/executor.ts +3 -2
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
- package/src/executors/cargo-zigbuild/executor.test.ts +129 -0
- package/src/executors/cargo-zigbuild/executor.ts +176 -0
- package/src/executors/cargo-zigbuild/schema.json +35 -0
- package/src/executors/cargo-zigbuild/schema.ts +14 -0
- package/src/index.test.ts +41 -63
- package/src/index.ts +73 -101
- package/src/nx-daemon-validity.test.ts +65 -0
- package/src/testing.ts +3 -2
- package/test-timings.json +2 -1
- package/dist/executors/napi-cross-toolchain/executor.d.ts +0 -42
- package/dist/executors/napi-cross-toolchain/executor.d.ts.map +0 -1
- package/dist/executors/napi-cross-toolchain/executor.js +0 -126
- package/dist/executors/napi-cross-toolchain/schema.d.ts +0 -4
- package/dist/executors/napi-cross-toolchain/schema.d.ts.map +0 -1
- package/src/executors/napi-cross-toolchain/executor.test.ts +0 -112
- package/src/executors/napi-cross-toolchain/executor.ts +0 -165
- package/src/executors/napi-cross-toolchain/schema.json +0 -15
- package/src/executors/napi-cross-toolchain/schema.ts +0 -3
- /package/dist/executors/{napi-cross-toolchain → cargo-zigbuild}/schema.js +0 -0
|
@@ -19,15 +19,23 @@ export type RunnerKind =
|
|
|
19
19
|
| 'nextest'
|
|
20
20
|
/** `bun test --parallel`: its worker processes. */
|
|
21
21
|
| 'bun-parallel'
|
|
22
|
+
/** A runner whose worker count is one flag of its own (`WORKER_FLAGS`). */
|
|
23
|
+
| WorkerFlagKind
|
|
24
|
+
/** `cargo mutants`: its own jobserver's tasks, `--jobserver-tasks`, shared by the `--jobs` cargo processes it starts. */
|
|
25
|
+
| 'cargo-mutants'
|
|
22
26
|
/** A cargo build (`cargo build`/`test`/`clippy`/…, `nextest archive`, `napi build`): its jobs. */
|
|
23
27
|
| 'cargo'
|
|
24
28
|
/** Anything else, one `bun test` process included: one process unless it declares more. */
|
|
25
29
|
| 'process';
|
|
26
30
|
|
|
27
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* What a runner asks the budget for. Cargo also carries its independent test-thread ceiling; cargo-mutants carries
|
|
33
|
+
* how many cargo processes it runs at once, which share its grant.
|
|
34
|
+
*/
|
|
28
35
|
export type Demand =
|
|
29
36
|
| { kind: 'cargo'; want: number; testThreads: number | null }
|
|
30
|
-
| { kind:
|
|
37
|
+
| { kind: 'cargo-mutants'; want: number; jobs: number }
|
|
38
|
+
| { kind: Exclude<RunnerKind, 'cargo' | 'cargo-mutants'>; want: number };
|
|
31
39
|
|
|
32
40
|
/** Every budgeted command learns its grant here, so a script that starts a runner can size it. */
|
|
33
41
|
export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
@@ -35,14 +43,75 @@ export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
|
35
43
|
const CARGO_BUILD =
|
|
36
44
|
/(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*\s+(?:build|test|check|clippy|rustc|doc|run|nextest\s+archive)|napi\s+build)(?=\s|$)/;
|
|
37
45
|
const TEST_THREADS = /--test-threads(?:=|\s+)(\d+)/g;
|
|
46
|
+
/** nextest's `--no-capture` (`--nocapture` past `--`): it runs tests serially and ignores `--test-threads`. */
|
|
47
|
+
const NO_CAPTURE = /(?<![\w-])--no-?capture(?![\w-])/;
|
|
38
48
|
const BUN_PARALLEL = /--parallel(?:=(\d+))?(?=\s|$)/g;
|
|
49
|
+
/** `cargo mutants`, cargo's global options allowed between, or the `cargo-mutants` binary itself. */
|
|
50
|
+
const CARGO_MUTANTS = /(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*|cargo-mutants)\s+mutants(?=\s|$)/;
|
|
51
|
+
/** cargo-mutants' own jobserver, one per run, whose tasks every cargo it starts shares: `--jobserver-tasks N`. */
|
|
52
|
+
const JOBSERVER_TASKS = /(?<![\w-])--jobserver-tasks(?:=|\s+)(\d+)(?![\w.])/g;
|
|
53
|
+
/** cargo-mutants' cargo build/test processes at once: `--jobs N` or `-j N`. */
|
|
54
|
+
const MUTANTS_JOBS = /(?<![\w-])(?:--jobs(?:=|\s+)|-j(?:=|\s*))(\d+)(?![\w.])/g;
|
|
55
|
+
/** A count, or a share of the cores (`N%`), optionally quoted. Any other word (`"$(nproc)"`) names no count. */
|
|
56
|
+
const WORKER_COUNT = /^(["']?)(\d+)(%?)\1$/;
|
|
39
57
|
const INHERITED_COUNT = /^[1-9]\d*$/;
|
|
40
58
|
|
|
59
|
+
type WorkerFlagKind = 'stryker' | 'vitest' | 'go-test';
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* A runner that starts its own workers, one per core or close to it unless one flag of its own names a count or a
|
|
63
|
+
* share of the cores. The grant always sizes it through that flag, which outranks the runner's config file.
|
|
64
|
+
*/
|
|
65
|
+
interface WorkerFlag {
|
|
66
|
+
/** How the runner is started; the flag counts only in this invocation's own arguments. */
|
|
67
|
+
readonly invocation: RegExp;
|
|
68
|
+
/** The flag with its value, one shell word, captured. */
|
|
69
|
+
readonly flag: RegExp;
|
|
70
|
+
/** The flag naming `count`. */
|
|
71
|
+
readonly sized: (count: number) => string;
|
|
72
|
+
/** The runner's own count when the flag names none, from the cores and the environment the command will see. */
|
|
73
|
+
readonly ownDefault: (cores: number, env: Readonly<NodeJS.ProcessEnv>) => number;
|
|
74
|
+
/** A flag with which the runner runs one worker whatever count it names. */
|
|
75
|
+
readonly serial?: RegExp;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** One shell word: quoted, a `$(…)` substitution, or bare. */
|
|
79
|
+
const WORD = String.raw`("[^"]*"|'[^']*'|\$\([^)]*\)|[^\s;&|()'"]+)`;
|
|
80
|
+
|
|
81
|
+
const WORKER_FLAGS: Readonly<Record<WorkerFlagKind, WorkerFlag>> = {
|
|
82
|
+
/** StrykerJS's `concurrency`: n - 1 workers past four cores, else n. */
|
|
83
|
+
stryker: {
|
|
84
|
+
invocation: /(?<=^|[\s;&|()/])stryker\s+run(?=\s|$)/,
|
|
85
|
+
flag: new RegExp(String.raw`(?<![\w-])(?:--concurrency|-c)(?:=|\s+)${WORD}`, 'g'),
|
|
86
|
+
sized: (count) => `--concurrency=${count}`,
|
|
87
|
+
ownDefault: (cores) => (cores > 4 ? cores - 1 : cores),
|
|
88
|
+
},
|
|
89
|
+
/** Vitest's `maxWorkers`: n - 1 workers outside watch mode, one with file parallelism off. */
|
|
90
|
+
vitest: {
|
|
91
|
+
invocation: /(?<=^|[\s;&|()/])vitest(?:\s+run)?(?=\s|$)/,
|
|
92
|
+
flag: new RegExp(String.raw`(?<![\w-])--max(?:W|-w)orkers(?:=|\s+)${WORD}`, 'g'),
|
|
93
|
+
sized: (count) => `--maxWorkers=${count}`,
|
|
94
|
+
ownDefault: (cores) => Math.max(cores - 1, 1),
|
|
95
|
+
serial: /(?<![\w-])(?:--no-file-parallelism|--file(?:P|-p)arallelism=false)(?![\w-])/,
|
|
96
|
+
},
|
|
97
|
+
/** `go test -p`: packages built and tested at once, `GOMAXPROCS` by default, normally one per core. */
|
|
98
|
+
'go-test': {
|
|
99
|
+
invocation: /(?<=^|[\s;&|()])go\s+test(?=\s|$)/,
|
|
100
|
+
flag: new RegExp(String.raw`(?<![\w-])-p(?:=|\s+)${WORD}`, 'g'),
|
|
101
|
+
sized: (count) => `-p=${count}`,
|
|
102
|
+
ownDefault: (cores, env) => parseInheritedTokens(env.GOMAXPROCS) ?? cores,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
|
|
41
106
|
/**
|
|
42
107
|
* What `command` asks for: its kind by invocation, and `declared` (the target's `parallelism`)
|
|
43
|
-
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N
|
|
44
|
-
* runner's own default — one thread per core for
|
|
45
|
-
*
|
|
108
|
+
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N`, a worker
|
|
109
|
+
* flag's N or N%, `--jobserver-tasks=N`), else the runner's own default — one thread per core for
|
|
110
|
+
* nextest, a bare `--parallel` and cargo; a worker-flag runner's own (`WORKER_FLAGS`); one per core
|
|
111
|
+
* for cargo-mutants, whose own jobserver defaults to one task per core whatever `--jobs` says; one
|
|
112
|
+
* process for anything else. A nextest run with `--no-capture` runs one test at a time whatever
|
|
113
|
+
* thread count it names, so it asks for one. Positive inherited runner counts are ceilings;
|
|
114
|
+
* configured counts may only lower them.
|
|
46
115
|
*/
|
|
47
116
|
export function runnerDemand(
|
|
48
117
|
command: string,
|
|
@@ -51,20 +120,15 @@ export function runnerDemand(
|
|
|
51
120
|
inherited: Readonly<NodeJS.ProcessEnv>,
|
|
52
121
|
configured?: Readonly<NodeJS.ProcessEnv>,
|
|
53
122
|
): Demand {
|
|
54
|
-
const kind
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
const named =
|
|
64
|
-
counted === null
|
|
65
|
-
? []
|
|
66
|
-
: [...command.matchAll(counted)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
67
|
-
const requested = declared ?? (named.length > 0 ? Math.max(...named) : kind === 'process' ? 1 : cores);
|
|
123
|
+
const kind = runnerKind(command);
|
|
124
|
+
// cargo-mutants' cargo processes at once: `--jobs`, else `CARGO_MUTANTS_JOBS` as the command will see it, else one.
|
|
125
|
+
const jobs =
|
|
126
|
+
kind === 'cargo-mutants'
|
|
127
|
+
? (namedCounts(invocationArgs(command, CARGO_MUTANTS), MUTANTS_JOBS).at(-1) ??
|
|
128
|
+
parseInheritedTokens(configured?.CARGO_MUTANTS_JOBS ?? inherited.CARGO_MUTANTS_JOBS) ??
|
|
129
|
+
1)
|
|
130
|
+
: 1;
|
|
131
|
+
const requested = declared ?? ownParallelism(kind, command, cores, jobs, { ...inherited, ...configured });
|
|
68
132
|
const ceiling =
|
|
69
133
|
kind === 'nextest'
|
|
70
134
|
? inheritedCeiling(inherited.NEXTEST_TEST_THREADS, configured?.NEXTEST_TEST_THREADS)
|
|
@@ -72,9 +136,126 @@ export function runnerDemand(
|
|
|
72
136
|
? inheritedCeiling(inherited.CARGO_BUILD_JOBS, configured?.CARGO_BUILD_JOBS)
|
|
73
137
|
: null;
|
|
74
138
|
const want = Math.max(1, Math.min(requested, ceiling ?? requested));
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
139
|
+
switch (kind) {
|
|
140
|
+
case 'cargo':
|
|
141
|
+
return { kind, want, testThreads: inheritedCeiling(inherited.RUST_TEST_THREADS, configured?.RUST_TEST_THREADS) };
|
|
142
|
+
case 'cargo-mutants':
|
|
143
|
+
return { kind, want, jobs };
|
|
144
|
+
default:
|
|
145
|
+
return { kind, want };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function runnerKind(command: string): RunnerKind {
|
|
150
|
+
if (command.search(NEXTEST_RUN) !== -1) {
|
|
151
|
+
return 'nextest';
|
|
152
|
+
}
|
|
153
|
+
// Stryker before Vitest: its vitest runner is Stryker's workers, not a `vitest` invocation of the command.
|
|
154
|
+
for (const kind of ['stryker', 'vitest', 'go-test'] as const) {
|
|
155
|
+
if (WORKER_FLAGS[kind].invocation.test(command)) {
|
|
156
|
+
return kind;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (CARGO_MUTANTS.test(command)) {
|
|
160
|
+
return 'cargo-mutants';
|
|
161
|
+
}
|
|
162
|
+
if (command.search(BUN_TEST) !== -1 && command.search(BUN_PARALLEL) !== -1) {
|
|
163
|
+
return 'bun-parallel';
|
|
164
|
+
}
|
|
165
|
+
return CARGO_BUILD.test(command) ? 'cargo' : 'process';
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** What the runner runs at once when the target declares nothing: the count it names, else its own default. */
|
|
169
|
+
function ownParallelism(
|
|
170
|
+
kind: RunnerKind,
|
|
171
|
+
command: string,
|
|
172
|
+
cores: number,
|
|
173
|
+
jobs: number,
|
|
174
|
+
env: Readonly<NodeJS.ProcessEnv>,
|
|
175
|
+
): number {
|
|
176
|
+
switch (kind) {
|
|
177
|
+
case 'nextest': {
|
|
178
|
+
const named = namedCounts(command, TEST_THREADS);
|
|
179
|
+
return NO_CAPTURE.test(command) ? 1 : named.length > 0 ? Math.max(...named) : cores;
|
|
180
|
+
}
|
|
181
|
+
case 'bun-parallel': {
|
|
182
|
+
const named = namedCounts(command, BUN_PARALLEL);
|
|
183
|
+
return named.length > 0 ? Math.max(...named) : cores;
|
|
184
|
+
}
|
|
185
|
+
case 'stryker':
|
|
186
|
+
case 'vitest':
|
|
187
|
+
case 'go-test': {
|
|
188
|
+
const runner = WORKER_FLAGS[kind];
|
|
189
|
+
const args = invocationArgs(command, runner.invocation);
|
|
190
|
+
if (runner.serial?.test(args)) {
|
|
191
|
+
return 1;
|
|
192
|
+
}
|
|
193
|
+
// The runner takes the last count it is given; a share is of the cores, between one worker and all of them.
|
|
194
|
+
const named = [...args.matchAll(runner.flag)].flatMap((match) => {
|
|
195
|
+
const count = WORKER_COUNT.exec(match[1] ?? '');
|
|
196
|
+
if (count === null) {
|
|
197
|
+
return [];
|
|
198
|
+
}
|
|
199
|
+
const value = Number(count[2]);
|
|
200
|
+
return [count[3] === '%' ? Math.max(1, Math.min(cores, Math.round((cores * value) / 100))) : value];
|
|
201
|
+
});
|
|
202
|
+
return named.at(-1) ?? runner.ownDefault(cores, env);
|
|
203
|
+
}
|
|
204
|
+
case 'cargo-mutants':
|
|
205
|
+
// Its compiles share its own jobserver (one task per core unless named); its `jobs` test runs need a thread each.
|
|
206
|
+
return Math.max(namedCounts(invocationArgs(command, CARGO_MUTANTS), JOBSERVER_TASKS).at(-1) ?? cores, jobs);
|
|
207
|
+
case 'cargo':
|
|
208
|
+
return cores;
|
|
209
|
+
case 'process':
|
|
210
|
+
return 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function namedCounts(text: string, flag: RegExp): number[] {
|
|
215
|
+
return [...text.matchAll(flag)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Where `invocation` ends in `command`, and where its own arguments end: the next `;`, `&`, `|`, `(` or `)` outside
|
|
220
|
+
* quotes and `$(…)`.
|
|
221
|
+
*/
|
|
222
|
+
function invocationSpan(command: string, invocation: RegExp): { end: number; argsEnd: number } {
|
|
223
|
+
const found = invocation.exec(command);
|
|
224
|
+
const end = found === null ? command.length : found.index + found[0].length;
|
|
225
|
+
let quote: string | null = null;
|
|
226
|
+
let substitutions = 0;
|
|
227
|
+
for (let at = end; at < command.length; at += 1) {
|
|
228
|
+
const char = command[at];
|
|
229
|
+
if (quote !== null) {
|
|
230
|
+
quote = char === quote ? null : quote;
|
|
231
|
+
} else if (char === '"' || char === "'") {
|
|
232
|
+
quote = char;
|
|
233
|
+
} else if (char === '$' && command[at + 1] === '(') {
|
|
234
|
+
substitutions += 1;
|
|
235
|
+
at += 1;
|
|
236
|
+
} else if (char === ')' && substitutions > 0) {
|
|
237
|
+
substitutions -= 1;
|
|
238
|
+
} else if (substitutions === 0 && (char === ';' || char === '&' || char === '|' || char === '(' || char === ')')) {
|
|
239
|
+
return { end, argsEnd: at };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return { end, argsEnd: command.length };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function invocationArgs(command: string, invocation: RegExp): string {
|
|
246
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
247
|
+
return command.slice(end, argsEnd);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* `command` with every `flag` in `invocation`'s own arguments replaced by `sized`; when it names none, `sized` is
|
|
252
|
+
* added right after the invocation if `add` says so.
|
|
253
|
+
*/
|
|
254
|
+
function withSizedFlag(command: string, invocation: RegExp, flag: RegExp, sized: string, add: boolean): string {
|
|
255
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
256
|
+
const args = command.slice(end, argsEnd);
|
|
257
|
+
const sizedArgs = args.search(flag) === -1 && add ? ` ${sized}${args}` : args.replace(flag, sized);
|
|
258
|
+
return `${command.slice(0, end)}${sizedArgs}${command.slice(argsEnd)}`;
|
|
78
259
|
}
|
|
79
260
|
|
|
80
261
|
/**
|
|
@@ -106,6 +287,32 @@ export function sizeRunner(
|
|
|
106
287
|
RUST_TEST_THREADS: String(Math.min(runnerCount, demand.testThreads ?? runnerCount)),
|
|
107
288
|
},
|
|
108
289
|
};
|
|
290
|
+
case 'stryker':
|
|
291
|
+
case 'vitest':
|
|
292
|
+
case 'go-test': {
|
|
293
|
+
const runner = WORKER_FLAGS[demand.kind];
|
|
294
|
+
return {
|
|
295
|
+
command: withSizedFlag(command, runner.invocation, runner.flag, runner.sized(runnerCount), true),
|
|
296
|
+
env: { [CPU_TOKENS_ENV]: held },
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
case 'cargo-mutants': {
|
|
300
|
+
// Its own jobserver bounds every compile it starts; each of its `jobs` cargo processes runs tests on its share.
|
|
301
|
+
// `--jobs` is lowered only where it was set: cargo-mutants refuses it beside `--in-place`.
|
|
302
|
+
const jobs = Math.min(demand.jobs, runnerCount);
|
|
303
|
+
const share = String(Math.max(1, Math.floor(runnerCount / jobs)));
|
|
304
|
+
const tasks = withSizedFlag(command, CARGO_MUTANTS, JOBSERVER_TASKS, `--jobserver-tasks=${count}`, true);
|
|
305
|
+
return {
|
|
306
|
+
command: withSizedFlag(tasks, CARGO_MUTANTS, MUTANTS_JOBS, `--jobs=${jobs}`, false),
|
|
307
|
+
env: {
|
|
308
|
+
[CPU_TOKENS_ENV]: held,
|
|
309
|
+
...(demand.jobs > 1 ? { CARGO_MUTANTS_JOBS: String(jobs) } : {}),
|
|
310
|
+
CARGO_BUILD_JOBS: share,
|
|
311
|
+
RUST_TEST_THREADS: share,
|
|
312
|
+
NEXTEST_TEST_THREADS: share,
|
|
313
|
+
},
|
|
314
|
+
};
|
|
315
|
+
}
|
|
109
316
|
case 'process':
|
|
110
317
|
return { command, env: { [CPU_TOKENS_ENV]: held } };
|
|
111
318
|
}
|
|
@@ -314,8 +314,9 @@ export async function runBoundedExec(
|
|
|
314
314
|
}
|
|
315
315
|
};
|
|
316
316
|
|
|
317
|
-
// The wait is one span line, as cowshed's own spans read.
|
|
318
|
-
//
|
|
317
|
+
// The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
|
|
318
|
+
// pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
|
|
319
|
+
// asked and nothing is said.
|
|
319
320
|
const cores = availableParallelism();
|
|
320
321
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
321
322
|
const asked = Date.now();
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { CPU_POOL_ENV, readInheritedCpuPool, takeParentCpuTokens } from './cpu-pool.js';
|
|
6
|
+
import { CPU_TOKENS_ENV } from './cpu-tokens.js';
|
|
7
|
+
import { holdLease } from './gateway-lease.js';
|
|
8
|
+
import { closeRunCpuPool, serveRunCpuPool } from './run-cpu-pool.js';
|
|
9
|
+
|
|
10
|
+
const directories: string[] = [];
|
|
11
|
+
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
for (const directory of directories.splice(0)) {
|
|
14
|
+
rmSync(directory, { recursive: true, force: true });
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
/** A gateway socket path in a fresh directory: absent unless the test makes it. */
|
|
19
|
+
function gatewayPath(): string {
|
|
20
|
+
const directory = mkdtempSync(join(tmpdir(), 'run-cpu-pool-'));
|
|
21
|
+
directories.push(directory);
|
|
22
|
+
return join(directory, 'gateway.sock');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe('a run without a gateway', () => {
|
|
26
|
+
it('serves its tasks one pool of one token per core, so a second runner waits for the first', async () => {
|
|
27
|
+
const env: NodeJS.ProcessEnv = {};
|
|
28
|
+
await serveRunCpuPool({ id: 'run-477' }, env, gatewayPath(), 4);
|
|
29
|
+
const pool = readInheritedCpuPool(env);
|
|
30
|
+
if (pool === null) {
|
|
31
|
+
throw new Error(`the run published no pool: ${JSON.stringify(env)}`);
|
|
32
|
+
}
|
|
33
|
+
expect(pool.tokens).toBe(4);
|
|
34
|
+
// Nightly 477's shape: two mutation sweeps, each asking for the whole host. The first holds the run's cores;
|
|
35
|
+
// the second queues for them instead of starting a second host's worth of workers beside it.
|
|
36
|
+
const first = await takeParentCpuTokens(pool, 31);
|
|
37
|
+
expect(first.granted && first.tokens).toBe(4);
|
|
38
|
+
const { promise: queued, resolve: queue } = Promise.withResolvers<void>();
|
|
39
|
+
const second = holdLease(
|
|
40
|
+
pool.socket,
|
|
41
|
+
{ op: 'cpu-pool', want: 31 },
|
|
42
|
+
'the run CPU pool',
|
|
43
|
+
{ ackMs: 2_000, afterCloseMs: 5_000, grantMs: null },
|
|
44
|
+
queue,
|
|
45
|
+
);
|
|
46
|
+
let secondSettled = false;
|
|
47
|
+
void second.then(() => {
|
|
48
|
+
secondSettled = true;
|
|
49
|
+
});
|
|
50
|
+
await queued;
|
|
51
|
+
expect(secondSettled).toBe(false);
|
|
52
|
+
if (first.granted) first.release();
|
|
53
|
+
const granted = await second;
|
|
54
|
+
expect(granted.granted && granted.answer.tokens).toBe(4);
|
|
55
|
+
if (granted.granted) granted.release();
|
|
56
|
+
|
|
57
|
+
await closeRunCpuPool({ id: 'run-477' });
|
|
58
|
+
// The run's end closes its pool: nothing is left to grant from it.
|
|
59
|
+
const late = await takeParentCpuTokens(pool, 1);
|
|
60
|
+
expect(late.granted === false && late.cause).toBe('absent');
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('leaves a run that inherits a pool to that pool', async () => {
|
|
64
|
+
const env: NodeJS.ProcessEnv = { [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' };
|
|
65
|
+
await serveRunCpuPool({ id: 'nested' }, env, gatewayPath(), 4);
|
|
66
|
+
expect(env).toEqual({ [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' });
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('leaves a run on a host whose gateway socket exists to the gateway, answering or not', async () => {
|
|
70
|
+
const gateway = gatewayPath();
|
|
71
|
+
writeFileSync(gateway, '');
|
|
72
|
+
const env: NodeJS.ProcessEnv = {};
|
|
73
|
+
await serveRunCpuPool({ id: 'gateway-host' }, env, gateway, 4);
|
|
74
|
+
expect(env).toEqual({});
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises';
|
|
2
|
+
import { availableParallelism } from 'node:os';
|
|
3
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
4
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
5
|
+
import { type CpuPool, createCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
6
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The CPU budget of an Nx run on a host without a gateway (specs/cowshed/05_gateway.md, "Host CPU budget", "A run
|
|
10
|
+
* without a gateway").
|
|
11
|
+
*
|
|
12
|
+
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
13
|
+
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
14
|
+
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
15
|
+
* task inherits it: `bounded-exec` and the `cargo` launcher take from it as from any parent pool, FIFO, the head
|
|
16
|
+
* taking `min(want, free)` once one token is free, so the runners of one run together stay within the cores.
|
|
17
|
+
*
|
|
18
|
+
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
19
|
+
* not exist. Only `ENOENT` proves the gateway absent, as for the launcher; a socket that exists but does not answer
|
|
20
|
+
* may be a live, saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
24
|
+
const pools = new Map<string, CpuPool>();
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
|
|
28
|
+
* inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
|
|
29
|
+
* unbudgeted, as a run without a gateway was before.
|
|
30
|
+
*/
|
|
31
|
+
export async function serveRunCpuPool(
|
|
32
|
+
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
33
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
34
|
+
gateway: string = GATEWAY_SOCKET,
|
|
35
|
+
cores: number = availableParallelism(),
|
|
36
|
+
): Promise<void> {
|
|
37
|
+
if (readInheritedCpuPool(env) !== null || !(await absent(gateway))) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
// The pool lives until the run's `postTasksExecution` closes it, or with this process.
|
|
41
|
+
const made = await createCpuPool(cores, new Promise<void>(() => {}), { begin() {}, end() {} });
|
|
42
|
+
if (!made.ok) {
|
|
43
|
+
logger.warn(
|
|
44
|
+
`bounded-exec: no gateway at ${gateway}, and this run's CPU pool failed (${made.error}); its tasks run unbudgeted`,
|
|
45
|
+
);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
pools.set(context.id, made.pool);
|
|
49
|
+
for (const [name, value] of Object.entries(made.pool.environment)) {
|
|
50
|
+
env[name] = value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Close the pool `context`'s run was served, if it was served one. */
|
|
55
|
+
export async function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
56
|
+
const pool = pools.get(context.id);
|
|
57
|
+
if (pool === undefined) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
pools.delete(context.id);
|
|
61
|
+
const closed = await pool.close();
|
|
62
|
+
if (!closed.ok) {
|
|
63
|
+
logger.warn(`bounded-exec: this run's CPU pool did not close cleanly: ${closed.error}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function absent(path: string): Promise<boolean> {
|
|
68
|
+
try {
|
|
69
|
+
await lstat(path);
|
|
70
|
+
return false;
|
|
71
|
+
} catch (error) {
|
|
72
|
+
const code = error instanceof Error && 'code' in error ? error.code : undefined;
|
|
73
|
+
return code === 'ENOENT' || code === 'ENOTDIR';
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
|
|
3
|
+
import { artifactFile, cargoTriple, zigbuildArgs, zigbuildEnv } from './executor.js';
|
|
4
|
+
|
|
5
|
+
const MANIFEST = '/repo/packages/cowshed/crates/cowshed-cli/Cargo.toml';
|
|
6
|
+
|
|
7
|
+
/** One `compiler-artifact` line in the shape cargo prints it, trimmed to stable fields. */
|
|
8
|
+
function artifactLine(fields: {
|
|
9
|
+
manifest: string;
|
|
10
|
+
kind: string[];
|
|
11
|
+
name: string;
|
|
12
|
+
filenames: string[];
|
|
13
|
+
executable: string | null;
|
|
14
|
+
}): string {
|
|
15
|
+
return JSON.stringify({
|
|
16
|
+
reason: 'compiler-artifact',
|
|
17
|
+
package_id: 'path+file:///repo/packages/cowshed/crates/cowshed-cli#0.0.1',
|
|
18
|
+
manifest_path: fields.manifest,
|
|
19
|
+
target: { kind: fields.kind, crate_types: fields.kind, name: fields.name, src_path: '/repo/src/main.rs' },
|
|
20
|
+
profile: { opt_level: '3', debuginfo: 0, debug_assertions: false, overflow_checks: false, test: false },
|
|
21
|
+
features: [],
|
|
22
|
+
filenames: fields.filenames,
|
|
23
|
+
executable: fields.executable,
|
|
24
|
+
fresh: false,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe('cargo-zigbuild executor', () => {
|
|
29
|
+
it('runs cargo-zigbuild directly, locked, with the pinned glibc suffix on the target', () => {
|
|
30
|
+
expect(
|
|
31
|
+
zigbuildArgs(
|
|
32
|
+
{ target: 'x86_64-unknown-linux-gnu.2.17', manifestPath: 'm', package: 'cowshed-napi', output: 'o' },
|
|
33
|
+
MANIFEST,
|
|
34
|
+
),
|
|
35
|
+
).toEqual([
|
|
36
|
+
'zigbuild',
|
|
37
|
+
'--locked',
|
|
38
|
+
'--message-format=json-render-diagnostics',
|
|
39
|
+
'--target',
|
|
40
|
+
'x86_64-unknown-linux-gnu.2.17',
|
|
41
|
+
'--manifest-path',
|
|
42
|
+
MANIFEST,
|
|
43
|
+
'--package',
|
|
44
|
+
'cowshed-napi',
|
|
45
|
+
'--lib',
|
|
46
|
+
]);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('selects the named binary and the release profile when asked', () => {
|
|
50
|
+
const args = zigbuildArgs(
|
|
51
|
+
{
|
|
52
|
+
target: 'aarch64-unknown-linux-musl',
|
|
53
|
+
manifestPath: 'm',
|
|
54
|
+
package: 'cowshed-cli',
|
|
55
|
+
bin: 'cowshed',
|
|
56
|
+
output: 'o',
|
|
57
|
+
release: true,
|
|
58
|
+
},
|
|
59
|
+
MANIFEST,
|
|
60
|
+
);
|
|
61
|
+
expect(args.slice(-3)).toEqual(['--release', '--bin', 'cowshed']);
|
|
62
|
+
expect(args).not.toContain('--lib');
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('names the bare triple cargo sees under a glibc suffix', () => {
|
|
66
|
+
expect(cargoTriple('x86_64-unknown-linux-gnu.2.17')).toBe('x86_64-unknown-linux-gnu');
|
|
67
|
+
expect(cargoTriple('aarch64-unknown-linux-musl')).toBe('aarch64-unknown-linux-musl');
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("drops the shell's compilers for the artifact's triple and keeps everything else", () => {
|
|
71
|
+
const env = zigbuildEnv(
|
|
72
|
+
{
|
|
73
|
+
CC_x86_64_unknown_linux_gnu: '/nix/store/cc-wrapper/bin/cc',
|
|
74
|
+
'CC_x86_64-unknown-linux-gnu': '/nix/store/cc-wrapper/bin/cc',
|
|
75
|
+
CXX_x86_64_unknown_linux_gnu: '/nix/store/cc-wrapper/bin/c++',
|
|
76
|
+
AR_x86_64_unknown_linux_gnu: 'ar',
|
|
77
|
+
RANLIB_x86_64_unknown_linux_gnu: 'ranlib',
|
|
78
|
+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: 'cc',
|
|
79
|
+
CC_aarch64_unknown_linux_gnu: 'kept-other-triple',
|
|
80
|
+
CC: 'kept-generic',
|
|
81
|
+
PATH: '/bin',
|
|
82
|
+
},
|
|
83
|
+
'x86_64-unknown-linux-gnu.2.17',
|
|
84
|
+
);
|
|
85
|
+
expect(env).toEqual({ CC_aarch64_unknown_linux_gnu: 'kept-other-triple', CC: 'kept-generic', PATH: '/bin' });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("reads the binary's executable from its artifact message", () => {
|
|
89
|
+
const line = artifactLine({
|
|
90
|
+
manifest: MANIFEST,
|
|
91
|
+
kind: ['bin'],
|
|
92
|
+
name: 'cowshed',
|
|
93
|
+
filenames: ['/repo/target/x86_64-unknown-linux-musl/release/cowshed'],
|
|
94
|
+
executable: '/repo/target/x86_64-unknown-linux-musl/release/cowshed',
|
|
95
|
+
});
|
|
96
|
+
expect(artifactFile(line, MANIFEST, { kind: 'bin', name: 'cowshed' })).toBe(
|
|
97
|
+
'/repo/target/x86_64-unknown-linux-musl/release/cowshed',
|
|
98
|
+
);
|
|
99
|
+
expect(artifactFile(line, MANIFEST, { kind: 'bin', name: 'other' })).toBeNull();
|
|
100
|
+
expect(artifactFile(line, '/repo/elsewhere/Cargo.toml', { kind: 'bin', name: 'cowshed' })).toBeNull();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("reads the cdylib's shared object, not the rlib beside it", () => {
|
|
104
|
+
const line = artifactLine({
|
|
105
|
+
manifest: MANIFEST,
|
|
106
|
+
kind: ['cdylib', 'rlib'],
|
|
107
|
+
name: 'cowshed_napi',
|
|
108
|
+
filenames: [
|
|
109
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.rlib',
|
|
110
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.so',
|
|
111
|
+
],
|
|
112
|
+
executable: null,
|
|
113
|
+
});
|
|
114
|
+
expect(artifactFile(line, MANIFEST, { kind: 'cdylib' })).toBe(
|
|
115
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.so',
|
|
116
|
+
);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('ignores the other messages cargo prints', () => {
|
|
120
|
+
expect(artifactFile('{"reason":"build-finished","success":true}', MANIFEST, { kind: 'cdylib' })).toBeNull();
|
|
121
|
+
expect(artifactFile('', MANIFEST, { kind: 'cdylib' })).toBeNull();
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('refuses an artifact message that breaks the contract instead of skipping it', () => {
|
|
125
|
+
expect(() =>
|
|
126
|
+
artifactFile('{"reason":"compiler-artifact","manifest_path":7}', MANIFEST, { kind: 'cdylib' }),
|
|
127
|
+
).toThrow('does not match the artifact contract');
|
|
128
|
+
});
|
|
129
|
+
});
|