@smoothbricks/nx-plugin 0.4.30 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -21
- package/dist/cargo-toolchain-policy.d.ts +6 -6
- package/dist/cargo-toolchain-policy.js +6 -6
- package/dist/cross-check-policy.d.ts +31 -14
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +43 -15
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +3 -2
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts +60 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -0
- package/dist/executors/cargo-zigbuild/executor.js +266 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts +15 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts.map +1 -0
- package/dist/index.d.ts +3 -16
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +69 -92
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3 -2
- package/executors.json +4 -4
- package/managed/raw/patches/nx@23.2.1.patch +177 -9
- package/managed/raw/tooling/direnv/devenv.smoo.nix +46 -46
- package/managed/raw/tooling/direnv/setup-environment.ts +13 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +41 -23
- package/package.json +1 -1
- package/src/cargo-closure-key.test.ts +3 -2
- package/src/cargo-toolchain-policy.ts +6 -6
- package/src/cross-check-policy.test.ts +90 -11
- package/src/cross-check-policy.ts +47 -17
- package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
- package/src/executors/bounded-exec/executor.ts +3 -2
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
- package/src/executors/cargo-zigbuild/executor.test.ts +129 -0
- package/src/executors/cargo-zigbuild/executor.ts +176 -0
- package/src/executors/cargo-zigbuild/schema.json +35 -0
- package/src/executors/cargo-zigbuild/schema.ts +14 -0
- package/src/index.test.ts +41 -63
- package/src/index.ts +73 -101
- package/src/nx-daemon-validity.test.ts +65 -0
- package/src/testing.ts +3 -2
- package/test-timings.json +2 -1
- package/dist/executors/napi-cross-toolchain/executor.d.ts +0 -42
- package/dist/executors/napi-cross-toolchain/executor.d.ts.map +0 -1
- package/dist/executors/napi-cross-toolchain/executor.js +0 -126
- package/dist/executors/napi-cross-toolchain/schema.d.ts +0 -4
- package/dist/executors/napi-cross-toolchain/schema.d.ts.map +0 -1
- package/src/executors/napi-cross-toolchain/executor.test.ts +0 -112
- package/src/executors/napi-cross-toolchain/executor.ts +0 -165
- package/src/executors/napi-cross-toolchain/schema.json +0 -15
- package/src/executors/napi-cross-toolchain/schema.ts +0 -3
- /package/dist/executors/{napi-cross-toolchain → cargo-zigbuild}/schema.js +0 -0
|
@@ -4,36 +4,181 @@ import { BUN_TEST, NEXTEST_RUN } from './verdict.js';
|
|
|
4
4
|
export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
5
5
|
const CARGO_BUILD = /(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*\s+(?:build|test|check|clippy|rustc|doc|run|nextest\s+archive)|napi\s+build)(?=\s|$)/;
|
|
6
6
|
const TEST_THREADS = /--test-threads(?:=|\s+)(\d+)/g;
|
|
7
|
+
/** nextest's `--no-capture` (`--nocapture` past `--`): it runs tests serially and ignores `--test-threads`. */
|
|
8
|
+
const NO_CAPTURE = /(?<![\w-])--no-?capture(?![\w-])/;
|
|
7
9
|
const BUN_PARALLEL = /--parallel(?:=(\d+))?(?=\s|$)/g;
|
|
10
|
+
/** `cargo mutants`, cargo's global options allowed between, or the `cargo-mutants` binary itself. */
|
|
11
|
+
const CARGO_MUTANTS = /(?<=^|[\s;&|()])(?:cargo(?:\s+-\S+)*|cargo-mutants)\s+mutants(?=\s|$)/;
|
|
12
|
+
/** cargo-mutants' own jobserver, one per run, whose tasks every cargo it starts shares: `--jobserver-tasks N`. */
|
|
13
|
+
const JOBSERVER_TASKS = /(?<![\w-])--jobserver-tasks(?:=|\s+)(\d+)(?![\w.])/g;
|
|
14
|
+
/** cargo-mutants' cargo build/test processes at once: `--jobs N` or `-j N`. */
|
|
15
|
+
const MUTANTS_JOBS = /(?<![\w-])(?:--jobs(?:=|\s+)|-j(?:=|\s*))(\d+)(?![\w.])/g;
|
|
16
|
+
/** A count, or a share of the cores (`N%`), optionally quoted. Any other word (`"$(nproc)"`) names no count. */
|
|
17
|
+
const WORKER_COUNT = /^(["']?)(\d+)(%?)\1$/;
|
|
8
18
|
const INHERITED_COUNT = /^[1-9]\d*$/;
|
|
19
|
+
/** One shell word: quoted, a `$(…)` substitution, or bare. */
|
|
20
|
+
const WORD = String.raw `("[^"]*"|'[^']*'|\$\([^)]*\)|[^\s;&|()'"]+)`;
|
|
21
|
+
const WORKER_FLAGS = {
|
|
22
|
+
/** StrykerJS's `concurrency`: n - 1 workers past four cores, else n. */
|
|
23
|
+
stryker: {
|
|
24
|
+
invocation: /(?<=^|[\s;&|()/])stryker\s+run(?=\s|$)/,
|
|
25
|
+
flag: new RegExp(String.raw `(?<![\w-])(?:--concurrency|-c)(?:=|\s+)${WORD}`, 'g'),
|
|
26
|
+
sized: (count) => `--concurrency=${count}`,
|
|
27
|
+
ownDefault: (cores) => (cores > 4 ? cores - 1 : cores),
|
|
28
|
+
},
|
|
29
|
+
/** Vitest's `maxWorkers`: n - 1 workers outside watch mode, one with file parallelism off. */
|
|
30
|
+
vitest: {
|
|
31
|
+
invocation: /(?<=^|[\s;&|()/])vitest(?:\s+run)?(?=\s|$)/,
|
|
32
|
+
flag: new RegExp(String.raw `(?<![\w-])--max(?:W|-w)orkers(?:=|\s+)${WORD}`, 'g'),
|
|
33
|
+
sized: (count) => `--maxWorkers=${count}`,
|
|
34
|
+
ownDefault: (cores) => Math.max(cores - 1, 1),
|
|
35
|
+
serial: /(?<![\w-])(?:--no-file-parallelism|--file(?:P|-p)arallelism=false)(?![\w-])/,
|
|
36
|
+
},
|
|
37
|
+
/** `go test -p`: packages built and tested at once, `GOMAXPROCS` by default, normally one per core. */
|
|
38
|
+
'go-test': {
|
|
39
|
+
invocation: /(?<=^|[\s;&|()])go\s+test(?=\s|$)/,
|
|
40
|
+
flag: new RegExp(String.raw `(?<![\w-])-p(?:=|\s+)${WORD}`, 'g'),
|
|
41
|
+
sized: (count) => `-p=${count}`,
|
|
42
|
+
ownDefault: (cores, env) => parseInheritedTokens(env.GOMAXPROCS) ?? cores,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
9
45
|
/**
|
|
10
46
|
* What `command` asks for: its kind by invocation, and `declared` (the target's `parallelism`)
|
|
11
|
-
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N
|
|
12
|
-
* runner's own default — one thread per core for
|
|
13
|
-
*
|
|
47
|
+
* when it says, else the count the command names (`--test-threads=N`, `--parallel=N`, a worker
|
|
48
|
+
* flag's N or N%, `--jobserver-tasks=N`), else the runner's own default — one thread per core for
|
|
49
|
+
* nextest, a bare `--parallel` and cargo; a worker-flag runner's own (`WORKER_FLAGS`); one per core
|
|
50
|
+
* for cargo-mutants, whose own jobserver defaults to one task per core whatever `--jobs` says; one
|
|
51
|
+
* process for anything else. A nextest run with `--no-capture` runs one test at a time whatever
|
|
52
|
+
* thread count it names, so it asks for one. Positive inherited runner counts are ceilings;
|
|
53
|
+
* configured counts may only lower them.
|
|
14
54
|
*/
|
|
15
55
|
export function runnerDemand(command, declared, cores, inherited, configured) {
|
|
16
|
-
const kind = command
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
const named = counted === null
|
|
25
|
-
? []
|
|
26
|
-
: [...command.matchAll(counted)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
27
|
-
const requested = declared ?? (named.length > 0 ? Math.max(...named) : kind === 'process' ? 1 : cores);
|
|
56
|
+
const kind = runnerKind(command);
|
|
57
|
+
// cargo-mutants' cargo processes at once: `--jobs`, else `CARGO_MUTANTS_JOBS` as the command will see it, else one.
|
|
58
|
+
const jobs = kind === 'cargo-mutants'
|
|
59
|
+
? (namedCounts(invocationArgs(command, CARGO_MUTANTS), MUTANTS_JOBS).at(-1) ??
|
|
60
|
+
parseInheritedTokens(configured?.CARGO_MUTANTS_JOBS ?? inherited.CARGO_MUTANTS_JOBS) ??
|
|
61
|
+
1)
|
|
62
|
+
: 1;
|
|
63
|
+
const requested = declared ?? ownParallelism(kind, command, cores, jobs, { ...inherited, ...configured });
|
|
28
64
|
const ceiling = kind === 'nextest'
|
|
29
65
|
? inheritedCeiling(inherited.NEXTEST_TEST_THREADS, configured?.NEXTEST_TEST_THREADS)
|
|
30
66
|
: kind === 'cargo'
|
|
31
67
|
? inheritedCeiling(inherited.CARGO_BUILD_JOBS, configured?.CARGO_BUILD_JOBS)
|
|
32
68
|
: null;
|
|
33
69
|
const want = Math.max(1, Math.min(requested, ceiling ?? requested));
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
70
|
+
switch (kind) {
|
|
71
|
+
case 'cargo':
|
|
72
|
+
return { kind, want, testThreads: inheritedCeiling(inherited.RUST_TEST_THREADS, configured?.RUST_TEST_THREADS) };
|
|
73
|
+
case 'cargo-mutants':
|
|
74
|
+
return { kind, want, jobs };
|
|
75
|
+
default:
|
|
76
|
+
return { kind, want };
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function runnerKind(command) {
|
|
80
|
+
if (command.search(NEXTEST_RUN) !== -1) {
|
|
81
|
+
return 'nextest';
|
|
82
|
+
}
|
|
83
|
+
// Stryker before Vitest: its vitest runner is Stryker's workers, not a `vitest` invocation of the command.
|
|
84
|
+
for (const kind of ['stryker', 'vitest', 'go-test']) {
|
|
85
|
+
if (WORKER_FLAGS[kind].invocation.test(command)) {
|
|
86
|
+
return kind;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (CARGO_MUTANTS.test(command)) {
|
|
90
|
+
return 'cargo-mutants';
|
|
91
|
+
}
|
|
92
|
+
if (command.search(BUN_TEST) !== -1 && command.search(BUN_PARALLEL) !== -1) {
|
|
93
|
+
return 'bun-parallel';
|
|
94
|
+
}
|
|
95
|
+
return CARGO_BUILD.test(command) ? 'cargo' : 'process';
|
|
96
|
+
}
|
|
97
|
+
/** What the runner runs at once when the target declares nothing: the count it names, else its own default. */
|
|
98
|
+
function ownParallelism(kind, command, cores, jobs, env) {
|
|
99
|
+
switch (kind) {
|
|
100
|
+
case 'nextest': {
|
|
101
|
+
const named = namedCounts(command, TEST_THREADS);
|
|
102
|
+
return NO_CAPTURE.test(command) ? 1 : named.length > 0 ? Math.max(...named) : cores;
|
|
103
|
+
}
|
|
104
|
+
case 'bun-parallel': {
|
|
105
|
+
const named = namedCounts(command, BUN_PARALLEL);
|
|
106
|
+
return named.length > 0 ? Math.max(...named) : cores;
|
|
107
|
+
}
|
|
108
|
+
case 'stryker':
|
|
109
|
+
case 'vitest':
|
|
110
|
+
case 'go-test': {
|
|
111
|
+
const runner = WORKER_FLAGS[kind];
|
|
112
|
+
const args = invocationArgs(command, runner.invocation);
|
|
113
|
+
if (runner.serial?.test(args)) {
|
|
114
|
+
return 1;
|
|
115
|
+
}
|
|
116
|
+
// The runner takes the last count it is given; a share is of the cores, between one worker and all of them.
|
|
117
|
+
const named = [...args.matchAll(runner.flag)].flatMap((match) => {
|
|
118
|
+
const count = WORKER_COUNT.exec(match[1] ?? '');
|
|
119
|
+
if (count === null) {
|
|
120
|
+
return [];
|
|
121
|
+
}
|
|
122
|
+
const value = Number(count[2]);
|
|
123
|
+
return [count[3] === '%' ? Math.max(1, Math.min(cores, Math.round((cores * value) / 100))) : value];
|
|
124
|
+
});
|
|
125
|
+
return named.at(-1) ?? runner.ownDefault(cores, env);
|
|
126
|
+
}
|
|
127
|
+
case 'cargo-mutants':
|
|
128
|
+
// Its compiles share its own jobserver (one task per core unless named); its `jobs` test runs need a thread each.
|
|
129
|
+
return Math.max(namedCounts(invocationArgs(command, CARGO_MUTANTS), JOBSERVER_TASKS).at(-1) ?? cores, jobs);
|
|
130
|
+
case 'cargo':
|
|
131
|
+
return cores;
|
|
132
|
+
case 'process':
|
|
133
|
+
return 1;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function namedCounts(text, flag) {
|
|
137
|
+
return [...text.matchAll(flag)].flatMap((match) => (match[1] === undefined ? [] : [Number(match[1])]));
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Where `invocation` ends in `command`, and where its own arguments end: the next `;`, `&`, `|`, `(` or `)` outside
|
|
141
|
+
* quotes and `$(…)`.
|
|
142
|
+
*/
|
|
143
|
+
function invocationSpan(command, invocation) {
|
|
144
|
+
const found = invocation.exec(command);
|
|
145
|
+
const end = found === null ? command.length : found.index + found[0].length;
|
|
146
|
+
let quote = null;
|
|
147
|
+
let substitutions = 0;
|
|
148
|
+
for (let at = end; at < command.length; at += 1) {
|
|
149
|
+
const char = command[at];
|
|
150
|
+
if (quote !== null) {
|
|
151
|
+
quote = char === quote ? null : quote;
|
|
152
|
+
}
|
|
153
|
+
else if (char === '"' || char === "'") {
|
|
154
|
+
quote = char;
|
|
155
|
+
}
|
|
156
|
+
else if (char === '$' && command[at + 1] === '(') {
|
|
157
|
+
substitutions += 1;
|
|
158
|
+
at += 1;
|
|
159
|
+
}
|
|
160
|
+
else if (char === ')' && substitutions > 0) {
|
|
161
|
+
substitutions -= 1;
|
|
162
|
+
}
|
|
163
|
+
else if (substitutions === 0 && (char === ';' || char === '&' || char === '|' || char === '(' || char === ')')) {
|
|
164
|
+
return { end, argsEnd: at };
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return { end, argsEnd: command.length };
|
|
168
|
+
}
|
|
169
|
+
function invocationArgs(command, invocation) {
|
|
170
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
171
|
+
return command.slice(end, argsEnd);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* `command` with every `flag` in `invocation`'s own arguments replaced by `sized`; when it names none, `sized` is
|
|
175
|
+
* added right after the invocation if `add` says so.
|
|
176
|
+
*/
|
|
177
|
+
function withSizedFlag(command, invocation, flag, sized, add) {
|
|
178
|
+
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
179
|
+
const args = command.slice(end, argsEnd);
|
|
180
|
+
const sizedArgs = args.search(flag) === -1 && add ? ` ${sized}${args}` : args.replace(flag, sized);
|
|
181
|
+
return `${command.slice(0, end)}${sizedArgs}${command.slice(argsEnd)}`;
|
|
37
182
|
}
|
|
38
183
|
/**
|
|
39
184
|
* Size the runner to the lesser of its bounded demand and grant, retaining Cargo's test-thread ceiling.
|
|
@@ -60,6 +205,32 @@ export function sizeRunner(demand, command, tokens) {
|
|
|
60
205
|
RUST_TEST_THREADS: String(Math.min(runnerCount, demand.testThreads ?? runnerCount)),
|
|
61
206
|
},
|
|
62
207
|
};
|
|
208
|
+
case 'stryker':
|
|
209
|
+
case 'vitest':
|
|
210
|
+
case 'go-test': {
|
|
211
|
+
const runner = WORKER_FLAGS[demand.kind];
|
|
212
|
+
return {
|
|
213
|
+
command: withSizedFlag(command, runner.invocation, runner.flag, runner.sized(runnerCount), true),
|
|
214
|
+
env: { [CPU_TOKENS_ENV]: held },
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
case 'cargo-mutants': {
|
|
218
|
+
// Its own jobserver bounds every compile it starts; each of its `jobs` cargo processes runs tests on its share.
|
|
219
|
+
// `--jobs` is lowered only where it was set: cargo-mutants refuses it beside `--in-place`.
|
|
220
|
+
const jobs = Math.min(demand.jobs, runnerCount);
|
|
221
|
+
const share = String(Math.max(1, Math.floor(runnerCount / jobs)));
|
|
222
|
+
const tasks = withSizedFlag(command, CARGO_MUTANTS, JOBSERVER_TASKS, `--jobserver-tasks=${count}`, true);
|
|
223
|
+
return {
|
|
224
|
+
command: withSizedFlag(tasks, CARGO_MUTANTS, MUTANTS_JOBS, `--jobs=${jobs}`, false),
|
|
225
|
+
env: {
|
|
226
|
+
[CPU_TOKENS_ENV]: held,
|
|
227
|
+
...(demand.jobs > 1 ? { CARGO_MUTANTS_JOBS: String(jobs) } : {}),
|
|
228
|
+
CARGO_BUILD_JOBS: share,
|
|
229
|
+
RUST_TEST_THREADS: share,
|
|
230
|
+
NEXTEST_TEST_THREADS: share,
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
}
|
|
63
234
|
case 'process':
|
|
64
235
|
return { command, env: { [CPU_TOKENS_ENV]: held } };
|
|
65
236
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAMA,OAAO,EAKL,KAAK,gBAAgB,EAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,SAAS,EAKf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAGhF,OAAO,EAML,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,eAAe,EAarB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAcD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAQpG;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED,mGAAmG;AACnG,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,GACzE,OAAO,CAAC,iBAAiB,CAAC,
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAMA,OAAO,EAKL,KAAK,gBAAgB,EAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,SAAS,EAKf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAGhF,OAAO,EAML,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,eAAe,EAarB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAcD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAQpG;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED,mGAAmG;AACnG,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,GACzE,OAAO,CAAC,iBAAiB,CAAC,CA8iB5B;AAED,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -191,8 +191,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
191
191
|
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
192
192
|
}
|
|
193
193
|
};
|
|
194
|
-
// The wait is one span line, as cowshed's own spans read.
|
|
195
|
-
//
|
|
194
|
+
// The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
|
|
195
|
+
// pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
|
|
196
|
+
// asked and nothing is said.
|
|
196
197
|
const cores = availableParallelism();
|
|
197
198
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
198
199
|
const asked = Date.now();
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
2
|
+
/**
|
|
3
|
+
* Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
|
|
4
|
+
* inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
|
|
5
|
+
* unbudgeted, as a run without a gateway was before.
|
|
6
|
+
*/
|
|
7
|
+
export declare function serveRunCpuPool(context: Pick<PreTasksExecutionContext, 'id'>, env?: NodeJS.ProcessEnv, gateway?: string, cores?: number): Promise<void>;
|
|
8
|
+
/** Close the pool `context`'s run was served, if it was served one. */
|
|
9
|
+
export declare function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void>;
|
|
10
|
+
//# sourceMappingURL=run-cpu-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/run-cpu-pool.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AAsBtH;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,CAAC,EAC7C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,OAAO,GAAE,MAAuB,EAChC,KAAK,GAAE,MAA+B,GACrC,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,uEAAuE;AACvE,wBAAsB,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnG"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises';
|
|
2
|
+
import { availableParallelism } from 'node:os';
|
|
3
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
4
|
+
import { createCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
5
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
6
|
+
/**
|
|
7
|
+
* The CPU budget of an Nx run on a host without a gateway (specs/cowshed/05_gateway.md, "Host CPU budget", "A run
|
|
8
|
+
* without a gateway").
|
|
9
|
+
*
|
|
10
|
+
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
11
|
+
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
12
|
+
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
13
|
+
* task inherits it: `bounded-exec` and the `cargo` launcher take from it as from any parent pool, FIFO, the head
|
|
14
|
+
* taking `min(want, free)` once one token is free, so the runners of one run together stay within the cores.
|
|
15
|
+
*
|
|
16
|
+
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
17
|
+
* not exist. Only `ENOENT` proves the gateway absent, as for the launcher; a socket that exists but does not answer
|
|
18
|
+
* may be a live, saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
19
|
+
*/
|
|
20
|
+
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
21
|
+
const pools = new Map();
|
|
22
|
+
/**
|
|
23
|
+
* Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
|
|
24
|
+
* inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
|
|
25
|
+
* unbudgeted, as a run without a gateway was before.
|
|
26
|
+
*/
|
|
27
|
+
export async function serveRunCpuPool(context, env = process.env, gateway = GATEWAY_SOCKET, cores = availableParallelism()) {
|
|
28
|
+
if (readInheritedCpuPool(env) !== null || !(await absent(gateway))) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
// The pool lives until the run's `postTasksExecution` closes it, or with this process.
|
|
32
|
+
const made = await createCpuPool(cores, new Promise(() => { }), { begin() { }, end() { } });
|
|
33
|
+
if (!made.ok) {
|
|
34
|
+
logger.warn(`bounded-exec: no gateway at ${gateway}, and this run's CPU pool failed (${made.error}); its tasks run unbudgeted`);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
pools.set(context.id, made.pool);
|
|
38
|
+
for (const [name, value] of Object.entries(made.pool.environment)) {
|
|
39
|
+
env[name] = value;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Close the pool `context`'s run was served, if it was served one. */
|
|
43
|
+
export async function closeRunCpuPool(context) {
|
|
44
|
+
const pool = pools.get(context.id);
|
|
45
|
+
if (pool === undefined) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
pools.delete(context.id);
|
|
49
|
+
const closed = await pool.close();
|
|
50
|
+
if (!closed.ok) {
|
|
51
|
+
logger.warn(`bounded-exec: this run's CPU pool did not close cleanly: ${closed.error}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async function absent(path) {
|
|
55
|
+
try {
|
|
56
|
+
await lstat(path);
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const code = error instanceof Error && 'code' in error ? error.code : undefined;
|
|
61
|
+
return code === 'ENOENT' || code === 'ENOTDIR';
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { CargoZigbuildOptions } from './schema.js';
|
|
2
|
+
interface CargoZigbuildContext {
|
|
3
|
+
root: string;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* The fields of cargo's `compiler-artifact` JSON message this executor reads.
|
|
7
|
+
* https://doc.rust-lang.org/cargo/reference/external-tools.html#artifact-messages
|
|
8
|
+
*/
|
|
9
|
+
export interface CargoArtifactMessage {
|
|
10
|
+
reason: 'compiler-artifact';
|
|
11
|
+
manifest_path: string;
|
|
12
|
+
target: {
|
|
13
|
+
kind: string[];
|
|
14
|
+
name: string;
|
|
15
|
+
};
|
|
16
|
+
filenames: string[];
|
|
17
|
+
executable: string | null;
|
|
18
|
+
}
|
|
19
|
+
/** What one build produces: a named binary, or the package's cdylib. */
|
|
20
|
+
export type ZigbuildArtifact = {
|
|
21
|
+
kind: 'bin';
|
|
22
|
+
name: string;
|
|
23
|
+
} | {
|
|
24
|
+
kind: 'cdylib';
|
|
25
|
+
};
|
|
26
|
+
export declare function zigbuildArtifact(options: Pick<CargoZigbuildOptions, 'bin'>): ZigbuildArtifact;
|
|
27
|
+
/**
|
|
28
|
+
* `cargo-zigbuild zigbuild`, not `cargo zigbuild`. Run through cargo, the
|
|
29
|
+
* subcommand inherits `CARGO` naming the real cargo binary and calls it
|
|
30
|
+
* directly, bypassing the `cargo` on PATH; run directly, its inner `cargo` is
|
|
31
|
+
* the PATH one every other cargo target runs. `--locked` keeps Cargo.lock
|
|
32
|
+
* exactly as committed. The glibc suffix stays on `--target`: it is how
|
|
33
|
+
* cargo-zigbuild pins the symbol versions, and cargo itself sees the bare
|
|
34
|
+
* triple.
|
|
35
|
+
*/
|
|
36
|
+
export declare function zigbuildArgs(options: CargoZigbuildOptions, manifestPath: string): string[];
|
|
37
|
+
/** `x86_64-unknown-linux-gnu.2.17` → `x86_64-unknown-linux-gnu`. */
|
|
38
|
+
export declare function cargoTriple(target: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* The environment without any compiler or linker named for this triple.
|
|
41
|
+
*
|
|
42
|
+
* cargo-zigbuild supplies the triple's `CC_`/`CXX_`/`AR_`/`RANLIB_` and
|
|
43
|
+
* linker only where none is set, and cc-rs reads the hyphenated spelling
|
|
44
|
+
* before cargo-zigbuild's underscored one. A shell that pins its host
|
|
45
|
+
* compiler for the same triple — a Linux devenv shell pins
|
|
46
|
+
* `CC_x86_64_unknown_linux_gnu` to Nix's cc wrapper for host builds — would
|
|
47
|
+
* otherwise compile the artifact's C against the build host's glibc headers,
|
|
48
|
+
* which is exactly the build-host leak the zig toolchain exists to remove.
|
|
49
|
+
*/
|
|
50
|
+
export declare function zigbuildEnv(env: NodeJS.ProcessEnv, target: string): NodeJS.ProcessEnv;
|
|
51
|
+
/**
|
|
52
|
+
* The artifact file one cargo JSON message names, or null when the message is
|
|
53
|
+
* about anything else. `manifestPath` must be canonical, as cargo reports it.
|
|
54
|
+
*/
|
|
55
|
+
export declare function artifactFile(line: string, manifestPath: string, artifact: ZigbuildArtifact): string | null;
|
|
56
|
+
export default function cargoZigbuildExecutor(options: CargoZigbuildOptions, context: CargoZigbuildContext): Promise<{
|
|
57
|
+
success: boolean;
|
|
58
|
+
}>;
|
|
59
|
+
export {};
|
|
60
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/cargo-zigbuild/executor.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAExD,UAAU,oBAAoB;IAC5B,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAKD,wEAAwE;AACxE,MAAM,MAAM,gBAAgB,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC;AAElF,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE,KAAK,CAAC,GAAG,gBAAgB,CAE7F;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,EAAE,CAe1F;AAED,oEAAoE;AACpE,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAGlD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,UAAU,CAQrF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAqB1G;AAED,wBAA8B,qBAAqB,CACjD,OAAO,EAAE,oBAAoB,EAC7B,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAyD/B"}
|