@smoothbricks/nx-plugin 0.4.31 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -46
- package/dist/cargo-toolchain-policy.d.ts +6 -6
- package/dist/cargo-toolchain-policy.js +6 -6
- package/dist/cross-check-policy.d.ts +31 -14
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +43 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +14 -9
- package/dist/executors/bounded-exec/cpu-pool.d.ts +14 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +297 -38
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +19 -7
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +21 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +12 -6
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +5 -4
- package/dist/executors/cargo-zigbuild/executor.d.ts +60 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -0
- package/dist/executors/cargo-zigbuild/executor.js +266 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts +15 -0
- package/dist/executors/cargo-zigbuild/schema.d.ts.map +1 -0
- package/dist/index.d.ts +0 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +56 -89
- package/dist/producer-receipts.d.ts +10 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +30 -5
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3 -2
- package/executors.json +4 -4
- package/managed/raw/patches/nx@23.2.1.patch +177 -9
- package/managed/raw/tooling/direnv/devenv.smoo.nix +47 -49
- package/managed/raw/tooling/direnv/linked-snapshots.ts +18 -8
- package/managed/raw/tooling/direnv/repo-path +1 -1
- package/managed/raw/tooling/git-hooks/pre-push.sh +41 -23
- package/package.json +1 -1
- package/src/__tests__/counted-cargo.ts +1 -1
- package/src/cargo-closure-input.test.ts +1 -1
- package/src/cargo-toolchain-policy.ts +6 -6
- package/src/cross-check-policy.test.ts +90 -11
- package/src/cross-check-policy.ts +47 -17
- package/src/ensure-built.test.ts +39 -0
- package/src/ensure-built.ts +23 -9
- package/src/executors/bounded-exec/cpu-pool.test.ts +132 -4
- package/src/executors/bounded-exec/cpu-pool.ts +300 -46
- package/src/executors/bounded-exec/cpu-tokens.test.ts +60 -15
- package/src/executors/bounded-exec/cpu-tokens.ts +34 -11
- package/src/executors/bounded-exec/executor.test.ts +72 -4
- package/src/executors/bounded-exec/executor.ts +12 -6
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +3 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/executors/cargo-zigbuild/executor.test.ts +129 -0
- package/src/executors/cargo-zigbuild/executor.ts +176 -0
- package/src/executors/cargo-zigbuild/schema.json +35 -0
- package/src/executors/cargo-zigbuild/schema.ts +14 -0
- package/src/index.test.ts +29 -61
- package/src/index.ts +60 -98
- package/src/nx-daemon-validity.test.ts +65 -0
- package/src/producer-receipts.test.ts +37 -15
- package/src/producer-receipts.ts +29 -5
- package/src/testing.ts +3 -2
- package/test-timings.json +1 -1
- package/dist/executors/napi-cross-toolchain/executor.d.ts +0 -42
- package/dist/executors/napi-cross-toolchain/executor.d.ts.map +0 -1
- package/dist/executors/napi-cross-toolchain/executor.js +0 -126
- package/dist/executors/napi-cross-toolchain/schema.d.ts +0 -4
- package/dist/executors/napi-cross-toolchain/schema.d.ts.map +0 -1
- package/src/executors/napi-cross-toolchain/executor.test.ts +0 -112
- package/src/executors/napi-cross-toolchain/executor.ts +0 -165
- package/src/executors/napi-cross-toolchain/schema.json +0 -15
- package/src/executors/napi-cross-toolchain/schema.ts +0 -3
- /package/dist/executors/{napi-cross-toolchain → cargo-zigbuild}/schema.js +0 -0
|
@@ -29,13 +29,22 @@ export type RunnerKind =
|
|
|
29
29
|
| 'process';
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
|
-
*
|
|
33
|
-
*
|
|
32
|
+
* The inherited ceilings, lowered by configuration, of test threads, which take no make jobserver token: libtest's
|
|
33
|
+
* `RUST_TEST_THREADS` and nextest's `NEXTEST_TEST_THREADS`. Null where neither environment names one.
|
|
34
|
+
*/
|
|
35
|
+
export interface TestThreadCeilings {
|
|
36
|
+
readonly rust: number | null;
|
|
37
|
+
readonly nextest: number | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* What a runner asks the budget for. A cargo build, and a process whose script may start Cargo, carry the test-thread
|
|
42
|
+
* ceilings their sizing keeps; cargo-mutants carries how many cargo processes it runs at once, which share its grant.
|
|
34
43
|
*/
|
|
35
44
|
export type Demand =
|
|
36
|
-
| { kind: 'cargo'; want: number; testThreads:
|
|
45
|
+
| { kind: 'cargo' | 'process'; want: number; testThreads: TestThreadCeilings }
|
|
37
46
|
| { kind: 'cargo-mutants'; want: number; jobs: number }
|
|
38
|
-
| { kind: Exclude<RunnerKind, 'cargo' | 'cargo-mutants'>; want: number };
|
|
47
|
+
| { kind: Exclude<RunnerKind, 'cargo' | 'cargo-mutants' | 'process'>; want: number };
|
|
39
48
|
|
|
40
49
|
/** Every budgeted command learns its grant here, so a script that starts a runner can size it. */
|
|
41
50
|
export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
|
|
@@ -138,7 +147,15 @@ export function runnerDemand(
|
|
|
138
147
|
const want = Math.max(1, Math.min(requested, ceiling ?? requested));
|
|
139
148
|
switch (kind) {
|
|
140
149
|
case 'cargo':
|
|
141
|
-
|
|
150
|
+
case 'process':
|
|
151
|
+
return {
|
|
152
|
+
kind,
|
|
153
|
+
want,
|
|
154
|
+
testThreads: {
|
|
155
|
+
rust: inheritedCeiling(inherited.RUST_TEST_THREADS, configured?.RUST_TEST_THREADS),
|
|
156
|
+
nextest: inheritedCeiling(inherited.NEXTEST_TEST_THREADS, configured?.NEXTEST_TEST_THREADS),
|
|
157
|
+
},
|
|
158
|
+
};
|
|
142
159
|
case 'cargo-mutants':
|
|
143
160
|
return { kind, want, jobs };
|
|
144
161
|
default:
|
|
@@ -259,8 +276,12 @@ function withSizedFlag(command: string, invocation: RegExp, flag: RegExp, sized:
|
|
|
259
276
|
}
|
|
260
277
|
|
|
261
278
|
/**
|
|
262
|
-
* Size the runner to the lesser of its bounded demand and grant,
|
|
263
|
-
*
|
|
279
|
+
* Size the runner to the lesser of its bounded demand and grant, keeping the test-thread ceilings. Rewrite named flags
|
|
280
|
+
* so they cannot outrank that size. `CPU_TOKENS_ENV` still reports the actual held grant.
|
|
281
|
+
*
|
|
282
|
+
* Cargo's rustc count is the pool's: the executor's sub-pool exports its jobserver fifo, so no `CARGO_BUILD_JOBS`
|
|
283
|
+
* comes from the grant (specs/cowshed/05_gateway.md, "The jobserver fifo"). Test threads take no jobserver token, so a
|
|
284
|
+
* cargo build's test threads are its size, and a process's, whose script may start Cargo, are its whole held grant.
|
|
264
285
|
*/
|
|
265
286
|
export function sizeRunner(
|
|
266
287
|
demand: Demand,
|
|
@@ -279,14 +300,18 @@ export function sizeRunner(
|
|
|
279
300
|
case 'bun-parallel':
|
|
280
301
|
return { command: command.replace(BUN_PARALLEL, `--parallel=${count}`), env: { [CPU_TOKENS_ENV]: held } };
|
|
281
302
|
case 'cargo':
|
|
303
|
+
case 'process': {
|
|
304
|
+
const threads = demand.kind === 'cargo' ? runnerCount : tokens;
|
|
305
|
+
const { rust, nextest } = demand.testThreads;
|
|
282
306
|
return {
|
|
283
307
|
command,
|
|
284
308
|
env: {
|
|
285
309
|
[CPU_TOKENS_ENV]: held,
|
|
286
|
-
|
|
287
|
-
|
|
310
|
+
RUST_TEST_THREADS: String(Math.min(threads, rust ?? threads)),
|
|
311
|
+
NEXTEST_TEST_THREADS: String(Math.min(threads, nextest ?? threads)),
|
|
288
312
|
},
|
|
289
313
|
};
|
|
314
|
+
}
|
|
290
315
|
case 'stryker':
|
|
291
316
|
case 'vitest':
|
|
292
317
|
case 'go-test': {
|
|
@@ -313,8 +338,6 @@ export function sizeRunner(
|
|
|
313
338
|
},
|
|
314
339
|
};
|
|
315
340
|
}
|
|
316
|
-
case 'process':
|
|
317
|
-
return { command, env: { [CPU_TOKENS_ENV]: held } };
|
|
318
341
|
}
|
|
319
342
|
}
|
|
320
343
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { afterEach, describe, expect, it, jest } from 'bun:test';
|
|
1
|
+
import { afterAll, afterEach, beforeAll, describe, expect, it, jest } from 'bun:test';
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
3
|
import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { createServer } from 'node:net';
|
|
@@ -8,7 +8,7 @@ import { onProgressClock } from '../../__tests__/progress-clock.js';
|
|
|
8
8
|
import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
|
|
9
9
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
10
10
|
import { CPU_POOL_ENV, type CpuPool, createCpuPool, type InheritedCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
11
|
-
import { CPU_TOKENS_ENV, type CpuBudget,
|
|
11
|
+
import { CPU_TOKENS_ENV, type CpuBudget, type TokenGrant } from './cpu-tokens.js';
|
|
12
12
|
import {
|
|
13
13
|
type BoundedExecContext,
|
|
14
14
|
type BoundedExecResult,
|
|
@@ -28,7 +28,26 @@ const ownedProcessGroups = new Set<number>();
|
|
|
28
28
|
const originalStdoutWrite = process.stdout.write;
|
|
29
29
|
const originalStderrWrite = process.stderr.write;
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
/** What the nextest cases declare as `parallelism`, and so what they ask: the suite runs under no inherited ceiling. */
|
|
32
|
+
const nextestWant = 8;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Every sizing variable a holder exports to its command (specs/cowshed/05_gateway.md, "Sized to the grant", "Cargo reads
|
|
36
|
+
* its holder's jobserver"). This suite's own process runs under them when a gate's `bounded-exec` started it, and the
|
|
37
|
+
* executor reads them as inherited ceilings and pool beside the ones its cases construct, so the suite clears them
|
|
38
|
+
* (specs/cowshed/08_testing.md: fixtures clear every inherited sizing variable before they set their own).
|
|
39
|
+
*/
|
|
40
|
+
const INHERITED_SIZING_ENV = [
|
|
41
|
+
CPU_POOL_ENV,
|
|
42
|
+
CPU_TOKENS_ENV,
|
|
43
|
+
'NEXTEST_TEST_THREADS',
|
|
44
|
+
'RUST_TEST_THREADS',
|
|
45
|
+
'CARGO_BUILD_JOBS',
|
|
46
|
+
'CARGO_MAKEFLAGS',
|
|
47
|
+
'MAKEFLAGS',
|
|
48
|
+
'MFLAGS',
|
|
49
|
+
] as const;
|
|
50
|
+
const inheritedSizing = INHERITED_SIZING_ENV.map((name) => [name, process.env[name]] as const);
|
|
32
51
|
|
|
33
52
|
/**
|
|
34
53
|
* Prints the budget variables it sees, then what a grandchild sees: a command's environment reaches its descendants,
|
|
@@ -41,6 +60,19 @@ console.log(execFileSync('node', ['-p', read]).toString().trim());
|
|
|
41
60
|
`;
|
|
42
61
|
|
|
43
62
|
describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
63
|
+
beforeAll(() => {
|
|
64
|
+
for (const name of INHERITED_SIZING_ENV) {
|
|
65
|
+
delete process.env[name];
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
afterAll(() => {
|
|
70
|
+
for (const [name, value] of inheritedSizing) {
|
|
71
|
+
if (value === undefined) delete process.env[name];
|
|
72
|
+
else process.env[name] = value;
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
44
76
|
afterEach(async () => {
|
|
45
77
|
process.stdout.write = originalStdoutWrite;
|
|
46
78
|
process.stderr.write = originalStderrWrite;
|
|
@@ -931,9 +963,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
931
963
|
return heldGrant(tokens, released);
|
|
932
964
|
},
|
|
933
965
|
};
|
|
966
|
+
// Cargo's ceiling bounds its ask; its rustc count is then the sub-pool's jobserver fifo, and its test threads
|
|
967
|
+
// are sized under their own ceiling, never to the target's 64.
|
|
934
968
|
for (const scenario of [
|
|
935
969
|
{ runner: 'nextest run', variable: 'NEXTEST_TEST_THREADS', cap: 6 },
|
|
936
|
-
{ runner: 'cargo test', variable: '
|
|
970
|
+
{ runner: 'cargo test', variable: 'RUST_TEST_THREADS', cap: 4 },
|
|
937
971
|
]) {
|
|
938
972
|
const command = `node -e "console.log('cap=' + process.env.${scenario.variable} + ' held=' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # ${scenario.runner}`;
|
|
939
973
|
const result = await runBoundedExec(
|
|
@@ -964,6 +998,40 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
964
998
|
}
|
|
965
999
|
});
|
|
966
1000
|
|
|
1001
|
+
it("names the run's own jobserver fifo to its command, never one the target's env names", async () => {
|
|
1002
|
+
const workspace = await createWorkspace();
|
|
1003
|
+
const released: number[] = [];
|
|
1004
|
+
const injected = '-j --jobserver-auth=fifo:/injected';
|
|
1005
|
+
const command = `node -e "console.log(JSON.stringify([process.env.CARGO_MAKEFLAGS, process.env.MAKEFLAGS, process.env.${CPU_POOL_ENV}]))"`;
|
|
1006
|
+
const result = await runBoundedExec(
|
|
1007
|
+
{
|
|
1008
|
+
command,
|
|
1009
|
+
timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
|
|
1010
|
+
env: { CARGO_MAKEFLAGS: injected, MAKEFLAGS: injected, MFLAGS: injected },
|
|
1011
|
+
},
|
|
1012
|
+
workspace.context,
|
|
1013
|
+
createProcessTreeKiller(),
|
|
1014
|
+
null,
|
|
1015
|
+
{
|
|
1016
|
+
async take() {
|
|
1017
|
+
return heldGrant(2, released);
|
|
1018
|
+
},
|
|
1019
|
+
},
|
|
1020
|
+
);
|
|
1021
|
+
expect(result.success).toBe(true);
|
|
1022
|
+
for (const name of ['CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS']) {
|
|
1023
|
+
expect(result.terminalOutput).toContain(
|
|
1024
|
+
`cowshed: ${name} in the target's env is ignored: the CPU budget sets it`,
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
// Cargo and make both read the fifo beside the sub-pool's socket that this run serves its descendants.
|
|
1028
|
+
expect(result.terminalOutput).toMatch(
|
|
1029
|
+
/^\["-j --jobserver-auth=fifo:(\/\S+)\/j","-j --jobserver-auth=fifo:\1\/j","\1\/p\.sock"\]$/m,
|
|
1030
|
+
);
|
|
1031
|
+
expect(result.terminalOutput).not.toContain('/injected');
|
|
1032
|
+
expect(released).toEqual([2]);
|
|
1033
|
+
});
|
|
1034
|
+
|
|
967
1035
|
it('starts the runner once granted, sized to its grant, and returns the tokens when it exits', async () => {
|
|
968
1036
|
const workspace = await createWorkspace();
|
|
969
1037
|
const asked: { want: number; checkout: string; command: string }[] = [];
|
|
@@ -64,8 +64,14 @@ const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
|
|
|
64
64
|
SIGTERM: 143,
|
|
65
65
|
};
|
|
66
66
|
|
|
67
|
-
/** The
|
|
68
|
-
const
|
|
67
|
+
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
68
|
+
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
69
|
+
/**
|
|
70
|
+
* The variables the CPU budget sets for the command it grants: no target's environment may set them. A jobserver the
|
|
71
|
+
* executor itself inherits, a parent make's, still reaches an ungranted command by Cargo's own rule; a granted one's
|
|
72
|
+
* sub-pool names its own fifo over it.
|
|
73
|
+
*/
|
|
74
|
+
const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
|
|
69
75
|
|
|
70
76
|
export interface BoundedExecContext {
|
|
71
77
|
root: string;
|
|
@@ -859,14 +865,14 @@ function buildCommand(options: BoundedExecOptions): string {
|
|
|
859
865
|
|
|
860
866
|
/** The lease's directory is the command's TMPDIR unless the target names one itself. */
|
|
861
867
|
function mergeEnv(lease: RamTempLease | null, env: Record<string, string>): NodeJS.ProcessEnv {
|
|
862
|
-
const inherited =
|
|
868
|
+
const inherited = withoutPoolEnv(process.env);
|
|
863
869
|
return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
|
|
864
870
|
}
|
|
865
871
|
|
|
866
|
-
/** The process's environment without the CPU
|
|
867
|
-
function
|
|
872
|
+
/** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
|
|
873
|
+
function withoutPoolEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
868
874
|
const kept = { ...env };
|
|
869
|
-
for (const name of
|
|
875
|
+
for (const name of POOL_ENV) {
|
|
870
876
|
delete kept[name];
|
|
871
877
|
}
|
|
872
878
|
return kept;
|
|
@@ -31,6 +31,9 @@ describe('a run without a gateway', () => {
|
|
|
31
31
|
throw new Error(`the run published no pool: ${JSON.stringify(env)}`);
|
|
32
32
|
}
|
|
33
33
|
expect(pool.tokens).toBe(4);
|
|
34
|
+
// A Cargo that Nx's own `nx:run-commands` starts reads the same count through the run's jobserver fifo.
|
|
35
|
+
expect(env.CARGO_MAKEFLAGS).toMatch(/^-j --jobserver-auth=fifo:\/\S+$/);
|
|
36
|
+
expect(env.MAKEFLAGS).toBe(env.CARGO_MAKEFLAGS);
|
|
34
37
|
// Nightly 477's shape: two mutation sweeps, each asking for the whole host. The first holds the run's cores;
|
|
35
38
|
// the second queues for them instead of starting a second host's worth of workers beside it.
|
|
36
39
|
const first = await takeParentCpuTokens(pool, 31);
|
|
@@ -12,12 +12,13 @@ import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
|
12
12
|
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
13
13
|
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
14
14
|
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
15
|
-
* task inherits it: `bounded-exec`
|
|
16
|
-
*
|
|
15
|
+
* task inherits it: `bounded-exec` takes from it as from any parent pool, FIFO, the head taking `min(want, free)` once
|
|
16
|
+
* one token is free, and a Cargo that Nx's own `nx:run-commands` starts reads its jobserver fifo through the
|
|
17
|
+
* `CARGO_MAKEFLAGS` the pool exports, so the runners of one run together stay within the cores.
|
|
17
18
|
*
|
|
18
19
|
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
19
|
-
* not exist. Only `ENOENT` proves the gateway absent
|
|
20
|
-
*
|
|
20
|
+
* not exist. Only `ENOENT` proves the gateway absent; a socket that exists but does not answer may be a live,
|
|
21
|
+
* saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
21
22
|
*/
|
|
22
23
|
|
|
23
24
|
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
|
|
3
|
+
import { artifactFile, cargoTriple, zigbuildArgs, zigbuildEnv } from './executor.js';
|
|
4
|
+
|
|
5
|
+
const MANIFEST = '/repo/packages/cowshed/crates/cowshed-cli/Cargo.toml';
|
|
6
|
+
|
|
7
|
+
/** One `compiler-artifact` line in the shape cargo prints it, trimmed to stable fields. */
|
|
8
|
+
function artifactLine(fields: {
|
|
9
|
+
manifest: string;
|
|
10
|
+
kind: string[];
|
|
11
|
+
name: string;
|
|
12
|
+
filenames: string[];
|
|
13
|
+
executable: string | null;
|
|
14
|
+
}): string {
|
|
15
|
+
return JSON.stringify({
|
|
16
|
+
reason: 'compiler-artifact',
|
|
17
|
+
package_id: 'path+file:///repo/packages/cowshed/crates/cowshed-cli#0.0.1',
|
|
18
|
+
manifest_path: fields.manifest,
|
|
19
|
+
target: { kind: fields.kind, crate_types: fields.kind, name: fields.name, src_path: '/repo/src/main.rs' },
|
|
20
|
+
profile: { opt_level: '3', debuginfo: 0, debug_assertions: false, overflow_checks: false, test: false },
|
|
21
|
+
features: [],
|
|
22
|
+
filenames: fields.filenames,
|
|
23
|
+
executable: fields.executable,
|
|
24
|
+
fresh: false,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe('cargo-zigbuild executor', () => {
|
|
29
|
+
it('runs cargo-zigbuild directly, locked, with the pinned glibc suffix on the target', () => {
|
|
30
|
+
expect(
|
|
31
|
+
zigbuildArgs(
|
|
32
|
+
{ target: 'x86_64-unknown-linux-gnu.2.17', manifestPath: 'm', package: 'cowshed-napi', output: 'o' },
|
|
33
|
+
MANIFEST,
|
|
34
|
+
),
|
|
35
|
+
).toEqual([
|
|
36
|
+
'zigbuild',
|
|
37
|
+
'--locked',
|
|
38
|
+
'--message-format=json-render-diagnostics',
|
|
39
|
+
'--target',
|
|
40
|
+
'x86_64-unknown-linux-gnu.2.17',
|
|
41
|
+
'--manifest-path',
|
|
42
|
+
MANIFEST,
|
|
43
|
+
'--package',
|
|
44
|
+
'cowshed-napi',
|
|
45
|
+
'--lib',
|
|
46
|
+
]);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('selects the named binary and the release profile when asked', () => {
|
|
50
|
+
const args = zigbuildArgs(
|
|
51
|
+
{
|
|
52
|
+
target: 'aarch64-unknown-linux-musl',
|
|
53
|
+
manifestPath: 'm',
|
|
54
|
+
package: 'cowshed-cli',
|
|
55
|
+
bin: 'cowshed',
|
|
56
|
+
output: 'o',
|
|
57
|
+
release: true,
|
|
58
|
+
},
|
|
59
|
+
MANIFEST,
|
|
60
|
+
);
|
|
61
|
+
expect(args.slice(-3)).toEqual(['--release', '--bin', 'cowshed']);
|
|
62
|
+
expect(args).not.toContain('--lib');
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('names the bare triple cargo sees under a glibc suffix', () => {
|
|
66
|
+
expect(cargoTriple('x86_64-unknown-linux-gnu.2.17')).toBe('x86_64-unknown-linux-gnu');
|
|
67
|
+
expect(cargoTriple('aarch64-unknown-linux-musl')).toBe('aarch64-unknown-linux-musl');
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("drops the shell's compilers for the artifact's triple and keeps everything else", () => {
|
|
71
|
+
const env = zigbuildEnv(
|
|
72
|
+
{
|
|
73
|
+
CC_x86_64_unknown_linux_gnu: '/nix/store/cc-wrapper/bin/cc',
|
|
74
|
+
'CC_x86_64-unknown-linux-gnu': '/nix/store/cc-wrapper/bin/cc',
|
|
75
|
+
CXX_x86_64_unknown_linux_gnu: '/nix/store/cc-wrapper/bin/c++',
|
|
76
|
+
AR_x86_64_unknown_linux_gnu: 'ar',
|
|
77
|
+
RANLIB_x86_64_unknown_linux_gnu: 'ranlib',
|
|
78
|
+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: 'cc',
|
|
79
|
+
CC_aarch64_unknown_linux_gnu: 'kept-other-triple',
|
|
80
|
+
CC: 'kept-generic',
|
|
81
|
+
PATH: '/bin',
|
|
82
|
+
},
|
|
83
|
+
'x86_64-unknown-linux-gnu.2.17',
|
|
84
|
+
);
|
|
85
|
+
expect(env).toEqual({ CC_aarch64_unknown_linux_gnu: 'kept-other-triple', CC: 'kept-generic', PATH: '/bin' });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("reads the binary's executable from its artifact message", () => {
|
|
89
|
+
const line = artifactLine({
|
|
90
|
+
manifest: MANIFEST,
|
|
91
|
+
kind: ['bin'],
|
|
92
|
+
name: 'cowshed',
|
|
93
|
+
filenames: ['/repo/target/x86_64-unknown-linux-musl/release/cowshed'],
|
|
94
|
+
executable: '/repo/target/x86_64-unknown-linux-musl/release/cowshed',
|
|
95
|
+
});
|
|
96
|
+
expect(artifactFile(line, MANIFEST, { kind: 'bin', name: 'cowshed' })).toBe(
|
|
97
|
+
'/repo/target/x86_64-unknown-linux-musl/release/cowshed',
|
|
98
|
+
);
|
|
99
|
+
expect(artifactFile(line, MANIFEST, { kind: 'bin', name: 'other' })).toBeNull();
|
|
100
|
+
expect(artifactFile(line, '/repo/elsewhere/Cargo.toml', { kind: 'bin', name: 'cowshed' })).toBeNull();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("reads the cdylib's shared object, not the rlib beside it", () => {
|
|
104
|
+
const line = artifactLine({
|
|
105
|
+
manifest: MANIFEST,
|
|
106
|
+
kind: ['cdylib', 'rlib'],
|
|
107
|
+
name: 'cowshed_napi',
|
|
108
|
+
filenames: [
|
|
109
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.rlib',
|
|
110
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.so',
|
|
111
|
+
],
|
|
112
|
+
executable: null,
|
|
113
|
+
});
|
|
114
|
+
expect(artifactFile(line, MANIFEST, { kind: 'cdylib' })).toBe(
|
|
115
|
+
'/repo/target/x86_64-unknown-linux-gnu/debug/libcowshed_napi.so',
|
|
116
|
+
);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('ignores the other messages cargo prints', () => {
|
|
120
|
+
expect(artifactFile('{"reason":"build-finished","success":true}', MANIFEST, { kind: 'cdylib' })).toBeNull();
|
|
121
|
+
expect(artifactFile('', MANIFEST, { kind: 'cdylib' })).toBeNull();
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('refuses an artifact message that breaks the contract instead of skipping it', () => {
|
|
125
|
+
expect(() =>
|
|
126
|
+
artifactFile('{"reason":"compiler-artifact","manifest_path":7}', MANIFEST, { kind: 'cdylib' }),
|
|
127
|
+
).toThrow('does not match the artifact contract');
|
|
128
|
+
});
|
|
129
|
+
});
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { realpathSync } from 'node:fs';
|
|
3
|
+
import { chmod, copyFile, mkdir, rename } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { createInterface } from 'node:readline';
|
|
6
|
+
import typia from 'typia';
|
|
7
|
+
|
|
8
|
+
import type { CargoZigbuildOptions } from './schema.js';
|
|
9
|
+
|
|
10
|
+
interface CargoZigbuildContext {
|
|
11
|
+
root: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The fields of cargo's `compiler-artifact` JSON message this executor reads.
|
|
16
|
+
* https://doc.rust-lang.org/cargo/reference/external-tools.html#artifact-messages
|
|
17
|
+
*/
|
|
18
|
+
export interface CargoArtifactMessage {
|
|
19
|
+
reason: 'compiler-artifact';
|
|
20
|
+
manifest_path: string;
|
|
21
|
+
target: { kind: string[]; name: string };
|
|
22
|
+
filenames: string[];
|
|
23
|
+
executable: string | null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const validateReason = typia.createValidate<{ reason: string }>();
|
|
27
|
+
const validateArtifact = typia.createValidate<CargoArtifactMessage>();
|
|
28
|
+
|
|
29
|
+
/** What one build produces: a named binary, or the package's cdylib. */
|
|
30
|
+
export type ZigbuildArtifact = { kind: 'bin'; name: string } | { kind: 'cdylib' };
|
|
31
|
+
|
|
32
|
+
export function zigbuildArtifact(options: Pick<CargoZigbuildOptions, 'bin'>): ZigbuildArtifact {
|
|
33
|
+
return options.bin === undefined ? { kind: 'cdylib' } : { kind: 'bin', name: options.bin };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* `cargo-zigbuild zigbuild`, not `cargo zigbuild`. Run through cargo, the
|
|
38
|
+
* subcommand inherits `CARGO` naming the real cargo binary and calls it
|
|
39
|
+
* directly, bypassing the `cargo` on PATH; run directly, its inner `cargo` is
|
|
40
|
+
* the PATH one every other cargo target runs. `--locked` keeps Cargo.lock
|
|
41
|
+
* exactly as committed. The glibc suffix stays on `--target`: it is how
|
|
42
|
+
* cargo-zigbuild pins the symbol versions, and cargo itself sees the bare
|
|
43
|
+
* triple.
|
|
44
|
+
*/
|
|
45
|
+
export function zigbuildArgs(options: CargoZigbuildOptions, manifestPath: string): string[] {
|
|
46
|
+
const artifact = zigbuildArtifact(options);
|
|
47
|
+
return [
|
|
48
|
+
'zigbuild',
|
|
49
|
+
'--locked',
|
|
50
|
+
'--message-format=json-render-diagnostics',
|
|
51
|
+
'--target',
|
|
52
|
+
options.target,
|
|
53
|
+
'--manifest-path',
|
|
54
|
+
manifestPath,
|
|
55
|
+
'--package',
|
|
56
|
+
options.package,
|
|
57
|
+
...(options.release === true ? ['--release'] : []),
|
|
58
|
+
...(artifact.kind === 'bin' ? ['--bin', artifact.name] : ['--lib']),
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** `x86_64-unknown-linux-gnu.2.17` → `x86_64-unknown-linux-gnu`. */
|
|
63
|
+
export function cargoTriple(target: string): string {
|
|
64
|
+
const glibcSuffix = /\.\d+\.\d+$/.exec(target);
|
|
65
|
+
return glibcSuffix === null ? target : target.slice(0, glibcSuffix.index);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The environment without any compiler or linker named for this triple.
|
|
70
|
+
*
|
|
71
|
+
* cargo-zigbuild supplies the triple's `CC_`/`CXX_`/`AR_`/`RANLIB_` and
|
|
72
|
+
* linker only where none is set, and cc-rs reads the hyphenated spelling
|
|
73
|
+
* before cargo-zigbuild's underscored one. A shell that pins its host
|
|
74
|
+
* compiler for the same triple — a Linux devenv shell pins
|
|
75
|
+
* `CC_x86_64_unknown_linux_gnu` to Nix's cc wrapper for host builds — would
|
|
76
|
+
* otherwise compile the artifact's C against the build host's glibc headers,
|
|
77
|
+
* which is exactly the build-host leak the zig toolchain exists to remove.
|
|
78
|
+
*/
|
|
79
|
+
export function zigbuildEnv(env: NodeJS.ProcessEnv, target: string): NodeJS.ProcessEnv {
|
|
80
|
+
const triple = cargoTriple(target);
|
|
81
|
+
const underscored = triple.replaceAll('-', '_');
|
|
82
|
+
const owned = new Set([
|
|
83
|
+
...['CC', 'CXX', 'AR', 'RANLIB'].flatMap((tool) => [`${tool}_${triple}`, `${tool}_${underscored}`]),
|
|
84
|
+
`CARGO_TARGET_${underscored.toUpperCase()}_LINKER`,
|
|
85
|
+
]);
|
|
86
|
+
return Object.fromEntries(Object.entries(env).filter(([name]) => !owned.has(name)));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The artifact file one cargo JSON message names, or null when the message is
|
|
91
|
+
* about anything else. `manifestPath` must be canonical, as cargo reports it.
|
|
92
|
+
*/
|
|
93
|
+
export function artifactFile(line: string, manifestPath: string, artifact: ZigbuildArtifact): string | null {
|
|
94
|
+
if (!line.startsWith('{')) return null;
|
|
95
|
+
const parsed: unknown = JSON.parse(line);
|
|
96
|
+
const header = validateReason(parsed);
|
|
97
|
+
if (!header.success) {
|
|
98
|
+
throw new Error(`cargo emitted a JSON message without a reason: ${line}`);
|
|
99
|
+
}
|
|
100
|
+
if (header.data.reason !== 'compiler-artifact') return null;
|
|
101
|
+
const message = validateArtifact(parsed);
|
|
102
|
+
if (!message.success) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`cargo compiler-artifact message does not match the artifact contract: ${JSON.stringify(message.errors)}`,
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
const { data } = message;
|
|
108
|
+
if (data.manifest_path !== manifestPath) return null;
|
|
109
|
+
if (artifact.kind === 'bin') {
|
|
110
|
+
return data.target.kind.includes('bin') && data.target.name === artifact.name ? data.executable : null;
|
|
111
|
+
}
|
|
112
|
+
if (!data.target.kind.includes('cdylib')) return null;
|
|
113
|
+
return data.filenames.find((file) => file.endsWith('.so')) ?? null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export default async function cargoZigbuildExecutor(
|
|
117
|
+
options: CargoZigbuildOptions,
|
|
118
|
+
context: CargoZigbuildContext,
|
|
119
|
+
): Promise<{ success: boolean }> {
|
|
120
|
+
const manifestPath = realpathSync(path.resolve(context.root, options.manifestPath));
|
|
121
|
+
const output = path.resolve(context.root, options.output);
|
|
122
|
+
const artifact = zigbuildArtifact(options);
|
|
123
|
+
const args = zigbuildArgs(options, manifestPath);
|
|
124
|
+
const command = `cargo-zigbuild ${args.join(' ')}`;
|
|
125
|
+
|
|
126
|
+
// Cargo discovers `.cargo/config.toml` from its working directory upward,
|
|
127
|
+
// so the crate's own directory sees every config a build of it would.
|
|
128
|
+
const child = spawn('cargo-zigbuild', args, {
|
|
129
|
+
cwd: path.dirname(manifestPath),
|
|
130
|
+
env: zigbuildEnv(process.env, options.target),
|
|
131
|
+
stdio: ['ignore', 'pipe', 'inherit'],
|
|
132
|
+
});
|
|
133
|
+
const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
|
|
134
|
+
child.once('error', reject);
|
|
135
|
+
child.once('close', (code, signal) => resolve({ code, signal }));
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
let built: string | null = null;
|
|
139
|
+
for await (const line of createInterface({ input: child.stdout })) {
|
|
140
|
+
built = artifactFile(line, manifestPath, artifact) ?? built;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
let status: { code: number | null; signal: NodeJS.Signals | null };
|
|
144
|
+
try {
|
|
145
|
+
status = await exited;
|
|
146
|
+
} catch (error) {
|
|
147
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
148
|
+
console.error(
|
|
149
|
+
`${command}: cargo-zigbuild is not on PATH. Linux artifacts build with the linux-cross toolchain; ` +
|
|
150
|
+
'on macOS run the target inside it: tooling/devenv -P linux-cross shell -- nx run <project>:<target>',
|
|
151
|
+
);
|
|
152
|
+
return { success: false };
|
|
153
|
+
}
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
if (status.code !== 0) {
|
|
157
|
+
console.error(`${command}: exited with ${status.signal ?? `status ${status.code}`}`);
|
|
158
|
+
return { success: false };
|
|
159
|
+
}
|
|
160
|
+
if (built === null) {
|
|
161
|
+
const wanted = artifact.kind === 'bin' ? `binary ${artifact.name}` : 'cdylib';
|
|
162
|
+
console.error(`${command}: cargo reported no ${wanted} artifact for ${options.package} at ${manifestPath}`);
|
|
163
|
+
return { success: false };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Copy then rename, so the output path never holds a partial file.
|
|
167
|
+
await mkdir(path.dirname(output), { recursive: true });
|
|
168
|
+
const staging = `${output}.partial`;
|
|
169
|
+
await copyFile(built, staging);
|
|
170
|
+
if (artifact.kind === 'bin') {
|
|
171
|
+
await chmod(staging, 0o755);
|
|
172
|
+
}
|
|
173
|
+
await rename(staging, output);
|
|
174
|
+
console.log(`${path.relative(context.root, built)} -> ${path.relative(context.root, output)}`);
|
|
175
|
+
return { success: true };
|
|
176
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/schema",
|
|
3
|
+
"version": 2,
|
|
4
|
+
"title": "Cargo Zigbuild Executor",
|
|
5
|
+
"description": "Build one Linux binary or cdylib with devenv's zig through `cargo-zigbuild` and copy it to its output file.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"properties": {
|
|
8
|
+
"target": {
|
|
9
|
+
"type": "string",
|
|
10
|
+
"description": "Rust target triple, with cargo-zigbuild's glibc suffix where it pins one, e.g. x86_64-unknown-linux-gnu.2.17 or x86_64-unknown-linux-musl."
|
|
11
|
+
},
|
|
12
|
+
"manifestPath": {
|
|
13
|
+
"type": "string",
|
|
14
|
+
"description": "The crate's Cargo.toml, relative to the workspace root."
|
|
15
|
+
},
|
|
16
|
+
"package": {
|
|
17
|
+
"type": "string",
|
|
18
|
+
"description": "Cargo package to build."
|
|
19
|
+
},
|
|
20
|
+
"bin": {
|
|
21
|
+
"type": "string",
|
|
22
|
+
"description": "Build this binary and copy its executable. Absent, build the package's cdylib."
|
|
23
|
+
},
|
|
24
|
+
"output": {
|
|
25
|
+
"type": "string",
|
|
26
|
+
"description": "Destination file for the artifact, relative to the workspace root."
|
|
27
|
+
},
|
|
28
|
+
"release": {
|
|
29
|
+
"type": "boolean",
|
|
30
|
+
"description": "Compile cargo's release profile instead of dev."
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"required": ["target", "manifestPath", "package", "output"],
|
|
34
|
+
"additionalProperties": false
|
|
35
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface CargoZigbuildOptions {
|
|
2
|
+
/** Rust target triple, with cargo-zigbuild's glibc suffix where it pins one: `x86_64-unknown-linux-gnu.2.17`. */
|
|
3
|
+
target: string;
|
|
4
|
+
/** The crate's Cargo.toml, relative to the workspace root. */
|
|
5
|
+
manifestPath: string;
|
|
6
|
+
/** Cargo package to build. */
|
|
7
|
+
package: string;
|
|
8
|
+
/** Build this binary and copy its executable; absent, build the package's cdylib. */
|
|
9
|
+
bin?: string;
|
|
10
|
+
/** Destination file for the artifact, relative to the workspace root. */
|
|
11
|
+
output: string;
|
|
12
|
+
/** Compile cargo's `release` profile instead of `dev`. */
|
|
13
|
+
release?: boolean;
|
|
14
|
+
}
|