@smoothbricks/nx-plugin 0.4.23 → 0.4.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -12
- package/dist/bin/smoo-nx-nextest-extract.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-extract.js +41 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +40 -15
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +6 -0
- package/dist/nextest-extraction.d.ts +101 -0
- package/dist/nextest-extraction.d.ts.map +1 -0
- package/dist/nextest-extraction.js +268 -0
- package/managed/raw/envrc +10 -0
- package/managed/raw/git-format-staged.yml +1 -0
- package/managed/raw/tooling/direnv/devenv.smoo.nix +4 -0
- package/managed/raw/tooling/direnv/inherited-devenv.ts +615 -0
- package/managed/raw/tooling/direnv/setup-environment.ts +44 -5
- package/package.json +42 -4
- package/src/bin/smoo-nx-nextest-extract.ts +43 -0
- package/src/bounded-test-policy.test.ts +4 -1
- package/src/index.test.ts +128 -22
- package/src/index.ts +49 -17
- package/src/managed-files/files.ts +6 -0
- package/src/nextest-extraction.test.ts +74 -0
- package/src/nextest-extraction.ts +329 -0
- package/test-timings.json +33 -0
|
@@ -35,6 +35,12 @@ import {
|
|
|
35
35
|
// above the repository and failing with "could not find a package.json".
|
|
36
36
|
const devenvRoot = process.env.DEVENV_ROOT;
|
|
37
37
|
|
|
38
|
+
/** What a setup command printed, as captured. */
|
|
39
|
+
interface CapturedOutput {
|
|
40
|
+
readonly stdout: Uint8Array;
|
|
41
|
+
readonly stderr: Uint8Array;
|
|
42
|
+
}
|
|
43
|
+
|
|
38
44
|
class CapturedCommandError extends Error {
|
|
39
45
|
constructor(
|
|
40
46
|
public readonly command: string,
|
|
@@ -163,12 +169,14 @@ try {
|
|
|
163
169
|
if (frozenError !== undefined) {
|
|
164
170
|
console.error('git diff after install:');
|
|
165
171
|
try {
|
|
166
|
-
|
|
172
|
+
// Captured and written to stderr, the one stream devenv shows for a failed entry.
|
|
173
|
+
await writeCapturedOutput(await runSetupCommand('git diff', $`git diff`));
|
|
167
174
|
} catch (diffError) {
|
|
168
175
|
// This diff is diagnostics for the install failure we are already
|
|
169
176
|
// reporting, so it must never become the failure: say it broke and keep
|
|
170
177
|
// the original exit path. Plain `git diff` exits 0 even when the
|
|
171
|
-
// lockfile drifted, so a
|
|
178
|
+
// lockfile drifted, so a failure here is git itself failing or stderr
|
|
179
|
+
// closing under the write.
|
|
172
180
|
console.error(`! git diff failed: ${diffError instanceof Error ? diffError.message : String(diffError)}`);
|
|
173
181
|
}
|
|
174
182
|
process.exit(1);
|
|
@@ -198,8 +206,13 @@ try {
|
|
|
198
206
|
ensureTypeScriptApiPackage(projectRoot);
|
|
199
207
|
}
|
|
200
208
|
// Under the same lock: the repository config and hooks are one more thing two shell
|
|
201
|
-
// entries would otherwise write at once.
|
|
202
|
-
|
|
209
|
+
// entries would otherwise write at once. Hooks this shell may not write — a cowshed
|
|
210
|
+
// job's sandbox refuses `.git/hooks` — take the shell down no more than an install
|
|
211
|
+
// does: say so and load it.
|
|
212
|
+
await applyWorkspaceGitConfig(projectRoot).catch((configError: unknown) => {
|
|
213
|
+
describeFailure('WARNING', configError);
|
|
214
|
+
console.error('Git hooks and repository config were not applied; the shell is loaded without them.\n---');
|
|
215
|
+
});
|
|
203
216
|
return error;
|
|
204
217
|
});
|
|
205
218
|
if (installError !== undefined) {
|
|
@@ -717,6 +730,9 @@ function assertTypescriptApiAt(typescriptRoot: string, expectedTarget: string):
|
|
|
717
730
|
* package graph exists — not the installer. Called under the install lock.
|
|
718
731
|
*/
|
|
719
732
|
async function applyWorkspaceGitConfig(root: string): Promise<void> {
|
|
733
|
+
// A workspace job inherits the installed hooks, but cannot mutate .git/config
|
|
734
|
+
// or .git/hooks. Git wiring belongs to setup outside the job's sandbox.
|
|
735
|
+
if (process.env.COWSHED_WORKSPACE_TOKEN) return;
|
|
720
736
|
await keepRepositoryConfig(root);
|
|
721
737
|
|
|
722
738
|
const gitDirResult = await $`git rev-parse --git-dir`.cwd(root).quiet().nothrow();
|
|
@@ -810,7 +826,7 @@ async function runSetupCommand(
|
|
|
810
826
|
command: string,
|
|
811
827
|
shell: ReturnType<typeof $>,
|
|
812
828
|
options: { quiet?: boolean; cwd?: string } = {},
|
|
813
|
-
): Promise<
|
|
829
|
+
): Promise<CapturedOutput> {
|
|
814
830
|
const result = await shell
|
|
815
831
|
.quiet(options.quiet ?? true)
|
|
816
832
|
.nothrow()
|
|
@@ -818,6 +834,7 @@ async function runSetupCommand(
|
|
|
818
834
|
if (result.exitCode !== 0) {
|
|
819
835
|
throw new CapturedCommandError(command, result.exitCode, result.stdout, result.stderr);
|
|
820
836
|
}
|
|
837
|
+
return result;
|
|
821
838
|
}
|
|
822
839
|
|
|
823
840
|
/**
|
|
@@ -999,3 +1016,25 @@ function replayCapturedOutput(error: unknown): void {
|
|
|
999
1016
|
process.stderr.write(stderr);
|
|
1000
1017
|
}
|
|
1001
1018
|
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Writes what a setup command printed to stderr and returns once the bytes have
|
|
1022
|
+
* left this process. Stderr, not stdout, because devenv reports a failed shell
|
|
1023
|
+
* entry as "Shell environment capture failed:" plus the shell's stderr and
|
|
1024
|
+
* discards the stdout it captured. Awaited, because a pipe write still queued
|
|
1025
|
+
* when `process.exit` runs is cut short (Bun 1.4.2: 5 MB written to a slow
|
|
1026
|
+
* reader arrived as 1.3 MB), and a diagnostic that ends mid-hunk is not one.
|
|
1027
|
+
* Redacts what a replay redacts: the values this script resolved itself. A CI
|
|
1028
|
+
* runner resolves none, so the secrets its store injects are the platform's to
|
|
1029
|
+
* mask in the job log, as for every other line of it.
|
|
1030
|
+
*/
|
|
1031
|
+
async function writeCapturedOutput(output: CapturedOutput): Promise<void> {
|
|
1032
|
+
for (const stream of [output.stdout, output.stderr]) {
|
|
1033
|
+
const masked = maskSecretValues(stream, resolvedSecretValues);
|
|
1034
|
+
if (masked.length > 0) {
|
|
1035
|
+
await new Promise<void>((resolve, reject) =>
|
|
1036
|
+
process.stderr.write(masked, (error) => (error ? reject(error) : resolve())),
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smoothbricks/nx-plugin",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.25",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./loader.js",
|
|
@@ -141,7 +141,8 @@
|
|
|
141
141
|
},
|
|
142
142
|
"bin": {
|
|
143
143
|
"smoo-nx-exec": "./dist/bin/smoo-nx-exec.js",
|
|
144
|
-
"smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js"
|
|
144
|
+
"smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js",
|
|
145
|
+
"smoo-nx-nextest-extract": "./dist/bin/smoo-nx-nextest-extract.js"
|
|
145
146
|
},
|
|
146
147
|
"generators": "./generators.json",
|
|
147
148
|
"executors": "./executors.json",
|
|
@@ -152,9 +153,45 @@
|
|
|
152
153
|
"targets": {
|
|
153
154
|
"lint": {},
|
|
154
155
|
"test": {
|
|
156
|
+
"executor": "nx:noop",
|
|
157
|
+
"dependsOn": [
|
|
158
|
+
"test-shard1",
|
|
159
|
+
"test-shard2",
|
|
160
|
+
"test-shard3",
|
|
161
|
+
"test-shard4"
|
|
162
|
+
]
|
|
163
|
+
},
|
|
164
|
+
"test-shard1": {
|
|
155
165
|
"executor": "@smoothbricks/nx-plugin:bounded-exec",
|
|
156
166
|
"options": {
|
|
157
|
-
"command": "bun test --timeout=30000",
|
|
167
|
+
"command": "bun test --timeout=30000 --timings=test-timings.json --shard=1/4",
|
|
168
|
+
"cwd": "{projectRoot}",
|
|
169
|
+
"timeoutMs": 120000,
|
|
170
|
+
"killAfterMs": 10000
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
"test-shard2": {
|
|
174
|
+
"executor": "@smoothbricks/nx-plugin:bounded-exec",
|
|
175
|
+
"options": {
|
|
176
|
+
"command": "bun test --timeout=30000 --timings=test-timings.json --shard=2/4",
|
|
177
|
+
"cwd": "{projectRoot}",
|
|
178
|
+
"timeoutMs": 120000,
|
|
179
|
+
"killAfterMs": 10000
|
|
180
|
+
}
|
|
181
|
+
},
|
|
182
|
+
"test-shard3": {
|
|
183
|
+
"executor": "@smoothbricks/nx-plugin:bounded-exec",
|
|
184
|
+
"options": {
|
|
185
|
+
"command": "bun test --timeout=30000 --timings=test-timings.json --shard=3/4",
|
|
186
|
+
"cwd": "{projectRoot}",
|
|
187
|
+
"timeoutMs": 120000,
|
|
188
|
+
"killAfterMs": 10000
|
|
189
|
+
}
|
|
190
|
+
},
|
|
191
|
+
"test-shard4": {
|
|
192
|
+
"executor": "@smoothbricks/nx-plugin:bounded-exec",
|
|
193
|
+
"options": {
|
|
194
|
+
"command": "bun test --timeout=30000 --timings=test-timings.json --shard=4/4",
|
|
158
195
|
"cwd": "{projectRoot}",
|
|
159
196
|
"timeoutMs": 120000,
|
|
160
197
|
"killAfterMs": 10000
|
|
@@ -170,6 +207,7 @@
|
|
|
170
207
|
"dist",
|
|
171
208
|
"src",
|
|
172
209
|
"nextest.toml",
|
|
210
|
+
"test-timings.json",
|
|
173
211
|
"README.md",
|
|
174
212
|
"loader.js",
|
|
175
213
|
"generators.json",
|
|
@@ -196,7 +234,7 @@
|
|
|
196
234
|
"yaml": "^2.9.0"
|
|
197
235
|
},
|
|
198
236
|
"devDependencies": {
|
|
199
|
-
"@smoothbricks/lmao": "0.5.
|
|
237
|
+
"@smoothbricks/lmao": "0.5.2",
|
|
200
238
|
"@smoothbricks/validation": "0.1.10",
|
|
201
239
|
"fast-check": "^4.4.0"
|
|
202
240
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { ensureNextestArchiveExtracted, extractWithNextest } from '../nextest-extraction.js';
|
|
3
|
+
|
|
4
|
+
// Prints the directory holding the archive's extraction on stdout, the only
|
|
5
|
+
// thing a runner's `$(...)` captures. Everything said to a person goes to stderr.
|
|
6
|
+
const args = process.argv.slice(2);
|
|
7
|
+
const archive = args[0];
|
|
8
|
+
if (archive === undefined || args.length !== 1 || archive.startsWith('-')) {
|
|
9
|
+
process.stderr.write('usage: smoo-nx-nextest-extract <archive.tar.zst>\n');
|
|
10
|
+
process.exit(2);
|
|
11
|
+
}
|
|
12
|
+
const say = (line: string) => process.stderr.write(`smoo-nx-nextest-extract: ${line}\n`);
|
|
13
|
+
const seconds = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
|
|
14
|
+
try {
|
|
15
|
+
const extraction = await ensureNextestArchiveExtracted(archive, extractWithNextest, (event) => {
|
|
16
|
+
switch (event.kind) {
|
|
17
|
+
case 'waiting':
|
|
18
|
+
say(`${event.holder} is extracting ${archive}; waiting for it`);
|
|
19
|
+
break;
|
|
20
|
+
case 'stale-lock':
|
|
21
|
+
say(`${event.holder} stopped refreshing its extraction lock ${seconds(event.ageMs)} ago; extracting instead`);
|
|
22
|
+
break;
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
const identity = `${archive} (sha256 ${extraction.key.slice(0, 12)})`;
|
|
26
|
+
switch (extraction.kind) {
|
|
27
|
+
case 'reused':
|
|
28
|
+
say(`running ${identity} from ${extraction.directory}`);
|
|
29
|
+
break;
|
|
30
|
+
case 'extracted':
|
|
31
|
+
say(`extracted ${identity} to ${extraction.directory} in ${seconds(extraction.elapsedMs)}`);
|
|
32
|
+
break;
|
|
33
|
+
case 'awaited':
|
|
34
|
+
say(
|
|
35
|
+
`running ${identity} from ${extraction.directory} after ${seconds(extraction.elapsedMs)} waiting on ${extraction.holder}`,
|
|
36
|
+
);
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
process.stdout.write(`${extraction.directory}\n`);
|
|
40
|
+
} catch (error) {
|
|
41
|
+
say(error instanceof Error ? error.message : String(error));
|
|
42
|
+
process.exit(1);
|
|
43
|
+
}
|
|
@@ -564,7 +564,10 @@ describe('cargo-test reachability policy', () => {
|
|
|
564
564
|
'cargo-test-rusty-core',
|
|
565
565
|
{
|
|
566
566
|
command:
|
|
567
|
-
"
|
|
567
|
+
'extracted="$(node ../../node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js target/nextest/archive.tar.zst)" && ' +
|
|
568
|
+
'cargo --frozen nextest run --binaries-metadata "$extracted/target/nextest/binaries-metadata.json" ' +
|
|
569
|
+
'--cargo-metadata "$extracted/target/nextest/cargo-metadata.json" --target-dir-remap "$extracted/target" ' +
|
|
570
|
+
"--workspace-remap . -E 'package(rusty-core)'",
|
|
568
571
|
},
|
|
569
572
|
],
|
|
570
573
|
]),
|
package/src/index.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from 'bun:test';
|
|
2
|
-
import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { chmod, mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
@@ -904,13 +904,15 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
904
904
|
}
|
|
905
905
|
expect(targets['cargo-test-ferris-core']?.dependsOn).toEqual(['cargo-fetch', 'cargo-test-archive']);
|
|
906
906
|
expect(targets['cargo-test-ferris-wasm']?.dependsOn).toEqual(['cargo-fetch', 'cargo-test-archive']);
|
|
907
|
-
// The runner executes the archive's binaries
|
|
908
|
-
//
|
|
909
|
-
//
|
|
910
|
-
//
|
|
911
|
-
//
|
|
907
|
+
// The runner executes the archive's binaries, unpacked once for every
|
|
908
|
+
// runner by the plugin's helper (through the root `node_modules` link,
|
|
909
|
+
// so the hashed command names no machine path), and remaps the
|
|
910
|
+
// workspace root: without `--workspace-remap`, a restored archive hands
|
|
911
|
+
// tests the PRODUCING tree's absolute CARGO_MANIFEST_DIR (measured on a
|
|
912
|
+
// moved tree). `--workspace` is not merely redundant here, nextest
|
|
913
|
+
// rejects it beside a reused build.
|
|
912
914
|
expect(targets['cargo-test-ferris-core']?.options?.command).toMatch(
|
|
913
|
-
/^cargo --frozen nextest run --
|
|
915
|
+
/^extracted="\$\(node \.\.\/\.\.\/node_modules\/@smoothbricks\/nx-plugin\/dist\/bin\/smoo-nx-nextest-extract\.js target\/nextest\/archive\.tar\.zst\)" && cargo --frozen nextest run --binaries-metadata "\$extracted\/target\/nextest\/binaries-metadata\.json" --cargo-metadata "\$extracted\/target\/nextest\/cargo-metadata\.json" --target-dir-remap "\$extracted\/target" --workspace-remap \. -E 'package\(ferris-core\)' --no-tests=pass --user-config-file none --tool-config-file "smoo:\$PWD\/.*nextest\.toml"$/,
|
|
914
916
|
);
|
|
915
917
|
expect(targets['cargo-test-ferris-core']?.inputs).toContain(
|
|
916
918
|
'{workspaceRoot}/packages/ferris/.config/nextest.toml',
|
|
@@ -1134,8 +1136,12 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
1134
1136
|
expect(runtime['cargo-test-runtime-core-shard2']?.dependsOn).toEqual([rootFetch, rootArchive]);
|
|
1135
1137
|
expect(runtime['cargo-test-runtime-core-exceptions']?.dependsOn).toEqual([rootFetch, rootArchive]);
|
|
1136
1138
|
expect(runtime['cargo-test-runtime-core-shard1']?.options).toMatchObject({ cwd: '.' });
|
|
1139
|
+
// At the root the helper is the root's own `node_modules` link, no `../`.
|
|
1140
|
+
expect(runtime['cargo-test-runtime-core-shard1']?.options?.command).toStartWith(
|
|
1141
|
+
'extracted="$(node node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js target/nextest/archive.tar.zst)" && ',
|
|
1142
|
+
);
|
|
1137
1143
|
expect(runtime['cargo-test-runtime-core-shard1']?.options?.command).toContain(
|
|
1138
|
-
|
|
1144
|
+
`--target-dir-remap "$extracted/target" --workspace-remap . -E 'package(runtime-core)`,
|
|
1139
1145
|
);
|
|
1140
1146
|
expect(native['cargo-test-native-napi']?.options?.command).toContain('--no-tests=pass');
|
|
1141
1147
|
expect(runtime['cargo-test']?.dependsOn).toEqual([
|
|
@@ -1377,12 +1383,12 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
1377
1383
|
// Scoped by filterset over one archived workspace build: exactly one
|
|
1378
1384
|
// crate runs, and no `-p` re-resolves its features.
|
|
1379
1385
|
expect(cargoPackageSelection(String(targets['cargo-test-ferris-core']?.options?.command ?? ''))).toEqual({
|
|
1380
|
-
|
|
1386
|
+
prebuiltRun: true,
|
|
1381
1387
|
packageFlags: [],
|
|
1382
1388
|
filtered: ['ferris-core'],
|
|
1383
1389
|
});
|
|
1384
1390
|
expect(cargoPackageSelection(String(targets['cargo-test-ferris-wasm']?.options?.command ?? ''))).toEqual({
|
|
1385
|
-
|
|
1391
|
+
prebuiltRun: true,
|
|
1386
1392
|
packageFlags: [],
|
|
1387
1393
|
filtered: ['ferris-wasm'],
|
|
1388
1394
|
});
|
|
@@ -1394,6 +1400,106 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
1394
1400
|
}
|
|
1395
1401
|
});
|
|
1396
1402
|
|
|
1403
|
+
it('extracts the test archive once for all per-crate runners, and again only when its bytes change', async () => {
|
|
1404
|
+
const workspace = await createWorkspace();
|
|
1405
|
+
const root = workspace.context.workspaceRoot;
|
|
1406
|
+
const crates = ['ferris-a', 'ferris-b', 'ferris-c'];
|
|
1407
|
+
try {
|
|
1408
|
+
await workspace.write('packages/ferris/package.json', '{"name":"ferris"}\n');
|
|
1409
|
+
await workspace.write('packages/ferris/Cargo.toml', '[workspace]\nmembers = ["crates/*"]\n');
|
|
1410
|
+
for (const crate of crates) {
|
|
1411
|
+
await workspace.write(`packages/ferris/crates/${crate}/Cargo.toml`, `[package]\nname = "${crate}"\n`);
|
|
1412
|
+
}
|
|
1413
|
+
const targets = await inferProjectTargets(workspace, 'packages/ferris/package.json');
|
|
1414
|
+
// Installed the way a consumer has it, so each command finds the helper it names.
|
|
1415
|
+
await mkdir(join(root, 'node_modules/@smoothbricks'), { recursive: true });
|
|
1416
|
+
await symlink(fileURLToPath(new URL('..', import.meta.url)), join(root, 'node_modules/@smoothbricks/nx-plugin'));
|
|
1417
|
+
// A cargo that does what nextest does with an archive, and says so: an
|
|
1418
|
+
// `--archive-file` unpacks it (into `--extract-to`, or a private temp
|
|
1419
|
+
// dir when a run is given the archive itself), and a run reports which
|
|
1420
|
+
// archive's binaries it executed.
|
|
1421
|
+
const log = join(root, 'nextest.log');
|
|
1422
|
+
await workspace.write(
|
|
1423
|
+
'bin/cargo',
|
|
1424
|
+
[
|
|
1425
|
+
'#!/bin/sh',
|
|
1426
|
+
'archive= dest= binaries=',
|
|
1427
|
+
'while [ $# -gt 0 ]; do',
|
|
1428
|
+
' case "$1" in',
|
|
1429
|
+
' --archive-file) archive=$2; shift ;;',
|
|
1430
|
+
' --extract-to) dest=$2; shift ;;',
|
|
1431
|
+
' --binaries-metadata) binaries=$2; shift ;;',
|
|
1432
|
+
' esac',
|
|
1433
|
+
' shift',
|
|
1434
|
+
'done',
|
|
1435
|
+
'if [ -n "$archive" ]; then',
|
|
1436
|
+
' echo "extract $(cat "$archive")" >> "$FAKE_NEXTEST_LOG"',
|
|
1437
|
+
' if [ -z "$dest" ]; then echo "run $(cat "$archive")" >> "$FAKE_NEXTEST_LOG"; exit 0; fi',
|
|
1438
|
+
' mkdir -p "$dest/target/nextest"',
|
|
1439
|
+
' cat "$archive" > "$dest/target/nextest/binaries-metadata.json"',
|
|
1440
|
+
' cat "$archive" > "$dest/target/nextest/cargo-metadata.json"',
|
|
1441
|
+
' exit 0',
|
|
1442
|
+
'fi',
|
|
1443
|
+
'echo "run $(cat "$binaries")" >> "$FAKE_NEXTEST_LOG"',
|
|
1444
|
+
'',
|
|
1445
|
+
].join('\n'),
|
|
1446
|
+
);
|
|
1447
|
+
await chmod(join(root, 'bin/cargo'), 0o755);
|
|
1448
|
+
// Every runner at once, through `sh -c` exactly as bounded-exec spawns it.
|
|
1449
|
+
const runAll = () =>
|
|
1450
|
+
Promise.all(
|
|
1451
|
+
crates.map(async (crate) => {
|
|
1452
|
+
const options = targets[`cargo-test-${crate}`]?.options;
|
|
1453
|
+
const child = Bun.spawn(['sh', '-c', String(options?.command)], {
|
|
1454
|
+
cwd: join(root, String(options?.cwd)),
|
|
1455
|
+
env: { ...process.env, PATH: `${join(root, 'bin')}:${process.env.PATH}`, FAKE_NEXTEST_LOG: log },
|
|
1456
|
+
stdout: 'pipe',
|
|
1457
|
+
stderr: 'pipe',
|
|
1458
|
+
});
|
|
1459
|
+
const [exitCode, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()]);
|
|
1460
|
+
return { crate, exitCode, stderr };
|
|
1461
|
+
}),
|
|
1462
|
+
);
|
|
1463
|
+
|
|
1464
|
+
await workspace.write('packages/ferris/target/nextest/archive.tar.zst', 'first archive');
|
|
1465
|
+
for (const run of await runAll()) {
|
|
1466
|
+
expect(run).toMatchObject({ exitCode: 0 });
|
|
1467
|
+
}
|
|
1468
|
+
expect((await readFile(log, 'utf8')).trim().split('\n').sort()).toEqual([
|
|
1469
|
+
'extract first archive',
|
|
1470
|
+
'run first archive',
|
|
1471
|
+
'run first archive',
|
|
1472
|
+
'run first archive',
|
|
1473
|
+
]);
|
|
1474
|
+
|
|
1475
|
+
// A rebuilt archive is new bytes at the same path: the runners must not
|
|
1476
|
+
// execute what the old one unpacked, and the old unpacking is garbage.
|
|
1477
|
+
await workspace.write('packages/ferris/target/nextest/archive.tar.zst', 'second archive');
|
|
1478
|
+
for (const run of await runAll()) {
|
|
1479
|
+
expect(run).toMatchObject({ exitCode: 0 });
|
|
1480
|
+
}
|
|
1481
|
+
expect((await readFile(log, 'utf8')).trim().split('\n').sort()).toEqual([
|
|
1482
|
+
'extract first archive',
|
|
1483
|
+
'extract second archive',
|
|
1484
|
+
'run first archive',
|
|
1485
|
+
'run first archive',
|
|
1486
|
+
'run first archive',
|
|
1487
|
+
'run second archive',
|
|
1488
|
+
'run second archive',
|
|
1489
|
+
'run second archive',
|
|
1490
|
+
]);
|
|
1491
|
+
const target = join(root, 'packages/ferris/target');
|
|
1492
|
+
const unpacked = (await readdir(target, { recursive: true })).filter((path) =>
|
|
1493
|
+
path.endsWith('binaries-metadata.json'),
|
|
1494
|
+
);
|
|
1495
|
+
expect(await Promise.all(unpacked.map((path) => readFile(join(target, path), 'utf8')))).toEqual([
|
|
1496
|
+
'second archive',
|
|
1497
|
+
]);
|
|
1498
|
+
} finally {
|
|
1499
|
+
await workspace.cleanup();
|
|
1500
|
+
}
|
|
1501
|
+
});
|
|
1502
|
+
|
|
1397
1503
|
it('infers a cached Cargo Wasm build directly from crate metadata', async () => {
|
|
1398
1504
|
const workspace = await createWorkspace();
|
|
1399
1505
|
try {
|
|
@@ -1702,7 +1808,7 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
1702
1808
|
'napi-debug',
|
|
1703
1809
|
]);
|
|
1704
1810
|
expect(targets['cargo-test-cowshed-napi']?.options?.command).toMatch(
|
|
1705
|
-
/^cargo --frozen nextest run --
|
|
1811
|
+
/^extracted="\$\(node \.\.\/\.\.\/node_modules\/@smoothbricks\/nx-plugin\/dist\/bin\/smoo-nx-nextest-extract\.js target\/nextest\/archive\.tar\.zst\)" && cargo --frozen nextest run --binaries-metadata "\$extracted\/target\/nextest\/binaries-metadata\.json" --cargo-metadata "\$extracted\/target\/nextest\/cargo-metadata\.json" --target-dir-remap "\$extracted\/target" --workspace-remap \. -E 'package\(cowshed-napi\)' --no-tests=pass --user-config-file none --tool-config-file "smoo:\$PWD\/.*nextest\.toml"$/,
|
|
1706
1812
|
);
|
|
1707
1813
|
expect(targets['napi-test']).toMatchObject({
|
|
1708
1814
|
executor: '@smoothbricks/nx-plugin:bounded-exec',
|
|
@@ -1804,7 +1910,7 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
1804
1910
|
// An unsharded crate keeps the bare name and takes no --partition, so
|
|
1805
1911
|
// declaring nothing is exactly the old single-target behaviour.
|
|
1806
1912
|
expect(targets['cargo-test-small']?.options?.command).toMatch(
|
|
1807
|
-
|
|
1913
|
+
/--workspace-remap \. -E 'package\(small\)' --no-tests=pass --user-config-file none/,
|
|
1808
1914
|
);
|
|
1809
1915
|
// The shards partition the crate MINUS the classes nextest.toml singles
|
|
1810
1916
|
// out, i of N. Those are lifted out because a test-group only holds
|
|
@@ -2491,17 +2597,17 @@ function resolveDeclaredOverInferred(
|
|
|
2491
2597
|
* How a cargo test command is scoped to one crate, split into the three facts
|
|
2492
2598
|
* that can independently go wrong.
|
|
2493
2599
|
*
|
|
2494
|
-
* `--
|
|
2495
|
-
* `cargo nextest archive --workspace` already built, so a per-crate
|
|
2496
|
-
* compile anything. Narrowing comes from the nextest filterset,
|
|
2497
|
-
* what RUNS. `--package` narrows too, but it re-resolves that
|
|
2498
|
-
* which is a build — 57.9s of a 120s bounded budget on a
|
|
2499
|
-
* runner when the runners still built — so its presence is
|
|
2500
|
-
* though it selects the same tests. nextest rejects it
|
|
2501
|
-
*
|
|
2600
|
+
* `--binaries-metadata` must be present: the runner executes binaries one
|
|
2601
|
+
* workspace `cargo nextest archive --workspace` already built, so a per-crate
|
|
2602
|
+
* run cannot compile anything. Narrowing comes from the nextest filterset,
|
|
2603
|
+
* which selects what RUNS. `--package` narrows too, but it re-resolves that
|
|
2604
|
+
* crate's features, which is a build — 57.9s of a 120s bounded budget on a
|
|
2605
|
+
* hosted 3-core macOS runner when the runners still built — so its presence is
|
|
2606
|
+
* a regression even though it selects the same tests. nextest rejects it
|
|
2607
|
+
* outright beside a reused build, and this assertion says so before cargo does.
|
|
2502
2608
|
*/
|
|
2503
2609
|
function cargoPackageSelection(command: string): {
|
|
2504
|
-
|
|
2610
|
+
prebuiltRun: boolean;
|
|
2505
2611
|
packageFlags: string[];
|
|
2506
2612
|
filtered: string[];
|
|
2507
2613
|
} {
|
|
@@ -2516,7 +2622,7 @@ function cargoPackageSelection(command: string): {
|
|
|
2516
2622
|
}
|
|
2517
2623
|
}
|
|
2518
2624
|
return {
|
|
2519
|
-
|
|
2625
|
+
prebuiltRun: tokens.includes('--binaries-metadata'),
|
|
2520
2626
|
packageFlags,
|
|
2521
2627
|
filtered: [...command.matchAll(/\bpackage\(([^)]+)\)/g)].flatMap((match) =>
|
|
2522
2628
|
match[1] === undefined ? [] : [match[1]],
|
package/src/index.ts
CHANGED
|
@@ -55,6 +55,12 @@ import {
|
|
|
55
55
|
cargoFrozen,
|
|
56
56
|
} from './cross-check-policy.js';
|
|
57
57
|
import { hashVersionlessCrateManifests } from './manifest-hash.js';
|
|
58
|
+
import {
|
|
59
|
+
EXTRACTED_BINARIES_METADATA,
|
|
60
|
+
EXTRACTED_CARGO_METADATA,
|
|
61
|
+
EXTRACTED_TARGET_DIR,
|
|
62
|
+
NEXTEST_EXTRACT_BIN,
|
|
63
|
+
} from './nextest-extraction.js';
|
|
58
64
|
import { isNonSourceDirectory } from './source-directories.js';
|
|
59
65
|
import {
|
|
60
66
|
isBuildOutputTargetName,
|
|
@@ -633,8 +639,8 @@ function createCargoTestCompileTarget(projectRoot: string): TargetConfiguration
|
|
|
633
639
|
* build with a filter: 32 such tasks on one repository each re-entered cargo,
|
|
634
640
|
* re-resolved features and chained behind one another so the flocked `target/`
|
|
635
641
|
* had a single writer. Reading from an archive removes the build from the run
|
|
636
|
-
* entirely — the
|
|
637
|
-
*
|
|
642
|
+
* entirely — the runners execute one extraction of it and never write cargo's
|
|
643
|
+
* `target/` build tree — which is what lets the runners fan out.
|
|
638
644
|
*
|
|
639
645
|
* Cached, unlike `cargo-test-compile`, because it produces a VALUE: one file,
|
|
640
646
|
* not a mutable build tree Nx would have to restore. `--workspace` matches the
|
|
@@ -2391,16 +2397,34 @@ async function resolveCargoWorkspaces(
|
|
|
2391
2397
|
* One bounded target per crate; a crate that declares `smoothbricks.test.shards`
|
|
2392
2398
|
* gets one per shard plus one for the tests nextest.toml singles out.
|
|
2393
2399
|
*
|
|
2394
|
-
* Every piece EXECUTES, it never builds:
|
|
2395
|
-
* `cargo-test-archive` compiled once for the whole workspace
|
|
2396
|
-
* the
|
|
2397
|
-
*
|
|
2398
|
-
*
|
|
2399
|
-
*
|
|
2400
|
-
*
|
|
2401
|
-
*
|
|
2402
|
-
*
|
|
2403
|
-
*
|
|
2400
|
+
* Every piece EXECUTES, it never builds: it runs the binaries
|
|
2401
|
+
* `cargo-test-archive` compiled once for the whole workspace. That is what
|
|
2402
|
+
* removed the two costs the old shape could not avoid. `nextest run --workspace
|
|
2403
|
+
* -E 'package(X)'` re-entered cargo once per crate, and the `-p X` form
|
|
2404
|
+
* re-resolved that crate's features so cargo rebuilt the divergent half inside
|
|
2405
|
+
* the bounded window — measured at 57.9s of a 120s budget on a hosted 3-core
|
|
2406
|
+
* macOS runner, killing the run with 264 tests still to go. Neither form is
|
|
2407
|
+
* reachable from a reused build: nextest rejects `--workspace` and `-p` beside
|
|
2408
|
+
* it, so the filterset is the only selector left, and it selects the same tests
|
|
2409
|
+
* it always did.
|
|
2410
|
+
*
|
|
2411
|
+
* The archive is unpacked ONCE per content, not once per piece. `nextest run
|
|
2412
|
+
* --archive-file` extracts the whole archive into a temporary directory of its
|
|
2413
|
+
* own on every run, so N pieces paid N extractions of every binary in the
|
|
2414
|
+
* workspace to run 1/N of the tests each: on a workspace with ~200 test
|
|
2415
|
+
* binaries, ~25 runners extracting side by side on a loaded host all hit the
|
|
2416
|
+
* 120s bound still "Extracting". `smoo-nx-nextest-extract` hashes the archive,
|
|
2417
|
+
* unpacks it under that hash unless a previous runner already did — one
|
|
2418
|
+
* extracts while the rest wait — and prints the directory; the run then reuses
|
|
2419
|
+
* it through nextest's own reuse-build flags, which is what `--archive-file`
|
|
2420
|
+
* does after unpacking. Keyed by the archive's bytes, so a rebuilt or restored
|
|
2421
|
+
* archive can never run an older extraction's binaries. The one difference
|
|
2422
|
+
* from `--archive-file` is the Rust standard library directory: nextest has no
|
|
2423
|
+
* flag to remap it, so a run uses the producing toolchain's own sysroot rather
|
|
2424
|
+
* than the archived copy of it — the same toolchain, because the archive's hash
|
|
2425
|
+
* covers the toolchain pin. The extraction lives under `target/nextest`, never
|
|
2426
|
+
* in an Nx output: it is a multi-GB tree for a cached verdict that has no
|
|
2427
|
+
* outputs.
|
|
2404
2428
|
*
|
|
2405
2429
|
* `--workspace-remap .` is not cosmetic. An archive records the ABSOLUTE paths
|
|
2406
2430
|
* of the tree that produced it; without the remap, a restored archive hands
|
|
@@ -2433,8 +2457,8 @@ async function resolveCargoWorkspaces(
|
|
|
2433
2457
|
* making valid empty crates and shards fail execution.
|
|
2434
2458
|
*
|
|
2435
2459
|
* Pieces fan out rather than chain. The chain existed because every runner was
|
|
2436
|
-
* a cargo writer on one flocked `target/`;
|
|
2437
|
-
* so the only serialization left is the one
|
|
2460
|
+
* a cargo writer on one flocked `target/`; a reused-build run writes nothing
|
|
2461
|
+
* there, so the only serialization left is the one extraction per archive.
|
|
2438
2462
|
*/
|
|
2439
2463
|
async function addCargoTestTargets(
|
|
2440
2464
|
targets: Record<string, TargetConfiguration>,
|
|
@@ -2445,6 +2469,12 @@ async function addCargoTestTargets(
|
|
|
2445
2469
|
): Promise<string[]> {
|
|
2446
2470
|
const targetNames: string[] = [];
|
|
2447
2471
|
const toolConfig = nextestToolConfigArg(workspaceRoot, workspace.projectRoot, PLUGIN_NEXTEST_CONFIG);
|
|
2472
|
+
// Relative to the runner's cwd, so the hashed command is the same in every checkout.
|
|
2473
|
+
const extractBin = posix.join(posix.relative(workspace.projectRoot, '.'), NEXTEST_EXTRACT_BIN);
|
|
2474
|
+
const reuseExtraction =
|
|
2475
|
+
`--binaries-metadata "$extracted/${EXTRACTED_BINARIES_METADATA}" ` +
|
|
2476
|
+
`--cargo-metadata "$extracted/${EXTRACTED_CARGO_METADATA}" ` +
|
|
2477
|
+
`--target-dir-remap "$extracted/${EXTRACTED_TARGET_DIR}"`;
|
|
2448
2478
|
for (const plan of workspace.packages) {
|
|
2449
2479
|
if (plan.package.projectRoot !== projectRoot) {
|
|
2450
2480
|
continue;
|
|
@@ -2487,9 +2517,11 @@ async function addCargoTestTargets(
|
|
|
2487
2517
|
inputs,
|
|
2488
2518
|
dependsOn: [archive],
|
|
2489
2519
|
options: {
|
|
2490
|
-
command:
|
|
2491
|
-
`
|
|
2492
|
-
|
|
2520
|
+
command:
|
|
2521
|
+
`extracted="$(node ${extractBin} ${CARGO_TEST_ARCHIVE_FILE})" && ` +
|
|
2522
|
+
cargoFrozen(
|
|
2523
|
+
`nextest run ${reuseExtraction} --workspace-remap . -E '${piece.selector}'${piece.extra} --no-tests=pass --user-config-file none ${toolConfig}`,
|
|
2524
|
+
),
|
|
2493
2525
|
cwd: workspace.projectRoot,
|
|
2494
2526
|
timeoutMs: BOUNDED_TEST_TIMEOUT_MS,
|
|
2495
2527
|
killAfterMs: BOUNDED_TEST_KILL_AFTER_MS,
|
|
@@ -135,6 +135,12 @@ const managedFiles: ManagedFileDescriptor[] = [
|
|
|
135
135
|
source: 'tooling/direnv/shared-caches.sh',
|
|
136
136
|
target: 'tooling/direnv/shared-caches.sh',
|
|
137
137
|
},
|
|
138
|
+
{
|
|
139
|
+
kind: 'raw',
|
|
140
|
+
source: 'tooling/direnv/inherited-devenv.ts',
|
|
141
|
+
target: 'tooling/direnv/inherited-devenv.ts',
|
|
142
|
+
executable: true,
|
|
143
|
+
},
|
|
138
144
|
{
|
|
139
145
|
kind: 'raw',
|
|
140
146
|
source: 'tooling/git-hooks/pre-commit.sh',
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import { mkdir, mkdtemp, readdir, rm, stat, utimes, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import {
|
|
6
|
+
EXTRACTED_BINARIES_METADATA,
|
|
7
|
+
EXTRACTED_CARGO_METADATA,
|
|
8
|
+
ensureNextestArchiveExtracted,
|
|
9
|
+
type NextestArchiveExtractor,
|
|
10
|
+
type NextestExtractionEvent,
|
|
11
|
+
nextestArchiveKey,
|
|
12
|
+
nextestExtractionPaths,
|
|
13
|
+
} from './nextest-extraction.js';
|
|
14
|
+
|
|
15
|
+
/** What nextest leaves behind: the two metadata files a reuse run reads. */
|
|
16
|
+
const unpack: NextestArchiveExtractor = async (_archive, destination) => {
|
|
17
|
+
await mkdir(join(destination, 'target/nextest'), { recursive: true });
|
|
18
|
+
await writeFile(join(destination, EXTRACTED_BINARIES_METADATA), '{}');
|
|
19
|
+
await writeFile(join(destination, EXTRACTED_CARGO_METADATA), '{}');
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const FAST = { heartbeatMs: 20, staleAfterMs: 400, pollMs: 10 };
|
|
23
|
+
|
|
24
|
+
describe('ensureNextestArchiveExtracted', () => {
|
|
25
|
+
it('takes over a lock its holder stopped refreshing instead of waiting on it forever', async () => {
|
|
26
|
+
const root = await mkdtemp(join(tmpdir(), 'nextest-extraction-'));
|
|
27
|
+
try {
|
|
28
|
+
const archive = join(root, 'archive.tar.zst');
|
|
29
|
+
await writeFile(archive, 'archive bytes');
|
|
30
|
+
const paths = nextestExtractionPaths(archive, await nextestArchiveKey(archive));
|
|
31
|
+
// A holder killed mid-extraction (a bounded run's SIGKILL) leaves its
|
|
32
|
+
// lock behind, and a pid in it that may since belong to anyone.
|
|
33
|
+
await mkdir(paths.root, { recursive: true });
|
|
34
|
+
await writeFile(paths.lock, `${process.pid} abandoned\n`);
|
|
35
|
+
const minuteAgo = new Date(Date.now() - 60_000);
|
|
36
|
+
await utimes(paths.lock, minuteAgo, minuteAgo);
|
|
37
|
+
|
|
38
|
+
const events: NextestExtractionEvent[] = [];
|
|
39
|
+
const extraction = await ensureNextestArchiveExtracted(archive, unpack, (event) => events.push(event), FAST);
|
|
40
|
+
|
|
41
|
+
expect(extraction).toMatchObject({ kind: 'extracted', directory: paths.directory });
|
|
42
|
+
expect(events).toEqual([{ kind: 'stale-lock', holder: `pid ${process.pid}`, ageMs: expect.any(Number) }]);
|
|
43
|
+
expect(await readdir(paths.root)).toEqual([paths.directory.slice(paths.root.length + 1)]);
|
|
44
|
+
} finally {
|
|
45
|
+
await rm(root, { recursive: true, force: true });
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('publishes nothing when the archive changes while it is being extracted', async () => {
|
|
50
|
+
const root = await mkdtemp(join(tmpdir(), 'nextest-extraction-'));
|
|
51
|
+
try {
|
|
52
|
+
const archive = join(root, 'archive.tar.zst');
|
|
53
|
+
await writeFile(archive, 'archive bytes');
|
|
54
|
+
const paths = nextestExtractionPaths(archive, await nextestArchiveKey(archive));
|
|
55
|
+
// Unpacked from the old bytes, but the directory would be named for
|
|
56
|
+
// them while the path now holds others: a later run hashing the new
|
|
57
|
+
// bytes would never find it, and one hashing the old bytes again — the
|
|
58
|
+
// archive restored from cache — must not find a mixture.
|
|
59
|
+
const rebuiltMidway: NextestArchiveExtractor = async (source, destination) => {
|
|
60
|
+
await unpack(source, destination);
|
|
61
|
+
await writeFile(archive, 'rebuilt archive bytes');
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
await expect(ensureNextestArchiveExtracted(archive, rebuiltMidway, () => {}, FAST)).rejects.toThrow(
|
|
65
|
+
/changed while it was being extracted/,
|
|
66
|
+
);
|
|
67
|
+
// No directory under the old key, no staging left over, no lock held.
|
|
68
|
+
expect(await readdir(paths.root)).toEqual([]);
|
|
69
|
+
await expect(stat(paths.directory)).rejects.toMatchObject({ code: 'ENOENT' });
|
|
70
|
+
} finally {
|
|
71
|
+
await rm(root, { recursive: true, force: true });
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
});
|