@smoothbricks/nx-plugin 0.4.23 → 0.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -12
- package/dist/bin/smoo-nx-nextest-extract.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-extract.js +41 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +40 -15
- package/dist/nextest-extraction.d.ts +101 -0
- package/dist/nextest-extraction.d.ts.map +1 -0
- package/dist/nextest-extraction.js +268 -0
- package/package.json +4 -3
- package/src/bin/smoo-nx-nextest-extract.ts +43 -0
- package/src/bounded-test-policy.test.ts +4 -1
- package/src/index.test.ts +128 -22
- package/src/index.ts +49 -17
- package/src/nextest-extraction.test.ts +74 -0
- package/src/nextest-extraction.ts +329 -0
package/README.md
CHANGED
|
@@ -273,18 +273,22 @@ Concrete targets come from concrete files:
|
|
|
273
273
|
`{workspaceRoot}`: Nx resolves a `{projectRoot}` glob against only the files its project owns, and a nested package
|
|
274
274
|
owns the crates under it, so a root workspace's archive spelled that way hashes none of its members and replays stale
|
|
275
275
|
binaries against edited crates. Each member crate gets a cached `cargo-test-<package>` run (30s per-test timeout like
|
|
276
|
-
`bun test --timeout=30000`) whose inputs include that crate and its path dependencies, and whose command
|
|
277
|
-
`cargo --frozen nextest run --
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
276
|
+
`bun test --timeout=30000`) whose inputs include that crate and its path dependencies, and whose command runs
|
|
277
|
+
`cargo --frozen nextest run --binaries-metadata … --cargo-metadata … --target-dir-remap … --workspace-remap . -E 'package(<crate>)'`
|
|
278
|
+
from the archive's one extraction. Runners therefore execute rather than build: they write nothing to cargo's build
|
|
279
|
+
tree, and fan out instead of chaining. They do not unpack the archive each: before the run, `smoo-nx-nextest-extract`
|
|
280
|
+
hashes it and extracts it into `target/nextest/archive.tar.zst.extracted/<sha256>` unless a runner already did — one
|
|
281
|
+
extracts, the others wait — so a workspace with ~200 test binaries and ~25 crates unpacks them once per archive
|
|
282
|
+
instead of once per crate. The directory is keyed by the archive's bytes, so a rebuilt or cache-restored archive never
|
|
283
|
+
runs an older extraction's binaries, and publishing one removes the others; it is never an Nx output.
|
|
284
|
+
`--workspace-remap` is required — an archive records the producing tree's absolute paths, and without it a restored
|
|
285
|
+
archive hands tests another checkout's `CARGO_MANIFEST_DIR`. With it the archive is relocatable, so one cache entry
|
|
286
|
+
serves every checkout of the same commit; nextest re-points `CARGO_BIN_EXE_<name>`/`NEXTEST_BIN_EXE_<name>` at the
|
|
287
|
+
extracted binaries at runtime, but a test that reads them through the compile-time `env!` macro keeps the producing
|
|
288
|
+
tree's path. Per-crate runners accept an empty nextest selection because a valid workspace member may have no tests
|
|
289
|
+
and a hash partition may legitimately be empty. `napi-debug` stays behind `cargo-test-compile`, and a crate in the
|
|
290
|
+
project that builds the debug cdylib runs after it. A crate declaring `[package.metadata.smoothbricks.wasm-bindgen]`
|
|
291
|
+
also receives the cacheable `cargo-wasm` output target in its owning project.
|
|
288
292
|
- The plugin's own `nextest.toml` is passed as `--tool-config-file "smoo:$PWD/<path>"`, so it is a layer BENEATH the
|
|
289
293
|
repository's `<cargo-workspace>/.config/nextest.toml` rather than a replacement for it: a repository can raise a
|
|
290
294
|
timeout, add a test group, or declare `archive.include` for a cdylib or fixture the archived test binaries need, and
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"smoo-nx-nextest-extract.d.ts","sourceRoot":"","sources":["../../src/bin/smoo-nx-nextest-extract.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { ensureNextestArchiveExtracted, extractWithNextest } from '../nextest-extraction.js';
|
|
3
|
+
// Prints the directory holding the archive's extraction on stdout, the only
|
|
4
|
+
// thing a runner's `$(...)` captures. Everything said to a person goes to stderr.
|
|
5
|
+
const args = process.argv.slice(2);
|
|
6
|
+
const archive = args[0];
|
|
7
|
+
if (archive === undefined || args.length !== 1 || archive.startsWith('-')) {
|
|
8
|
+
process.stderr.write('usage: smoo-nx-nextest-extract <archive.tar.zst>\n');
|
|
9
|
+
process.exit(2);
|
|
10
|
+
}
|
|
11
|
+
const say = (line) => process.stderr.write(`smoo-nx-nextest-extract: ${line}\n`);
|
|
12
|
+
const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
|
|
13
|
+
try {
|
|
14
|
+
const extraction = await ensureNextestArchiveExtracted(archive, extractWithNextest, (event) => {
|
|
15
|
+
switch (event.kind) {
|
|
16
|
+
case 'waiting':
|
|
17
|
+
say(`${event.holder} is extracting ${archive}; waiting for it`);
|
|
18
|
+
break;
|
|
19
|
+
case 'stale-lock':
|
|
20
|
+
say(`${event.holder} stopped refreshing its extraction lock ${seconds(event.ageMs)} ago; extracting instead`);
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
const identity = `${archive} (sha256 ${extraction.key.slice(0, 12)})`;
|
|
25
|
+
switch (extraction.kind) {
|
|
26
|
+
case 'reused':
|
|
27
|
+
say(`running ${identity} from ${extraction.directory}`);
|
|
28
|
+
break;
|
|
29
|
+
case 'extracted':
|
|
30
|
+
say(`extracted ${identity} to ${extraction.directory} in ${seconds(extraction.elapsedMs)}`);
|
|
31
|
+
break;
|
|
32
|
+
case 'awaited':
|
|
33
|
+
say(`running ${identity} from ${extraction.directory} after ${seconds(extraction.elapsedMs)} waiting on ${extraction.holder}`);
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
process.stdout.write(`${extraction.directory}\n`);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
say(error instanceof Error ? error.message : String(error));
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,KAAK,aAAa,EAGnB,MAAM,0BAA0B,CAAC;AAalC,OAAO,EAKL,yBAAyB,EAa1B,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,KAAK,aAAa,EAGnB,MAAM,0BAA0B,CAAC;AAalC,OAAO,EAKL,yBAAyB,EAa1B,MAAM,sBAAsB,CAAC;AA4B9B,OAAO,EAAE,yBAAyB,EAAE,CAAC;AAErC,gEAAgE;AAChE,eAAO,MAAM,aAAa,WAAW,CAAC;AACtC,uEAAuE;AACvE,eAAO,MAAM,mBAAmB,iBAAiB,CAAC;AAqQlD,KAAK,gBAAgB,GAAG,OAAO,GAAG,KAAK,CAAC;AACxC,KAAK,gBAAgB,GAAG,OAAO,GAAG,OAAO,CAAC;AAE1C,UAAU,YAAY;IACpB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,YAAY,EAAE,gBAAgB,CAAC;CAChC;AAmID,UAAU,oBAAqB,SAAQ,YAAY;IACjD,UAAU,EAAE,MAAM,CAAC;CACpB;AA0CD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAEhF;AA+XD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAEvG;AAED,eAAO,MAAM,aAAa,EAAE,aAG3B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -12,6 +12,7 @@ import { CARGO_TOOLCHAIN_NAMED_INPUT, CARGO_TOOLCHAIN_PIN_INPUTS } from './cargo
|
|
|
12
12
|
import { attributeCargoWorkspacePackages, CARGO_TEST_ARCHIVE_FILE, CARGO_TEST_ARCHIVE_TARGET, CARGO_TEST_COMPILE_TARGET, CARGO_TEST_EXCEPTIONS_SUFFIX, CARGO_TEST_TARGET, cargoPackageTestInputs, cargoTestPackageTargetName, createCargoInputsCache, exceptionalTestFilter, listCargoCrossTestTargets, listCargoWorkspacePackages, NEXTEST_REPO_CONFIG_PATH, nextestToolConfigArg, } from './cargo-workspace.js';
|
|
13
13
|
import { CARGO_CROSS_LINT_COMMAND, CARGO_CROSS_LINT_TARGET, CARGO_FETCH_COMMAND, CARGO_FETCH_TARGET, CARGO_FROZEN_PREFIX, CARGO_LINT_CLIPPY_COMMAND, cargoCrossTestArchiveFile, cargoCrossTestArchiveTargetName, cargoCrossTestTargetName, cargoFrozen, } from './cross-check-policy.js';
|
|
14
14
|
import { hashVersionlessCrateManifests } from './manifest-hash.js';
|
|
15
|
+
import { EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA, EXTRACTED_TARGET_DIR, NEXTEST_EXTRACT_BIN, } from './nextest-extraction.js';
|
|
15
16
|
import { isNonSourceDirectory } from './source-directories.js';
|
|
16
17
|
import { isBuildOutputTargetName, isTestRunnerTargetName, PLATFORM_TARGET_GLOBS, RELEASE_CONFIGURATION, } from './workspace-config-policy.js';
|
|
17
18
|
export { CARGO_TEST_COMPILE_TARGET };
|
|
@@ -502,8 +503,8 @@ function createCargoTestCompileTarget(projectRoot) {
|
|
|
502
503
|
* build with a filter: 32 such tasks on one repository each re-entered cargo,
|
|
503
504
|
* re-resolved features and chained behind one another so the flocked `target/`
|
|
504
505
|
* had a single writer. Reading from an archive removes the build from the run
|
|
505
|
-
* entirely — the
|
|
506
|
-
*
|
|
506
|
+
* entirely — the runners execute one extraction of it and never write cargo's
|
|
507
|
+
* `target/` build tree — which is what lets the runners fan out.
|
|
507
508
|
*
|
|
508
509
|
* Cached, unlike `cargo-test-compile`, because it produces a VALUE: one file,
|
|
509
510
|
* not a mutable build tree Nx would have to restore. `--workspace` matches the
|
|
@@ -1962,16 +1963,34 @@ async function resolveCargoWorkspaces(packageJsonPaths, workspaceRoot) {
|
|
|
1962
1963
|
* One bounded target per crate; a crate that declares `smoothbricks.test.shards`
|
|
1963
1964
|
* gets one per shard plus one for the tests nextest.toml singles out.
|
|
1964
1965
|
*
|
|
1965
|
-
* Every piece EXECUTES, it never builds:
|
|
1966
|
-
* `cargo-test-archive` compiled once for the whole workspace
|
|
1967
|
-
* the
|
|
1968
|
-
*
|
|
1969
|
-
*
|
|
1970
|
-
*
|
|
1971
|
-
*
|
|
1972
|
-
*
|
|
1973
|
-
*
|
|
1974
|
-
*
|
|
1966
|
+
* Every piece EXECUTES, it never builds: it runs the binaries
|
|
1967
|
+
* `cargo-test-archive` compiled once for the whole workspace. That is what
|
|
1968
|
+
* removed the two costs the old shape could not avoid. `nextest run --workspace
|
|
1969
|
+
* -E 'package(X)'` re-entered cargo once per crate, and the `-p X` form
|
|
1970
|
+
* re-resolved that crate's features so cargo rebuilt the divergent half inside
|
|
1971
|
+
* the bounded window — measured at 57.9s of a 120s budget on a hosted 3-core
|
|
1972
|
+
* macOS runner, killing the run with 264 tests still to go. Neither form is
|
|
1973
|
+
* reachable from a reused build: nextest rejects `--workspace` and `-p` beside
|
|
1974
|
+
* it, so the filterset is the only selector left, and it selects the same tests
|
|
1975
|
+
* it always did.
|
|
1976
|
+
*
|
|
1977
|
+
* The archive is unpacked ONCE per content, not once per piece. `nextest run
|
|
1978
|
+
* --archive-file` extracts the whole archive into a temporary directory of its
|
|
1979
|
+
* own on every run, so N pieces paid N extractions of every binary in the
|
|
1980
|
+
* workspace to run 1/N of the tests each: on a workspace with ~200 test
|
|
1981
|
+
* binaries, ~25 runners extracting side by side on a loaded host all hit the
|
|
1982
|
+
* 120s bound still "Extracting". `smoo-nx-nextest-extract` hashes the archive,
|
|
1983
|
+
* unpacks it under that hash unless a previous runner already did — one
|
|
1984
|
+
* extracts while the rest wait — and prints the directory; the run then reuses
|
|
1985
|
+
* it through nextest's own reuse-build flags, which is what `--archive-file`
|
|
1986
|
+
* does after unpacking. Keyed by the archive's bytes, so a rebuilt or restored
|
|
1987
|
+
* archive can never run an older extraction's binaries. The one difference
|
|
1988
|
+
* from `--archive-file` is the Rust standard library directory: nextest has no
|
|
1989
|
+
* flag to remap it, so a run uses the producing toolchain's own sysroot rather
|
|
1990
|
+
* than the archived copy of it — the same toolchain, because the archive's hash
|
|
1991
|
+
* covers the toolchain pin. The extraction lives under `target/nextest`, never
|
|
1992
|
+
* in an Nx output: it is a multi-GB tree for a cached verdict that has no
|
|
1993
|
+
* outputs.
|
|
1975
1994
|
*
|
|
1976
1995
|
* `--workspace-remap .` is not cosmetic. An archive records the ABSOLUTE paths
|
|
1977
1996
|
* of the tree that produced it; without the remap, a restored archive hands
|
|
@@ -2004,12 +2023,17 @@ async function resolveCargoWorkspaces(packageJsonPaths, workspaceRoot) {
|
|
|
2004
2023
|
* making valid empty crates and shards fail execution.
|
|
2005
2024
|
*
|
|
2006
2025
|
* Pieces fan out rather than chain. The chain existed because every runner was
|
|
2007
|
-
* a cargo writer on one flocked `target/`;
|
|
2008
|
-
* so the only serialization left is the one
|
|
2026
|
+
* a cargo writer on one flocked `target/`; a reused-build run writes nothing
|
|
2027
|
+
* there, so the only serialization left is the one extraction per archive.
|
|
2009
2028
|
*/
|
|
2010
2029
|
async function addCargoTestTargets(targets, currentProjectName, projectRoot, workspaceRoot, workspace) {
|
|
2011
2030
|
const targetNames = [];
|
|
2012
2031
|
const toolConfig = nextestToolConfigArg(workspaceRoot, workspace.projectRoot, PLUGIN_NEXTEST_CONFIG);
|
|
2032
|
+
// Relative to the runner's cwd, so the hashed command is the same in every checkout.
|
|
2033
|
+
const extractBin = posix.join(posix.relative(workspace.projectRoot, '.'), NEXTEST_EXTRACT_BIN);
|
|
2034
|
+
const reuseExtraction = `--binaries-metadata "$extracted/${EXTRACTED_BINARIES_METADATA}" ` +
|
|
2035
|
+
`--cargo-metadata "$extracted/${EXTRACTED_CARGO_METADATA}" ` +
|
|
2036
|
+
`--target-dir-remap "$extracted/${EXTRACTED_TARGET_DIR}"`;
|
|
2013
2037
|
for (const plan of workspace.packages) {
|
|
2014
2038
|
if (plan.package.projectRoot !== projectRoot) {
|
|
2015
2039
|
continue;
|
|
@@ -2052,7 +2076,8 @@ async function addCargoTestTargets(targets, currentProjectName, projectRoot, wor
|
|
|
2052
2076
|
inputs,
|
|
2053
2077
|
dependsOn: [archive],
|
|
2054
2078
|
options: {
|
|
2055
|
-
command:
|
|
2079
|
+
command: `extracted="$(node ${extractBin} ${CARGO_TEST_ARCHIVE_FILE})" && ` +
|
|
2080
|
+
cargoFrozen(`nextest run ${reuseExtraction} --workspace-remap . -E '${piece.selector}'${piece.extra} --no-tests=pass --user-config-file none ${toolConfig}`),
|
|
2056
2081
|
cwd: workspace.projectRoot,
|
|
2057
2082
|
timeoutMs: BOUNDED_TEST_TIMEOUT_MS,
|
|
2058
2083
|
killAfterMs: BOUNDED_TEST_KILL_AFTER_MS,
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The helper a per-crate runner calls before `nextest run`, as the root
|
|
3
|
+
* `node_modules` links it. Spelled through the package link rather than this
|
|
4
|
+
* file's real location: under an isolated install that location is a store
|
|
5
|
+
* path outside the workspace, which would differ per machine in a command Nx
|
|
6
|
+
* hashes.
|
|
7
|
+
*/
|
|
8
|
+
export declare const NEXTEST_EXTRACT_BIN = "node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js";
|
|
9
|
+
/**
|
|
10
|
+
* Where nextest puts things inside an extracted archive, relative to the
|
|
11
|
+
* extraction directory (nextest-runner's `reuse_build` constants). A run given
|
|
12
|
+
* these three paths reuses that build exactly as `--archive-file` would after
|
|
13
|
+
* unpacking it.
|
|
14
|
+
*/
|
|
15
|
+
export declare const EXTRACTED_TARGET_DIR = "target";
|
|
16
|
+
export declare const EXTRACTED_BINARIES_METADATA = "target/nextest/binaries-metadata.json";
|
|
17
|
+
export declare const EXTRACTED_CARGO_METADATA = "target/nextest/cargo-metadata.json";
|
|
18
|
+
/** Unpacks `archive` into the empty, existing directory `destination`. */
|
|
19
|
+
export type NextestArchiveExtractor = (archive: string, destination: string) => Promise<void>;
|
|
20
|
+
export type NextestExtraction = {
|
|
21
|
+
kind: 'reused';
|
|
22
|
+
key: string;
|
|
23
|
+
directory: string;
|
|
24
|
+
} | {
|
|
25
|
+
kind: 'extracted';
|
|
26
|
+
key: string;
|
|
27
|
+
directory: string;
|
|
28
|
+
elapsedMs: number;
|
|
29
|
+
} | {
|
|
30
|
+
kind: 'awaited';
|
|
31
|
+
key: string;
|
|
32
|
+
directory: string;
|
|
33
|
+
elapsedMs: number;
|
|
34
|
+
holder: string;
|
|
35
|
+
};
|
|
36
|
+
/** What a caller is told while it does not hold the extraction itself. */
|
|
37
|
+
export type NextestExtractionEvent = {
|
|
38
|
+
kind: 'waiting';
|
|
39
|
+
holder: string;
|
|
40
|
+
} | {
|
|
41
|
+
kind: 'stale-lock';
|
|
42
|
+
holder: string;
|
|
43
|
+
ageMs: number;
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* How a lock proves its holder is alive: it rewrites the lock's mtime every
|
|
47
|
+
* `heartbeatMs`, and a lock older than `staleAfterMs` belongs to nobody. A pid
|
|
48
|
+
* cannot answer that — a holder SIGKILLed at its bound leaves a pid the system
|
|
49
|
+
* may already have handed to something else.
|
|
50
|
+
*/
|
|
51
|
+
export interface NextestExtractionTiming {
|
|
52
|
+
heartbeatMs: number;
|
|
53
|
+
staleAfterMs: number;
|
|
54
|
+
pollMs: number;
|
|
55
|
+
}
|
|
56
|
+
/** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
|
|
57
|
+
export declare function nextestArchiveKey(archive: string): Promise<string>;
|
|
58
|
+
/**
|
|
59
|
+
* One archive path's extractions live beside it, each under the hash of the
|
|
60
|
+
* bytes it came from; the lock and staging directories of a key carry that
|
|
61
|
+
* key as their prefix, so cleanup can tell them from another key's.
|
|
62
|
+
*/
|
|
63
|
+
export interface NextestExtractionPaths {
|
|
64
|
+
root: string;
|
|
65
|
+
directory: string;
|
|
66
|
+
lock: string;
|
|
67
|
+
stagingPrefix: string;
|
|
68
|
+
}
|
|
69
|
+
export declare function nextestExtractionPaths(archive: string, key: string): NextestExtractionPaths;
|
|
70
|
+
/**
|
|
71
|
+
* The directory holding `archive`'s extraction, extracting it first if no
|
|
72
|
+
* caller has yet.
|
|
73
|
+
*
|
|
74
|
+
* Correctness does not rest on the lock. An extraction lands under the hash of
|
|
75
|
+
* the bytes it came from by renaming a private staging directory into place,
|
|
76
|
+
* so a directory under a key is a complete extraction of exactly those bytes
|
|
77
|
+
* or it does not exist: concurrent extractors cannot corrupt it, the loser of
|
|
78
|
+
* a race discards its own copy, and an archive rewritten mid-extraction is
|
|
79
|
+
* caught by hashing it again before publishing. The lock only stops every
|
|
80
|
+
* runner of one Nx run from unpacking the same bytes side by side, which is the
|
|
81
|
+
* cost it exists to remove.
|
|
82
|
+
*
|
|
83
|
+
* Publishing a key removes every other key's extraction of this archive path,
|
|
84
|
+
* including stale locks and staging left by a holder that died. Leftovers of
|
|
85
|
+
* the CURRENT key are not touched: a holder that took over a stale lock may
|
|
86
|
+
* still be extracting into one.
|
|
87
|
+
*/
|
|
88
|
+
export declare function ensureNextestArchiveExtracted(archive: string, extract: NextestArchiveExtractor, onEvent: (event: NextestExtractionEvent) => void, timing?: NextestExtractionTiming): Promise<NextestExtraction>;
|
|
89
|
+
/**
|
|
90
|
+
* nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
|
|
91
|
+
* archive and executes nothing from it.
|
|
92
|
+
*
|
|
93
|
+
* `list` loads nextest configuration after extracting, and a repository's
|
|
94
|
+
* `.config/nextest.toml` may name test groups only the plugin's tool config
|
|
95
|
+
* defines, so it is given an empty config of its own instead: unpacking bytes
|
|
96
|
+
* does not depend on how tests are scheduled. `--workspace-remap .` because
|
|
97
|
+
* nextest insists the workspace root exist, and the archive's own is the
|
|
98
|
+
* producing tree's path.
|
|
99
|
+
*/
|
|
100
|
+
export declare const extractWithNextest: NextestArchiveExtractor;
|
|
101
|
+
//# sourceMappingURL=nextest-extraction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nextest-extraction.d.ts","sourceRoot":"","sources":["../src/nextest-extraction.ts"],"names":[],"mappings":"AAQA;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,6EAA6E,CAAC;AAE9G;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,WAAW,CAAC;AAC7C,eAAO,MAAM,2BAA2B,0CAA0C,CAAC;AACnF,eAAO,MAAM,wBAAwB,uCAAuC,CAAC;AAE7E,0EAA0E;AAC1E,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9F,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,MAAM,sBAAsB,GAC9B;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,WAAW,uBAAuB;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAID,gGAAgG;AAChG,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,sBAAsB,CAQ3F;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,uBAAuB,EAChC,OAAO,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChD,MAAM,GAAE,uBAAwC,GAC/C,OAAO,CAAC,iBAAiB,CAAC,CAiD5B;AA4GD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAiChC,CAAC"}
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { once } from 'node:events';
|
|
4
|
+
import { createReadStream } from 'node:fs';
|
|
5
|
+
import { mkdir, mkdtemp, open, readdir, readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
8
|
+
/**
|
|
9
|
+
* The helper a per-crate runner calls before `nextest run`, as the root
|
|
10
|
+
* `node_modules` links it. Spelled through the package link rather than this
|
|
11
|
+
* file's real location: under an isolated install that location is a store
|
|
12
|
+
* path outside the workspace, which would differ per machine in a command Nx
|
|
13
|
+
* hashes.
|
|
14
|
+
*/
|
|
15
|
+
export const NEXTEST_EXTRACT_BIN = 'node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js';
|
|
16
|
+
/**
|
|
17
|
+
* Where nextest puts things inside an extracted archive, relative to the
|
|
18
|
+
* extraction directory (nextest-runner's `reuse_build` constants). A run given
|
|
19
|
+
* these three paths reuses that build exactly as `--archive-file` would after
|
|
20
|
+
* unpacking it.
|
|
21
|
+
*/
|
|
22
|
+
export const EXTRACTED_TARGET_DIR = 'target';
|
|
23
|
+
export const EXTRACTED_BINARIES_METADATA = 'target/nextest/binaries-metadata.json';
|
|
24
|
+
export const EXTRACTED_CARGO_METADATA = 'target/nextest/cargo-metadata.json';
|
|
25
|
+
const DEFAULT_TIMING = { heartbeatMs: 1_000, staleAfterMs: 10_000, pollMs: 100 };
|
|
26
|
+
/** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
|
|
27
|
+
export async function nextestArchiveKey(archive) {
|
|
28
|
+
const hash = createHash('sha256');
|
|
29
|
+
for await (const chunk of createReadStream(archive, { highWaterMark: 1 << 20 })) {
|
|
30
|
+
hash.update(chunk);
|
|
31
|
+
}
|
|
32
|
+
return hash.digest('hex');
|
|
33
|
+
}
|
|
34
|
+
export function nextestExtractionPaths(archive, key) {
|
|
35
|
+
const root = `${archive}.extracted`;
|
|
36
|
+
return {
|
|
37
|
+
root,
|
|
38
|
+
directory: join(root, key),
|
|
39
|
+
lock: join(root, `${key}.lock`),
|
|
40
|
+
stagingPrefix: join(root, `${key}.staging-`),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The directory holding `archive`'s extraction, extracting it first if no
|
|
45
|
+
* caller has yet.
|
|
46
|
+
*
|
|
47
|
+
* Correctness does not rest on the lock. An extraction lands under the hash of
|
|
48
|
+
* the bytes it came from by renaming a private staging directory into place,
|
|
49
|
+
* so a directory under a key is a complete extraction of exactly those bytes
|
|
50
|
+
* or it does not exist: concurrent extractors cannot corrupt it, the loser of
|
|
51
|
+
* a race discards its own copy, and an archive rewritten mid-extraction is
|
|
52
|
+
* caught by hashing it again before publishing. The lock only stops every
|
|
53
|
+
* runner of one Nx run from unpacking the same bytes side by side, which is the
|
|
54
|
+
* cost it exists to remove.
|
|
55
|
+
*
|
|
56
|
+
* Publishing a key removes every other key's extraction of this archive path,
|
|
57
|
+
* including stale locks and staging left by a holder that died. Leftovers of
|
|
58
|
+
* the CURRENT key are not touched: a holder that took over a stale lock may
|
|
59
|
+
* still be extracting into one.
|
|
60
|
+
*/
|
|
61
|
+
export async function ensureNextestArchiveExtracted(archive, extract, onEvent, timing = DEFAULT_TIMING) {
|
|
62
|
+
const started = performance.now();
|
|
63
|
+
const key = await nextestArchiveKey(archive);
|
|
64
|
+
const paths = nextestExtractionPaths(archive, key);
|
|
65
|
+
if (await isDirectory(paths.directory)) {
|
|
66
|
+
return { kind: 'reused', key, directory: paths.directory };
|
|
67
|
+
}
|
|
68
|
+
await mkdir(paths.root, { recursive: true });
|
|
69
|
+
let holder;
|
|
70
|
+
const settled = () => holder === undefined
|
|
71
|
+
? { kind: 'reused', key, directory: paths.directory }
|
|
72
|
+
: { kind: 'awaited', key, directory: paths.directory, elapsedMs: performance.now() - started, holder };
|
|
73
|
+
for (;;) {
|
|
74
|
+
const lock = await acquireLock(paths.lock, timing.heartbeatMs);
|
|
75
|
+
if (lock !== undefined) {
|
|
76
|
+
try {
|
|
77
|
+
if (await isDirectory(paths.directory)) {
|
|
78
|
+
return settled();
|
|
79
|
+
}
|
|
80
|
+
const published = await extractAndPublish(archive, key, paths, extract);
|
|
81
|
+
if (published) {
|
|
82
|
+
await removeOtherKeys(paths.root, key);
|
|
83
|
+
return { kind: 'extracted', key, directory: paths.directory, elapsedMs: performance.now() - started };
|
|
84
|
+
}
|
|
85
|
+
return settled();
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
await lock.release();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (await isDirectory(paths.directory)) {
|
|
92
|
+
return settled();
|
|
93
|
+
}
|
|
94
|
+
const current = await readLock(paths.lock);
|
|
95
|
+
if (current === undefined) {
|
|
96
|
+
// Released between our attempt and this read: try again at once.
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (current.ageMs > timing.staleAfterMs) {
|
|
100
|
+
onEvent({ kind: 'stale-lock', holder: current.holder, ageMs: current.ageMs });
|
|
101
|
+
await rm(paths.lock, { force: true });
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (holder === undefined) {
|
|
105
|
+
holder = current.holder;
|
|
106
|
+
onEvent({ kind: 'waiting', holder });
|
|
107
|
+
}
|
|
108
|
+
await sleep(timing.pollMs);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Extract into a private staging directory and rename it to `paths.directory`.
|
|
113
|
+
* False when another extractor published the same key first, which leaves a
|
|
114
|
+
* complete extraction of the same bytes in place.
|
|
115
|
+
*/
|
|
116
|
+
async function extractAndPublish(archive, key, paths, extract) {
|
|
117
|
+
const staging = await mkdtemp(paths.stagingPrefix);
|
|
118
|
+
try {
|
|
119
|
+
await extract(archive, staging);
|
|
120
|
+
for (const file of [EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA]) {
|
|
121
|
+
if (!(await statOrUndefined(join(staging, file)))?.isFile()) {
|
|
122
|
+
throw new Error(`extracting ${archive} produced no ${file}; a reuse run cannot read this layout`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const after = await nextestArchiveKey(archive);
|
|
126
|
+
if (after !== key) {
|
|
127
|
+
throw new Error(`${archive} changed while it was being extracted (sha256 ${key} before, ${after} after); ` +
|
|
128
|
+
'rerun once whatever is rewriting it has finished');
|
|
129
|
+
}
|
|
130
|
+
await rename(staging, paths.directory);
|
|
131
|
+
return true;
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
await rm(staging, { recursive: true, force: true });
|
|
135
|
+
const code = errorCode(error);
|
|
136
|
+
if ((code === 'ENOTEMPTY' || code === 'EEXIST') && (await isDirectory(paths.directory))) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
async function removeOtherKeys(root, key) {
|
|
143
|
+
for (const entry of await readdir(root)) {
|
|
144
|
+
if (entry !== key && !entry.startsWith(`${key}.`)) {
|
|
145
|
+
await rm(join(root, entry), { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/** Create the lock exclusively, or report it held. A held lock is refreshed until released. */
|
|
150
|
+
async function acquireLock(path, heartbeatMs) {
|
|
151
|
+
const token = `${process.pid} ${randomUUID()}`;
|
|
152
|
+
try {
|
|
153
|
+
await writeFile(path, `${token}\n`, { flag: 'wx' });
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
if (errorCode(error) === 'EEXIST') {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
const heartbeat = setInterval(() => {
|
|
162
|
+
const now = new Date();
|
|
163
|
+
// ENOENT: a waiter judged this lock stale and removed it. The extraction
|
|
164
|
+
// carries on; publishing is atomic whoever else extracts beside it.
|
|
165
|
+
utimes(path, now, now).catch((error) => {
|
|
166
|
+
if (errorCode(error) !== 'ENOENT')
|
|
167
|
+
throw error;
|
|
168
|
+
});
|
|
169
|
+
}, heartbeatMs);
|
|
170
|
+
return {
|
|
171
|
+
async release() {
|
|
172
|
+
clearInterval(heartbeat);
|
|
173
|
+
// Remove it only while it is still ours: a waiter that took it over owns it now.
|
|
174
|
+
const content = await readFile(path, 'utf8').catch((error) => {
|
|
175
|
+
if (errorCode(error) === 'ENOENT')
|
|
176
|
+
return undefined;
|
|
177
|
+
throw error;
|
|
178
|
+
});
|
|
179
|
+
if (content?.trim() === token) {
|
|
180
|
+
await rm(path, { force: true });
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
async function readLock(path) {
|
|
186
|
+
try {
|
|
187
|
+
const handle = await open(path, 'r');
|
|
188
|
+
try {
|
|
189
|
+
const [content, metadata] = await Promise.all([handle.readFile('utf8'), handle.stat()]);
|
|
190
|
+
const pid = content.trim().split(' ')[0];
|
|
191
|
+
return {
|
|
192
|
+
// Empty while its creator is between creating and writing it.
|
|
193
|
+
holder: pid === undefined || pid.length === 0 ? 'a process that has not yet named itself' : `pid ${pid}`,
|
|
194
|
+
ageMs: Date.now() - metadata.mtimeMs,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
await handle.close();
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
if (errorCode(error) === 'ENOENT') {
|
|
203
|
+
return undefined;
|
|
204
|
+
}
|
|
205
|
+
throw error;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
|
|
210
|
+
* archive and executes nothing from it.
|
|
211
|
+
*
|
|
212
|
+
* `list` loads nextest configuration after extracting, and a repository's
|
|
213
|
+
* `.config/nextest.toml` may name test groups only the plugin's tool config
|
|
214
|
+
* defines, so it is given an empty config of its own instead: unpacking bytes
|
|
215
|
+
* does not depend on how tests are scheduled. `--workspace-remap .` because
|
|
216
|
+
* nextest insists the workspace root exist, and the archive's own is the
|
|
217
|
+
* producing tree's path.
|
|
218
|
+
*/
|
|
219
|
+
export const extractWithNextest = async (archive, destination) => {
|
|
220
|
+
const config = join(destination, 'nextest-extract.toml');
|
|
221
|
+
await writeFile(config, '');
|
|
222
|
+
try {
|
|
223
|
+
const args = [
|
|
224
|
+
'--frozen',
|
|
225
|
+
'nextest',
|
|
226
|
+
'list',
|
|
227
|
+
'--archive-file',
|
|
228
|
+
archive,
|
|
229
|
+
'--extract-to',
|
|
230
|
+
destination,
|
|
231
|
+
'--workspace-remap',
|
|
232
|
+
'.',
|
|
233
|
+
'--list-type',
|
|
234
|
+
'binaries-only',
|
|
235
|
+
'--user-config-file',
|
|
236
|
+
'none',
|
|
237
|
+
'--config-file',
|
|
238
|
+
config,
|
|
239
|
+
];
|
|
240
|
+
// The binary list on stdout is not wanted; nextest's progress and errors are on stderr.
|
|
241
|
+
const child = spawn('cargo', args, { stdio: ['ignore', 'ignore', 'inherit'] });
|
|
242
|
+
// Rejects with the spawn error when cargo cannot be started at all.
|
|
243
|
+
const [code, signal] = await once(child, 'exit');
|
|
244
|
+
if (code !== 0) {
|
|
245
|
+
throw new Error(`cargo ${args.join(' ')} ${code === null ? `died of ${String(signal)}` : `exited with status ${String(code)}`}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
finally {
|
|
249
|
+
await rm(config, { force: true });
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
async function isDirectory(path) {
|
|
253
|
+
return (await statOrUndefined(path))?.isDirectory() ?? false;
|
|
254
|
+
}
|
|
255
|
+
async function statOrUndefined(path) {
|
|
256
|
+
try {
|
|
257
|
+
return await stat(path);
|
|
258
|
+
}
|
|
259
|
+
catch (error) {
|
|
260
|
+
if (errorCode(error) === 'ENOENT') {
|
|
261
|
+
return undefined;
|
|
262
|
+
}
|
|
263
|
+
throw error;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function errorCode(error) {
|
|
267
|
+
return error instanceof Error && 'code' in error ? error.code : undefined;
|
|
268
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smoothbricks/nx-plugin",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.24",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./loader.js",
|
|
@@ -141,7 +141,8 @@
|
|
|
141
141
|
},
|
|
142
142
|
"bin": {
|
|
143
143
|
"smoo-nx-exec": "./dist/bin/smoo-nx-exec.js",
|
|
144
|
-
"smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js"
|
|
144
|
+
"smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js",
|
|
145
|
+
"smoo-nx-nextest-extract": "./dist/bin/smoo-nx-nextest-extract.js"
|
|
145
146
|
},
|
|
146
147
|
"generators": "./generators.json",
|
|
147
148
|
"executors": "./executors.json",
|
|
@@ -196,7 +197,7 @@
|
|
|
196
197
|
"yaml": "^2.9.0"
|
|
197
198
|
},
|
|
198
199
|
"devDependencies": {
|
|
199
|
-
"@smoothbricks/lmao": "0.5.
|
|
200
|
+
"@smoothbricks/lmao": "0.5.2",
|
|
200
201
|
"@smoothbricks/validation": "0.1.10",
|
|
201
202
|
"fast-check": "^4.4.0"
|
|
202
203
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { ensureNextestArchiveExtracted, extractWithNextest } from '../nextest-extraction.js';
|
|
3
|
+
|
|
4
|
+
// Prints the directory holding the archive's extraction on stdout, the only
|
|
5
|
+
// thing a runner's `$(...)` captures. Everything said to a person goes to stderr.
|
|
6
|
+
const args = process.argv.slice(2);
|
|
7
|
+
const archive = args[0];
|
|
8
|
+
if (archive === undefined || args.length !== 1 || archive.startsWith('-')) {
|
|
9
|
+
process.stderr.write('usage: smoo-nx-nextest-extract <archive.tar.zst>\n');
|
|
10
|
+
process.exit(2);
|
|
11
|
+
}
|
|
12
|
+
const say = (line: string) => process.stderr.write(`smoo-nx-nextest-extract: ${line}\n`);
|
|
13
|
+
const seconds = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
|
|
14
|
+
try {
|
|
15
|
+
const extraction = await ensureNextestArchiveExtracted(archive, extractWithNextest, (event) => {
|
|
16
|
+
switch (event.kind) {
|
|
17
|
+
case 'waiting':
|
|
18
|
+
say(`${event.holder} is extracting ${archive}; waiting for it`);
|
|
19
|
+
break;
|
|
20
|
+
case 'stale-lock':
|
|
21
|
+
say(`${event.holder} stopped refreshing its extraction lock ${seconds(event.ageMs)} ago; extracting instead`);
|
|
22
|
+
break;
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
const identity = `${archive} (sha256 ${extraction.key.slice(0, 12)})`;
|
|
26
|
+
switch (extraction.kind) {
|
|
27
|
+
case 'reused':
|
|
28
|
+
say(`running ${identity} from ${extraction.directory}`);
|
|
29
|
+
break;
|
|
30
|
+
case 'extracted':
|
|
31
|
+
say(`extracted ${identity} to ${extraction.directory} in ${seconds(extraction.elapsedMs)}`);
|
|
32
|
+
break;
|
|
33
|
+
case 'awaited':
|
|
34
|
+
say(
|
|
35
|
+
`running ${identity} from ${extraction.directory} after ${seconds(extraction.elapsedMs)} waiting on ${extraction.holder}`,
|
|
36
|
+
);
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
process.stdout.write(`${extraction.directory}\n`);
|
|
40
|
+
} catch (error) {
|
|
41
|
+
say(error instanceof Error ? error.message : String(error));
|
|
42
|
+
process.exit(1);
|
|
43
|
+
}
|
|
@@ -564,7 +564,10 @@ describe('cargo-test reachability policy', () => {
|
|
|
564
564
|
'cargo-test-rusty-core',
|
|
565
565
|
{
|
|
566
566
|
command:
|
|
567
|
-
"
|
|
567
|
+
'extracted="$(node ../../node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js target/nextest/archive.tar.zst)" && ' +
|
|
568
|
+
'cargo --frozen nextest run --binaries-metadata "$extracted/target/nextest/binaries-metadata.json" ' +
|
|
569
|
+
'--cargo-metadata "$extracted/target/nextest/cargo-metadata.json" --target-dir-remap "$extracted/target" ' +
|
|
570
|
+
"--workspace-remap . -E 'package(rusty-core)'",
|
|
568
571
|
},
|
|
569
572
|
],
|
|
570
573
|
]),
|