@smoothbricks/nx-plugin 0.4.23 → 0.4.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -273,18 +273,22 @@ Concrete targets come from concrete files:
273
273
  `{workspaceRoot}`: Nx resolves a `{projectRoot}` glob against only the files its project owns, and a nested package
274
274
  owns the crates under it, so a root workspace's archive spelled that way hashes none of its members and replays stale
275
275
  binaries against edited crates. Each member crate gets a cached `cargo-test-<package>` run (30s per-test timeout like
276
- `bun test --timeout=30000`) whose inputs include that crate and its path dependencies, and whose command is
277
- `cargo --frozen nextest run --archive-file <archive> --workspace-remap . -E 'package(<crate>)'`. Runners therefore
278
- execute rather than build: they extract binaries into their own temporary directory, write nothing to cargo's flocked
279
- `target/`, and fan out instead of chaining. `--workspace-remap` is required — an archive records the producing tree's
280
- absolute paths, and without it a restored archive hands tests another checkout's `CARGO_MANIFEST_DIR`. With it the
281
- archive is relocatable, so one cache entry serves every checkout of the same commit; nextest re-points
282
- `CARGO_BIN_EXE_<name>`/`NEXTEST_BIN_EXE_<name>` at the extracted binaries at runtime, but a test that reads them
283
- through the compile-time `env!` macro keeps the producing tree's path. Per-crate runners accept an empty nextest
284
- selection because a valid workspace member may have no tests and a hash partition may legitimately be empty.
285
- `napi-debug` stays behind `cargo-test-compile`, and a crate in the project that builds the debug cdylib runs after it.
286
- A crate declaring `[package.metadata.smoothbricks.wasm-bindgen]` also receives the cacheable `cargo-wasm` output
287
- target in its owning project.
276
+ `bun test --timeout=30000`) whose inputs include that crate and its path dependencies, and whose command runs
277
+ `cargo --frozen nextest run --binaries-metadata … --cargo-metadata … --target-dir-remap … --workspace-remap . -E 'package(<crate>)'`
278
+ from the archive's one extraction. Runners therefore execute rather than build: they write nothing to cargo's build
279
+ tree, and fan out instead of chaining. They do not unpack the archive each: before the run, `smoo-nx-nextest-extract`
280
+ hashes it and extracts it into `target/nextest/archive.tar.zst.extracted/<sha256>` unless a runner already did — one
281
+ extracts, the others wait — so a workspace with ~200 test binaries and ~25 crates unpacks them once per archive
282
+ instead of once per crate. The directory is keyed by the archive's bytes, so a rebuilt or cache-restored archive never
283
+ runs an older extraction's binaries, and publishing one removes the others; it is never an Nx output.
284
+ `--workspace-remap` is required — an archive records the producing tree's absolute paths, and without it a restored
285
+ archive hands tests another checkout's `CARGO_MANIFEST_DIR`. With it the archive is relocatable, so one cache entry
286
+ serves every checkout of the same commit; nextest re-points `CARGO_BIN_EXE_<name>`/`NEXTEST_BIN_EXE_<name>` at the
287
+ extracted binaries at runtime, but a test that reads them through the compile-time `env!` macro keeps the producing
288
+ tree's path. Per-crate runners accept an empty nextest selection because a valid workspace member may have no tests
289
+ and a hash partition may legitimately be empty. `napi-debug` stays behind `cargo-test-compile`, and a crate in the
290
+ project that builds the debug cdylib runs after it. A crate declaring `[package.metadata.smoothbricks.wasm-bindgen]`
291
+ also receives the cacheable `cargo-wasm` output target in its owning project.
288
292
  - The plugin's own `nextest.toml` is passed as `--tool-config-file "smoo:$PWD/<path>"`, so it is a layer BENEATH the
289
293
  repository's `<cargo-workspace>/.config/nextest.toml` rather than a replacement for it: a repository can raise a
290
294
  timeout, add a test group, or declare `archive.include` for a cdylib or fixture the archived test binaries need, and
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env node
2
+ export {};
3
+ //# sourceMappingURL=smoo-nx-nextest-extract.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"smoo-nx-nextest-extract.d.ts","sourceRoot":"","sources":["../../src/bin/smoo-nx-nextest-extract.ts"],"names":[],"mappings":""}
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env node
2
+ import { ensureNextestArchiveExtracted, extractWithNextest } from '../nextest-extraction.js';
3
+ // Prints the directory holding the archive's extraction on stdout, the only
4
+ // thing a runner's `$(...)` captures. Everything said to a person goes to stderr.
5
+ const args = process.argv.slice(2);
6
+ const archive = args[0];
7
+ if (archive === undefined || args.length !== 1 || archive.startsWith('-')) {
8
+ process.stderr.write('usage: smoo-nx-nextest-extract <archive.tar.zst>\n');
9
+ process.exit(2);
10
+ }
11
+ const say = (line) => process.stderr.write(`smoo-nx-nextest-extract: ${line}\n`);
12
+ const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
13
+ try {
14
+ const extraction = await ensureNextestArchiveExtracted(archive, extractWithNextest, (event) => {
15
+ switch (event.kind) {
16
+ case 'waiting':
17
+ say(`${event.holder} is extracting ${archive}; waiting for it`);
18
+ break;
19
+ case 'stale-lock':
20
+ say(`${event.holder} stopped refreshing its extraction lock ${seconds(event.ageMs)} ago; extracting instead`);
21
+ break;
22
+ }
23
+ });
24
+ const identity = `${archive} (sha256 ${extraction.key.slice(0, 12)})`;
25
+ switch (extraction.kind) {
26
+ case 'reused':
27
+ say(`running ${identity} from ${extraction.directory}`);
28
+ break;
29
+ case 'extracted':
30
+ say(`extracted ${identity} to ${extraction.directory} in ${seconds(extraction.elapsedMs)}`);
31
+ break;
32
+ case 'awaited':
33
+ say(`running ${identity} from ${extraction.directory} after ${seconds(extraction.elapsedMs)} waiting on ${extraction.holder}`);
34
+ break;
35
+ }
36
+ process.stdout.write(`${extraction.directory}\n`);
37
+ }
38
+ catch (error) {
39
+ say(error instanceof Error ? error.message : String(error));
40
+ process.exit(1);
41
+ }
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,KAAK,aAAa,EAGnB,MAAM,0BAA0B,CAAC;AAalC,OAAO,EAKL,yBAAyB,EAa1B,MAAM,sBAAsB,CAAC;AAsB9B,OAAO,EAAE,yBAAyB,EAAE,CAAC;AAErC,gEAAgE;AAChE,eAAO,MAAM,aAAa,WAAW,CAAC;AACtC,uEAAuE;AACvE,eAAO,MAAM,mBAAmB,iBAAiB,CAAC;AAqQlD,KAAK,gBAAgB,GAAG,OAAO,GAAG,KAAK,CAAC;AACxC,KAAK,gBAAgB,GAAG,OAAO,GAAG,OAAO,CAAC;AAE1C,UAAU,YAAY;IACpB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,YAAY,EAAE,gBAAgB,CAAC;CAChC;AAmID,UAAU,oBAAqB,SAAQ,YAAY;IACjD,UAAU,EAAE,MAAM,CAAC;CACpB;AA0CD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAEhF;AA+XD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAEvG;AAED,eAAO,MAAM,aAAa,EAAE,aAG3B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAGL,KAAK,aAAa,EAGnB,MAAM,0BAA0B,CAAC;AAalC,OAAO,EAKL,yBAAyB,EAa1B,MAAM,sBAAsB,CAAC;AA4B9B,OAAO,EAAE,yBAAyB,EAAE,CAAC;AAErC,gEAAgE;AAChE,eAAO,MAAM,aAAa,WAAW,CAAC;AACtC,uEAAuE;AACvE,eAAO,MAAM,mBAAmB,iBAAiB,CAAC;AAqQlD,KAAK,gBAAgB,GAAG,OAAO,GAAG,KAAK,CAAC;AACxC,KAAK,gBAAgB,GAAG,OAAO,GAAG,OAAO,CAAC;AAE1C,UAAU,YAAY;IACpB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,YAAY,EAAE,gBAAgB,CAAC;CAChC;AAmID,UAAU,oBAAqB,SAAQ,YAAY;IACjD,UAAU,EAAE,MAAM,CAAC;CACpB;AA0CD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,oBAAoB,GAAG,MAAM,CAEhF;AA+XD,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAEvG;AAED,eAAO,MAAM,aAAa,EAAE,aAG3B,CAAC"}
package/dist/index.js CHANGED
@@ -12,6 +12,7 @@ import { CARGO_TOOLCHAIN_NAMED_INPUT, CARGO_TOOLCHAIN_PIN_INPUTS } from './cargo
12
12
  import { attributeCargoWorkspacePackages, CARGO_TEST_ARCHIVE_FILE, CARGO_TEST_ARCHIVE_TARGET, CARGO_TEST_COMPILE_TARGET, CARGO_TEST_EXCEPTIONS_SUFFIX, CARGO_TEST_TARGET, cargoPackageTestInputs, cargoTestPackageTargetName, createCargoInputsCache, exceptionalTestFilter, listCargoCrossTestTargets, listCargoWorkspacePackages, NEXTEST_REPO_CONFIG_PATH, nextestToolConfigArg, } from './cargo-workspace.js';
13
13
  import { CARGO_CROSS_LINT_COMMAND, CARGO_CROSS_LINT_TARGET, CARGO_FETCH_COMMAND, CARGO_FETCH_TARGET, CARGO_FROZEN_PREFIX, CARGO_LINT_CLIPPY_COMMAND, cargoCrossTestArchiveFile, cargoCrossTestArchiveTargetName, cargoCrossTestTargetName, cargoFrozen, } from './cross-check-policy.js';
14
14
  import { hashVersionlessCrateManifests } from './manifest-hash.js';
15
+ import { EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA, EXTRACTED_TARGET_DIR, NEXTEST_EXTRACT_BIN, } from './nextest-extraction.js';
15
16
  import { isNonSourceDirectory } from './source-directories.js';
16
17
  import { isBuildOutputTargetName, isTestRunnerTargetName, PLATFORM_TARGET_GLOBS, RELEASE_CONFIGURATION, } from './workspace-config-policy.js';
17
18
  export { CARGO_TEST_COMPILE_TARGET };
@@ -502,8 +503,8 @@ function createCargoTestCompileTarget(projectRoot) {
502
503
  * build with a filter: 32 such tasks on one repository each re-entered cargo,
503
504
  * re-resolved features and chained behind one another so the flocked `target/`
504
505
  * had a single writer. Reading from an archive removes the build from the run
505
- * entirely — the runner extracts binaries into its own temporary directory and
506
- * touches no shared state — which is what lets the runners fan out.
506
+ * entirely — the runners execute one extraction of it and never write cargo's
507
+ * `target/` build tree — which is what lets the runners fan out.
507
508
  *
508
509
  * Cached, unlike `cargo-test-compile`, because it produces a VALUE: one file,
509
510
  * not a mutable build tree Nx would have to restore. `--workspace` matches the
@@ -1962,16 +1963,34 @@ async function resolveCargoWorkspaces(packageJsonPaths, workspaceRoot) {
1962
1963
  * One bounded target per crate; a crate that declares `smoothbricks.test.shards`
1963
1964
  * gets one per shard plus one for the tests nextest.toml singles out.
1964
1965
  *
1965
- * Every piece EXECUTES, it never builds: `--archive-file` runs the binaries
1966
- * `cargo-test-archive` compiled once for the whole workspace, extracted into
1967
- * the runner's own temporary directory. That is what removed the two costs the
1968
- * old shape could not avoid. `nextest run --workspace -E 'package(X)'` re-entered
1969
- * cargo once per crate, and the `-p X` form re-resolved that crate's features so
1970
- * cargo rebuilt the divergent half inside the bounded window — measured at 57.9s
1971
- * of a 120s budget on a hosted 3-core macOS runner, killing the run with 264
1972
- * tests still to go. Neither form is reachable from an archive: nextest rejects
1973
- * `--workspace` and `-p` beside `--archive-file`, so the filterset is the only
1974
- * selector left, and it selects the same tests it always did.
1966
+ * Every piece EXECUTES, it never builds: it runs the binaries
1967
+ * `cargo-test-archive` compiled once for the whole workspace. That is what
1968
+ * removed the two costs the old shape could not avoid. `nextest run --workspace
1969
+ * -E 'package(X)'` re-entered cargo once per crate, and the `-p X` form
1970
+ * re-resolved that crate's features so cargo rebuilt the divergent half inside
1971
+ * the bounded window — measured at 57.9s of a 120s budget on a hosted 3-core
1972
+ * macOS runner, killing the run with 264 tests still to go. Neither form is
1973
+ * reachable from a reused build: nextest rejects `--workspace` and `-p` beside
1974
+ * it, so the filterset is the only selector left, and it selects the same tests
1975
+ * it always did.
1976
+ *
1977
+ * The archive is unpacked ONCE per content, not once per piece. `nextest run
1978
+ * --archive-file` extracts the whole archive into a temporary directory of its
1979
+ * own on every run, so N pieces paid N extractions of every binary in the
1980
+ * workspace to run 1/N of the tests each: on a workspace with ~200 test
1981
+ * binaries, ~25 runners extracting side by side on a loaded host all hit the
1982
+ * 120s bound still "Extracting". `smoo-nx-nextest-extract` hashes the archive,
1983
+ * unpacks it under that hash unless a previous runner already did — one
1984
+ * extracts while the rest wait — and prints the directory; the run then reuses
1985
+ * it through nextest's own reuse-build flags, which is what `--archive-file`
1986
+ * does after unpacking. Keyed by the archive's bytes, so a rebuilt or restored
1987
+ * archive can never run an older extraction's binaries. The one difference
1988
+ * from `--archive-file` is the Rust standard library directory: nextest has no
1989
+ * flag to remap it, so a run uses the producing toolchain's own sysroot rather
1990
+ * than the archived copy of it — the same toolchain, because the archive's hash
1991
+ * covers the toolchain pin. The extraction lives under `target/nextest`, never
1992
+ * in an Nx output: it is a multi-GB tree for a cached verdict that has no
1993
+ * outputs.
1975
1994
  *
1976
1995
  * `--workspace-remap .` is not cosmetic. An archive records the ABSOLUTE paths
1977
1996
  * of the tree that produced it; without the remap, a restored archive hands
@@ -2004,12 +2023,17 @@ async function resolveCargoWorkspaces(packageJsonPaths, workspaceRoot) {
2004
2023
  * making valid empty crates and shards fail execution.
2005
2024
  *
2006
2025
  * Pieces fan out rather than chain. The chain existed because every runner was
2007
- * a cargo writer on one flocked `target/`; an archive run writes nothing there,
2008
- * so the only serialization left is the one the machine's own scheduler does.
2026
+ * a cargo writer on one flocked `target/`; a reused-build run writes nothing
2027
+ * there, so the only serialization left is the one extraction per archive.
2009
2028
  */
2010
2029
  async function addCargoTestTargets(targets, currentProjectName, projectRoot, workspaceRoot, workspace) {
2011
2030
  const targetNames = [];
2012
2031
  const toolConfig = nextestToolConfigArg(workspaceRoot, workspace.projectRoot, PLUGIN_NEXTEST_CONFIG);
2032
+ // Relative to the runner's cwd, so the hashed command is the same in every checkout.
2033
+ const extractBin = posix.join(posix.relative(workspace.projectRoot, '.'), NEXTEST_EXTRACT_BIN);
2034
+ const reuseExtraction = `--binaries-metadata "$extracted/${EXTRACTED_BINARIES_METADATA}" ` +
2035
+ `--cargo-metadata "$extracted/${EXTRACTED_CARGO_METADATA}" ` +
2036
+ `--target-dir-remap "$extracted/${EXTRACTED_TARGET_DIR}"`;
2013
2037
  for (const plan of workspace.packages) {
2014
2038
  if (plan.package.projectRoot !== projectRoot) {
2015
2039
  continue;
@@ -2052,7 +2076,8 @@ async function addCargoTestTargets(targets, currentProjectName, projectRoot, wor
2052
2076
  inputs,
2053
2077
  dependsOn: [archive],
2054
2078
  options: {
2055
- command: cargoFrozen(`nextest run --archive-file ${CARGO_TEST_ARCHIVE_FILE} --workspace-remap . -E '${piece.selector}'${piece.extra} --no-tests=pass --user-config-file none ${toolConfig}`),
2079
+ command: `extracted="$(node ${extractBin} ${CARGO_TEST_ARCHIVE_FILE})" && ` +
2080
+ cargoFrozen(`nextest run ${reuseExtraction} --workspace-remap . -E '${piece.selector}'${piece.extra} --no-tests=pass --user-config-file none ${toolConfig}`),
2056
2081
  cwd: workspace.projectRoot,
2057
2082
  timeoutMs: BOUNDED_TEST_TIMEOUT_MS,
2058
2083
  killAfterMs: BOUNDED_TEST_KILL_AFTER_MS,
@@ -0,0 +1,101 @@
1
+ /**
2
+ * The helper a per-crate runner calls before `nextest run`, as the root
3
+ * `node_modules` links it. Spelled through the package link rather than this
4
+ * file's real location: under an isolated install that location is a store
5
+ * path outside the workspace, which would differ per machine in a command Nx
6
+ * hashes.
7
+ */
8
+ export declare const NEXTEST_EXTRACT_BIN = "node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js";
9
+ /**
10
+ * Where nextest puts things inside an extracted archive, relative to the
11
+ * extraction directory (nextest-runner's `reuse_build` constants). A run given
12
+ * these three paths reuses that build exactly as `--archive-file` would after
13
+ * unpacking it.
14
+ */
15
+ export declare const EXTRACTED_TARGET_DIR = "target";
16
+ export declare const EXTRACTED_BINARIES_METADATA = "target/nextest/binaries-metadata.json";
17
+ export declare const EXTRACTED_CARGO_METADATA = "target/nextest/cargo-metadata.json";
18
+ /** Unpacks `archive` into the empty, existing directory `destination`. */
19
+ export type NextestArchiveExtractor = (archive: string, destination: string) => Promise<void>;
20
+ export type NextestExtraction = {
21
+ kind: 'reused';
22
+ key: string;
23
+ directory: string;
24
+ } | {
25
+ kind: 'extracted';
26
+ key: string;
27
+ directory: string;
28
+ elapsedMs: number;
29
+ } | {
30
+ kind: 'awaited';
31
+ key: string;
32
+ directory: string;
33
+ elapsedMs: number;
34
+ holder: string;
35
+ };
36
+ /** What a caller is told while it does not hold the extraction itself. */
37
+ export type NextestExtractionEvent = {
38
+ kind: 'waiting';
39
+ holder: string;
40
+ } | {
41
+ kind: 'stale-lock';
42
+ holder: string;
43
+ ageMs: number;
44
+ };
45
+ /**
46
+ * How a lock proves its holder is alive: it rewrites the lock's mtime every
47
+ * `heartbeatMs`, and a lock older than `staleAfterMs` belongs to nobody. A pid
48
+ * cannot answer that — a holder SIGKILLed at its bound leaves a pid the system
49
+ * may already have handed to something else.
50
+ */
51
+ export interface NextestExtractionTiming {
52
+ heartbeatMs: number;
53
+ staleAfterMs: number;
54
+ pollMs: number;
55
+ }
56
+ /** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
57
+ export declare function nextestArchiveKey(archive: string): Promise<string>;
58
+ /**
59
+ * One archive path's extractions live beside it, each under the hash of the
60
+ * bytes it came from; the lock and staging directories of a key carry that
61
+ * key as their prefix, so cleanup can tell them from another key's.
62
+ */
63
+ export interface NextestExtractionPaths {
64
+ root: string;
65
+ directory: string;
66
+ lock: string;
67
+ stagingPrefix: string;
68
+ }
69
+ export declare function nextestExtractionPaths(archive: string, key: string): NextestExtractionPaths;
70
+ /**
71
+ * The directory holding `archive`'s extraction, extracting it first if no
72
+ * caller has yet.
73
+ *
74
+ * Correctness does not rest on the lock. An extraction lands under the hash of
75
+ * the bytes it came from by renaming a private staging directory into place,
76
+ * so a directory under a key is a complete extraction of exactly those bytes
77
+ * or it does not exist: concurrent extractors cannot corrupt it, the loser of
78
+ * a race discards its own copy, and an archive rewritten mid-extraction is
79
+ * caught by hashing it again before publishing. The lock only stops every
80
+ * runner of one Nx run from unpacking the same bytes side by side, which is the
81
+ * cost it exists to remove.
82
+ *
83
+ * Publishing a key removes every other key's extraction of this archive path,
84
+ * including stale locks and staging left by a holder that died. Leftovers of
85
+ * the CURRENT key are not touched: a holder that took over a stale lock may
86
+ * still be extracting into one.
87
+ */
88
+ export declare function ensureNextestArchiveExtracted(archive: string, extract: NextestArchiveExtractor, onEvent: (event: NextestExtractionEvent) => void, timing?: NextestExtractionTiming): Promise<NextestExtraction>;
89
+ /**
90
+ * nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
91
+ * archive and executes nothing from it.
92
+ *
93
+ * `list` loads nextest configuration after extracting, and a repository's
94
+ * `.config/nextest.toml` may name test groups only the plugin's tool config
95
+ * defines, so it is given an empty config of its own instead: unpacking bytes
96
+ * does not depend on how tests are scheduled. `--workspace-remap .` because
97
+ * nextest insists the workspace root exist, and the archive's own is the
98
+ * producing tree's path.
99
+ */
100
+ export declare const extractWithNextest: NextestArchiveExtractor;
101
+ //# sourceMappingURL=nextest-extraction.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"nextest-extraction.d.ts","sourceRoot":"","sources":["../src/nextest-extraction.ts"],"names":[],"mappings":"AAQA;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,6EAA6E,CAAC;AAE9G;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,WAAW,CAAC;AAC7C,eAAO,MAAM,2BAA2B,0CAA0C,CAAC;AACnF,eAAO,MAAM,wBAAwB,uCAAuC,CAAC;AAE7E,0EAA0E;AAC1E,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9F,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,MAAM,sBAAsB,GAC9B;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,WAAW,uBAAuB;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAID,gGAAgG;AAChG,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMxE;AAED;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,sBAAsB,CAQ3F;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,uBAAuB,EAChC,OAAO,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChD,MAAM,GAAE,uBAAwC,GAC/C,OAAO,CAAC,iBAAiB,CAAC,CAiD5B;AA4GD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAiChC,CAAC"}
@@ -0,0 +1,268 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
+ import { once } from 'node:events';
4
+ import { createReadStream } from 'node:fs';
5
+ import { mkdir, mkdtemp, open, readdir, readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
6
+ import { join } from 'node:path';
7
+ import { setTimeout as sleep } from 'node:timers/promises';
8
+ /**
9
+ * The helper a per-crate runner calls before `nextest run`, as the root
10
+ * `node_modules` links it. Spelled through the package link rather than this
11
+ * file's real location: under an isolated install that location is a store
12
+ * path outside the workspace, which would differ per machine in a command Nx
13
+ * hashes.
14
+ */
15
+ export const NEXTEST_EXTRACT_BIN = 'node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js';
16
+ /**
17
+ * Where nextest puts things inside an extracted archive, relative to the
18
+ * extraction directory (nextest-runner's `reuse_build` constants). A run given
19
+ * these three paths reuses that build exactly as `--archive-file` would after
20
+ * unpacking it.
21
+ */
22
+ export const EXTRACTED_TARGET_DIR = 'target';
23
+ export const EXTRACTED_BINARIES_METADATA = 'target/nextest/binaries-metadata.json';
24
+ export const EXTRACTED_CARGO_METADATA = 'target/nextest/cargo-metadata.json';
25
+ const DEFAULT_TIMING = { heartbeatMs: 1_000, staleAfterMs: 10_000, pollMs: 100 };
26
+ /** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
27
+ export async function nextestArchiveKey(archive) {
28
+ const hash = createHash('sha256');
29
+ for await (const chunk of createReadStream(archive, { highWaterMark: 1 << 20 })) {
30
+ hash.update(chunk);
31
+ }
32
+ return hash.digest('hex');
33
+ }
34
+ export function nextestExtractionPaths(archive, key) {
35
+ const root = `${archive}.extracted`;
36
+ return {
37
+ root,
38
+ directory: join(root, key),
39
+ lock: join(root, `${key}.lock`),
40
+ stagingPrefix: join(root, `${key}.staging-`),
41
+ };
42
+ }
43
+ /**
44
+ * The directory holding `archive`'s extraction, extracting it first if no
45
+ * caller has yet.
46
+ *
47
+ * Correctness does not rest on the lock. An extraction lands under the hash of
48
+ * the bytes it came from by renaming a private staging directory into place,
49
+ * so a directory under a key is a complete extraction of exactly those bytes
50
+ * or it does not exist: concurrent extractors cannot corrupt it, the loser of
51
+ * a race discards its own copy, and an archive rewritten mid-extraction is
52
+ * caught by hashing it again before publishing. The lock only stops every
53
+ * runner of one Nx run from unpacking the same bytes side by side, which is the
54
+ * cost it exists to remove.
55
+ *
56
+ * Publishing a key removes every other key's extraction of this archive path,
57
+ * including stale locks and staging left by a holder that died. Leftovers of
58
+ * the CURRENT key are not touched: a holder that took over a stale lock may
59
+ * still be extracting into one.
60
+ */
61
+ export async function ensureNextestArchiveExtracted(archive, extract, onEvent, timing = DEFAULT_TIMING) {
62
+ const started = performance.now();
63
+ const key = await nextestArchiveKey(archive);
64
+ const paths = nextestExtractionPaths(archive, key);
65
+ if (await isDirectory(paths.directory)) {
66
+ return { kind: 'reused', key, directory: paths.directory };
67
+ }
68
+ await mkdir(paths.root, { recursive: true });
69
+ let holder;
70
+ const settled = () => holder === undefined
71
+ ? { kind: 'reused', key, directory: paths.directory }
72
+ : { kind: 'awaited', key, directory: paths.directory, elapsedMs: performance.now() - started, holder };
73
+ for (;;) {
74
+ const lock = await acquireLock(paths.lock, timing.heartbeatMs);
75
+ if (lock !== undefined) {
76
+ try {
77
+ if (await isDirectory(paths.directory)) {
78
+ return settled();
79
+ }
80
+ const published = await extractAndPublish(archive, key, paths, extract);
81
+ if (published) {
82
+ await removeOtherKeys(paths.root, key);
83
+ return { kind: 'extracted', key, directory: paths.directory, elapsedMs: performance.now() - started };
84
+ }
85
+ return settled();
86
+ }
87
+ finally {
88
+ await lock.release();
89
+ }
90
+ }
91
+ if (await isDirectory(paths.directory)) {
92
+ return settled();
93
+ }
94
+ const current = await readLock(paths.lock);
95
+ if (current === undefined) {
96
+ // Released between our attempt and this read: try again at once.
97
+ continue;
98
+ }
99
+ if (current.ageMs > timing.staleAfterMs) {
100
+ onEvent({ kind: 'stale-lock', holder: current.holder, ageMs: current.ageMs });
101
+ await rm(paths.lock, { force: true });
102
+ continue;
103
+ }
104
+ if (holder === undefined) {
105
+ holder = current.holder;
106
+ onEvent({ kind: 'waiting', holder });
107
+ }
108
+ await sleep(timing.pollMs);
109
+ }
110
+ }
111
+ /**
112
+ * Extract into a private staging directory and rename it to `paths.directory`.
113
+ * False when another extractor published the same key first, which leaves a
114
+ * complete extraction of the same bytes in place.
115
+ */
116
+ async function extractAndPublish(archive, key, paths, extract) {
117
+ const staging = await mkdtemp(paths.stagingPrefix);
118
+ try {
119
+ await extract(archive, staging);
120
+ for (const file of [EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA]) {
121
+ if (!(await statOrUndefined(join(staging, file)))?.isFile()) {
122
+ throw new Error(`extracting ${archive} produced no ${file}; a reuse run cannot read this layout`);
123
+ }
124
+ }
125
+ const after = await nextestArchiveKey(archive);
126
+ if (after !== key) {
127
+ throw new Error(`${archive} changed while it was being extracted (sha256 ${key} before, ${after} after); ` +
128
+ 'rerun once whatever is rewriting it has finished');
129
+ }
130
+ await rename(staging, paths.directory);
131
+ return true;
132
+ }
133
+ catch (error) {
134
+ await rm(staging, { recursive: true, force: true });
135
+ const code = errorCode(error);
136
+ if ((code === 'ENOTEMPTY' || code === 'EEXIST') && (await isDirectory(paths.directory))) {
137
+ return false;
138
+ }
139
+ throw error;
140
+ }
141
+ }
142
+ async function removeOtherKeys(root, key) {
143
+ for (const entry of await readdir(root)) {
144
+ if (entry !== key && !entry.startsWith(`${key}.`)) {
145
+ await rm(join(root, entry), { recursive: true, force: true });
146
+ }
147
+ }
148
+ }
149
+ /** Create the lock exclusively, or report it held. A held lock is refreshed until released. */
150
+ async function acquireLock(path, heartbeatMs) {
151
+ const token = `${process.pid} ${randomUUID()}`;
152
+ try {
153
+ await writeFile(path, `${token}\n`, { flag: 'wx' });
154
+ }
155
+ catch (error) {
156
+ if (errorCode(error) === 'EEXIST') {
157
+ return undefined;
158
+ }
159
+ throw error;
160
+ }
161
+ const heartbeat = setInterval(() => {
162
+ const now = new Date();
163
+ // ENOENT: a waiter judged this lock stale and removed it. The extraction
164
+ // carries on; publishing is atomic whoever else extracts beside it.
165
+ utimes(path, now, now).catch((error) => {
166
+ if (errorCode(error) !== 'ENOENT')
167
+ throw error;
168
+ });
169
+ }, heartbeatMs);
170
+ return {
171
+ async release() {
172
+ clearInterval(heartbeat);
173
+ // Remove it only while it is still ours: a waiter that took it over owns it now.
174
+ const content = await readFile(path, 'utf8').catch((error) => {
175
+ if (errorCode(error) === 'ENOENT')
176
+ return undefined;
177
+ throw error;
178
+ });
179
+ if (content?.trim() === token) {
180
+ await rm(path, { force: true });
181
+ }
182
+ },
183
+ };
184
+ }
185
+ async function readLock(path) {
186
+ try {
187
+ const handle = await open(path, 'r');
188
+ try {
189
+ const [content, metadata] = await Promise.all([handle.readFile('utf8'), handle.stat()]);
190
+ const pid = content.trim().split(' ')[0];
191
+ return {
192
+ // Empty while its creator is between creating and writing it.
193
+ holder: pid === undefined || pid.length === 0 ? 'a process that has not yet named itself' : `pid ${pid}`,
194
+ ageMs: Date.now() - metadata.mtimeMs,
195
+ };
196
+ }
197
+ finally {
198
+ await handle.close();
199
+ }
200
+ }
201
+ catch (error) {
202
+ if (errorCode(error) === 'ENOENT') {
203
+ return undefined;
204
+ }
205
+ throw error;
206
+ }
207
+ }
208
+ /**
209
+ * nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
210
+ * archive and executes nothing from it.
211
+ *
212
+ * `list` loads nextest configuration after extracting, and a repository's
213
+ * `.config/nextest.toml` may name test groups only the plugin's tool config
214
+ * defines, so it is given an empty config of its own instead: unpacking bytes
215
+ * does not depend on how tests are scheduled. `--workspace-remap .` because
216
+ * nextest insists the workspace root exist, and the archive's own is the
217
+ * producing tree's path.
218
+ */
219
+ export const extractWithNextest = async (archive, destination) => {
220
+ const config = join(destination, 'nextest-extract.toml');
221
+ await writeFile(config, '');
222
+ try {
223
+ const args = [
224
+ '--frozen',
225
+ 'nextest',
226
+ 'list',
227
+ '--archive-file',
228
+ archive,
229
+ '--extract-to',
230
+ destination,
231
+ '--workspace-remap',
232
+ '.',
233
+ '--list-type',
234
+ 'binaries-only',
235
+ '--user-config-file',
236
+ 'none',
237
+ '--config-file',
238
+ config,
239
+ ];
240
+ // The binary list on stdout is not wanted; nextest's progress and errors are on stderr.
241
+ const child = spawn('cargo', args, { stdio: ['ignore', 'ignore', 'inherit'] });
242
+ // Rejects with the spawn error when cargo cannot be started at all.
243
+ const [code, signal] = await once(child, 'exit');
244
+ if (code !== 0) {
245
+ throw new Error(`cargo ${args.join(' ')} ${code === null ? `died of ${String(signal)}` : `exited with status ${String(code)}`}`);
246
+ }
247
+ }
248
+ finally {
249
+ await rm(config, { force: true });
250
+ }
251
+ };
252
+ async function isDirectory(path) {
253
+ return (await statOrUndefined(path))?.isDirectory() ?? false;
254
+ }
255
+ async function statOrUndefined(path) {
256
+ try {
257
+ return await stat(path);
258
+ }
259
+ catch (error) {
260
+ if (errorCode(error) === 'ENOENT') {
261
+ return undefined;
262
+ }
263
+ throw error;
264
+ }
265
+ }
266
+ function errorCode(error) {
267
+ return error instanceof Error && 'code' in error ? error.code : undefined;
268
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@smoothbricks/nx-plugin",
3
- "version": "0.4.23",
3
+ "version": "0.4.24",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "main": "./loader.js",
@@ -141,7 +141,8 @@
141
141
  },
142
142
  "bin": {
143
143
  "smoo-nx-exec": "./dist/bin/smoo-nx-exec.js",
144
- "smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js"
144
+ "smoo-nx-cargo-hash": "./dist/bin/smoo-nx-cargo-hash.js",
145
+ "smoo-nx-nextest-extract": "./dist/bin/smoo-nx-nextest-extract.js"
145
146
  },
146
147
  "generators": "./generators.json",
147
148
  "executors": "./executors.json",
@@ -196,7 +197,7 @@
196
197
  "yaml": "^2.9.0"
197
198
  },
198
199
  "devDependencies": {
199
- "@smoothbricks/lmao": "0.5.1",
200
+ "@smoothbricks/lmao": "0.5.2",
200
201
  "@smoothbricks/validation": "0.1.10",
201
202
  "fast-check": "^4.4.0"
202
203
  }
@@ -0,0 +1,43 @@
1
+ #!/usr/bin/env node
2
+ import { ensureNextestArchiveExtracted, extractWithNextest } from '../nextest-extraction.js';
3
+
4
+ // Prints the directory holding the archive's extraction on stdout, the only
5
+ // thing a runner's `$(...)` captures. Everything said to a person goes to stderr.
6
+ const args = process.argv.slice(2);
7
+ const archive = args[0];
8
+ if (archive === undefined || args.length !== 1 || archive.startsWith('-')) {
9
+ process.stderr.write('usage: smoo-nx-nextest-extract <archive.tar.zst>\n');
10
+ process.exit(2);
11
+ }
12
+ const say = (line: string) => process.stderr.write(`smoo-nx-nextest-extract: ${line}\n`);
13
+ const seconds = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
14
+ try {
15
+ const extraction = await ensureNextestArchiveExtracted(archive, extractWithNextest, (event) => {
16
+ switch (event.kind) {
17
+ case 'waiting':
18
+ say(`${event.holder} is extracting ${archive}; waiting for it`);
19
+ break;
20
+ case 'stale-lock':
21
+ say(`${event.holder} stopped refreshing its extraction lock ${seconds(event.ageMs)} ago; extracting instead`);
22
+ break;
23
+ }
24
+ });
25
+ const identity = `${archive} (sha256 ${extraction.key.slice(0, 12)})`;
26
+ switch (extraction.kind) {
27
+ case 'reused':
28
+ say(`running ${identity} from ${extraction.directory}`);
29
+ break;
30
+ case 'extracted':
31
+ say(`extracted ${identity} to ${extraction.directory} in ${seconds(extraction.elapsedMs)}`);
32
+ break;
33
+ case 'awaited':
34
+ say(
35
+ `running ${identity} from ${extraction.directory} after ${seconds(extraction.elapsedMs)} waiting on ${extraction.holder}`,
36
+ );
37
+ break;
38
+ }
39
+ process.stdout.write(`${extraction.directory}\n`);
40
+ } catch (error) {
41
+ say(error instanceof Error ? error.message : String(error));
42
+ process.exit(1);
43
+ }
@@ -564,7 +564,10 @@ describe('cargo-test reachability policy', () => {
564
564
  'cargo-test-rusty-core',
565
565
  {
566
566
  command:
567
- "cargo --frozen nextest run --archive-file target/nextest/archive.tar.zst --workspace-remap . -E 'package(rusty-core)'",
567
+ 'extracted="$(node ../../node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js target/nextest/archive.tar.zst)" && ' +
568
+ 'cargo --frozen nextest run --binaries-metadata "$extracted/target/nextest/binaries-metadata.json" ' +
569
+ '--cargo-metadata "$extracted/target/nextest/cargo-metadata.json" --target-dir-remap "$extracted/target" ' +
570
+ "--workspace-remap . -E 'package(rusty-core)'",
568
571
  },
569
572
  ],
570
573
  ]),