@smoothbricks/nx-plugin 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +107 -46
- package/dist/bin/smoo-nx-test-clocks.js +2 -2
- package/dist/bin/smoo-nx-test-walks.js +6 -9
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +25 -15
- package/dist/ensure-built-witness.d.ts +33 -5
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +84 -18
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +33 -14
- package/dist/executors/bounded-exec/cpu-pool.d.ts +14 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +297 -38
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +19 -7
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +21 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +12 -6
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/owned-installation.d.ts +6 -5
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +324 -201
- package/dist/producer-receipts.d.ts +10 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +30 -5
- package/dist/test-walk-policy.d.ts +8 -0
- package/dist/test-walk-policy.d.ts.map +1 -1
- package/dist/test-walk-policy.js +34 -0
- package/dist/workspace-manifest.d.ts +5 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/managed/raw/patches/nx@23.2.1.patch +14 -3
- package/managed/raw/tooling/direnv/developer-links.ts +37 -10
- package/managed/raw/tooling/direnv/devenv.smoo.nix +1 -3
- package/managed/raw/tooling/direnv/linked-snapshots.ts +49 -12
- package/managed/raw/tooling/direnv/repo-path +1 -1
- package/managed/raw/tooling/direnv/setup-environment.ts +2 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +34 -11
- package/package.json +3 -3
- package/runtime-input.sh +21 -9
- package/src/__tests__/cargo-closure-filesets.ts +94 -0
- package/src/__tests__/counted-cargo.ts +1 -1
- package/src/__tests__/owned-installation.ts +5 -6
- package/src/bin/smoo-nx-test-clocks.ts +2 -2
- package/src/bin/smoo-nx-test-walks.ts +6 -9
- package/src/cargo-closure-filesets.test.ts +63 -0
- package/src/cargo-closure-input.test.ts +2 -2
- package/src/cargo-closure-input.ts +23 -15
- package/src/cargo-closure-key.test.ts +1 -1
- package/src/cargo-resolve-cache.test.ts +1 -1
- package/src/cross-check-policy.test.ts +53 -12
- package/src/ensure-built-witness.test.ts +53 -2
- package/src/ensure-built-witness.ts +103 -18
- package/src/ensure-built.test.ts +79 -5
- package/src/ensure-built.ts +51 -15
- package/src/executors/bounded-exec/cpu-pool.test.ts +132 -4
- package/src/executors/bounded-exec/cpu-pool.ts +300 -46
- package/src/executors/bounded-exec/cpu-tokens.test.ts +60 -15
- package/src/executors/bounded-exec/cpu-tokens.ts +34 -11
- package/src/executors/bounded-exec/executor.test.ts +72 -4
- package/src/executors/bounded-exec/executor.ts +12 -6
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +3 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/index.ts +4 -0
- package/src/linked-snapshots.nx.test.ts +1 -1
- package/src/nextest-partitions.test.ts +1 -1
- package/src/nx-daemon-validity.test.ts +51 -0
- package/src/owned-installation.test.ts +105 -0
- package/src/owned-installation.ts +335 -93
- package/src/producer-receipts.test.ts +37 -15
- package/src/producer-receipts.ts +29 -5
- package/src/runtime-input.test.ts +22 -1
- package/src/task-results.test.ts +1 -1
- package/src/test-walk-policy.test.ts +35 -0
- package/src/test-walk-policy.ts +32 -0
- package/src/workspace-manifest.ts +5 -5
- package/test-timings.json +94 -92
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* A fixture's own copy of a workspace's Nx installation, and the check that nothing wrote into it.
|
|
2
|
+
* A fixture's own copy of a workspace's Nx installation, kept between runs, and the check that nothing wrote into it.
|
|
3
3
|
*
|
|
4
4
|
* The graph witness binds node stamps (ctime included) of every file in the closure of Nx, this plugin and
|
|
5
5
|
* TypeScript (README "Ancestor directories bind..."). Linked from a workspace, that closure is the host's: on CI
|
|
@@ -13,11 +13,25 @@
|
|
|
13
13
|
* its copy as Bun's isolated layout does, the top-level packages' bins in `.bin`. Every copy is a new inode, cloned
|
|
14
14
|
* where the volume allows and written otherwise, so no link count is shared with anything outside its owner.
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
16
|
+
* The copy is kept between runs, in the workspace's own tool cache, because making it is most of what a test file
|
|
17
|
+
* costs: about 21,000 files, which clonefile cannot share across the volume between Bun's global store and any
|
|
18
|
+
* directory a cowshed job may write (specs/cowshed/03_caches.md "Clone-materializing caches"). It is trusted only as
|
|
19
|
+
* its census proves it: on first use in a run, the census of the host closure and the proven census of the copy are
|
|
20
|
+
* compared package by package, by what each holds (resolution edges, node types and modes, file bytes), and a package
|
|
21
|
+
* whose copy no longer holds what the host's holds is copied again. Both censuses take the hashes of files whose
|
|
22
|
+
* stamps have not moved from the ones recorded beside the copy, so a run that changes nothing reads no file bytes.
|
|
23
|
+
*
|
|
24
|
+
* The copy lies beneath `censusRoot`'s `node_modules`, so a census of it that resolves any package outside the copy
|
|
25
|
+
* proves nothing about it, and the copy is made again, whole, in a directory of its own (`installation-<uuid>`, named
|
|
26
|
+
* in the record beside it). A package in the copy that requires a dependency it does not
|
|
27
|
+
* declare still reaches the host's top-level `node_modules` the same way; the census binds declared edges only.
|
|
28
|
+
*
|
|
29
|
+
* A test file uses its copy, on first use, for all its fixtures; retiring it fails, naming the changed nodes, if any
|
|
30
|
+
* fixture wrote into it.
|
|
18
31
|
*/
|
|
32
|
+
import { randomUUID } from 'node:crypto';
|
|
19
33
|
import { constants } from 'node:fs';
|
|
20
|
-
import { cp, mkdir, readFile, rm, symlink } from 'node:fs/promises';
|
|
34
|
+
import { cp, mkdir, readdir, readFile, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises';
|
|
21
35
|
import { basename, dirname, join, relative } from 'node:path';
|
|
22
36
|
import { expandOutputs } from 'nx/src/native';
|
|
23
37
|
import { readModulePackageJson } from 'nx/src/utils/package-json';
|
|
@@ -25,129 +39,356 @@ import typia from 'typia';
|
|
|
25
39
|
import {
|
|
26
40
|
type GraphInstallationCensus,
|
|
27
41
|
graphInstallationCensus,
|
|
28
|
-
|
|
42
|
+
installationIdentityOf,
|
|
43
|
+
provenInstallationCensus,
|
|
44
|
+
stampMode,
|
|
29
45
|
} from './ensure-built-witness.js';
|
|
30
46
|
|
|
31
47
|
/** The configuration the census resolves: Nx, this plugin and TypeScript. */
|
|
32
48
|
const CENSUS_NX_JSON = { plugins: ['@smoothbricks/nx-plugin'] };
|
|
33
49
|
|
|
34
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* Where a workspace keeps its test files' copies: its tool cache, `node_modules/.cache`. Nothing sweeps it. A cowshed
|
|
52
|
+
* fork clones it with the build volume and main adopts it at land (specs/cowshed/16_build_volumes.md); it goes cold
|
|
53
|
+
* only when `node_modules` is reinstalled or the build volume reseeded.
|
|
54
|
+
*/
|
|
55
|
+
const KEPT = join('node_modules', '.cache', 'smoothbricks-nx-plugin', 'owned-installations');
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* What is recorded beside a kept copy: where it is, the host closure's last census and the copy's last proven census.
|
|
59
|
+
* Only a hash source for the next censuses; never a proof that the copy is current.
|
|
60
|
+
*/
|
|
61
|
+
interface KeptCensuses {
|
|
62
|
+
readonly version: 2;
|
|
63
|
+
/** The directory, beside this record, that holds the copy. */
|
|
64
|
+
readonly installation: string;
|
|
65
|
+
readonly host: GraphInstallationCensus;
|
|
66
|
+
readonly copy: GraphInstallationCensus;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** An installation kept under `root`, and its proven census when this run took it. */
|
|
35
70
|
interface OwnedInstallation {
|
|
36
71
|
readonly root: string;
|
|
37
|
-
readonly
|
|
72
|
+
readonly census: GraphInstallationCensus;
|
|
38
73
|
}
|
|
39
74
|
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
75
|
+
/** Each package root of the closure, where the copy keeps it, and the bins its manifest declares. */
|
|
76
|
+
interface Placement {
|
|
77
|
+
/** `node_modules/.store/<index>/node_modules`, numbered in resolution order: also where its dependencies link. */
|
|
78
|
+
readonly store: string;
|
|
79
|
+
/** `<store>/<name>`: the same for host and copy. */
|
|
80
|
+
readonly place: string;
|
|
81
|
+
readonly bin: string | Record<string, string> | undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** What a census says the closure holds, named by place in the copy: its resolution edges, and each package's nodes. */
|
|
85
|
+
interface Holdings {
|
|
86
|
+
readonly edges: string;
|
|
87
|
+
readonly packages: ReadonlyMap<string, string>;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The proven installation census resolved from `root`, taking unmoved files' hashes from `previous`. */
|
|
91
|
+
function provenCensus(
|
|
92
|
+
root: string,
|
|
93
|
+
previous: GraphInstallationCensus | undefined,
|
|
94
|
+
onUnproved: (detail: string) => void,
|
|
95
|
+
): GraphInstallationCensus | null {
|
|
96
|
+
return provenInstallationCensus(
|
|
44
97
|
root,
|
|
45
98
|
CENSUS_NX_JSON,
|
|
46
99
|
readModulePackageJson,
|
|
47
100
|
expandOutputs,
|
|
48
101
|
null,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
},
|
|
102
|
+
onUnproved,
|
|
103
|
+
previous,
|
|
52
104
|
);
|
|
53
|
-
if (identity === null) throw new Error(`the Nx installation resolved from ${root} is unproved: ${unproved}`);
|
|
54
|
-
return identity;
|
|
55
105
|
}
|
|
56
106
|
|
|
57
|
-
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
107
|
+
/**
|
|
108
|
+
* The proven census of the copy under `root`, or `null`, with `onUnproved` told why, for one that is unproved or
|
|
109
|
+
* that resolves a package outside the copy. The copy lies in `censusRoot`'s `node_modules/.cache`, so where that is a
|
|
110
|
+
* real directory, as on a fresh CI checkout, Node's ancestor lookup from the copy reaches the host's own
|
|
111
|
+
* `node_modules`: a copy that is missing, or lacks a link, would otherwise be proved by a census of host packages.
|
|
112
|
+
* Node also keeps every resolution for the life of the process, so once a census has resolved a host package from a
|
|
113
|
+
* copy's directory, no copy made there again can be proved in that process: a copy made whole goes to a new directory.
|
|
63
114
|
*/
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const copies = new Map<
|
|
81
|
-
string,
|
|
82
|
-
{ readonly store: string; readonly copy: string; readonly bin: string | Record<string, string> | undefined }
|
|
83
|
-
>();
|
|
115
|
+
function copyCensus(
|
|
116
|
+
root: string,
|
|
117
|
+
previous: GraphInstallationCensus | undefined,
|
|
118
|
+
onUnproved: (detail: string) => void,
|
|
119
|
+
): GraphInstallationCensus | null {
|
|
120
|
+
const census = provenCensus(root, previous, onUnproved);
|
|
121
|
+
const escaped = census?.resolutions.find(([, , manifest]) => manifest !== null && !manifest.startsWith(`${root}/`));
|
|
122
|
+
if (escaped === undefined) return census;
|
|
123
|
+
const [from, name, manifest] = escaped;
|
|
124
|
+
onUnproved(`it resolves ${name} from ${from} to ${manifest}, outside the copy`);
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Where the copy keeps each package root the census of `censusRoot` resolves. */
|
|
129
|
+
async function placements({ resolutions }: GraphInstallationCensus): Promise<Map<string, Placement>> {
|
|
130
|
+
const placed = new Map<string, Placement>();
|
|
84
131
|
for (const [, , manifest] of resolutions) {
|
|
85
|
-
if (manifest === null ||
|
|
132
|
+
if (manifest === null || placed.has(dirname(manifest))) continue;
|
|
86
133
|
const { name, bin } = typia.json.assertParse<{ name: string; bin?: string | Record<string, string> }>(
|
|
87
134
|
await readFile(manifest, 'utf8'),
|
|
88
135
|
);
|
|
89
|
-
const store = join(
|
|
90
|
-
|
|
136
|
+
const store = join('node_modules', '.store', String(placed.size), 'node_modules');
|
|
137
|
+
placed.set(dirname(manifest), { store, place: join(store, name), bin });
|
|
91
138
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
139
|
+
return placed;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* What `census` holds, read from `root` with each package root in `places` named by its place in the copy. A node
|
|
144
|
+
* outside every package root keeps its path, in the place `''` with the root's own edges.
|
|
145
|
+
*/
|
|
146
|
+
function holdings(census: GraphInstallationCensus, root: string, places: ReadonlyMap<string, string>): Holdings {
|
|
147
|
+
const named = (path: string): readonly [string, string] => {
|
|
148
|
+
for (let at = path; ; at = dirname(at)) {
|
|
149
|
+
const place = at === root ? '' : places.get(at);
|
|
150
|
+
if (place !== undefined) return [place, relative(at, path)];
|
|
151
|
+
if (dirname(at) === at) return ['', path];
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
const packages = new Map<string, unknown[]>();
|
|
155
|
+
const hold = (path: string, value: string) => {
|
|
156
|
+
const [place, inner] = named(path);
|
|
157
|
+
const held = packages.get(place);
|
|
158
|
+
if (held === undefined) packages.set(place, [[inner, value]]);
|
|
159
|
+
else held.push([inner, value]);
|
|
96
160
|
};
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
161
|
+
for (const { path, stamp } of census.nodes) hold(path, stampMode(stamp));
|
|
162
|
+
for (const { path, sha256 } of census.files) hold(path, sha256);
|
|
163
|
+
return {
|
|
164
|
+
edges: JSON.stringify(
|
|
165
|
+
census.resolutions.map(([from, name, manifest]) => [
|
|
166
|
+
named(from),
|
|
167
|
+
name,
|
|
168
|
+
manifest === null ? null : named(manifest),
|
|
169
|
+
]),
|
|
105
170
|
),
|
|
106
|
-
|
|
171
|
+
packages: new Map([...packages].map(([place, held]) => [place, JSON.stringify(held)])),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Each package of `packages` (the first eight) with the first entry in which what the copy holds is not what the host
|
|
177
|
+
* closure holds: a path within the package and its mode or SHA-256, on each side.
|
|
178
|
+
*/
|
|
179
|
+
function differences(packages: readonly (readonly [string, Placement])[], host: Holdings, copy: Holdings): string {
|
|
180
|
+
const entries = (held: string | undefined) => typia.json.assertParse<[string, string][]>(held ?? '[]');
|
|
181
|
+
const entry = (held: [string, string] | undefined) =>
|
|
182
|
+
held === undefined ? 'nothing' : `${held[0] || '.'} ${held[1]}`;
|
|
183
|
+
const named = packages.slice(0, 8).map(([source, { place }]) => {
|
|
184
|
+
const wanted = entries(host.packages.get(place));
|
|
185
|
+
const found = entries(copy.packages.get(place));
|
|
186
|
+
let at = 0;
|
|
187
|
+
while (at < Math.max(wanted.length, found.length) && entry(wanted[at]) === entry(found[at])) at += 1;
|
|
188
|
+
return `${source} (host: ${entry(wanted[at])}; copy: ${entry(found[at])})`;
|
|
189
|
+
});
|
|
190
|
+
return `${named.join(', ')}${packages.length > 8 ? ` and ${packages.length - 8} more` : ''}`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function readKept(path: string): Promise<KeptCensuses | null> {
|
|
194
|
+
try {
|
|
195
|
+
return typia.json.isParse<KeptCensuses>(await readFile(path, 'utf8'));
|
|
196
|
+
} catch (error) {
|
|
197
|
+
if (error instanceof SyntaxError || (error instanceof Error && 'code' in error && error.code === 'ENOENT')) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
throw error;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function writeKept(path: string, kept: KeptCensuses): Promise<void> {
|
|
205
|
+
const temporary = `${path}.${randomUUID()}`;
|
|
206
|
+
try {
|
|
207
|
+
await writeFile(temporary, JSON.stringify(kept), { flag: 'wx', mode: 0o600 });
|
|
208
|
+
await rename(temporary, path);
|
|
209
|
+
} finally {
|
|
210
|
+
await rm(temporary, { force: true });
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Copy the host package root `source` to its place under `root`, with the census's own exclusions. */
|
|
215
|
+
function copyPackage(source: string, root: string, { place }: Placement): Promise<void> {
|
|
216
|
+
return cp(source, join(root, place), {
|
|
217
|
+
recursive: true,
|
|
218
|
+
verbatimSymlinks: true,
|
|
219
|
+
mode: constants.COPYFILE_FICLONE,
|
|
220
|
+
filter: (path) => path === source || !['node_modules', '.git', '.nx'].includes(basename(path)),
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Link every resolution edge of `host` beside its copy under `root`, and the top-level packages' bins in `.bin`. */
|
|
225
|
+
async function linkCopies(
|
|
226
|
+
censusRoot: string,
|
|
227
|
+
root: string,
|
|
228
|
+
{ resolutions }: GraphInstallationCensus,
|
|
229
|
+
placed: ReadonlyMap<string, Placement>,
|
|
230
|
+
): Promise<void> {
|
|
231
|
+
const placementOf = (source: string) => {
|
|
232
|
+
const placement = placed.get(source);
|
|
233
|
+
if (placement === undefined) throw new Error(`${source} resolves dependencies but is not in the census`);
|
|
234
|
+
return placement;
|
|
235
|
+
};
|
|
236
|
+
const modules = join(root, 'node_modules');
|
|
107
237
|
for (const [from, name, manifest] of resolutions) {
|
|
108
238
|
if (manifest === null) continue;
|
|
109
|
-
const link = join(from === censusRoot ? modules :
|
|
239
|
+
const link = join(from === censusRoot ? modules : join(root, placementOf(from).store), name);
|
|
240
|
+
const { place, bin } = placementOf(dirname(manifest));
|
|
110
241
|
await mkdir(dirname(link), { recursive: true });
|
|
111
|
-
await symlink(relative(dirname(link),
|
|
242
|
+
await symlink(relative(dirname(link), join(root, place)), link, 'dir');
|
|
112
243
|
if (from !== censusRoot) continue;
|
|
113
|
-
const { bin } = copyOf(dirname(manifest));
|
|
114
244
|
const commands = typeof bin === 'string' ? { [basename(name)]: bin } : (bin ?? {});
|
|
115
245
|
for (const [command, path] of Object.entries(commands)) {
|
|
116
246
|
await mkdir(join(modules, '.bin'), { recursive: true });
|
|
117
247
|
await symlink(relative(join(modules, '.bin'), join(link, path)), join(modules, '.bin', command));
|
|
118
248
|
}
|
|
119
249
|
}
|
|
120
|
-
return { root, identity: installationIdentity(root) };
|
|
121
250
|
}
|
|
122
251
|
|
|
123
252
|
/**
|
|
124
|
-
* Remove
|
|
125
|
-
* with
|
|
253
|
+
* Remove whatever sits in the copy's `node_modules` beside its store, bins and top-level links: a census binds none
|
|
254
|
+
* of it, and a run starts with the layout alone, as a fresh copy would.
|
|
126
255
|
*/
|
|
127
|
-
async function
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
...files.map(({ path, sha256 }) => `${path} ${sha256}`),
|
|
136
|
-
]);
|
|
137
|
-
};
|
|
138
|
-
const made = entries(identity);
|
|
139
|
-
const now = entries(retired);
|
|
140
|
-
const changed = [
|
|
141
|
-
...[...made].filter((entry) => !now.has(entry)).map((entry) => `- ${entry}`),
|
|
142
|
-
...[...now].filter((entry) => !made.has(entry)).map((entry) => `+ ${entry}`),
|
|
143
|
-
];
|
|
144
|
-
throw new Error(`a fixture changed the installation it shares, in ${root}:\n${changed.slice(0, 20).join('\n')}`);
|
|
145
|
-
} finally {
|
|
146
|
-
await rm(root, { recursive: true, force: true });
|
|
256
|
+
async function clearLeftovers(
|
|
257
|
+
censusRoot: string,
|
|
258
|
+
root: string,
|
|
259
|
+
{ resolutions }: GraphInstallationCensus,
|
|
260
|
+
): Promise<void> {
|
|
261
|
+
const own = new Set(['.store', '.bin']);
|
|
262
|
+
for (const [from, name, manifest] of resolutions) {
|
|
263
|
+
if (from === censusRoot && manifest !== null) own.add(name.split('/')[0]);
|
|
147
264
|
}
|
|
265
|
+
const modules = join(root, 'node_modules');
|
|
266
|
+
for (const entry of await readdir(modules)) {
|
|
267
|
+
if (!own.has(entry)) await rm(join(modules, entry), { recursive: true, force: true });
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* The copy of the closure `censusRoot` resolves kept as `name`, proved to hold what the host closure holds, copying
|
|
273
|
+
* again each package whose copy does not, and every package when the resolution edges moved or the copy is unproved.
|
|
274
|
+
* Says so on stderr when a copy that was there fails its proof: a closure that keeps changing is itself worth knowing.
|
|
275
|
+
*
|
|
276
|
+
* Only the closure is read from `censusRoot`, never its proof: that installation is the host's, whose shared store
|
|
277
|
+
* other tasks relink while the census reads it, which changes node stamps and would make every test that copies it
|
|
278
|
+
* fail for the host's work rather than its own. The copy, which the fixture owns, is what gets proved.
|
|
279
|
+
*/
|
|
280
|
+
async function keptInstallation(censusRoot: string, name: string): Promise<OwnedInstallation> {
|
|
281
|
+
await mkdir(join(censusRoot, KEPT, name), { recursive: true });
|
|
282
|
+
const kept = await realpath(join(censusRoot, KEPT, name));
|
|
283
|
+
const record = join(kept, 'censuses.json');
|
|
284
|
+
const recorded = await readKept(record);
|
|
285
|
+
let unread = 'no detail';
|
|
286
|
+
const host = graphInstallationCensus(
|
|
287
|
+
censusRoot,
|
|
288
|
+
CENSUS_NX_JSON,
|
|
289
|
+
readModulePackageJson,
|
|
290
|
+
expandOutputs,
|
|
291
|
+
null,
|
|
292
|
+
(detail) => {
|
|
293
|
+
unread = detail;
|
|
294
|
+
},
|
|
295
|
+
recorded?.host,
|
|
296
|
+
);
|
|
297
|
+
if (host === null) throw new Error(`the Nx installation closure of ${censusRoot} cannot be read: ${unread}`);
|
|
298
|
+
const placed = await placements(host);
|
|
299
|
+
const wanted = holdings(host, censusRoot, new Map([...placed].map(([source, { place }]) => [source, place])));
|
|
300
|
+
const placesIn = (root: string) => new Map([...placed.values()].map(({ place }) => [join(root, place), place]));
|
|
301
|
+
// Only a recorded copy is censused: a census of none would resolve the host's packages through the ancestor lookup.
|
|
302
|
+
const current = recorded === null ? null : join(kept, recorded.installation);
|
|
303
|
+
let unproved = 'no detail';
|
|
304
|
+
const found =
|
|
305
|
+
current === null
|
|
306
|
+
? null
|
|
307
|
+
: copyCensus(current, recorded?.copy, (detail) => {
|
|
308
|
+
unproved = detail;
|
|
309
|
+
});
|
|
310
|
+
const held = current === null || found === null ? null : holdings(found, current, placesIn(current));
|
|
311
|
+
const stale =
|
|
312
|
+
held === null || held.edges !== wanted.edges || held.packages.get('') !== wanted.packages.get('')
|
|
313
|
+
? null
|
|
314
|
+
: [...placed].filter(([, { place }]) => held.packages.get(place) !== wanted.packages.get(place));
|
|
315
|
+
if (current !== null && stale !== null) {
|
|
316
|
+
await clearLeftovers(censusRoot, current, host);
|
|
317
|
+
if (stale.length === 0 && found !== null) {
|
|
318
|
+
await writeKept(record, { version: 2, installation: basename(current), host, copy: found });
|
|
319
|
+
return { root: current, census: found };
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (current !== null) {
|
|
323
|
+
const why =
|
|
324
|
+
held === null
|
|
325
|
+
? `its census is unproved (${unproved})`
|
|
326
|
+
: stale === null
|
|
327
|
+
? 'its resolution edges are not the host closure’s'
|
|
328
|
+
: `${stale.length} of ${placed.size} packages no longer hold what the host closure holds: ${differences(
|
|
329
|
+
stale,
|
|
330
|
+
wanted,
|
|
331
|
+
held,
|
|
332
|
+
)}`;
|
|
333
|
+
process.stderr.write(`kept Nx installation ${current} failed its proof, copying again: ${why}\n`);
|
|
334
|
+
}
|
|
335
|
+
// A copy made whole goes to a directory no census in this process has resolved from (copyCensus).
|
|
336
|
+
const root = current !== null && stale !== null ? current : join(kept, `installation-${randomUUID()}`);
|
|
337
|
+
if (root === current && stale !== null) {
|
|
338
|
+
for (const [, { place }] of stale) await rm(join(root, place), { recursive: true, force: true });
|
|
339
|
+
await Promise.all(stale.map(([source, placement]) => copyPackage(source, root, placement)));
|
|
340
|
+
} else {
|
|
341
|
+
for (const entry of await readdir(kept)) {
|
|
342
|
+
if (entry !== 'censuses.json') await rm(join(kept, entry), { recursive: true, force: true });
|
|
343
|
+
}
|
|
344
|
+
await Promise.all([...placed].map(([source, placement]) => copyPackage(source, root, placement)));
|
|
345
|
+
await linkCopies(censusRoot, root, host, placed);
|
|
346
|
+
}
|
|
347
|
+
const census = copyCensus(root, root === current ? (found ?? undefined) : undefined, (detail) => {
|
|
348
|
+
unproved = detail;
|
|
349
|
+
});
|
|
350
|
+
if (census === null) throw new Error(`the Nx installation copied to ${root} is unproved: ${unproved}`);
|
|
351
|
+
const copied = holdings(census, root, placesIn(root));
|
|
352
|
+
const moved = [...placed].filter(([, { place }]) => copied.packages.get(place) !== wanted.packages.get(place));
|
|
353
|
+
if (copied.edges !== wanted.edges || moved.length > 0) {
|
|
354
|
+
// The host's own installs relink its shared store while it is read: the copy is proved as made, and the next run
|
|
355
|
+
// compares it with the host closure again.
|
|
356
|
+
process.stderr.write(
|
|
357
|
+
`the Nx installation closure of ${censusRoot} moved while ${root} copied it: ${differences(moved, wanted, copied)}\n`,
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
await writeKept(record, { version: 2, installation: basename(root), host, copy: census });
|
|
361
|
+
return { root, census };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* End a test file's use of its installation, failing, with the changed nodes named, if its identity is no longer the
|
|
366
|
+
* one it was taken with: the proof that its sharers only read it. A write moves a node's stamp, which the identity
|
|
367
|
+
* binds, so the census that proves it reads again only the files whose stamps moved. The copy stays for the next run,
|
|
368
|
+
* which proves it against the host closure before using it.
|
|
369
|
+
*/
|
|
370
|
+
function retireOwnedInstallation({ root, census }: OwnedInstallation): void {
|
|
371
|
+
let unproved = 'no detail';
|
|
372
|
+
const retired = provenCensus(root, census, (detail) => {
|
|
373
|
+
unproved = detail;
|
|
374
|
+
});
|
|
375
|
+
if (retired === null) throw new Error(`a fixture left the installation it shares unproved, in ${root}: ${unproved}`);
|
|
376
|
+
if (installationIdentityOf(retired, null) === installationIdentityOf(census, null)) return;
|
|
377
|
+
const entries = ({ nodes, files }: GraphInstallationCensus) =>
|
|
378
|
+
new Set([
|
|
379
|
+
...nodes.map(({ path, stamp }) => `${path} ${stamp}`),
|
|
380
|
+
...files.map(({ path, sha256 }) => `${path} ${sha256}`),
|
|
381
|
+
]);
|
|
382
|
+
const taken = entries(census);
|
|
383
|
+
const now = entries(retired);
|
|
384
|
+
const changed = [
|
|
385
|
+
...[...taken].filter((entry) => !now.has(entry)).map((entry) => `- ${entry}`),
|
|
386
|
+
...[...now].filter((entry) => !taken.has(entry)).map((entry) => `+ ${entry}`),
|
|
387
|
+
];
|
|
388
|
+
throw new Error(`a fixture changed the installation it shares, in ${root}:\n${changed.slice(0, 20).join('\n')}`);
|
|
148
389
|
}
|
|
149
390
|
|
|
150
|
-
/** One test file's installation:
|
|
391
|
+
/** One test file's installation: taken on first use and shared by its fixtures, retired by the file's `afterAll`. */
|
|
151
392
|
export interface TestFileInstallation {
|
|
152
393
|
/** The `node_modules` a fixture workspace links. */
|
|
153
394
|
readonly nodeModules: () => Promise<string>;
|
|
@@ -156,19 +397,20 @@ export interface TestFileInstallation {
|
|
|
156
397
|
}
|
|
157
398
|
|
|
158
399
|
/**
|
|
159
|
-
* The installation one test file owns:
|
|
160
|
-
* TypeScript the fixtures stand for) resolves,
|
|
161
|
-
* per file, at module scope: Bun runs many test files in one process, and
|
|
400
|
+
* The installation one test file owns: the copy of the closure `censusRoot` (the workspace whose Nx, plugin and
|
|
401
|
+
* TypeScript the fixtures stand for) resolves, kept as `name` in `censusRoot`'s tool cache and proved on first use.
|
|
402
|
+
* Call once per file, at module scope, with a name no other file uses: Bun runs many test files in one process, and
|
|
403
|
+
* each must own its copy. Two runs of one file at once share it, and a copy one of them makes again fails the other.
|
|
162
404
|
*/
|
|
163
|
-
export function ownedInstallation(censusRoot: string,
|
|
164
|
-
let
|
|
405
|
+
export function ownedInstallation(censusRoot: string, name: string): TestFileInstallation {
|
|
406
|
+
let taken: Promise<OwnedInstallation> | undefined;
|
|
165
407
|
return {
|
|
166
408
|
nodeModules: async () => {
|
|
167
|
-
|
|
168
|
-
return join((await
|
|
409
|
+
taken ??= keptInstallation(censusRoot, name);
|
|
410
|
+
return join((await taken).root, 'node_modules');
|
|
169
411
|
},
|
|
170
412
|
retire: async () => {
|
|
171
|
-
if (
|
|
413
|
+
if (taken !== undefined) retireOwnedInstallation(await taken);
|
|
172
414
|
},
|
|
173
415
|
};
|
|
174
416
|
}
|
|
@@ -9,6 +9,7 @@ import type { TaskResults } from 'nx/src/tasks-runner/life-cycle';
|
|
|
9
9
|
import type { EnsureBuiltProducerReceipt } from './ensure-built.js';
|
|
10
10
|
import {
|
|
11
11
|
completedProducerTasks,
|
|
12
|
+
PRODUCER_RECORDS,
|
|
12
13
|
postTasksExecution,
|
|
13
14
|
preTasksExecution,
|
|
14
15
|
producerEnvironmentIdentity,
|
|
@@ -116,16 +117,14 @@ describe('actual producer completion evidence', () => {
|
|
|
116
117
|
});
|
|
117
118
|
it('publishes a refusal beside a published receipt, never over it, and names it in the miss', async () => {
|
|
118
119
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'producer-refusal-')));
|
|
119
|
-
const inherited = process.env.NX_WORKSPACE_DATA_DIRECTORY;
|
|
120
|
-
delete process.env.NX_WORKSPACE_DATA_DIRECTORY;
|
|
121
120
|
try {
|
|
122
121
|
await writeFile(join(root, 'package.json'), JSON.stringify({ name: 'refusal-fixture' }));
|
|
123
122
|
await writeFile(join(root, 'nx.json'), '{}');
|
|
124
123
|
await symlink(join(repoRoot, 'node_modules'), join(root, 'node_modules'), 'dir');
|
|
125
|
-
const
|
|
126
|
-
await mkdir(
|
|
124
|
+
const records = join(root, PRODUCER_RECORDS);
|
|
125
|
+
await mkdir(records, { recursive: true });
|
|
127
126
|
const name = createHash('sha256').update(JSON.stringify(task.id)).digest('hex');
|
|
128
|
-
const receiptPath = join(
|
|
127
|
+
const receiptPath = join(records, `producer-receipt-${name}.json`);
|
|
129
128
|
// What an earlier run published; this run's refusal is no evidence about it.
|
|
130
129
|
const published = JSON.stringify({ version: 1, disposition: 'published', receipt: 'an earlier run' });
|
|
131
130
|
await writeFile(receiptPath, published);
|
|
@@ -143,21 +142,44 @@ describe('actual producer completion evidence', () => {
|
|
|
143
142
|
},
|
|
144
143
|
);
|
|
145
144
|
expect(await readFile(receiptPath, 'utf8')).toBe(published);
|
|
146
|
-
expect((await readdir(
|
|
145
|
+
expect((await readdir(records)).sort()).toEqual([
|
|
146
|
+
'.gitignore',
|
|
147
|
+
`producer-receipt-${name}.json`,
|
|
148
|
+
`producer-refusal-${name}.json`,
|
|
149
|
+
]);
|
|
150
|
+
// The records directory ignores itself, so no checkout tracks or hashes a record.
|
|
151
|
+
expect(await readFile(join(records, '.gitignore'), 'utf8')).toBe('*\n');
|
|
147
152
|
expect(readProducerReceipt(root, task.id)).toEqual({
|
|
148
153
|
ok: false,
|
|
149
|
-
detail:
|
|
150
|
-
"Missing/corrupt producer completion contract; the producer's latest run refused: No bound producer pre-execution frame",
|
|
154
|
+
detail: `Corrupt producer completion contract at ${receiptPath}; the producer's latest run refused: No bound producer pre-execution frame`,
|
|
151
155
|
});
|
|
152
|
-
// With no receipt at all, the miss
|
|
156
|
+
// With no receipt at all, the miss names where the receipt would be and why the latest run published none.
|
|
153
157
|
await rm(receiptPath);
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
);
|
|
158
|
+
expect(readProducerReceipt(root, task.id)).toEqual({
|
|
159
|
+
ok: false,
|
|
160
|
+
detail: `No producer completion receipt at ${receiptPath}; the producer's latest run refused: No bound producer pre-execution frame`,
|
|
161
|
+
});
|
|
162
|
+
} finally {
|
|
163
|
+
await rm(root, { recursive: true, force: true });
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
it('tells a receipt it cannot read from a missing one, naming the path either way', async () => {
|
|
167
|
+
const root = await realpath(await mkdtemp(join(tmpdir(), 'producer-unreadable-')));
|
|
168
|
+
try {
|
|
169
|
+
const records = join(root, PRODUCER_RECORDS);
|
|
170
|
+
await mkdir(records, { recursive: true });
|
|
171
|
+
const name = createHash('sha256').update(JSON.stringify(task.id)).digest('hex');
|
|
172
|
+
const receiptPath = join(records, `producer-receipt-${name}.json`);
|
|
173
|
+
expect(readProducerReceipt(root, task.id)).toEqual({
|
|
174
|
+
ok: false,
|
|
175
|
+
detail: `No producer completion receipt at ${receiptPath}`,
|
|
176
|
+
});
|
|
177
|
+
// A sandbox that may not read the receipt answers EPERM; an unreadable file answers EACCES the same way.
|
|
178
|
+
await writeFile(receiptPath, '{}', { mode: 0 });
|
|
179
|
+
const denied = readProducerReceipt(root, task.id);
|
|
180
|
+
expect(denied.ok ? 'hit' : denied.detail).toStartWith('Cannot read producer completion receipt: EACCES');
|
|
181
|
+
expect(denied.ok ? 'hit' : denied.detail).toContain(receiptPath);
|
|
158
182
|
} finally {
|
|
159
|
-
if (inherited === undefined) delete process.env.NX_WORKSPACE_DATA_DIRECTORY;
|
|
160
|
-
else process.env.NX_WORKSPACE_DATA_DIRECTORY = inherited;
|
|
161
183
|
await rm(root, { recursive: true, force: true });
|
|
162
184
|
}
|
|
163
185
|
});
|
package/src/producer-receipts.ts
CHANGED
|
@@ -46,6 +46,16 @@ import { graphInputContractIdentity } from './graph-input-contract.js';
|
|
|
46
46
|
export interface ProducerReceiptsOptions {
|
|
47
47
|
readonly producerReceipts?: { readonly targets: readonly string[] };
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Where a checkout keeps its producer records, relative to its root: in its own source tree, never in Nx's
|
|
51
|
+
* workspace-data. Under cowshed that directory is build state on the checkout's build volume
|
|
52
|
+
* (specs/cowshed/16_build_volumes.md), and a consumer's read grant of the provider checkout does not reach a volume
|
|
53
|
+
* the checkout only links to (specs/cowshed/04_sandbox.md, "Read grants have one narrow meaning"), so a consumer
|
|
54
|
+
* could never read the receipt it is asked to validate. The directory ignores itself (a `.gitignore` of `*` written
|
|
55
|
+
* with its first record), so in any checkout git never tracks a record and Nx's file walk never hashes one: a record
|
|
56
|
+
* is no source or input of the run that publishes it.
|
|
57
|
+
*/
|
|
58
|
+
export const PRODUCER_RECORDS = join('.cache', 'producer-receipts');
|
|
49
59
|
interface ToolResolution {
|
|
50
60
|
readonly name: string;
|
|
51
61
|
readonly from: string;
|
|
@@ -95,16 +105,18 @@ function configured(options: ProducerReceiptsOptions | undefined): readonly stri
|
|
|
95
105
|
* its products never replaces the receipt an earlier run published: readers validate that receipt on its own evidence.
|
|
96
106
|
*/
|
|
97
107
|
function recordPath(root: string, target: string, disposition: ProducerRecord['disposition']): string {
|
|
98
|
-
const load = createWorkspaceNxRequire(root);
|
|
99
108
|
return join(
|
|
100
|
-
|
|
109
|
+
root,
|
|
110
|
+
PRODUCER_RECORDS,
|
|
101
111
|
`producer-${disposition === 'published' ? 'receipt' : 'refusal'}-${sha(target)}.json`,
|
|
102
112
|
);
|
|
103
113
|
}
|
|
104
114
|
function publish(root: string, target: string, record: ProducerRecord): ReceiptEvidence<true> {
|
|
105
115
|
try {
|
|
106
116
|
const path = recordPath(root, target, record.disposition);
|
|
107
|
-
|
|
117
|
+
const records = dirname(path);
|
|
118
|
+
mkdirSync(records, { recursive: true });
|
|
119
|
+
if (!existsSync(join(records, '.gitignore'))) writeFileSync(join(records, '.gitignore'), '*\n');
|
|
108
120
|
const temporary = `${path}.${randomUUID()}`;
|
|
109
121
|
try {
|
|
110
122
|
writeFileSync(temporary, JSON.stringify(record), { flag: 'wx', mode: 0o600 });
|
|
@@ -550,10 +562,22 @@ function latestRefusal(root: string, target: string): string | null {
|
|
|
550
562
|
}
|
|
551
563
|
}
|
|
552
564
|
function validatedReceipt(root: string, target: string): ReceiptEvidence<EnsureBuiltReceipt> {
|
|
565
|
+
const path = recordPath(root, target, 'published');
|
|
566
|
+
let text: string;
|
|
567
|
+
try {
|
|
568
|
+
text = readFileSync(path, 'utf8');
|
|
569
|
+
} catch (error) {
|
|
570
|
+
// No receipt is a producer that never published one; any other failure is a receipt this reader cannot reach.
|
|
571
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
572
|
+
return { ok: false, detail: `No producer completion receipt at ${path}` };
|
|
573
|
+
if (error instanceof Error)
|
|
574
|
+
return { ok: false, detail: `Cannot read producer completion receipt: ${error.message}` };
|
|
575
|
+
throw error;
|
|
576
|
+
}
|
|
553
577
|
try {
|
|
554
|
-
const record = typia.json.isParse<PublishedReceipt>(
|
|
578
|
+
const record = typia.json.isParse<PublishedReceipt>(text);
|
|
555
579
|
if (record === null || record.version !== 1)
|
|
556
|
-
return { ok: false, detail:
|
|
580
|
+
return { ok: false, detail: `Corrupt producer completion contract at ${path}` };
|
|
557
581
|
const receipt = record.receipt;
|
|
558
582
|
if (
|
|
559
583
|
receipt.workspaceRoot !== realpathSync(root) ||
|