@smoothbricks/nx-plugin 0.4.23 → 0.4.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,329 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
+ import { once } from 'node:events';
4
+ import { createReadStream } from 'node:fs';
5
+ import { mkdir, mkdtemp, open, readdir, readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
6
+ import { join } from 'node:path';
7
+ import { setTimeout as sleep } from 'node:timers/promises';
8
+
9
+ /**
10
+ * The helper a per-crate runner calls before `nextest run`, as the root
11
+ * `node_modules` links it. Spelled through the package link rather than this
12
+ * file's real location: under an isolated install that location is a store
13
+ * path outside the workspace, which would differ per machine in a command Nx
14
+ * hashes.
15
+ */
16
+ export const NEXTEST_EXTRACT_BIN = 'node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js';
17
+
18
+ /**
19
+ * Where nextest puts things inside an extracted archive, relative to the
20
+ * extraction directory (nextest-runner's `reuse_build` constants). A run given
21
+ * these three paths reuses that build exactly as `--archive-file` would after
22
+ * unpacking it.
23
+ */
24
+ export const EXTRACTED_TARGET_DIR = 'target';
25
+ export const EXTRACTED_BINARIES_METADATA = 'target/nextest/binaries-metadata.json';
26
+ export const EXTRACTED_CARGO_METADATA = 'target/nextest/cargo-metadata.json';
27
+
28
+ /** Unpacks `archive` into the empty, existing directory `destination`. */
29
+ export type NextestArchiveExtractor = (archive: string, destination: string) => Promise<void>;
30
+
31
+ export type NextestExtraction =
32
+ | { kind: 'reused'; key: string; directory: string }
33
+ | { kind: 'extracted'; key: string; directory: string; elapsedMs: number }
34
+ | { kind: 'awaited'; key: string; directory: string; elapsedMs: number; holder: string };
35
+
36
+ /** What a caller is told while it does not hold the extraction itself. */
37
+ export type NextestExtractionEvent =
38
+ | { kind: 'waiting'; holder: string }
39
+ | { kind: 'stale-lock'; holder: string; ageMs: number };
40
+
41
+ /**
42
+ * How a lock proves its holder is alive: it rewrites the lock's mtime every
43
+ * `heartbeatMs`, and a lock older than `staleAfterMs` belongs to nobody. A pid
44
+ * cannot answer that — a holder SIGKILLed at its bound leaves a pid the system
45
+ * may already have handed to something else.
46
+ */
47
+ export interface NextestExtractionTiming {
48
+ heartbeatMs: number;
49
+ staleAfterMs: number;
50
+ pollMs: number;
51
+ }
52
+
53
+ const DEFAULT_TIMING: NextestExtractionTiming = { heartbeatMs: 1_000, staleAfterMs: 10_000, pollMs: 100 };
54
+
55
+ /** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
56
+ export async function nextestArchiveKey(archive: string): Promise<string> {
57
+ const hash = createHash('sha256');
58
+ for await (const chunk of createReadStream(archive, { highWaterMark: 1 << 20 })) {
59
+ hash.update(chunk);
60
+ }
61
+ return hash.digest('hex');
62
+ }
63
+
64
+ /**
65
+ * One archive path's extractions live beside it, each under the hash of the
66
+ * bytes it came from; the lock and staging directories of a key carry that
67
+ * key as their prefix, so cleanup can tell them from another key's.
68
+ */
69
+ export interface NextestExtractionPaths {
70
+ root: string;
71
+ directory: string;
72
+ lock: string;
73
+ stagingPrefix: string;
74
+ }
75
+
76
+ export function nextestExtractionPaths(archive: string, key: string): NextestExtractionPaths {
77
+ const root = `${archive}.extracted`;
78
+ return {
79
+ root,
80
+ directory: join(root, key),
81
+ lock: join(root, `${key}.lock`),
82
+ stagingPrefix: join(root, `${key}.staging-`),
83
+ };
84
+ }
85
+
86
+ /**
87
+ * The directory holding `archive`'s extraction, extracting it first if no
88
+ * caller has yet.
89
+ *
90
+ * Correctness does not rest on the lock. An extraction lands under the hash of
91
+ * the bytes it came from by renaming a private staging directory into place,
92
+ * so a directory under a key is a complete extraction of exactly those bytes
93
+ * or it does not exist: concurrent extractors cannot corrupt it, the loser of
94
+ * a race discards its own copy, and an archive rewritten mid-extraction is
95
+ * caught by hashing it again before publishing. The lock only stops every
96
+ * runner of one Nx run from unpacking the same bytes side by side, which is the
97
+ * cost it exists to remove.
98
+ *
99
+ * Publishing a key removes every other key's extraction of this archive path,
100
+ * including stale locks and staging left by a holder that died. Leftovers of
101
+ * the CURRENT key are not touched: a holder that took over a stale lock may
102
+ * still be extracting into one.
103
+ */
104
+ export async function ensureNextestArchiveExtracted(
105
+ archive: string,
106
+ extract: NextestArchiveExtractor,
107
+ onEvent: (event: NextestExtractionEvent) => void,
108
+ timing: NextestExtractionTiming = DEFAULT_TIMING,
109
+ ): Promise<NextestExtraction> {
110
+ const started = performance.now();
111
+ const key = await nextestArchiveKey(archive);
112
+ const paths = nextestExtractionPaths(archive, key);
113
+ if (await isDirectory(paths.directory)) {
114
+ return { kind: 'reused', key, directory: paths.directory };
115
+ }
116
+ await mkdir(paths.root, { recursive: true });
117
+ let holder: string | undefined;
118
+ const settled = (): NextestExtraction =>
119
+ holder === undefined
120
+ ? { kind: 'reused', key, directory: paths.directory }
121
+ : { kind: 'awaited', key, directory: paths.directory, elapsedMs: performance.now() - started, holder };
122
+ for (;;) {
123
+ const lock = await acquireLock(paths.lock, timing.heartbeatMs);
124
+ if (lock !== undefined) {
125
+ try {
126
+ if (await isDirectory(paths.directory)) {
127
+ return settled();
128
+ }
129
+ const published = await extractAndPublish(archive, key, paths, extract);
130
+ if (published) {
131
+ await removeOtherKeys(paths.root, key);
132
+ return { kind: 'extracted', key, directory: paths.directory, elapsedMs: performance.now() - started };
133
+ }
134
+ return settled();
135
+ } finally {
136
+ await lock.release();
137
+ }
138
+ }
139
+ if (await isDirectory(paths.directory)) {
140
+ return settled();
141
+ }
142
+ const current = await readLock(paths.lock);
143
+ if (current === undefined) {
144
+ // Released between our attempt and this read: try again at once.
145
+ continue;
146
+ }
147
+ if (current.ageMs > timing.staleAfterMs) {
148
+ onEvent({ kind: 'stale-lock', holder: current.holder, ageMs: current.ageMs });
149
+ await rm(paths.lock, { force: true });
150
+ continue;
151
+ }
152
+ if (holder === undefined) {
153
+ holder = current.holder;
154
+ onEvent({ kind: 'waiting', holder });
155
+ }
156
+ await sleep(timing.pollMs);
157
+ }
158
+ }
159
+
160
+ /**
161
+ * Extract into a private staging directory and rename it to `paths.directory`.
162
+ * False when another extractor published the same key first, which leaves a
163
+ * complete extraction of the same bytes in place.
164
+ */
165
+ async function extractAndPublish(
166
+ archive: string,
167
+ key: string,
168
+ paths: NextestExtractionPaths,
169
+ extract: NextestArchiveExtractor,
170
+ ): Promise<boolean> {
171
+ const staging = await mkdtemp(paths.stagingPrefix);
172
+ try {
173
+ await extract(archive, staging);
174
+ for (const file of [EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA]) {
175
+ if (!(await statOrUndefined(join(staging, file)))?.isFile()) {
176
+ throw new Error(`extracting ${archive} produced no ${file}; a reuse run cannot read this layout`);
177
+ }
178
+ }
179
+ const after = await nextestArchiveKey(archive);
180
+ if (after !== key) {
181
+ throw new Error(
182
+ `${archive} changed while it was being extracted (sha256 ${key} before, ${after} after); ` +
183
+ 'rerun once whatever is rewriting it has finished',
184
+ );
185
+ }
186
+ await rename(staging, paths.directory);
187
+ return true;
188
+ } catch (error) {
189
+ await rm(staging, { recursive: true, force: true });
190
+ const code = errorCode(error);
191
+ if ((code === 'ENOTEMPTY' || code === 'EEXIST') && (await isDirectory(paths.directory))) {
192
+ return false;
193
+ }
194
+ throw error;
195
+ }
196
+ }
197
+
198
+ async function removeOtherKeys(root: string, key: string): Promise<void> {
199
+ for (const entry of await readdir(root)) {
200
+ if (entry !== key && !entry.startsWith(`${key}.`)) {
201
+ await rm(join(root, entry), { recursive: true, force: true });
202
+ }
203
+ }
204
+ }
205
+
206
+ interface HeldLock {
207
+ release(): Promise<void>;
208
+ }
209
+
210
+ /** Create the lock exclusively, or report it held. A held lock is refreshed until released. */
211
+ async function acquireLock(path: string, heartbeatMs: number): Promise<HeldLock | undefined> {
212
+ const token = `${process.pid} ${randomUUID()}`;
213
+ try {
214
+ await writeFile(path, `${token}\n`, { flag: 'wx' });
215
+ } catch (error) {
216
+ if (errorCode(error) === 'EEXIST') {
217
+ return undefined;
218
+ }
219
+ throw error;
220
+ }
221
+ const heartbeat = setInterval(() => {
222
+ const now = new Date();
223
+ // ENOENT: a waiter judged this lock stale and removed it. The extraction
224
+ // carries on; publishing is atomic whoever else extracts beside it.
225
+ utimes(path, now, now).catch((error: unknown) => {
226
+ if (errorCode(error) !== 'ENOENT') throw error;
227
+ });
228
+ }, heartbeatMs);
229
+ return {
230
+ async release() {
231
+ clearInterval(heartbeat);
232
+ // Remove it only while it is still ours: a waiter that took it over owns it now.
233
+ const content = await readFile(path, 'utf8').catch((error: unknown) => {
234
+ if (errorCode(error) === 'ENOENT') return undefined;
235
+ throw error;
236
+ });
237
+ if (content?.trim() === token) {
238
+ await rm(path, { force: true });
239
+ }
240
+ },
241
+ };
242
+ }
243
+
244
+ async function readLock(path: string): Promise<{ holder: string; ageMs: number } | undefined> {
245
+ try {
246
+ const handle = await open(path, 'r');
247
+ try {
248
+ const [content, metadata] = await Promise.all([handle.readFile('utf8'), handle.stat()]);
249
+ const pid = content.trim().split(' ')[0];
250
+ return {
251
+ // Empty while its creator is between creating and writing it.
252
+ holder: pid === undefined || pid.length === 0 ? 'a process that has not yet named itself' : `pid ${pid}`,
253
+ ageMs: Date.now() - metadata.mtimeMs,
254
+ };
255
+ } finally {
256
+ await handle.close();
257
+ }
258
+ } catch (error) {
259
+ if (errorCode(error) === 'ENOENT') {
260
+ return undefined;
261
+ }
262
+ throw error;
263
+ }
264
+ }
265
+
266
+ /**
267
+ * nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
268
+ * archive and executes nothing from it.
269
+ *
270
+ * `list` loads nextest configuration after extracting, and a repository's
271
+ * `.config/nextest.toml` may name test groups only the plugin's tool config
272
+ * defines, so it is given an empty config of its own instead: unpacking bytes
273
+ * does not depend on how tests are scheduled. `--workspace-remap .` because
274
+ * nextest insists the workspace root exist, and the archive's own is the
275
+ * producing tree's path.
276
+ */
277
+ export const extractWithNextest: NextestArchiveExtractor = async (archive, destination) => {
278
+ const config = join(destination, 'nextest-extract.toml');
279
+ await writeFile(config, '');
280
+ try {
281
+ const args = [
282
+ '--frozen',
283
+ 'nextest',
284
+ 'list',
285
+ '--archive-file',
286
+ archive,
287
+ '--extract-to',
288
+ destination,
289
+ '--workspace-remap',
290
+ '.',
291
+ '--list-type',
292
+ 'binaries-only',
293
+ '--user-config-file',
294
+ 'none',
295
+ '--config-file',
296
+ config,
297
+ ];
298
+ // The binary list on stdout is not wanted; nextest's progress and errors are on stderr.
299
+ const child = spawn('cargo', args, { stdio: ['ignore', 'ignore', 'inherit'] });
300
+ // Rejects with the spawn error when cargo cannot be started at all.
301
+ const [code, signal]: unknown[] = await once(child, 'exit');
302
+ if (code !== 0) {
303
+ throw new Error(
304
+ `cargo ${args.join(' ')} ${code === null ? `died of ${String(signal)}` : `exited with status ${String(code)}`}`,
305
+ );
306
+ }
307
+ } finally {
308
+ await rm(config, { force: true });
309
+ }
310
+ };
311
+
312
+ async function isDirectory(path: string): Promise<boolean> {
313
+ return (await statOrUndefined(path))?.isDirectory() ?? false;
314
+ }
315
+
316
+ async function statOrUndefined(path: string) {
317
+ try {
318
+ return await stat(path);
319
+ } catch (error) {
320
+ if (errorCode(error) === 'ENOENT') {
321
+ return undefined;
322
+ }
323
+ throw error;
324
+ }
325
+ }
326
+
327
+ function errorCode(error: unknown): unknown {
328
+ return error instanceof Error && 'code' in error ? error.code : undefined;
329
+ }