@specific.dev/spectest 0.81.1 → 0.83.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/coverage.ts CHANGED
@@ -13,11 +13,11 @@
13
13
  // a report in the directory.
14
14
  //
15
15
  // After every adapter ran, the harness reads the directory the same way
16
- // it does for `coverage: true`: at least one well-formed report (lcov or
17
- // V8 JSON), shipped verbatim. Nothing is converted in the VM: `node()`
18
- // compacts its V8 dumps (the app's own scripts, each source map once per
19
- // branch) and ships them as they are. The control plane derives what it
20
- // needs from the stored bytes, off the test run.
16
+ // it does for `coverage: true`: at least one well-formed report (lcov,
17
+ // V8 JSON, or a node() script record), shipped verbatim. Nothing is
18
+ // converted, parsed or looked up in the VM: `node()`'s hook collects
19
+ // inside the process and writes what ships. The control plane derives
20
+ // what it needs from the stored bytes, off the test run.
21
21
  //
22
22
  // Imported from `@specific.dev/spectest/coverage`. The three shipped
23
23
  // adapters: `node()` (V8 JSON through a hook spectest mounts), `browser()`
@@ -27,8 +27,6 @@
27
27
  // import * as coverage from "@specific.dev/spectest/coverage";
28
28
  // coverage: { adapters: [coverage.node(), coverage.browser()] }
29
29
 
30
- import { createHash } from "node:crypto";
31
-
32
30
  import type { ServiceConfig } from "./index.js";
33
31
  import { harvestAllBrowserCoverage, takeBrowserCoverageReports } from "./browser-coverage.js";
34
32
 
@@ -45,11 +43,6 @@ export const NODE_COVERAGE_EMPTY_REPORT = "coverage-spectest-empty.json";
45
43
  * to the coverage dir: `.ctl-<pid>`. */
46
44
  export const NODE_COVERAGE_SOCKET_PREFIX = ".ctl-";
47
45
 
48
- /** Key of the content hash (`sha256:<hex>`) a shipped `source-map-cache`
49
- * entry carries, and key of the stub that stands in for an entry that
50
- * shipped earlier on the same branch. See {@link compactV8Document}. */
51
- export const SOURCE_MAP_HASH_KEY = "spectestHash";
52
- export const SOURCE_MAP_REF_KEY = "spectestRef";
53
46
 
54
47
  /** What `configure` learns about the service it rewrites. */
55
48
  export interface CoverageConfigureInfo {
@@ -90,9 +83,6 @@ export interface CoverageCaptureContext {
90
83
  exec(command: string): Promise<{ stdout: string; stderr: string }>;
91
84
  /** Write one report file (name relative to the directory). */
92
85
  writeReport(name: string, content: string): Promise<void>;
93
- /** Read one file out of the container (`docker cp`). `null` when it
94
- * does not exist. Absent on a context that has no container. */
95
- readContainerFile?(containerPath: string): Promise<Buffer | null>;
96
86
  /** Aborts when the per-service capture budget runs out. */
97
87
  signal: AbortSignal;
98
88
  }
@@ -221,90 +211,310 @@ export function appendEnvFlag(
221
211
 
222
212
  // ── node ──────────────────────────────────────────────────────────
223
213
 
214
+ /** Env var the hook reads: the coverage directory inside the container. */
215
+ export const NODE_COVERAGE_DIR_ENV = "SPECTEST_COVERAGE_DIR";
216
+
217
+ /** Subdirectory of the coverage dir where the hook writes one record per
218
+ * script (`scripts/<id>.json`, {@link NODE_COVERAGE_HOOK}). Shipped
219
+ * once, like a dump. */
220
+ export const NODE_COVERAGE_SCRIPTS_SUBDIR = "scripts";
221
+
222
+ /** Hidden subdirectory holding one empty marker per script record ever
223
+ * written on this branch. The hook's "written already?" check, which
224
+ * must survive the harness shipping and removing the record; hidden,
225
+ * so the harness never reads it. */
226
+ export const NODE_COVERAGE_MARKERS_SUBDIR = ".spectest/scripts";
227
+
228
+ /** Name prefix of the dumps the hook writes: `node-<pid>-<thread>-<ms>-<n>.json`. */
229
+ export const NODE_COVERAGE_DUMP_PREFIX = "node-";
230
+
224
231
  /**
225
- * The hook `node()` mounts and `--require`s into every node process of
226
- * the container. Node writes V8 coverage JSON into `NODE_V8_COVERAGE`
227
- * when a process exits; a long-lived server never exits, so the hook
228
- * binds a Unix socket in the coverage directory and calls
229
- * `v8.takeCoverage()` on request. **Every** process binds its own socket
230
- * (`.ctl-<pid>`) — there is no "first process owns it" rule, because the
231
- * first node process is often a wrapper (`pnpm exec`, the `tsx` binary,
232
- * `npm run`) whose child is the real server; a single socket on the
233
- * wrapper dumped the wrapper and silently never the server (reported by
234
- * a user 2026-08-27). At capture spectest asks every live socket and
235
- * unlinks the stale ones. Unref'd, so a short-lived process still exits.
236
- * Main thread only: a worker thread inherits `NODE_OPTIONS`, and tsx's
237
- * ESM loader thread bound the shared per-pid path last, so every
238
- * `--import tsx` server answered with the loader's coverage — the app's
239
- * main thread was never dumped (found in the same user's first full run).
240
- * No signal is used: signals are claimed by frameworks (SIGUSR2 stops a
241
- * Temporal worker, restarts nodemon), a socket is nobody's.
232
+ * The hook `node()` mounts at {@link NODE_COVERAGE_HOOK_PATH} and
233
+ * `--require`s into every node process of the service. **It is the
234
+ * collector**: nothing outside the process reads, converts or looks
235
+ * anything up. Runs in every thread (a worker thread inherits
236
+ * `NODE_OPTIONS`); reads one env var, {@link NODE_COVERAGE_DIR_ENV}; uses
237
+ * node built-ins only, so it runs in any image that runs node.
238
+ *
239
+ * What it does:
240
+ *
241
+ * - Starts V8 precise (block, counted) coverage through the in-process
242
+ * inspector session. `NODE_V8_COVERAGE` is **not** set: Node would then
243
+ * write everything the process loaded at every exit — `node:`
244
+ * internals, every `node_modules` file, and the whole source map of
245
+ * every script that carries a `sourceMappingURL` comment, with or
246
+ * without `--enable-source-maps`. For a bundled CLI that was 12 MB per
247
+ * invocation, inside the test's own time, parsed again at capture.
248
+ * - **Takes** on request — the main thread binds a Unix socket
249
+ * `.ctl-<pid>` in the coverage directory and takes on `dump\n`; every
250
+ * process binds its own (the first node process is often a wrapper,
251
+ * `pnpm exec`, `tsx`, `npm run`) — and in every thread's `exit`
252
+ * handler, which is how a short-lived CLI and a worker thread report.
253
+ * The in-process session answers synchronously, so the exit handler
254
+ * has the result before the process is gone. A take resets V8's
255
+ * counters: every dump is the delta since the previous take. No
256
+ * signal is used: signals are claimed by frameworks (SIGUSR2 stops a
257
+ * Temporal worker, restarts nodemon), a socket is nobody's.
258
+ * - Writes a **dump** `node-<pid>-<thread>-<ms>-<n>.json`: the app's own
259
+ * scripts (`file://`, outside `node_modules` and `.cache`, not the
260
+ * hook) in which some range count is above zero, and of each only the
261
+ * functions that ran, each one flat array: `[start, length, count]`
262
+ * for its extent (V8's first range), then `[offset from start,
263
+ * length, count]` per block range in V8's order. Name and block flag
264
+ * are in the record, joined on the extent. Nothing when nothing ran.
265
+ * `{ spectestVersion: 2, result: [{ url, spectestScript, functions:
266
+ * [[s, l, c, o, l, c, …], …] }] }`.
267
+ * - Writes, **once per script per branch**, a **script record**
268
+ * `scripts/<id>.json` (`id` = sha256 of the URL): the length of every
269
+ * line of the source V8 compiled (`Debugger.getScriptSource`, so a
270
+ * transpiled module — tsx, ts-node, a bundler's register hook — is
271
+ * described by the text V8 ran, not the file on disk), every function
272
+ * with its extent (the skeleton: what a dump's "ran" functions are a
273
+ * subset of, so a never-run function is known to exist), and the
274
+ * source map: the inline `data:` map in the text, the file the last
275
+ * `sourceMappingURL` comment names, or `<script>.map` next to the
276
+ * script; `sourcesContent` dropped (the sources are the repository).
277
+ * "Once per branch" is a marker file under `.spectest/scripts/`: the
278
+ * directory is shared by every process of the service and forks with
279
+ * the environment, so a record written at bring-up or by an ancestor
280
+ * is never written again, whichever process comes next.
281
+ *
282
+ * The harness ships new dumps and new records as they are. It parses
283
+ * nothing and touches no container; the control plane derives lines
284
+ * from the stored documents (`SUBSET_RUNS.md`).
242
285
  */
243
286
  export const NODE_COVERAGE_HOOK = `"use strict";
244
287
  // spectest coverage hook (coverage.node() adapter). See \`spectest docs /services/coverage\`.
245
- // Main thread only: worker threads inherit NODE_OPTIONS and would bind
246
- // the same per-pid socket last — tsx's ESM loader runs on one, and it
247
- // stole the socket from every \`--import tsx\` server, so a dump was the
248
- // loader's isolate and never the app's.
249
- if (process.env.NODE_V8_COVERAGE && require("node:worker_threads").isMainThread) {
250
- const net = require("node:net");
251
- const fs = require("node:fs");
252
- const v8 = require("node:v8");
253
- const SOCK = require("node:path").join(process.env.NODE_V8_COVERAGE, ${JSON.stringify(NODE_COVERAGE_SOCKET_PREFIX)} + process.pid);
254
- const server = net.createServer((conn) => {
255
- // A dump is taken only on an explicit "dump" request.
256
- let buf = "";
257
- conn.on("data", (chunk) => {
258
- buf += chunk;
259
- if (!buf.includes("\\n")) return;
260
- let reply;
288
+ (function () {
289
+ const dir = process.env[${JSON.stringify(NODE_COVERAGE_DIR_ENV)}];
290
+ if (!dir) return;
291
+ let inspector, fs, path, crypto, url, wt, net;
292
+ try {
293
+ inspector = require("node:inspector");
294
+ fs = require("node:fs");
295
+ path = require("node:path");
296
+ crypto = require("node:crypto");
297
+ url = require("node:url");
298
+ wt = require("node:worker_threads");
299
+ net = require("node:net");
300
+ } catch (e) {
301
+ return;
302
+ }
303
+ const session = new inspector.Session();
304
+ try {
305
+ session.connect();
306
+ } catch (e) {
307
+ return;
308
+ }
309
+ // An in-process session answers before post() returns.
310
+ function post(method, params) {
311
+ let out = null;
312
+ session.post(method, params || {}, function (err, res) {
313
+ out = err ? null : res;
314
+ });
315
+ return out;
316
+ }
317
+ if (post("Profiler.enable") === null) return;
318
+ post("Profiler.startPreciseCoverage", { callCount: true, detailed: true });
319
+
320
+ const SELF = url.pathToFileURL(__filename).href;
321
+ const SCRIPTS = path.join(dir, ${JSON.stringify(NODE_COVERAGE_SCRIPTS_SUBDIR)});
322
+ const MARKERS = path.join(dir, ${JSON.stringify(NODE_COVERAGE_MARKERS_SUBDIR)});
323
+ const known = new Set();
324
+ let seq = 0;
325
+
326
+ function isApp(u) {
327
+ return typeof u === "string" && u.startsWith("file://") && u.indexOf("/node_modules/") < 0 && u.indexOf("/.cache/") < 0 && u !== SELF;
328
+ }
329
+ function ran(f) {
330
+ return f.ranges.some(function (r) { return r.count > 0; });
331
+ }
332
+ function idOf(u) {
333
+ return crypto.createHash("sha256").update(u).digest("hex").slice(0, 32);
334
+ }
335
+ function lineLengths(text) {
336
+ const out = [];
337
+ let len = 0;
338
+ for (let i = 0; i < text.length; i++, len++) {
339
+ const c = text.charCodeAt(i);
340
+ if (c === 0x0a || c === 0x2028 || c === 0x2029) {
341
+ out.push(len);
342
+ len = -1;
343
+ }
344
+ }
345
+ out.push(len);
346
+ return out;
347
+ }
348
+ function readMap(scriptPath, text) {
349
+ const re = /\\/\\/[#@]\\s*sourceMappingURL=(\\S+)/g;
350
+ let ref = null;
351
+ let m;
352
+ while ((m = re.exec(text)) !== null) ref = m[1];
353
+ let mapText = null;
354
+ let mapUrl = null;
355
+ try {
356
+ if (ref === null) {
357
+ const p = scriptPath + ".map";
358
+ mapText = fs.readFileSync(p, "utf8");
359
+ mapUrl = url.pathToFileURL(p).href;
360
+ } else if (ref.startsWith("data:")) {
361
+ const c = ref.indexOf(",");
362
+ if (c < 0) return null;
363
+ const head = ref.slice(5, c);
364
+ const body = ref.slice(c + 1);
365
+ mapText = /;base64$/i.test(head) ? Buffer.from(body, "base64").toString("utf8") : decodeURIComponent(body);
366
+ } else if (/^[a-z][a-z0-9+.-]*:/i.test(ref) && !ref.startsWith("file:")) {
367
+ return null;
368
+ } else {
369
+ const p = ref.startsWith("file:") ? url.fileURLToPath(ref) : path.resolve(path.dirname(scriptPath), decodeURIComponent(ref));
370
+ mapText = fs.readFileSync(p, "utf8");
371
+ mapUrl = url.pathToFileURL(p).href;
372
+ }
373
+ } catch (e) {
374
+ return null;
375
+ }
376
+ let data;
377
+ try {
378
+ data = JSON.parse(mapText);
379
+ } catch (e) {
380
+ return null;
381
+ }
382
+ if (!data || typeof data !== "object" || !("mappings" in data)) return null;
383
+ delete data.sourcesContent;
384
+ return { url: mapUrl, data: data };
385
+ }
386
+ function writeAtomic(file, text) {
387
+ const tmp = file + "." + process.pid + "-" + wt.threadId + ".tmp";
388
+ fs.writeFileSync(tmp, text);
389
+ fs.renameSync(tmp, file);
390
+ }
391
+ function describe(s, source) {
392
+ const id = idOf(s.url);
393
+ let scriptPath = null;
394
+ try {
395
+ scriptPath = url.fileURLToPath(s.url);
396
+ } catch (e) {}
397
+ const text = typeof source === "string" ? source : null;
398
+ const record = {
399
+ url: s.url,
400
+ lineLengths: text !== null ? lineLengths(text) : null,
401
+ functions: s.functions.map(function (f) {
402
+ return [f.functionName, f.ranges[0].startOffset, f.ranges[0].endOffset];
403
+ }),
404
+ sourceMap: text !== null && scriptPath !== null ? readMap(scriptPath, text) : null,
405
+ };
406
+ writeAtomic(path.join(SCRIPTS, id + ".json"), JSON.stringify({ spectestScript: record }));
407
+ try {
408
+ fs.writeFileSync(path.join(MARKERS, id), "");
409
+ } catch (e) {}
410
+ }
411
+ function take() {
412
+ const res = post("Profiler.takePreciseCoverage");
413
+ if (!res || !Array.isArray(res.result)) return;
414
+ const kept = [];
415
+ for (const s of res.result) {
416
+ if (isApp(s.url) && Array.isArray(s.functions) && s.functions.some(ran)) kept.push(s);
417
+ }
418
+ if (kept.length === 0) return;
419
+ const fresh = [];
420
+ for (const s of kept) {
421
+ if (known.has(s.url)) continue;
422
+ known.add(s.url);
423
+ if (!fs.existsSync(path.join(MARKERS, idOf(s.url)))) fresh.push(s);
424
+ }
425
+ if (fresh.length > 0) {
261
426
  try {
262
- if (!buf.startsWith("dump")) throw new Error("unknown request");
263
- v8.takeCoverage();
264
- reply = "ok\\n";
265
- } catch (err) {
266
- reply = "error " + (err && err.message ? err.message : String(err)) + "\\n";
427
+ fs.mkdirSync(SCRIPTS, { recursive: true });
428
+ fs.mkdirSync(MARKERS, { recursive: true });
429
+ } catch (e) {}
430
+ post("Debugger.enable");
431
+ for (const s of fresh) {
432
+ const r = post("Debugger.getScriptSource", { scriptId: s.scriptId });
433
+ try {
434
+ describe(s, r && r.scriptSource);
435
+ } catch (e) {}
267
436
  }
268
- conn.end(reply);
437
+ post("Debugger.disable");
438
+ }
439
+ // A ran function is one flat array: [start, length, count] for its
440
+ // extent (V8's first range), then [offset from start, length, count]
441
+ // per block range in V8's order. Small numbers, so the dump gzips to
442
+ // less than half of V8's own JSON; name and block flag live in the
443
+ // script record, joined on the extent.
444
+ const doc = {
445
+ spectestVersion: 2,
446
+ result: kept.map(function (s) {
447
+ return {
448
+ url: s.url,
449
+ spectestScript: idOf(s.url),
450
+ functions: s.functions.filter(ran).map(function (f) {
451
+ const first = f.ranges[0];
452
+ const out = [first.startOffset, first.endOffset - first.startOffset, first.count];
453
+ for (let i = 1; i < f.ranges.length; i++) {
454
+ const r = f.ranges[i];
455
+ out.push(r.startOffset - first.startOffset, r.endOffset - r.startOffset, r.count);
456
+ }
457
+ return out;
458
+ }),
459
+ };
460
+ }),
461
+ };
462
+ const name = ${JSON.stringify(NODE_COVERAGE_DUMP_PREFIX)} + process.pid + "-" + wt.threadId + "-" + Date.now() + "-" + seq++ + ".json";
463
+ writeAtomic(path.join(dir, name), JSON.stringify(doc));
464
+ }
465
+
466
+ if (wt.isMainThread) {
467
+ const SOCK = path.join(dir, ${JSON.stringify(NODE_COVERAGE_SOCKET_PREFIX)} + process.pid);
468
+ const server = net.createServer(function (conn) {
469
+ let buf = "";
470
+ conn.on("data", function (chunk) {
471
+ buf += chunk;
472
+ if (buf.indexOf("\\n") < 0) return;
473
+ let reply;
474
+ try {
475
+ if (!buf.startsWith("dump")) throw new Error("unknown request");
476
+ take();
477
+ reply = "ok\\n";
478
+ } catch (err) {
479
+ reply = "error " + (err && err.message ? err.message : String(err)) + "\\n";
480
+ }
481
+ conn.end(reply);
482
+ });
483
+ });
484
+ server.unref();
485
+ server.on("error", function () {});
486
+ try {
487
+ fs.unlinkSync(SOCK);
488
+ } catch (e) {}
489
+ server.listen(SOCK);
490
+ process.on("exit", function () {
491
+ try {
492
+ fs.unlinkSync(SOCK);
493
+ } catch (e) {}
269
494
  });
495
+ }
496
+ process.on("exit", function () {
497
+ try {
498
+ take();
499
+ } catch (e) {}
270
500
  });
271
- server.unref();
272
- server.on("error", () => {});
273
- try { fs.unlinkSync(SOCK); } catch {}
274
- server.listen(SOCK);
275
- process.on("exit", () => { try { fs.unlinkSync(SOCK); } catch {} });
276
- }
501
+ })();
277
502
  `;
278
503
 
279
504
  /**
280
505
  * Coverage for a Node service — a long-lived server, or a container whose
281
506
  * node processes are short-lived CLIs run by `ctx.exec`. Sets
282
- * `NODE_V8_COVERAGE` to the coverage directory (every node process in the
283
- * container then writes V8 coverage JSON when it exits) and `--require`s
284
- * a hook that lets spectest ask every live node process for a dump at
285
- * capture time — the server, and any wrapper it sits behind (`pnpm exec`,
286
- * `tsx`). Nothing for the app to write.
287
- *
288
- * What ships is the V8 documents themselves, compacted
289
- * ({@link compactV8Document}): the app's own scripts that ran, and each
290
- * script's source map once per branch. A map comes from Node's own
291
- * `source-map-cache` when Node found one through the script's
292
- * `sourceMappingURL` (it caches maps under NODE_V8_COVERAGE with or
293
- * without `--enable-source-maps`), else from `<script>.map` next to the
294
- * script, read out of the container once
295
- * ({@link resolveSourceMapsFromContainer}). The second way is the one to
296
- * build a short-lived bundled process in: a map Node finds is serialized
297
- * whole into every exit dump — 12 MB per invocation for a bundled CLI,
298
- * inside the test's own time — and parsed again by the harness; a map
299
- * with no comment costs nothing there. Nothing is converted in the VM.
300
- * A dump is a **delta** by construction: V8 resets its counters at every
301
- * `takeCoverage()`, so a live server's dump after a test is that test's
302
- * own execution, and the boot dump (everything loaded) lands in the
303
- * bring-up capture and nowhere else. The control plane derives lines
304
- * from the stored documents, off the test run. Converting to lcov in
305
- * the container (`c8`, SDK 0.60 to 0.79) took seconds per capture per
306
- * service on a real project, inside the test run; a real project turned
307
- * coverage off because of it, and the conversion went.
507
+ * {@link NODE_COVERAGE_DIR_ENV} to the coverage directory and
508
+ * `--require`s {@link NODE_COVERAGE_HOOK}, which collects everything in
509
+ * the process. Nothing for the app to write, and nothing the harness
510
+ * reads out of the container: at capture it asks every live process for
511
+ * a take over its socket and lists what appeared in the directory —
512
+ * dumps and script records — for the harness to ship as they are. A
513
+ * dump is a delta by construction (V8 resets its counters at every
514
+ * take), so the report after a test is that test's own execution and
515
+ * the boot dump lands in the bring-up capture alone. Each capture ships
516
+ * only its own files: what an earlier capture on this branch shipped is
517
+ * removed first (module memory, forks with the environment).
308
518
  */
309
519
  export function node(): CoverageAdapter {
310
520
  return {
@@ -312,7 +522,7 @@ export function node(): CoverageAdapter {
312
522
  reports: "delta",
313
523
  configure(svc) {
314
524
  const env = appendEnvFlag(svc.env, "NODE_OPTIONS", `--require ${NODE_COVERAGE_HOOK_PATH}`);
315
- env.NODE_V8_COVERAGE = COVERAGE_CONTAINER_DIR;
525
+ env[NODE_COVERAGE_DIR_ENV] = COVERAGE_CONTAINER_DIR;
316
526
  return {
317
527
  ...svc,
318
528
  env,
@@ -329,24 +539,18 @@ export function node(): CoverageAdapter {
329
539
  // rather than failed: an empty report is a report. A server whose
330
540
  // hook never loaded (NODE_OPTIONS not reaching it) then shows as
331
541
  // empty reports after bring-up, which the boot log warns about.
332
- //
333
- // Each capture ships only its own dumps: what an earlier capture on
334
- // this branch shipped is removed first (module memory, forks with
335
- // the environment).
336
542
  const t0 = performance.now();
337
- await removeShippedV8Reports(ctx.reportDir);
543
+ await removeShippedNodeReports(ctx.reportDir);
338
544
  const live = await dumpAllNodeProcesses(ctx.reportDir, ctx.signal);
339
545
  const t1 = performance.now();
340
- const stats = await prepareV8Reports(ctx.reportDir, (urls) => resolveSourceMapsFromContainer(ctx, urls));
546
+ const stats = await collectNodeReports(ctx.reportDir);
341
547
  if (stats.dumps === 0) {
342
548
  await ctx.writeReport(NODE_COVERAGE_EMPTY_REPORT, '{"result":[]}\n');
343
- SHIPPED_V8_REPORTS.add((await import("node:path")).join(ctx.reportDir, NODE_COVERAGE_EMPTY_REPORT));
549
+ SHIPPED_NODE_REPORTS.add((await import("node:path")).join(ctx.reportDir, NODE_COVERAGE_EMPTY_REPORT));
344
550
  }
345
- const t2 = performance.now();
346
551
  console.log(
347
552
  `[coverage] node/${ctx.service}: ${live} live process(es) dumped in ${Math.round(t1 - t0)} ms; ` +
348
- `${stats.dumps} dump(s), ${stats.scripts} script(s), ${stats.bytesIn} → ${stats.bytesOut} bytes compacted in ${Math.round(t2 - t1)} ms; ` +
349
- `${stats.mapsFromContainer} map(s) read from the container`,
553
+ `${stats.dumps} dump(s) (${stats.dumpBytes} B), ${stats.scripts} new script record(s) (${stats.scriptBytes} B)`,
350
554
  );
351
555
  },
352
556
  };
@@ -402,387 +606,84 @@ export async function dumpAllNodeProcesses(dir: string, signal: AbortSignal): Pr
402
606
  return live;
403
607
  }
404
608
 
405
- /** Report files shipped at an earlier capture on this branch, by host
406
- * path. Module memory: forks with the environment, so a forked child
407
- * removes its ancestors' reports at its first capture and ships only
408
- * its own. */
409
- const SHIPPED_V8_REPORTS = new Set<string>();
410
-
411
- /** Source-map entries shipped earlier on this branch: script URL → hash
412
- * of the entry. Module memory, forks with the environment. A later dump
413
- * of the same script carries a stub that refers to the hash instead of
414
- * the map again. */
415
- const SHIPPED_SOURCE_MAPS = new Map<string, string>();
416
-
417
- /** Scripts looked up in the container and found to have no source map
418
- * (no `sourceMappingURL`, or a map file that is not there). Module
419
- * memory, forks with the environment: one lookup per script per branch. */
420
- const NO_SOURCE_MAP = new Set<string>();
609
+ /** Files shipped at an earlier capture on this branch, by host path:
610
+ * dumps, script records, the empty report. Module memory: forks with
611
+ * the environment, so a forked child removes its ancestors' files at
612
+ * its first capture and ships only its own. The hook's own "written
613
+ * already" markers are elsewhere and stay. */
614
+ const SHIPPED_NODE_REPORTS = new Set<string>();
421
615
 
422
616
  /** Forget the branch memory (tests). */
423
617
  export function resetNodeCoverageMemory(): void {
424
- SHIPPED_V8_REPORTS.clear();
425
- SHIPPED_SOURCE_MAPS.clear();
426
- NO_SOURCE_MAP.clear();
618
+ SHIPPED_NODE_REPORTS.clear();
427
619
  }
428
620
 
429
- /** Remove from `dir` the reports an earlier capture shipped. */
430
- export async function removeShippedV8Reports(dir: string): Promise<void> {
621
+ /** Remove from `dir` the files an earlier capture shipped. */
622
+ export async function removeShippedNodeReports(dir: string): Promise<void> {
431
623
  const fs = await import("node:fs/promises");
432
- for (const file of [...SHIPPED_V8_REPORTS]) {
624
+ for (const file of [...SHIPPED_NODE_REPORTS]) {
433
625
  if (!file.startsWith(dir + "/")) continue;
434
626
  await fs.unlink(file).catch(() => {});
435
- SHIPPED_V8_REPORTS.delete(file);
436
- }
437
- }
438
-
439
- /** A script is the app's own when it is a file outside node_modules —
440
- * and not our hook, which every dump would otherwise carry. */
441
- export function isAppScriptUrl(url: string): boolean {
442
- return (
443
- url.startsWith("file://") &&
444
- !url.includes("/node_modules/") &&
445
- // Package managers run from a cache, not node_modules: corepack's pnpm
446
- // is `/root/.cache/node/corepack/…`, 2 MiB of dump per `pnpm run`.
447
- !url.includes("/.cache/") &&
448
- !url.endsWith("/" + NODE_COVERAGE_HOOK_PATH.split("/").pop())
449
- );
450
- }
451
-
452
- interface V8Range {
453
- count?: unknown;
454
- }
455
- interface V8Function {
456
- ranges?: V8Range[];
457
- }
458
- interface V8Script {
459
- url?: unknown;
460
- functions?: V8Function[];
461
- }
462
-
463
- /** True when any range of any function of the script has a count above
464
- * zero: something in it ran since the previous take. A script whose
465
- * every count is zero is one V8 still lists after a reset; it says
466
- * nothing and is dropped. */
467
- export function scriptExecuted(script: V8Script): boolean {
468
- const fns = Array.isArray(script.functions) ? script.functions : [];
469
- return fns.some((f) => Array.isArray(f.ranges) && f.ranges.some((r) => typeof r.count === "number" && r.count > 0));
470
- }
471
-
472
- /**
473
- * Compact one V8 coverage document to what the derivation needs. What
474
- * Node writes is everything the process loaded: `node:` internals, every
475
- * `node_modules` file, and — under `--enable-source-maps` — a
476
- * `source-map-cache` with each file's full map **and its sources**,
477
- * repeated in every dump. Measured on a real project: a 12–20 MiB dump
478
- * per capture, of which the app's own coverage was under 0.5 MiB, and a
479
- * suite that hit the 64 MiB cap on its third test. Kept: `file://`
480
- * scripts outside `node_modules` in which something ran, and the map
481
- * entries of exactly those scripts, minus `sourcesContent` (the sources
482
- * are the repo). Each kept map entry carries its content hash under
483
- * {@link SOURCE_MAP_HASH_KEY}; when `shipped` holds the same hash for the
484
- * URL — the map went out with an earlier dump on this branch — a stub
485
- * `{ [SOURCE_MAP_REF_KEY]: hash }` stands in for it, and the reader finds
486
- * the map in an earlier capture of the branch. `shipped` is updated in
487
- * place. Still a V8 document — nothing is converted here.
488
- */
489
- export function compactV8Document(
490
- doc: Record<string, unknown>,
491
- shipped?: Map<string, string>,
492
- ): Record<string, unknown> {
493
- const result = Array.isArray(doc.result) ? (doc.result as V8Script[]) : [];
494
- const kept = result.filter((s) => typeof s.url === "string" && isAppScriptUrl(s.url) && scriptExecuted(s));
495
- const out: Record<string, unknown> = { ...doc, result: kept };
496
- const cache = doc["source-map-cache"];
497
- if (cache && typeof cache === "object") {
498
- const urls = new Set(kept.map((s) => s.url as string));
499
- const slim: Record<string, unknown> = {};
500
- for (const [url, entry] of Object.entries(cache as Record<string, unknown>)) {
501
- if (!urls.has(url) || !entry || typeof entry !== "object") continue;
502
- const e = { ...(entry as Record<string, unknown>) };
503
- if (e.data && typeof e.data === "object") {
504
- const { sourcesContent: _dropped, ...data } = e.data as Record<string, unknown>;
505
- e.data = data;
506
- }
507
- const hash = sourceMapEntryHash(e);
508
- if (shipped?.get(url) === hash) {
509
- slim[url] = { [SOURCE_MAP_REF_KEY]: hash };
510
- } else {
511
- slim[url] = { ...e, [SOURCE_MAP_HASH_KEY]: hash };
512
- shipped?.set(url, hash);
513
- }
514
- }
515
- if (Object.keys(slim).length > 0) out["source-map-cache"] = slim;
516
- else delete out["source-map-cache"];
627
+ SHIPPED_NODE_REPORTS.delete(file);
517
628
  }
518
- return out;
519
629
  }
520
630
 
521
- /** What {@link prepareV8Reports} did, for the capture's log line. */
522
- export interface PrepareV8Stats {
631
+ /** What {@link collectNodeReports} found, for the capture's log line. */
632
+ export interface NodeReportStats {
523
633
  dumps: number;
634
+ dumpBytes: number;
524
635
  scripts: number;
525
- bytesIn: number;
526
- bytesOut: number;
527
- /** Source maps read out of the container this capture. */
528
- mapsFromContainer: number;
529
- }
530
-
531
- /** One `source-map-cache` entry in the shape Node writes: the map (its
532
- * `sourcesContent` dropped), the generated script's line lengths, and
533
- * the map's own URL. */
534
- export interface SourceMapEntry {
535
- url: string | null;
536
- data: Record<string, unknown>;
537
- lineLengths: number[];
636
+ scriptBytes: number;
538
637
  }
539
638
 
540
- /** Finds the source maps of the scripts a dump carries no map for, in
541
- * one go. A script absent from the result has none. */
542
- export type SourceMapResolver = (scriptUrls: string[]) => Promise<Map<string, SourceMapEntry>>;
543
-
544
- /** Shell-quote one argument for `sh -c`. */
545
- function shQuote(s: string): string {
546
- return `'${s.replace(/'/g, `'\\''`)}'`;
639
+ /** True for a dump the hook wrote (`node-*.json`). */
640
+ export function isNodeDumpName(name: string): boolean {
641
+ return name.startsWith(NODE_COVERAGE_DUMP_PREFIX) && name.endsWith(".json");
547
642
  }
548
643
 
549
644
  /**
550
- * The probe {@link resolveSourceMapsFromContainer} runs in the container:
551
- * one `sh` loop over every script path, printing `<path>\t<ref>` — the
552
- * last `sourceMappingURL` value in the file, else `<path>.map` when that
553
- * file exists, else `-`; `!` for a script that is not there. One exec per
554
- * capture per service, whatever the number of scripts: a Next.js server
555
- * loads hundreds of chunk scripts with no map, and a `docker cp` per
556
- * script (the first cut) took a capture past its budget on a busy host.
645
+ * Mark shipped every dump at the root of `dir` and every script record
646
+ * under `scripts/` that no earlier capture shipped, and count them.
647
+ * Nothing is read: the hook wrote each file whole (write + rename), and
648
+ * the harness ships them as they are.
557
649
  */
558
- export function sourceMapProbeCommand(scriptPaths: string[]): string {
559
- const script =
560
- 'for f in "$@"; do ' +
561
- 'if [ -f "$f" ]; then ' +
562
- "ref=$(grep -o 'sourceMappingURL=[^[:space:]]*' -- \"$f\" 2>/dev/null | tail -n 1 | cut -c18-); " +
563
- 'if [ -n "$ref" ]; then printf \'%s\\t%s\\n\' "$f" "$ref"; ' +
564
- 'elif [ -f "$f.map" ]; then printf \'%s\\t%s.map\\n\' "$f" "$f"; ' +
565
- 'else printf \'%s\\t-\\n\' "$f"; fi; ' +
566
- 'else printf \'%s\\t!\\n\' "$f"; fi; ' +
567
- "done";
568
- return `sh -c ${shQuote(script)} sh ${scriptPaths.map(shQuote).join(" ")}`;
569
- }
570
-
571
- /** The length of every line of `text`, the way Node computes it for
572
- * `source-map-cache`: split on `\n` (and U+2028/2029), `\r` kept, the
573
- * last line included. Lengths in UTF-16 units, which is what V8's byte
574
- * offsets count in. */
575
- export function lineLengthsOf(text: string): number[] {
576
- const out: number[] = [];
577
- let len = 0;
578
- for (let i = 0; i < text.length; i++, len++) {
579
- const c = text.charCodeAt(i);
580
- if (c === 0x0a || c === 0x2028 || c === 0x2029) {
581
- out.push(len);
582
- len = -1;
583
- }
584
- }
585
- out.push(len);
586
- return out;
587
- }
588
-
589
- /**
590
- * The source maps of `scriptUrls`, read out of the container. One probe
591
- * exec ({@link sourceMapProbeCommand}) finds, per script, the map a
592
- * `sourceMappingURL` comment names or `<script>.map` next to it; only a
593
- * script that has one is then read (`docker cp`, the script for its line
594
- * lengths and the map file, or the inline `data:` map decoded). A map's
595
- * `sourcesContent` is dropped. Scripts that are not `file://` URLs, have
596
- * no map, or whose map is not there or not JSON are absent from the
597
- * result. The caller remembers every outcome, so this runs once per
598
- * script per branch.
599
- *
600
- * With no comment the map next to the script is the shape to build a
601
- * short-lived bundled process in: Node caches a map it finds through a
602
- * comment into every dump it writes under NODE_V8_COVERAGE, flag or no
603
- * flag, and a bundled CLI's map is tens of MB per exit; a map it does not
604
- * find costs nothing there.
605
- */
606
- export async function resolveSourceMapsFromContainer(
607
- ctx: CoverageCaptureContext,
608
- scriptUrls: string[],
609
- ): Promise<Map<string, SourceMapEntry>> {
610
- const out = new Map<string, SourceMapEntry>();
611
- if (!ctx.readContainerFile) return out;
612
- const { fileURLToPath, pathToFileURL } = await import("node:url");
650
+ export async function collectNodeReports(dir: string): Promise<NodeReportStats> {
651
+ const fs = await import("node:fs/promises");
613
652
  const path = await import("node:path");
614
- const byPath = new Map<string, string>();
615
- for (const url of scriptUrls) {
616
- if (!url.startsWith("file://")) continue;
653
+ const stats: NodeReportStats = { dumps: 0, dumpBytes: 0, scripts: 0, scriptBytes: 0 };
654
+ const seen = async (d: string, keep: (name: string) => boolean, onNew: (bytes: number) => void): Promise<void> => {
655
+ let names: string[];
617
656
  try {
618
- byPath.set(fileURLToPath(url), url);
657
+ names = await fs.readdir(d);
619
658
  } catch {
620
- // not a path
659
+ return;
621
660
  }
622
- }
623
- if (byPath.size === 0) return out;
624
- let probe: string;
625
- try {
626
- probe = (await ctx.exec(sourceMapProbeCommand([...byPath.keys()]))).stdout;
627
- } catch {
628
- return out; // no shell, no grep: no maps from this container
629
- }
630
- for (const line of probe.split("\n")) {
631
- const tab = line.indexOf("\t");
632
- if (tab < 0) continue;
633
- const scriptPath = line.slice(0, tab);
634
- const ref = line.slice(tab + 1).trim();
635
- const url = byPath.get(scriptPath);
636
- if (!url || ref === "-" || ref === "!" || ref === "") continue;
637
- let mapText: string;
638
- let mapUrl: string | null;
639
- if (ref.startsWith("data:")) {
640
- const comma = ref.indexOf(",");
641
- if (comma < 0) continue;
642
- const head = ref.slice(5, comma);
643
- const body = ref.slice(comma + 1);
661
+ for (const name of names.sort()) {
662
+ if (!keep(name)) continue;
663
+ const file = path.join(d, name);
664
+ if (SHIPPED_NODE_REPORTS.has(file)) continue;
665
+ let size: number;
644
666
  try {
645
- mapText = /;base64$/i.test(head) ? Buffer.from(body, "base64").toString("utf8") : decodeURIComponent(body);
667
+ size = (await fs.stat(file)).size;
646
668
  } catch {
647
669
  continue;
648
670
  }
649
- mapUrl = null;
650
- } else {
651
- let mapPath: string;
652
- if (ref.startsWith("file://")) {
653
- try {
654
- mapPath = fileURLToPath(ref);
655
- } catch {
656
- continue;
657
- }
658
- } else if (/^[a-z]+:/i.test(ref)) {
659
- continue; // an http(s) map is not in the container
660
- } else {
661
- let rel = ref;
662
- try {
663
- rel = decodeURIComponent(ref);
664
- } catch {
665
- // keep as written
666
- }
667
- mapPath = path.resolve(path.dirname(scriptPath), rel);
668
- }
669
- const map = await ctx.readContainerFile(mapPath);
670
- if (!map) continue;
671
- mapText = map.toString("utf8");
672
- mapUrl = pathToFileURL(mapPath).href;
673
- }
674
- let data: Record<string, unknown>;
675
- try {
676
- data = JSON.parse(mapText) as Record<string, unknown>;
677
- } catch {
678
- continue;
679
- }
680
- if (!data || typeof data !== "object" || !("mappings" in data)) continue;
681
- const script = await ctx.readContainerFile(scriptPath);
682
- if (!script) continue;
683
- const { sourcesContent: _dropped, ...slim } = data;
684
- out.set(url, { url: mapUrl, data: slim, lineLengths: lineLengthsOf(script.toString("utf8")) });
685
- }
686
- return out;
687
- }
688
-
689
- /** The content hash of a map entry, over its JSON with the spectest
690
- * keys removed. */
691
- function sourceMapEntryHash(entry: Record<string, unknown>): string {
692
- const e = { ...entry };
693
- delete e[SOURCE_MAP_HASH_KEY];
694
- delete e[SOURCE_MAP_REF_KEY];
695
- return "sha256:" + createHash("sha256").update(JSON.stringify(e)).digest("hex");
696
- }
697
-
698
- /**
699
- * Give every kept script of a compacted document a `source-map-cache`
700
- * entry it lacks: a stub when the map shipped earlier on this branch,
701
- * else the map `resolve` finds (asked once for all such scripts; shipped
702
- * whole, with its hash, and remembered), else nothing — and that outcome
703
- * is remembered too, so a script with no map is looked up once per
704
- * branch. Returns how many maps `resolve` supplied.
705
- */
706
- export async function attachSourceMaps(
707
- doc: Record<string, unknown>,
708
- resolve: SourceMapResolver,
709
- shipped: Map<string, string> = SHIPPED_SOURCE_MAPS,
710
- missing: Set<string> = NO_SOURCE_MAP,
711
- ): Promise<number> {
712
- const result = Array.isArray(doc.result) ? (doc.result as V8Script[]) : [];
713
- const cache = ((doc["source-map-cache"] as Record<string, unknown> | undefined) ?? {}) as Record<string, unknown>;
714
- const unknown: string[] = [];
715
- for (const s of result) {
716
- const url = s.url;
717
- if (typeof url !== "string" || url in cache) continue;
718
- const prior = shipped.get(url);
719
- if (prior) {
720
- cache[url] = { [SOURCE_MAP_REF_KEY]: prior };
721
- continue;
722
- }
723
- if (missing.has(url) || unknown.includes(url)) continue;
724
- unknown.push(url);
725
- }
726
- let found = 0;
727
- if (unknown.length > 0) {
728
- const entries = await resolve(unknown);
729
- for (const url of unknown) {
730
- const entry = entries.get(url);
731
- if (!entry) {
732
- missing.add(url);
733
- continue;
734
- }
735
- const hash = sourceMapEntryHash(entry as unknown as Record<string, unknown>);
736
- cache[url] = { ...entry, [SOURCE_MAP_HASH_KEY]: hash };
737
- shipped.set(url, hash);
738
- found++;
671
+ SHIPPED_NODE_REPORTS.add(file);
672
+ onNew(size);
739
673
  }
740
- }
741
- if (Object.keys(cache).length > 0) doc["source-map-cache"] = cache;
742
- return found;
743
- }
744
-
745
- /**
746
- * Compact, in place, every `coverage-*.json` in `dir` that no earlier
747
- * capture shipped, attach the maps the dump lacks through `resolve`
748
- * ({@link attachSourceMaps}), and mark it shipped. A dump mid-write (not
749
- * yet valid JSON) is left for the next capture.
750
- */
751
- export async function prepareV8Reports(dir: string, resolve?: SourceMapResolver): Promise<PrepareV8Stats> {
752
- const fs = await import("node:fs/promises");
753
- const path = await import("node:path");
754
- const stats: PrepareV8Stats = { dumps: 0, scripts: 0, bytesIn: 0, bytesOut: 0, mapsFromContainer: 0 };
755
- let names: string[];
756
- try {
757
- names = await fs.readdir(dir);
758
- } catch {
759
- return stats;
760
- }
761
- for (const name of names.sort()) {
762
- if (!name.startsWith("coverage-") || !name.endsWith(".json") || name === NODE_COVERAGE_EMPTY_REPORT) continue;
763
- const file = path.join(dir, name);
764
- if (SHIPPED_V8_REPORTS.has(file)) continue;
765
- let text: string;
766
- let doc: Record<string, unknown>;
767
- try {
768
- text = await fs.readFile(file, "utf8");
769
- doc = JSON.parse(text) as Record<string, unknown>;
770
- } catch {
771
- continue; // a dump mid-write, or not ours; the read path judges it
772
- }
773
- if (!Array.isArray(doc.result)) continue;
774
- const compact = compactV8Document(doc, SHIPPED_SOURCE_MAPS);
775
- if (resolve) stats.mapsFromContainer += await attachSourceMaps(compact, resolve);
776
- const out = JSON.stringify(compact);
777
- const tmp = path.join(dir, `.${name}.compact`);
778
- await fs.writeFile(tmp, out);
779
- await fs.rename(tmp, file);
780
- SHIPPED_V8_REPORTS.add(file);
674
+ };
675
+ await seen(dir, isNodeDumpName, (b) => {
781
676
  stats.dumps++;
782
- stats.scripts += (doc.result as unknown[]).length;
783
- stats.bytesIn += text.length;
784
- stats.bytesOut += out.length;
785
- }
677
+ stats.dumpBytes += b;
678
+ });
679
+ await seen(
680
+ path.join(dir, NODE_COVERAGE_SCRIPTS_SUBDIR),
681
+ (n) => n.endsWith(".json"),
682
+ (b) => {
683
+ stats.scripts++;
684
+ stats.scriptBytes += b;
685
+ },
686
+ );
786
687
  return stats;
787
688
  }
788
689