@specific.dev/spectest 0.59.3 → 0.60.0

package/src/daemon.ts

@@ -41,6 +41,7 @@ import {
   encodeCoverageBundle,
   isEmptyReport,
   readCoverageDir,
+  applyCoverageDelta,
   type CoverageBundleRef,
   type CoverageReport,
   type ServiceCoverageCapture,
@@ -49,7 +50,11 @@ import { configureBrowserCoverage } from "./browser-coverage.js";
 import {
   applyCoverageAdapters,
   coverageAdapters,
+  coverageReportsMode,
   validateCoverage,
+  nodeCoverageToolsAvailable,
+  nodeCoverageToolsDir,
+  NODE_COVERAGE_TOOLS_CONTAINER_DIR,
   type CoverageAdapter,
   type CoverageCaptureContext,
 } from "./coverage.js";
@@ -790,7 +795,16 @@ async function ensureCoverage(svc: NamedService): Promise<string[]> {
   const host = coverageHostDir(WORKSPACE, svc.name);
   await fs.mkdir(host, { recursive: true });
   await fs.chmod(host, 0o777);
-  return [`--volume=${host}:${COVERAGE_CONTAINER_DIR}`];
+  const flags = [`--volume=${host}:${COVERAGE_CONTAINER_DIR}`];
+  // The conversion tools golden ships (`c8` for `coverage.node()`),
+  // read-only, on the same existence check the adapter makes at capture
+  // — so the VM and the container agree on whether they are there. A
+  // plain `--volume`, never a config `volumes` entry: an absolute-source
+  // volume dir is listed in the delta-restore manifest and wiped.
+  if (await nodeCoverageToolsAvailable()) {
+    flags.push(`--volume=${nodeCoverageToolsDir()}:${NODE_COVERAGE_TOOLS_CONTAINER_DIR}:ro`);
+  }
+  return flags;
 }
 
 /** A `user`/`group` pair resolved to the numeric ids the kernel wants.
@@ -3667,8 +3681,14 @@ function rebuildBrowserCoverageResolver(): void {
   });
 }
 
-/** The context one adapter's `capture` runs against. */
-function coverageCaptureContext(svc: NamedService, signal: AbortSignal): CoverageCaptureContext {
+/** The context one adapter's `capture` runs against. Every report it
+ *  writes through `writeReport` is added to `written`, so the delta step
+ *  knows which files came from which adapter. */
+function coverageCaptureContext(
+  svc: NamedService,
+  signal: AbortSignal,
+  written: Set<string>,
+): CoverageCaptureContext {
   const reportDir = coverageHostDir(WORKSPACE, svc.name);
   return {
     service: svc.name,
@@ -3691,20 +3711,27 @@ function coverageCaptureContext(svc: NamedService, signal: AbortSignal): Coverag
       const tmp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
       await fs.writeFile(tmp, content);
       await fs.rename(tmp, file);
+      written.add(path.relative(reportDir, file));
     },
   };
 }
 
 /** Run one service's adapters in order under one budget. Returns the
- *  error to fail the capture with, naming the adapter. */
-async function runCoverageAdapters(svc: NamedService): Promise<string | undefined> {
+ *  error to fail the capture with, naming the adapter, and the reports
+ *  written by adapters that declare `reports: "delta"`. */
+async function runCoverageAdapters(
+  svc: NamedService,
+): Promise<{ error?: string; deltaReports: Set<string> }> {
+  const deltaReports = new Set<string>();
   const adapters = coverageAdapters(svc.coverage).filter((a) => a.capture);
-  if (adapters.length === 0) return undefined;
+  if (adapters.length === 0) return { deltaReports };
   const ac = new AbortController();
   const timer = setTimeout(() => ac.abort(), COVERAGE_CAPTURE_TIMEOUT_MS);
-  const ctx = coverageCaptureContext(svc, ac.signal);
+  const written = new Set<string>();
+  const ctx = coverageCaptureContext(svc, ac.signal, written);
   try {
     for (const a of adapters) {
+      written.clear();
       try {
         await Promise.race([
           (a as Required<Pick<CoverageAdapter, "capture">>).capture(ctx),
@@ -3717,12 +3744,16 @@ async function runCoverageAdapters(svc: NamedService): Promise<string | undefine
           }),
         ]);
       } catch (err) {
-        return `coverage adapter "${a.name}" on service "${svc.name}" failed: ${
-          err instanceof Error ? err.message : String(err)
-        }`;
+        return {
+          error: `coverage adapter "${a.name}" on service "${svc.name}" failed: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+          deltaReports,
+        };
       }
+      if (a.reports === "delta") for (const n of written) deltaReports.add(n);
     }
-    return undefined;
+    return { deltaReports };
   } finally {
     clearTimeout(timer);
   }
@@ -3737,9 +3768,14 @@ async function captureServiceCoverage(): Promise<ServiceCoverageCapture[]> {
   const services = [...byName.values()].filter((s) => s.coverage !== undefined);
   return Promise.all(
     services.map(async (svc): Promise<ServiceCoverageCapture> => {
-      const error = await runCoverageAdapters(svc);
+      const { error, deltaReports } = await runCoverageAdapters(svc);
       if (error) return { service: svc.name, reports: [], error };
-      return readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
+      const read = await readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
+      // What ships is what ran during this capture, whatever the tool
+      // counts: cumulative lcov is subtracted against the previous
+      // capture (module memory, forks with the env); a delta adapter's
+      // files and V8 JSON pass through.
+      return applyCoverageDelta(read, coverageReportsMode(svc.coverage), deltaReports);
     }),
   );
 }

package/src/harness/browser-coverage.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 import {
   decodeSourceMap,
   decodeVlq,
+  isDocumentScriptPath,
   isForeignSource,
   lcovDocument,
   mapToOriginal,
@@ -92,3 +93,19 @@ describe("serviceForHost", () => {
     expect(serviceForHost("example.com", services, table)).toBeUndefined();
   });
 });
+
+describe("turbopack + inline scripts", () => {
+  test("[project] root token is stripped, [turbopack] shims are foreign", () => {
+    expect(normalizeSourcePath("/[project]/dashboard/app/page.tsx")).toBe("dashboard/app/page.tsx");
+    expect(normalizeSourcePath("[project]/dashboard/app/page.tsx")).toBe("dashboard/app/page.tsx");
+    expect(isForeignSource("/[turbopack]/browser/runtime/base/build-base.ts")).toBe(true);
+    expect(isForeignSource("[turbopack]/shared/runtime-utils.ts")).toBe(true);
+    expect(isForeignSource("dashboard/app/page.tsx")).toBe(false);
+  });
+  test("an unmapped script at a page URL is a document script, a .js file is not", () => {
+    expect(isDocumentScriptPath("/")).toBe(true);
+    expect(isDocumentScriptPath("/projects/proj_1")).toBe(true);
+    expect(isDocumentScriptPath("/_next/static/chunks/main.js")).toBe(false);
+    expect(isDocumentScriptPath("/app.js")).toBe(false);
+  });
+});

package/src/harness/browser-coverage.ts

@@ -136,6 +136,10 @@ export function normalizeSourcePath(src: string, sourceRoot?: string): string {
   if (m) s = m[1];
   s = s.replace(/^\/@fs\//, "/");
   s = s.replace(/\?.*$/, "");
+  // Turbopack (Next.js) roots every source at the `[project]` token —
+  // `/[project]/dashboard/app/page.tsx` — which is the workspace root,
+  // i.e. repo-relative once stripped.
+  s = s.replace(/^\/?\[project\]\//, "");
   while (s.startsWith("./")) s = s.slice(2);
   return s;
 }
@@ -150,10 +154,24 @@ export function isForeignSource(path: string): boolean {
     path.startsWith("(webpack)") ||
     path.startsWith("\0") ||
     path.startsWith("vite/") ||
-    path.startsWith("@vite/")
+    path.startsWith("@vite/") ||
+    path.startsWith("[turbopack]/") ||
+    path.startsWith("/[turbopack]/")
   );
 }
 
+/**
+ * A script with no source map is attributed to its served path — unless
+ * that path is a page, not a file: an inline `<script>` in an HTML
+ * document reports the document's URL (`/`, `/projects/123`), and no
+ * repository file can ever be named by it. Extension-less paths are
+ * treated as pages.
+ */
+export function isDocumentScriptPath(pathname: string): boolean {
+  const last = pathname.split("/").pop() ?? "";
+  return !last.includes(".");
+}
+
 /**
  * Decode a source map document. Index maps (`sections`) are not
  * supported and decode to `null`, as does anything malformed — the

package/src/harness/coverage.test.ts

@@ -131,3 +131,75 @@ test("bundle round-trips and the ref summarises it", () => {
 test("coverageHostDir sits under the workspace's .spectest", () => {
   expect(coverageHostDir("/workspace", "api")).toBe("/workspace/.spectest/coverage/api");
 });
+
+import {
+  applyCoverageDelta,
+  parseLcovCounts,
+  renderLcovCounts,
+  resetCoverageDeltaMemory,
+  subtractLcovCounts,
+} from "./coverage.js";
+
+describe("coverage delta", () => {
+  const boot = "TN:\nSF:/app/a.py\nFN:1,main\nFNDA:1,main\nDA:1,1\nDA:2,1\nDA:3,0\nend_of_record\nSF:/app/b.py\nDA:1,5\nend_of_record\n";
+  const later = "TN:\nSF:/app/a.py\nFN:1,main\nFNDA:1,main\nDA:1,1\nDA:2,1\nDA:3,4\nend_of_record\nSF:/app/b.py\nDA:1,5\nend_of_record\nSF:/app/c.py\nDA:1,2\nend_of_record\n";
+
+  test("parse merges records for one path and reads DA/FNDA/FN", () => {
+    const c = parseLcovCounts("SF:/x\nDA:1,1\nend_of_record\nSF:/x\nDA:1,2\nFN:3,f\nFNDA:4,f\nend_of_record\nSF:\nDA:9,9\n");
+    expect([...c.keys()]).toEqual(["/x"]);
+    expect(c.get("/x")!.lines.get(1)).toBe(3);
+    expect(c.get("/x")!.functions.get("f")).toBe(4);
+    expect(c.get("/x")!.functionLines.get("f")).toBe(3);
+  });
+
+  test("subtract keeps only what ran since; a dropped counter is a restart; a new file counts whole", () => {
+    const d = subtractLcovCounts(parseLcovCounts(later), parseLcovCounts(boot));
+    expect([...d.keys()].sort()).toEqual(["/app/a.py", "/app/c.py"]); // b.py: nothing new
+    expect(d.get("/app/a.py")!.lines.get(3)).toBe(4);
+    expect(d.get("/app/a.py")!.lines.get(1)).toBe(0);
+    expect(d.get("/app/a.py")!.functions.get("main")).toBe(0);
+    expect(d.get("/app/c.py")!.lines.get(1)).toBe(2);
+    const restarted = subtractLcovCounts(parseLcovCounts("SF:/app/b.py\nDA:1,2\nend_of_record\n"), parseLcovCounts(boot));
+    expect(restarted.get("/app/b.py")!.lines.get(1)).toBe(2);
+    // A record with no counts at all: presence is its signal, new only once.
+    const bare = "SF:/app/bare.py\nend_of_record\n";
+    expect(subtractLcovCounts(parseLcovCounts(bare), new Map()).has("/app/bare.py")).toBe(true);
+    expect(subtractLcovCounts(parseLcovCounts(bare), parseLcovCounts(bare)).has("/app/bare.py")).toBe(false);
+  });
+
+  test("render round-trips and totals the header counts", () => {
+    const text = renderLcovCounts(subtractLcovCounts(parseLcovCounts(later), parseLcovCounts(boot)));
+    expect(text).toBe(
+      "TN:\nSF:/app/a.py\nFN:1,main\nFNDA:0,main\nFNF:1\nFNH:0\nDA:1,0\nDA:2,0\nDA:3,4\nLF:3\nLH:1\nend_of_record\nSF:/app/c.py\nFNF:0\nFNH:0\nDA:1,2\nLF:1\nLH:1\nend_of_record\n",
+    );
+    expect(parseLcovCounts(text).get("/app/a.py")!.lines.get(3)).toBe(4);
+  });
+
+  test("applyCoverageDelta: cumulative reports are subtracted per service and name; delta and V8 reports pass", () => {
+    resetCoverageDeltaMemory();
+    const first = applyCoverageDelta(
+      { service: "api", reports: [{ name: "cov.lcov", content: boot }, { name: "node.lcov", content: later }, { name: "coverage-1.json", content: '{"result":[]}' }] },
+      "cumulative",
+      new Set(["node.lcov"]),
+    );
+    // First capture: no previous, the whole boot set.
+    expect(first.reports[0]!.content).toContain("SF:/app/a.py");
+    expect(first.reports[0]!.content).toContain("SF:/app/b.py");
+    expect(first.reports[1]!.content).toBe(later); // delta adapter's own file, untouched
+    expect(first.reports[2]!.content).toBe('{"result":[]}');
+    const second = applyCoverageDelta({ service: "api", reports: [{ name: "cov.lcov", content: later }] }, "cumulative", new Set());
+    expect(second.reports[0]!.content).not.toContain("SF:/app/b.py");
+    expect(second.reports[0]!.content).toContain("DA:3,4");
+    // Nothing new: an lcov with no records.
+    const third = applyCoverageDelta({ service: "api", reports: [{ name: "cov.lcov", content: later }] }, "cumulative", new Set());
+    expect(third.reports[0]!.content).toBe("TN:\n");
+    // Another service has its own memory; a delta-mode service is never touched.
+    const other = applyCoverageDelta({ service: "web", reports: [{ name: "cov.lcov", content: later }] }, "cumulative", new Set());
+    expect(other.reports[0]!.content).toContain("SF:/app/b.py");
+    const asIs = applyCoverageDelta({ service: "api", reports: [{ name: "cov.lcov", content: later }] }, "delta", new Set());
+    expect(asIs.reports[0]!.content).toBe(later);
+    // An errored capture passes through.
+    const err = applyCoverageDelta({ service: "api", reports: [], error: "x" }, "cumulative", new Set());
+    expect(err.error).toBe("x");
+  });
+});

package/src/harness/coverage.ts

@@ -15,6 +15,17 @@
 // stored bytes, so a change in that derivation is a re-read of stored
 // runs, never a hole in the history.
 //
+// The one transformation done here is the **delta**: a stored report
+// describes what ran between the previous capture and this one, for
+// every source alike. A tool that resets its own counters (V8) writes
+// deltas already; a cumulative tool (coverage.py, Go, JaCoCo — most)
+// gets subtracted against the previous capture, which lives in module
+// memory and so forks with the environment: a forked child's "previous"
+// is its parent's last capture, exactly the state it started from. Which
+// of the two a file is must be declared (an adapter's `reports`, or the
+// service's) — nothing in the bytes tells a delta that happens to equal
+// the previous one from a cumulative report with no new execution.
+//
 // The container side (mount flag, `command` exec) is in `daemon.ts`.
 
 import { promises as fs } from "node:fs";
@@ -211,3 +222,159 @@ export function coverageBundleRef(
     })),
   };
 }
+
+// ── delta ─────────────────────────────────────────────────────────
+
+/** How a report's counters behave across captures. */
+export type CoverageReportsMode = "delta" | "cumulative";
+
+/** One lcov record's execution counts: `DA:` by line, `FNDA:` by function
+ *  name. Branch data is dropped: file and line are what the derivation
+ *  reads. */
+export interface LcovRecordCounts {
+  lines: Map<number, number>;
+  functions: Map<string, number>;
+  /** `FN:<line>,<name>` declarations, kept so the delta names them. */
+  functionLines: Map<string, number>;
+}
+
+/** An lcov document parsed to counts, by `SF:` path. Records for one
+ *  path are merged (a tool may split a file across records). */
+export function parseLcovCounts(text: string): Map<string, LcovRecordCounts> {
+  const out = new Map<string, LcovRecordCounts>();
+  let cur: LcovRecordCounts | null = null;
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (line.startsWith("SF:")) {
+      const file = line.slice(3).trim();
+      if (!file) {
+        cur = null;
+        continue;
+      }
+      cur = out.get(file) ?? { lines: new Map(), functions: new Map(), functionLines: new Map() };
+      out.set(file, cur);
+    } else if (!cur) {
+      continue;
+    } else if (line.startsWith("DA:")) {
+      const [ln, hits] = line.slice(3).split(",");
+      const l = Number(ln), h = Number(hits);
+      if (Number.isFinite(l) && Number.isFinite(h)) cur.lines.set(l, (cur.lines.get(l) ?? 0) + h);
+    } else if (line.startsWith("FNDA:")) {
+      const i = line.indexOf(",", 5);
+      if (i < 0) continue;
+      const h = Number(line.slice(5, i)), name = line.slice(i + 1);
+      if (Number.isFinite(h)) cur.functions.set(name, (cur.functions.get(name) ?? 0) + h);
+    } else if (line.startsWith("FN:")) {
+      const i = line.indexOf(",", 3);
+      if (i < 0) continue;
+      const l = Number(line.slice(3, i)), name = line.slice(i + 1);
+      if (Number.isFinite(l)) cur.functionLines.set(name, l);
+    } else if (line === "end_of_record") {
+      cur = null;
+    }
+  }
+  return out;
+}
+
+/**
+ * `now` minus `prev`, per line and per function. A count that went
+ * *down* is a counter that restarted (the process was replaced): the
+ * new value stands on its own. A file not in `prev` is new: everything
+ * in it counts. Only records with at least one execution are kept — a
+ * file with nothing new in it did not run.
+ */
+export function subtractLcovCounts(
+  now: Map<string, LcovRecordCounts>,
+  prev: Map<string, LcovRecordCounts>,
+): Map<string, LcovRecordCounts> {
+  const out = new Map<string, LcovRecordCounts>();
+  for (const [file, rec] of now) {
+    const before = prev.get(file);
+    const lines = new Map<number, number>();
+    const functions = new Map<string, number>();
+    let executed = false;
+    for (const [l, h] of rec.lines) {
+      const b = before?.lines.get(l) ?? 0;
+      const d = h < b ? h : h - b;
+      lines.set(l, d);
+      if (d > 0) executed = true;
+    }
+    for (const [name, h] of rec.functions) {
+      const b = before?.functions.get(name) ?? 0;
+      const d = h < b ? h : h - b;
+      functions.set(name, d);
+      if (d > 0) executed = true;
+    }
+    // A record with no counts at all carries presence as its only
+    // signal; it is new only when it was not there before.
+    if (rec.lines.size === 0 && rec.functions.size === 0 && !before) executed = true;
+    if (executed) out.set(file, { lines, functions, functionLines: rec.functionLines });
+  }
+  return out;
+}
+
+/** Counts rendered back to lcov, records in path order. */
+export function renderLcovCounts(counts: Map<string, LcovRecordCounts>): string {
+  const parts: string[] = ["TN:"];
+  for (const file of [...counts.keys()].sort()) {
+    const rec = counts.get(file)!;
+    parts.push(`SF:${file}`);
+    for (const [name, l] of [...rec.functionLines].sort((a, b) => a[1] - b[1])) parts.push(`FN:${l},${name}`);
+    let fnh = 0;
+    for (const [name, h] of rec.functions) {
+      parts.push(`FNDA:${h},${name}`);
+      if (h > 0) fnh++;
+    }
+    parts.push(`FNF:${rec.functions.size}`, `FNH:${fnh}`);
+    let lh = 0;
+    for (const [l, h] of [...rec.lines].sort((a, b) => a[0] - b[0])) {
+      parts.push(`DA:${l},${h}`);
+      if (h > 0) lh++;
+    }
+    parts.push(`LF:${rec.lines.size}`, `LH:${lh}`, "end_of_record");
+  }
+  return parts.join("\n") + "\n";
+}
+
+/** The previous capture's counts: service → report name → parsed lcov.
+ *  Module memory, so it forks with the environment — a forked child's
+ *  previous capture is its parent's last one. */
+const PREVIOUS_CAPTURE = new Map<string, Map<string, Map<string, LcovRecordCounts>>>();
+
+/** Forget every previous capture (tests). */
+export function resetCoverageDeltaMemory(): void {
+  PREVIOUS_CAPTURE.clear();
+}
+
+/**
+ * Turn a capture's cumulative lcov reports into deltas against the
+ * previous capture of the same service and report name, and remember
+ * this capture as the new previous. Reports named in `deltaReports`
+ * (written by an adapter that declares `reports: "delta"`) and
+ * non-lcov reports (V8 JSON is a delta by construction) pass through
+ * untouched. With `mode === "delta"` every report passes through.
+ */
+export function applyCoverageDelta(
+  capture: ServiceCoverageCapture,
+  mode: CoverageReportsMode,
+  deltaReports: ReadonlySet<string>,
+): ServiceCoverageCapture {
+  if (capture.error) return capture;
+  let prevByName = PREVIOUS_CAPTURE.get(capture.service);
+  if (!prevByName) {
+    prevByName = new Map();
+    PREVIOUS_CAPTURE.set(capture.service, prevByName);
+  }
+  const reports: CoverageReport[] = [];
+  for (const r of capture.reports) {
+    if (mode === "delta" || deltaReports.has(r.name) || !isLcovShaped(r.content)) {
+      reports.push(r);
+      continue;
+    }
+    const now = parseLcovCounts(r.content);
+    const prev = prevByName.get(r.name) ?? new Map<string, LcovRecordCounts>();
+    prevByName.set(r.name, now);
+    reports.push({ name: r.name, content: renderLcovCounts(subtractLcovCounts(now, prev)) });
+  }
+  return { ...capture, reports };
+}