@specific.dev/spectest 0.57.0 → 0.59.0

package/src/harness/coverage.test.ts

@@ -8,6 +8,10 @@ import {
   coverageHostDir,
   encodeCoverageBundle,
   hasLcovSourceFile,
+  isEmptyReport,
+  isLcovShaped,
+  isV8CoverageJson,
+  isWellFormedReport,
   listReportFiles,
   readCoverageDir,
 } from "./coverage";
@@ -24,7 +28,48 @@ describe("hasLcovSourceFile", () => {
   });
 });
 
+describe("report shapes", () => {
+  test("lcov: every non-blank line a record; an SF-less report is well-formed and empty", () => {
+    expect(isLcovShaped("TN:\nSF:/a.js\nDA:1,1\nend_of_record\n")).toBe(true);
+    expect(isLcovShaped("TN:\n")).toBe(true);
+    expect(isEmptyReport("TN:\n")).toBe(true);
+    expect(isEmptyReport("TN:\nSF:/a.js\n")).toBe(false);
+    expect(isLcovShaped("")).toBe(false);
+    expect(isLcovShaped("hello world\n")).toBe(false);
+    expect(isLcovShaped("SF:/a.js\ngarbage\n")).toBe(false);
+  });
+  test("V8 JSON: shape check only; an empty result is empty", () => {
+    const v8 = '{"result":[{"scriptId":"1","url":"file:///app/a.js","functions":[]}]}';
+    expect(isV8CoverageJson(v8)).toBe(true);
+    expect(isEmptyReport(v8)).toBe(false);
+    expect(isEmptyReport('{"result":[]}')).toBe(true);
+    expect(isV8CoverageJson("{}")).toBe(false);
+    expect(isWellFormedReport(v8)).toBe(true);
+    expect(isWellFormedReport("TN:\n")).toBe(true);
+    expect(isWellFormedReport("nope")).toBe(false);
+  });
+});
+
 describe("readCoverageDir", () => {
+  test("a V8 coverage JSON file satisfies the contract", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-cov-"));
+    await fs.writeFile(
+      path.join(dir, "coverage-1-2-3.json"),
+      '{"result":[{"scriptId":"1","url":"file:///app/a.js","functions":[]}]}',
+    );
+    const r = await readCoverageDir("web", dir);
+    expect(r.error).toBeUndefined();
+    expect(r.reports.length).toBe(1);
+  });
+  test("a malformed file next to a good one is an error naming the file", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-cov-"));
+    await fs.writeFile(path.join(dir, "lcov.info"), "TN:\nSF:/a.js\nend_of_record\n");
+    await fs.writeFile(path.join(dir, "junk.txt"), "not a report\n");
+    const r = await readCoverageDir("web", dir);
+    expect(r.error).toContain("junk.txt");
+    expect(r.reports.length).toBe(2);
+  });
+
   test("ships every report verbatim, nested, with relative names", async () => {
     const dir = await tmp();
     await fs.mkdir(path.join(dir, "sub"));
@@ -56,7 +101,7 @@ describe("readCoverageDir", () => {
     await fs.writeFile(path.join(dir, "coverage.json"), "{}");
     const r = await readCoverageDir("web", dir);
     expect(r.reports.length).toBe(1);
-    expect(r.error).toContain("SF:");
+    expect(r.error).toContain("none is an lcov report or a V8 coverage JSON document");
   });
 
   test("listReportFiles skips dotfiles", async () => {

package/src/harness/coverage.ts

@@ -2,12 +2,15 @@
 //
 // A service that opts in (`coverage: true | { command }`) gets a writable
 // directory bind-mounted at `/spectest/coverage/` in its container. The
-// service's own tool writes lcov reports there. The daemon reads the
+// service's own tool, or its coverage adapters, write reports there —
+// lcov, or the V8 coverage JSON that `NODE_V8_COVERAGE` writes. The daemon reads the
 // directory from the VM side (no `docker exec` to fetch) after bring-up
 // and after each test, and ships the reports **verbatim** to the control
 // plane as a gzipped bundle. Nothing is interpreted here beyond the one
-// contract the harness must fail a case on: at least one report, and at
-// least one lcov `SF:` record among them. The server derives whatever it
+// contract the harness must fail a case on: at least one report, every
+// one well-formed. Counters may reset between dumps and a dump may be a
+// new file: the directory forks with the environment, so it accumulates
+// along the ancestor chain by itself. The server derives whatever it
 // needs (today the set of source files; later, finer things) from the
 // stored bytes, so a change in that derivation is a re-read of stored
 // runs, never a hole in the history.
@@ -22,7 +25,7 @@ import { gzipSync } from "node:zlib";
 export const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
 
 /** Raw bytes of reports one service may ship per capture. Reports are
- *  cumulative and bounded by code size, so anything past this is a
+ *  bounded by code size times chain depth, so anything past this is a
  *  runaway (a tool writing a new file per request), not coverage. */
 export const COVERAGE_MAX_BYTES_PER_SERVICE = 64 * 1024 * 1024;
 
@@ -76,6 +79,37 @@ export function hasLcovSourceFile(text: string): boolean {
   return /^SF:\S/m.test(text);
 }
 
+/** True when `text` is lcov-shaped: at least one line, every non-blank
+ *  line an `XX:` record or `end_of_record`. An lcov with no `SF:` record
+ *  at all (`TN:` alone) is well-formed — it is a report that says
+ *  nothing ran. */
+export function isLcovShaped(text: string): boolean {
+  const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0);
+  return lines.length > 0 && lines.every((l) => /^[A-Z]{2,5}:/.test(l) || l.trim() === "end_of_record");
+}
+
+/**
+ * True when `text` looks like a V8 coverage JSON document — what
+ * `NODE_V8_COVERAGE` / `v8.takeCoverage()` write: an object with a
+ * `result` array of scripts. A cheap shape check, not a parse (a report
+ * can be megabytes); the control plane parses for real.
+ */
+export function isV8CoverageJson(text: string): boolean {
+  return /^\s*\{/.test(text) && /"result"\s*:\s*\[/.test(text);
+}
+
+/** True when `text` is a report spectest stores: lcov or V8 JSON. */
+export function isWellFormedReport(text: string): boolean {
+  return isLcovShaped(text) || isV8CoverageJson(text);
+}
+
+/** True when a well-formed report names no source file at all (an lcov
+ *  with no `SF:`, a V8 document with an empty `result`). */
+export function isEmptyReport(text: string): boolean {
+  if (isV8CoverageJson(text)) return /"result"\s*:\s*\[\s*\]/.test(text);
+  return !hasLcovSourceFile(text);
+}
+
 /** Every regular file under `dir`, recursively, as paths relative to it.
  *  Hidden files are skipped (a tool's own bookkeeping — `.seq`, `.lock`,
  *  a tmp file mid-write — is not a report). */
@@ -100,9 +134,11 @@ export async function listReportFiles(dir: string): Promise<string[]> {
 }
 
 /**
- * Read every report under `dir`, verbatim. An empty directory, or one with
- * no `SF:` record anywhere, is an error rather than an empty capture: the
- * contract is that an opted-in service always has a report to read.
+ * Read every report under `dir`, verbatim. An empty directory, or a file
+ * that is neither lcov nor V8 JSON, is an error rather than an empty
+ * capture: the contract is that an opted-in service always has a
+ * well-formed report to read. A report that names no file is fine — it
+ * says nothing ran.
  */
 export async function readCoverageDir(
   service: string,
@@ -113,12 +149,13 @@ export async function readCoverageDir(
     return {
       service,
       reports: [],
-      error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov file there`,
+      error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov or V8 coverage JSON file there`,
     };
   }
   const reports: CoverageReport[] = [];
   let total = 0;
-  let anyLcov = false;
+  let anyReport = false;
+  const malformed: string[] = [];
   for (const name of names) {
     let content: string;
     try {
@@ -131,17 +168,20 @@ export async function readCoverageDir(
       return {
         service,
         reports: [],
-        error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — reports must be cumulative, not one file per dump`,
+        error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — that is a runaway, not coverage`,
       };
     }
-    if (hasLcovSourceFile(content)) anyLcov = true;
+    if (isWellFormedReport(content)) anyReport = true;
+    else malformed.push(name);
     reports.push({ name, content });
   }
-  if (!anyLcov) {
+  if (!anyReport || malformed.length > 0) {
     return {
       service,
       reports,
-      error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but none holds an lcov \`SF:\` record — only lcov reports are read`,
+      error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but ${
+        anyReport ? `${malformed.slice(0, 5).join(", ")} ${malformed.length === 1 ? "is" : "are"} neither` : "none is"
+      } an lcov report or a V8 coverage JSON document — only those two formats are read`,
     };
   }
   return { service, reports };

package/src/index.ts

@@ -132,6 +132,19 @@ export type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
 
 import type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
 
+// Coverage adapter types. The adapters themselves (`node()`, `browser()`,
+// `command()`, `defineAdapter`) are imported from
+// `@specific.dev/spectest/coverage` — see `coverage.ts` and
+// `spectest docs /services/coverage`.
+export {
+  type CoverageAdapter,
+  type CoverageCaptureContext,
+  type CoverageConfigureInfo,
+  type CoverageLoadContext,
+  type ServiceCoverage,
+} from "./coverage.js";
+import { applyCoverageAdapters, validateCoverage, type ServiceCoverage } from "./coverage.js";
+
 // Low-level ingress primitives + the framework lowering that the friendly
 // `tls` / `hostnames` fields and `defineFake(...)` are built on. See
 // `ingress.ts`.
@@ -256,20 +269,24 @@ export interface ServiceConfig {
    * Opt this service in to **code coverage collection** (experimental).
    *
    * The service gets a writable directory bind-mounted at
-   * `/spectest/coverage/`. The service's own coverage tool writes **lcov**
-   * reports there — cumulative from process start, never reset. After
-   * bring-up and after every test, spectest runs `command` (if given) inside
-   * the container, then reads every file in the directory and records the
-   * set of source files that ran. Only the `SF:` records are read.
+   * `/spectest/coverage/`. After bring-up and after every test, spectest
+   * reads every file in it — **lcov** or **V8 coverage JSON**, verbatim —
+   * and stores them with the test; the control plane derives what ran.
    *
-   * - `true`: the service writes its reports by itself (a dump on a timer,
-   *   an exit-time dump for short-lived processes).
-   * - `{ command }`: run this (via `sh -c`) in the container to make the
-   *   service write a report, before the directory is read.
+   * - `true`: the program writes its own reports there (an exit-time dump
+   *   for a short-lived process, a dump on a timer, its own hook).
+   * - `{ adapters: [...] }`: spectest gets the reports out. From
+   *   `@specific.dev/spectest/coverage`: `node()` (V8 JSON from a node
+   *   service, through a hook spectest mounts), `browser()` (the frontend
+   *   this service serves, collected from the guest browser),
+   *   `command(cmd)` (run a command in the container first), or your own
+   *   via `defineAdapter`.
    *
-   * The directory must hold at least one lcov report each time it is read;
-   * an empty directory or a failed command fails the test with a clear
-   * error, so a hole in the coverage data is never silent. See
+   * The directory must hold at least one well-formed report each time it
+   * is read; an empty directory or a failing adapter fails the test with a
+   * clear error, so a hole in the coverage data is never silent. Reports
+   * accumulate along the test's ancestor chain by themselves (the
+   * directory forks with the environment). See
    * `spectest docs /services/coverage`.
    */
   coverage?: ServiceCoverage;
@@ -1038,12 +1055,8 @@ export type ServiceImage =
       exclude?: readonly string[];
     };
 
-/**
- * A service's opt-in to coverage collection — see `ServiceConfig.coverage`.
- * `true` when the service writes its own reports; `{ command }` when
- * spectest must run a command in the container to produce one.
- */
-export type ServiceCoverage = true | { command: string };
+// Coverage adapters live in `coverage.ts`; the type is re-declared here
+// through the import below so `ServiceConfig.coverage` reads in one place.
 
 export interface VolumeMount {
   /**
@@ -1264,25 +1277,6 @@ export interface FakeContext {
   certificate(hostnames: readonly string[]): Promise<CertificateMaterial>;
 }
 
-/**
- * Check one service's `coverage` field. Exported so the daemon applies the
- * same rule to a runtime `startService` spec.
- */
-export function validateCoverage(service: string, cov: unknown): void {
-  if (cov === undefined || cov === true) return;
-  if (
-    typeof cov === "object" &&
-    cov !== null &&
-    typeof (cov as { command?: unknown }).command === "string" &&
-    (cov as { command: string }).command.trim().length > 0
-  ) {
-    return;
-  }
-  throw new Error(
-    `service "${service}" has an invalid \`coverage\` value — use \`true\` or \`{ command: "<shell command>" }\``,
-  );
-}
-
 function validateEnvironmentConfig<S extends ServicesMap>(
   config: EnvironmentConfig<S>,
 ): void {
@@ -2203,9 +2197,13 @@ export function defineEnvironment<
   // { name, services, timeoutSecs } — with plain, fully-expanded services —
   // is stored as `config` and shipped to the control plane.
   const { fakes, services: rawServices, ...rest } = input;
+  const expanded = expandServiceGroups(rawServices);
+  for (const [key, svc] of Object.entries(expanded)) {
+    expanded[key] = applyCoverageAdapters(key, svc);
+  }
   const config: EnvironmentConfig<S> = {
     ...rest,
-    services: expandServiceGroups(rawServices) as S,
+    services: expanded as S,
   };
   validateEnvironmentConfig(config);
   if (fakes) validateFakes(config, fakes);