@specific.dev/spectest 0.86.1 → 0.87.0

package/src/harness/coverage.ts

@@ -40,9 +40,13 @@ export const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
  *  project's reach (bring-up) and the project files the harness read. */
 export const ENVIRONMENT_SERVICE = "__environment__";
 
-/** The bring-up report listing the project's reach, one project-relative
- *  path per line (`harness/reach.ts`). */
-export const REACH_REPORT = "reach.paths";
+/** The bring-up report carrying the project's reach as rules plus a
+ *  compacted prefix list (`harness/reach.ts`, `ReachRules`). */
+export const REACH_REPORT = "reach.json";
+
+/** The per-capture report naming the project files the harness read
+ *  through the SDK since the previous capture, as whole lcov records. */
+export const READS_REPORT = "reads.lcov";
 
 /** Raw bytes of reports one service may ship per capture. Reports are
  *  bounded by code size times chain depth, so anything past this is a
@@ -145,13 +149,26 @@ export function isUnitsReport(text: string): boolean {
  *  script record, or (by name) a `.units` list. */
 export function isWellFormedReport(text: string, name = ""): boolean {
   if (name.endsWith(".units")) return isUnitsReport(text);
+  if (name === REACH_REPORT) return isReachRules(text);
   return isLcovShaped(text) || isV8CoverageJson(text) || isSpectestScriptRecord(text);
 }
 
+/** True when `text` is a `reach.json` document: a JSON object with a
+ *  `sources` array and a `prefixes` array. */
+export function isReachRules(text: string): boolean {
+  try {
+    const v = JSON.parse(text) as { sources?: unknown; prefixes?: unknown };
+    return typeof v === "object" && v !== null && Array.isArray(v.sources) && Array.isArray(v.prefixes);
+  } catch {
+    return false;
+  }
+}
+
 /** True when a well-formed report names no source file at all (an lcov
  *  with no `SF:`, a V8 document with an empty `result`, a units list
  *  with no unit). */
 export function isEmptyReport(text: string): boolean {
+  if (isReachRules(text)) return false;
   if (isV8CoverageJson(text)) return /"result"\s*:\s*\[\s*\]/.test(text);
   if (isSpectestScriptRecord(text)) return false;
   if (isUnitsReport(text) && !isLcovShaped(text)) return text.trim().length === 0;

package/src/harness/reach.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { compileDockerignore, isExcluded, projectReach, renderReadsLcov } from "./reach.js";
+import { compactReach, compileDockerignore, isExcluded, projectReach, reachContains, renderReadsLcov, sourceOf } from "./reach.js";
 
 describe("dockerignore semantics", () => {
   const ex = (text: string, rel: string) => isExcluded(compileDockerignore(text), rel);
@@ -67,3 +67,69 @@ describe("projectReach", () => {
     expect(renderReadsLcov(["docs/x.md"])).toBe("TN:\nSF:docs/x.md\nend_of_record\n");
   });
 });
+
+describe("reach rules", () => {
+  const files = [
+    "package.json",
+    "packages/api/src/a.ts",
+    "packages/api/src/b.ts",
+    "packages/api/Dockerfile",
+    "packages/api/README.md",
+    "packages/dashboard/src/b.tsx",
+    "site/app/page.tsx",
+    ".github/workflows/ci.yml",
+    "docs/x.md",
+    "spectest/index.ts",
+    "data/seed.sql",
+  ];
+  const sources = [
+    { service: "api", context: ".", ignore: "site/**\ndocs/**\n**/*.md\n", extraFiles: ["packages/api/Dockerfile"] },
+    { service: "db", volumeSources: ["data", "/var/run/docker.sock"] },
+  ];
+  const reach = projectReach(files, sources);
+  const rules = { version: 1 as const, sources, prefixes: compactReach(files, reach) };
+
+  test("compactReach folds a wholly reached directory into one prefix and keeps the odd file", () => {
+    // packages/api holds README.md, which the ignore drops, so the
+    // directory is not whole; its src/ subdirectory is.
+    expect(rules.prefixes).toEqual([
+      ".github/",
+      "data/",
+      "package.json",
+      "packages/api/Dockerfile",
+      "packages/api/src/",
+      "packages/dashboard/",
+      "spectest/",
+    ]);
+    expect(compactReach(files, files)).toEqual(["."]);
+    expect(compactReach(files, [])).toEqual([]);
+  });
+
+  test("reachContains answers from the prefixes, then from the rules for a path the prefixes do not know", () => {
+    expect(reachContains(rules, "packages/api/src/a.ts")).toBe(true);
+    expect(reachContains(rules, "packages/api/src/new.ts")).toBe(true);
+    expect(reachContains(rules, "site/app/page.tsx")).toBe(false);
+    expect(reachContains(rules, "site/new/thing.tsx")).toBe(false);
+    expect(reachContains(rules, "docs/new.md")).toBe(false);
+    // A new file the ignore's *.md rule drops, in a directory no prefix
+    // covers whole: the rules decide, and they say out.
+    expect(reachContains(rules, "packages/api/NOTES.md")).toBe(false);
+    // A new file under a whole prefix that the rules would drop: the
+    // prefix answers first, on the safe side (in).
+    expect(reachContains(rules, "packages/api/src/NOTES.md")).toBe(true);
+    // A volume source subtree, and an absolute source that is no project path.
+    expect(reachContains(rules, "data/new.sql")).toBe(true);
+    // The root context carries data/ too, and it is asked first.
+    expect(sourceOf(sources, "data/new.sql")?.service).toBe("api");
+    expect(sourceOf([sources[1]!], "data/new.sql")?.service).toBe("db");
+    expect(sourceOf([sources[1]!], "var/run/docker.sock")).toBeUndefined();
+    // The Dockerfile named by path, and the one at the context root.
+    expect(sourceOf(sources, "packages/api/Dockerfile")?.service).toBe("api");
+    expect(reachContains({ version: 1, sources: [{ context: "packages/x", ignore: "**" }], prefixes: [] }, "packages/x/Dockerfile")).toBe(true);
+  });
+
+  test("the whole project as one prefix", () => {
+    expect(reachContains({ version: 1, sources: [], prefixes: ["."] }, "anything/at/all")).toBe(true);
+    expect(reachContains({ version: 1, sources: [], prefixes: [] }, "anything/at/all")).toBe(false);
+  });
+});

package/src/harness/reach.ts

@@ -4,14 +4,24 @@
 // a service's **build context** (so an image may carry it), it is the
 // **source of a volume** a service mounts, or a test **reads** it through
 // the SDK. The first two are known from the environment config before any
-// test runs; the harness computes their union here — the project's
-// **reach** — and ships it with the bring-up capture (`reach.paths`, one
-// project-relative path per line, under the synthetic `__environment__`
-// service). The subset selector then treats a changed file outside the
-// reach as one that cannot affect the suite: `site/**` behind a
+// test runs; the harness ships them with the bring-up capture as
+// **rules** (`reach.json`, under the synthetic `__environment__`
+// service): per service its context directory, the composed
+// `.dockerignore` text, the Dockerfile it reads and the volume sources it
+// mounts. Rules, not a file list: the control plane evaluates any changed
+// path against them, a file that did not exist at the base included, and
+// a project of ten thousand files ships a few lines. Beside the rules
+// rides a **compacted** view of the current file reach — a directory
+// every file of which is in the reach as one `dir/` prefix, the odd file
+// alone — which the control plane uses as a fast path and the shadow page
+// as the summary. The subset selector then treats a changed file outside
+// the reach as one that cannot affect the suite: `site/**` behind a
 // `.dockerignore`, `.github/**` outside every context. The third door is
 // recorded as it happens (`project-files.ts`, `takeProjectReads`).
 //
+// The rules are shipped for every project, coverage adapters or not: the
+// reach is a property of the environment config alone.
+//
 // The context of a build is what `docker build` would send: every file
 // under the context directory minus what the composed `.dockerignore`
 // (the project's own rules plus the service's `exclude`) matches, with
@@ -102,6 +112,8 @@ export function isExcluded(patterns: Pattern[], rel: string): boolean {
 
 /** One service's contribution to the reach. */
 export interface ReachSource {
+  /** The service the source belongs to (for the shadow page's verdicts). */
+  service?: string;
   /** Build context directory, project-relative (`.` for the root); absent
    *  for a registry image. */
   context?: string;
@@ -156,9 +168,102 @@ export function projectReach(files: readonly string[], sources: readonly ReachSo
   return [...out].sort();
 }
 
-/** The `reach.paths` report: one path per line. */
-export function renderReach(paths: readonly string[]): string {
-  return paths.join("\n") + (paths.length ? "\n" : "");
+/** The `reach.json` report. Mirrors `reach::ReachRules` on the server. */
+export interface ReachRules {
+  version: 1;
+  /** Every service's contribution, as rules. */
+  sources: ReachSource[];
+  /** The current file reach, compacted: `dir/` for a directory every
+   *  project file of which is in the reach, a bare path for a file on its
+   *  own, `.` when the whole project is. */
+  prefixes: string[];
+}
+
+/**
+ * Compact a file reach against the project's file list: a directory
+ * whose every file is in the reach becomes one `dir/` prefix (the root
+ * becomes `.`), and a file whose directory is not wholly reached stays a
+ * path. Only directories that hold a project file exist here; an empty
+ * directory has nothing to be in or out of the reach.
+ */
+export function compactReach(files: readonly string[], reach: readonly string[]): string[] {
+  const inReach = new Set(reach);
+  // Per directory: how many project files live below it, how many of
+  // those the reach holds. "" is the root.
+  const total = new Map<string, number>();
+  const hit = new Map<string, number>();
+  const children = new Map<string, Set<string>>();
+  const filesIn = new Map<string, string[]>();
+  for (const f of files) {
+    const parts = f.split("/");
+    let dir = "";
+    for (let i = 0; i < parts.length; i += 1) {
+      total.set(dir, (total.get(dir) ?? 0) + 1);
+      if (inReach.has(f)) hit.set(dir, (hit.get(dir) ?? 0) + 1);
+      if (i === parts.length - 1) {
+        let list = filesIn.get(dir);
+        if (!list) filesIn.set(dir, (list = []));
+        list.push(f);
+      } else {
+        const child = dir === "" ? parts[i]! : `${dir}/${parts[i]!}`;
+        let set = children.get(dir);
+        if (!set) children.set(dir, (set = new Set()));
+        set.add(child);
+        dir = child;
+      }
+    }
+  }
+  const out: string[] = [];
+  const walk = (dir: string): void => {
+    const n = total.get(dir) ?? 0;
+    if (n > 0 && (hit.get(dir) ?? 0) === n) {
+      out.push(dir === "" ? "." : `${dir}/`);
+      return;
+    }
+    for (const f of filesIn.get(dir) ?? []) if (inReach.has(f)) out.push(f);
+    for (const c of children.get(dir) ?? []) walk(c);
+  };
+  walk("");
+  return out.sort();
+}
+
+/** Is `path` (project-relative) in the reach the rules describe? The
+ *  compacted prefixes answer first; a path outside them is evaluated
+ *  against every source, so a file that did not exist when the prefixes
+ *  were computed gets the same answer the build would give. */
+export function reachContains(rules: ReachRules, path: string): boolean {
+  const p = normRel(path);
+  for (const pre of rules.prefixes) {
+    if (pre === "." || pre === p || (pre.endsWith("/") && p.startsWith(pre))) return true;
+  }
+  return sourceOf(rules.sources, p) !== undefined;
+}
+
+/** The first source whose rules carry `path`, or `undefined`. */
+export function sourceOf(sources: readonly ReachSource[], path: string): ReachSource | undefined {
+  const p = normRel(path);
+  for (const s of sources) {
+    if (s.context !== undefined) {
+      const ctx = normRel(s.context);
+      const prefix = ctx === "" ? "" : `${ctx}/`;
+      if (!prefix || p.startsWith(prefix)) {
+        const rel = p.slice(prefix.length);
+        if (rel === ".dockerignore" || rel === "Dockerfile" || !isExcluded(compileDockerignore(s.ignore ?? ""), rel)) return s;
+      }
+    }
+    for (const e of s.extraFiles ?? []) if (normRel(e) === p) return s;
+    for (const v of s.volumeSources ?? []) {
+      if (v.startsWith("/")) continue;
+      const src = normRel(v);
+      if (p === src || p.startsWith(`${src}/`)) return s;
+    }
+  }
+  return undefined;
+}
+
+/** The `reach.json` report text. */
+export function renderReachRules(rules: ReachRules): string {
+  return JSON.stringify(rules);
 }
 
 /** An lcov document that names `paths` as files with no line data —

package/src/harness/wrapper-rules.test.ts

@@ -10,6 +10,8 @@ import {
   splitUnion,
   startOfNode,
   truthyMessage,
+  CODE_RAW_FS,
+  fsUsageDiagnostics,
 } from "./wrapper-rules.js";
 
 describe("splitUnion", () => {
@@ -168,3 +170,27 @@ describe("startOfNode", () => {
     expect(startOfNode("a;\n/* open", 2)).toBe(10);
   });
 });
+
+describe("raw fs reads in tests (SPECTEST2003, advisory)", () => {
+  test("imports, requires, dynamic imports and Bun.file are found; other modules are not", () => {
+    const text = [
+      'import { readFileSync } from "node:fs";',
+      'import fsp from "fs/promises";',
+      'import path from "node:path";',
+      'const fs = require("fs");',
+      'const p = await import("node:fs/promises");',
+      'const t = await Bun.file("x").text();',
+      'import { z } from "zod";',
+    ].join("\n");
+    const d = fsUsageDiagnostics("tests/a.ts", text);
+    expect(d.map((x) => [x.line, x.code, x.blocking])).toEqual([
+      [1, CODE_RAW_FS, false],
+      [2, CODE_RAW_FS, false],
+      [4, CODE_RAW_FS, false],
+      [5, CODE_RAW_FS, false],
+      [6, CODE_RAW_FS, false],
+    ]);
+    expect(d[0]!.message).toContain("ctx.readProjectFile");
+    expect(fsUsageDiagnostics("tests/b.ts", 'import x from "./fs-helpers.js";\nconst fsLike = 1;')).toEqual([]);
+  });
+});

package/src/harness/wrapper-rules.ts

@@ -56,6 +56,45 @@
 // that happens to be called `Carrier<T>` would also be flagged — and would
 // also be an object at runtime, so the finding stays true.
 
+/**
+ * A test that opens a project file through `node:fs` (or `Bun.file`)
+ * instead of `ctx.readProjectFile`. **Advisory.** The subset selector
+ * learns which project files a test depends on from the reads the SDK
+ * serves (`project-files.ts`); a raw read is a dependence it never sees,
+ * so a change to that file can skip the test. The rule is textual — an
+ * import or `require` of the fs modules, or a `Bun.file(` call — and is
+ * applied to the test tree only, where the alternative exists. It turns
+ * blocking when the reach gate enforces (`SUBSET_RUNS.md`).
+ */
+export const CODE_RAW_FS = "SPECTEST2003";
+
+const FS_MODULES = new Set(["fs", "node:fs", "fs/promises", "node:fs/promises"]);
+
+/** The raw-fs findings of one test file, from its text alone. */
+export function fsUsageDiagnostics(file: string, text: string): WrapperDiagnostic[] {
+  const out: WrapperDiagnostic[] = [];
+  const push = (pos: number, what: string) => {
+    const { line, column } = lineColumnAt(text, pos);
+    out.push({
+      file,
+      line,
+      column,
+      code: CODE_RAW_FS,
+      message:
+        `${what} reads project files behind the SDK's back; use ctx.readProjectFile (or ctx.readProjectFileBytes) ` +
+        "so the subset selector knows the test depends on the file",
+      blocking: false,
+    });
+  };
+  const importRe = /\b(?:import|export)\b[^;'"]*?\bfrom\s*['"]([^'"]+)['"]|\bimport\s*['"]([^'"]+)['"]|\brequire\(\s*['"]([^'"]+)['"]\s*\)|\bimport\(\s*['"]([^'"]+)['"]\s*\)/g;
+  for (const m of text.matchAll(importRe)) {
+    const spec = m[1] ?? m[2] ?? m[3] ?? m[4] ?? "";
+    if (FS_MODULES.has(spec)) push(m.index ?? 0, `an import of "${spec}"`);
+  }
+  for (const m of text.matchAll(/\bBun\.file\(/g)) push(m.index ?? 0, "Bun.file()");
+  return out;
+}
+
 /** A finding, shaped like the `TypecheckError` the report already carries. */
 export interface WrapperDiagnostic {
   /** Path relative to the app dir, matching the advisory diagnostics. */