module-spec 0.1.0 → 0.2.1

package/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Changelog
 
-## Unreleased
+## 0.2.1
 
+- Under vitest, a live mutant of a module-level constant or regex runs again in a fresh vitest
+  process. Stryker's vitest runner, given the module's test files, ran its tests without the
+  mutant switched on, so it read as live whatever the tests checked.
+
+## 0.2.0
+
+- `mutate` counts only the module's core, shell and door tests. An adapter's test and a
+  contract test prove the adapter against the outside and are slow, so a mutant that only
+  they catch now lives until a test through the door catches it. On factory's chat module
+  those tests were 39 of 40 seconds of test time, rerun for every core mutant.
+- Under vitest, `mutate` keeps each module's verdicts in
+  `node_modules/.cache/module-spec/mutate/`, and a rerun tests again only the mutants whose
+  code or covering tests changed.
 - `mutate` changes only `core/`, `shell/` and the door, as the spec says. Before,
   it also changed files in no part, such as an app's `main.ts` or a
   `vitest.config.ts`.

package/lib/languages/typescript/mutate.js

@@ -11,10 +11,18 @@ const OUTPUT_LIMIT = 512 * 1024 * 1024;
 const CAUGHT = new Set(["Killed", "Timeout"]);
 const LIVE = new Set(["Survived", "NoCoverage"]);
 
+/** The project's own vitest, as Stryker's vitest runner loads it, or else the kit's. */
+function vitestBin(root) {
+  const local = path.join(root, "node_modules", "vitest");
+  const dir = fs.existsSync(path.join(local, "package.json")) ? local : kitPackageDir("vitest");
+  return path.join(dir, JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")).bin.vitest);
+}
+
 /** The shell command that runs only `tests`, with the project's own runner. Stryker runs it once per mutant. */
 function testCommand(root, tests) {
   const files = tests.map((file) => JSON.stringify(file)).join(" ");
-  return testRunnerOf(root) === "vitest" ? `node_modules/.bin/vitest run ${files}` : `${JSON.stringify(process.execPath)} --test ${files}`;
+  if (testRunnerOf(root) !== "vitest") return `${JSON.stringify(process.execPath)} --test ${files}`;
+  return `${JSON.stringify(process.execPath)} ${JSON.stringify(vitestBin(root))} run ${files}`;
 }
 
 const oneLine = (text) => {
@@ -40,37 +48,70 @@ function runnerConfig(root, tests) {
   };
 }
 
+/**
+ * Under vitest, Stryker keeps each mutant's verdict and the tests that covered it, so a rerun tests again only the
+ * mutants whose code or covering tests changed. The command runner reports no tests, so a weakened test would keep
+ * its old verdict there, and it reruns everything. The file sits under node_modules/.cache, which no project commits.
+ */
+function incrementalConfig(root, module) {
+  if (testRunnerOf(root) !== "vitest") return {};
+  const file = `${module.dir.replaceAll("/", "_")}.json`;
+  return { incremental: true, incrementalFile: path.join(root, "node_modules", ".cache", "module-spec", "mutate", file) };
+}
+
+/** One Stryker run with `config`, in its own folder under `work`, and the mutants it reported. */
+function strykerRun(root, work, name, config) {
+  const report = path.join(work, `${name}.json`);
+  const configFile = path.join(work, `${name}.config.json`);
+  fs.writeFileSync(configFile, JSON.stringify({
+    ...config,
+    reporters: ["json"],
+    jsonReporter: { fileName: report },
+    tempDirName: path.join(work, `${name}-sandbox`),
+    cleanTempDir: "always",
+    checkers: [],
+    // Stryker rewrites tsconfig.json with the project's own TypeScript, which from version 7 has no JS API. The kit's
+    // runners read TypeScript without it, so the rewrite is skipped by naming a file that does not exist.
+    tsconfigFile: "module-spec-skips-the-tsconfig-rewrite.json",
+    logLevel: "error",
+  }));
+  const dir = kitPackageDir("@stryker-mutator/core");
+  const bin = path.join(dir, JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")).bin.stryker);
+  const run = spawnSync(process.execPath, [bin, "run", configFile], { cwd: root, env: environmentForTests(), encoding: "utf8", maxBuffer: OUTPUT_LIMIT });
+  if (run.status !== 0 || !fs.existsSync(report)) throw new Error(`Stryker failed:\n${run.stderr || run.stdout}`);
+  return Object.entries(JSON.parse(fs.readFileSync(report, "utf8")).files).flatMap(([file, { mutants }]) => mutants.map((mutant) => ({ ...mutant, file })));
+}
+
+const mutantKey = (mutant) => `${mutant.file}:${mutant.location.start.line}:${mutant.location.start.column} ${mutant.mutatorName} ${mutant.replacement ?? ""}`;
+/** Stryker reads a range's columns from another origin than its report writes them, so a range names whole lines. */
+const linesOf = ({ file, location: { start, end } }) => `${file}:${start.line}-${end.line}`;
+
+/**
+ * Stryker's vitest runner, given the module's test files, runs a static mutant's tests without the mutant switched
+ * on, so a module-level constant or regex reads as live whatever its tests check. A command run starts vitest anew
+ * for each mutant, so the module loads with the mutant on. Only the live static mutants run again that way.
+ */
+function withStaticMutantsRetested(root, work, tests, mutants) {
+  const doubtful = mutants.filter((mutant) => mutant.static && LIVE.has(mutant.status));
+  if (doubtful.length === 0) return mutants;
+  const retested = new Map(
+    strykerRun(root, work, "static", {
+      testRunner: "command",
+      commandRunner: { command: testCommand(root, tests) },
+      coverageAnalysis: "off",
+      plugins: [],
+      mutate: [...new Set(doubtful.map(linesOf))],
+    }).map((mutant) => [mutantKey(mutant), mutant.status]),
+  );
+  return mutants.map((mutant) => (retested.has(mutantKey(mutant)) ? { ...mutant, status: retested.get(mutantKey(mutant)) } : mutant));
+}
+
 /** Mutation testing by the kit's own StrykerJS, in a sandbox under the system temp folder. */
-export async function mutate({ root, files, tests }) {
+export async function mutate({ root, module, files, tests }) {
   const work = fs.mkdtempSync(path.join(os.tmpdir(), "module-spec-stryker-"));
   try {
-    const report = path.join(work, "report.json");
-    const configFile = path.join(work, "stryker.config.json");
-    fs.writeFileSync(configFile, JSON.stringify({
-      ...runnerConfig(root, tests),
-      mutate: files,
-      reporters: ["json"],
-      jsonReporter: { fileName: report },
-      tempDirName: path.join(work, "sandbox"),
-      cleanTempDir: "always",
-      checkers: [],
-      // Stryker rewrites tsconfig.json with the project's own TypeScript, which from version 7 has no JS API. The kit's
-      // runners read TypeScript without it, so the rewrite is skipped by naming a file that does not exist.
-      tsconfigFile: "module-spec-skips-the-tsconfig-rewrite.json",
-      logLevel: "error",
-    }));
-    const dir = kitPackageDir("@stryker-mutator/core");
-    const bin = path.join(dir, JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")).bin.stryker);
-    const run = spawnSync(process.execPath, [bin, "run", configFile], {
-      cwd: root,
-      env: environmentForTests(),
-      encoding: "utf8",
-      maxBuffer: OUTPUT_LIMIT,
-    });
-    if (run.status !== 0 || !fs.existsSync(report)) throw new Error(`Stryker failed:\n${run.stderr || run.stdout}`);
-    const mutants = Object.entries(JSON.parse(fs.readFileSync(report, "utf8")).files).flatMap(([file, { mutants }]) =>
-      mutants.map((mutant) => ({ ...mutant, file })),
-    );
+    const first = strykerRun(root, work, "run", { ...runnerConfig(root, tests), mutate: files, ...incrementalConfig(root, module) });
+    const mutants = testRunnerOf(root) === "vitest" ? withStaticMutantsRetested(root, work, tests, first) : first;
     return {
       killed: mutants.filter((mutant) => CAUGHT.has(mutant.status)).length,
       survivors: mutants

package/lib/mutation.js

@@ -15,13 +15,19 @@ export const MUTATION_REPORT = "mutation-report.txt";
  */
 const MUTATED = new Set(["core", "shell"]);
 
-/** The files mutation testing changes in a module, and the module's own tests, which must notice. */
+/**
+ * The files mutation testing changes in a module, and the module's own tests, which must notice. An adapter's test
+ * proves the adapter against the outside and is slow, while the code it reaches is proven through the door, so it
+ * does not count.
+ */
 export function mutationTargets(module) {
   const files = [];
   const tests = [];
   for (const file of module.files) {
     const place = placeOf(module, file);
-    if (place.test) tests.push(file);
+    if (place.test) {
+      if (place.part !== "adapters" && !place.contractTest) tests.push(file);
+    }
     else if (place.door || MUTATED.has(place.part)) files.push(file);
   }
   return { files, tests };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "module-spec",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "How every project splits its code into modules, with the checks that enforce it. The why behind each rule: spec/.",
   "type": "module",
   "packageManager": "pnpm@12.10.1",

package/spec/tests.md

@@ -130,8 +130,10 @@ code does. Mutation testing catches it, because every mutant lives.
   hold only types, not `adapters/`, whose contract tests need the real
   service, and not `view/`, which holds no rules
   ([F3](frontend.md#f3-a-view-holds-no-rules)).
-- **Which tests count:** only the module's own. A mutant that only another
-  module's test catches still lives.
+- **Which tests count:** only the module's own, and not its adapters' tests
+  or contract tests. Those prove an adapter against the outside and are slow.
+  The core they reach is proven through the door. A mutant that only another
+  module's test, or an adapter's test, catches still lives.
 - **Zero live mutants:** `module-spec mutate` fails on any live mutant, and
   names it, even on a module's first run.
 - **When no test can tell:** some mutants behave exactly like the code, such