@avi2dg/checks 0.2.0 → 0.3.0

package/README.md

@@ -5,8 +5,9 @@ Deterministic checks shared across my TypeScript repos. One package,
 Effect language-service block, the shared commitlint config, the shared
 dependency-cruiser base, the test-layout check with its bunfig preset,
 the commit-identity check with its workflow, the comment gate with its
-workflow and backtest, and the Effect error-channel plugin compiled to
-JavaScript.
+workflow and backtest, the Stryker mutation-testing preset with its
+no-regression comparator, and the Effect error-channel plugin compiled
+to JavaScript.
 
 Published as `@avi2dg/checks` on the public npm registry.
 
@@ -64,6 +65,18 @@ cp node_modules/@avi2dg/checks/bunfig.toml bunfig.toml
 `commit-identity.ts` refuses a commit with an author other than the
 repository owner; see "Commit identity" below.
 
+A repo that runs mutation testing installs `@stryker-mutator/core` and
+`@hughescr/stryker-bun-runner`, then spreads the shipped preset in
+`stryker.conf.mjs`:
+
+```js
+import preset from "@avi2dg/checks/stryker.preset.js";
+
+export default {
+  ...preset,
+};
+```
+
 The registry version is pinned by the consumer's lockfile; bump
 `@avi2dg/checks` to adopt a new release.
 
@@ -277,6 +290,52 @@ comment text as a share of added lines.
 `generated/`, `vendor/`, `repos/`, `node_modules/` and `dist/` are out
 of reach, so the figures are authored code.
 
+## Mutation compare
+
+`checks-mutation-compare` gates a pull request on no-regression rather
+than an absolute threshold: the head mutation score may not fall below
+the score at the merge-base.
+
+```sh
+checks-mutation-compare <base-report> <head-report> [--advisory]
+```
+
+Both reports are Stryker `mutation.json` files. It prints the overall
+score of each report and the per-file scores that differ, and exits 1
+when the head score is below the base score. The score is Stryker's:
+`Killed` and `Timeout` over those plus `Survived` and `NoCoverage`, so
+`CompileError`, `RuntimeError`, `Ignored` and `Pending` mutants leave
+it. Every file in a report counts toward that report's score, including
+files present in only one of the two.
+
+`--advisory` prints the same verdict and always exits 0. That is how
+consumers run it for the first month; after that they drop the flag and
+it blocks.
+
+Enforcement runs on pull requests, comparing the head report against a
+report built at the merge-base:
+
+```yaml
+jobs:
+  mutation-compare:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: bunx stryker run
+      - run: |
+          base="$(git merge-base HEAD origin/main)"
+          git worktree add /tmp/mutation-base "$base"
+          (cd /tmp/mutation-base && bun install --frozen-lockfile && bunx stryker run)
+      - run: bun run checks-mutation-compare --advisory /tmp/mutation-base/reports/mutation/mutation.json reports/mutation/mutation.json
+```
+
+The shared Stryker preset's `json` reporter writes
+`reports/mutation/mutation.json` in each worktree.
+
 ## Why it is shaped this way
 
 - `plugins` does not inherit through oxlint `extends`. `rules`,
@@ -307,6 +366,10 @@ of reach, so the figures are authored code.
   consumer's copy is compared key by key against the installed one
   instead. `[test] pathIgnorePatterns` is a real bunfig key, and an empty
   `--path-ignore-patterns` flag overrides the file's own list.
+- The Stryker preset is a JavaScript module, not JSON: Stryker 10 does
+  not resolve `extends` in a JSON config, but a `.mjs` config that
+  spreads an imported object consumes it. Keys the consumer sets after
+  the spread win.
 - Each runnable script ships a `checks-` bin entry, so consumer
   `package.json` scripts call the short name, which the package manager
   puts on `PATH` only there; a shell runs it through `bun run`, which
@@ -346,7 +409,7 @@ a tag off `main` and reruns the build, `dist/` check, lint, typecheck
 and tests before it publishes:
 
 ```sh
-git tag v0.2.0 && git push origin v0.2.0
+git tag v0.3.0 && git push origin v0.3.0
 ```
 
 The `release` workflow publishes the tagged version through npm

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avi2dg/checks",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Deterministic checks shared across the captain's TypeScript repos",
   "license": "MIT",
   "repository": {
@@ -18,10 +18,12 @@
     "scripts/lint-coverage.sh",
     "scripts/test-layout.ts",
     "scripts/commit-identity.ts",
+    "scripts/mutation-compare.ts",
     "scripts/comments.ts",
     "scripts/comment-gate.ts",
     "scripts/backtest.ts",
     "oxlintrc.json",
+    "stryker.preset.js",
     "tsconfig.effect.json",
     "dist/index.js"
   ],
@@ -32,10 +34,12 @@
     "./scripts/lint-coverage.sh": "./scripts/lint-coverage.sh",
     "./scripts/test-layout.ts": "./scripts/test-layout.ts",
     "./scripts/commit-identity.ts": "./scripts/commit-identity.ts",
+    "./scripts/mutation-compare.ts": "./scripts/mutation-compare.ts",
     "./scripts/comments.ts": "./scripts/comments.ts",
     "./scripts/comment-gate.ts": "./scripts/comment-gate.ts",
     "./scripts/backtest.ts": "./scripts/backtest.ts",
     "./oxlintrc.json": "./oxlintrc.json",
+    "./stryker.preset.js": "./stryker.preset.js",
     "./tsconfig.effect.json": "./tsconfig.effect.json",
     "./dist/index.js": "./dist/index.js"
   },
@@ -43,12 +47,13 @@
     "checks-lint-coverage": "scripts/lint-coverage.sh",
     "checks-test-layout": "scripts/test-layout.ts",
     "checks-commit-identity": "scripts/commit-identity.ts",
+    "checks-mutation-compare": "scripts/mutation-compare.ts",
     "checks-comment-gate": "scripts/comment-gate.ts",
     "checks-backtest": "scripts/backtest.ts"
   },
   "scripts": {
     "build": "bun build effect-channel/index.ts --outdir dist --target node --format esm",
-    "lint": "oxlint --type-aware && ./scripts/lint-coverage.sh && bun scripts/test-layout.ts && bun scripts/commit-identity.ts HEAD && bun scripts/comment-gate.ts HEAD && depcruise --config .dependency-cruiser.cjs effect-channel scripts tests commitlint.config.js dependency-cruiser.config.js .dependency-cruiser.cjs",
+    "lint": "oxlint --type-aware && ./scripts/lint-coverage.sh && bun scripts/test-layout.ts && bun scripts/commit-identity.ts HEAD && bun scripts/comment-gate.ts HEAD && depcruise --config .dependency-cruiser.cjs effect-channel scripts tests commitlint.config.js dependency-cruiser.config.js stryker.preset.js .dependency-cruiser.cjs",
     "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
     "test": "bun test --randomize"
   },

package/scripts/mutation-compare.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env bun
+import { readFile } from "node:fs/promises";
+
+export type Tally = {
+  readonly killed: number;
+  readonly total: number;
+};
+
+export type FileComparison = {
+  readonly path: string;
+  readonly base: Tally;
+  readonly head: Tally;
+};
+
+export type Comparison = {
+  readonly base: Tally;
+  readonly head: Tally;
+  readonly files: readonly FileComparison[];
+  readonly regression: boolean;
+};
+
+export type Options = {
+  readonly basePath: string;
+  readonly headPath: string;
+  readonly advisory: boolean;
+};
+
+export class ReportError extends Error {}
+
+const DETECTED = new Set(["Killed", "Timeout"]);
+const UNDETECTED = new Set(["Survived", "NoCoverage"]);
+const USAGE = "usage: mutation-compare.ts [--advisory] <base-report> <head-report>";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+export function parseReport(text: string): Map<string, readonly string[]> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new ReportError("mutation-compare: report is not valid JSON");
+  }
+  if (!isRecord(parsed) || !isRecord(parsed["files"])) {
+    throw new ReportError("mutation-compare: report has no files table");
+  }
+  const files = new Map<string, readonly string[]>();
+  for (const [path, entry] of Object.entries(parsed["files"])) {
+    if (!isRecord(entry) || !Array.isArray(entry["mutants"])) {
+      throw new ReportError(`mutation-compare: ${path} has no mutants list`);
+    }
+    const statuses: string[] = [];
+    for (const mutant of entry["mutants"]) {
+      if (!isRecord(mutant) || typeof mutant["status"] !== "string") {
+        throw new ReportError(`mutation-compare: ${path} carries a mutant without a status`);
+      }
+      statuses.push(mutant["status"]);
+    }
+    files.set(path, statuses);
+  }
+  return files;
+}
+
+function tally(statuses: readonly string[]): Tally {
+  let killed = 0;
+  let total = 0;
+  for (const status of statuses) {
+    if (DETECTED.has(status)) killed += 1;
+    if (DETECTED.has(status) || UNDETECTED.has(status)) total += 1;
+  }
+  return { killed, total };
+}
+
+const EMPTY: Tally = { killed: 0, total: 0 };
+
+export function compareReports(
+  baseFiles: ReadonlyMap<string, readonly string[]>,
+  headFiles: ReadonlyMap<string, readonly string[]>,
+): Comparison {
+  const paths = [...new Set([...baseFiles.keys(), ...headFiles.keys()])].sort();
+  const files = paths.map((path) => {
+    const baseStatuses = baseFiles.get(path);
+    const headStatuses = headFiles.get(path);
+    return {
+      path,
+      base: baseStatuses === undefined ? EMPTY : tally(baseStatuses),
+      head: headStatuses === undefined ? EMPTY : tally(headStatuses),
+    };
+  });
+  const base = tally([...baseFiles.values()].flat());
+  const head = tally([...headFiles.values()].flat());
+  return { base, head, files, regression: regressed(base, head) };
+}
+
+export function regressed(base: Tally, head: Tally): boolean {
+  if (base.total === 0) return head.total > 0 && head.killed < head.total;
+  if (head.total === 0) return false;
+  return head.killed * base.total < base.killed * head.total;
+}
+
+function points(count: Tally): number {
+  return count.total === 0 ? 100 : (100 * count.killed) / count.total;
+}
+
+function percent(count: Tally): string {
+  return count.total === 0 ? "n/a" : `${points(count).toFixed(2)}%`;
+}
+
+function describe(count: Tally): string {
+  return `${percent(count)} (${count.killed}/${count.total})`;
+}
+
+function deltaPoints(base: Tally, head: Tally): string {
+  const delta = points(head) - points(base);
+  return `${delta < 0 ? "-" : "+"}${Math.abs(delta).toFixed(2)}pp`;
+}
+
+export function formatComparison(comparison: Comparison, advisory: boolean): string {
+  const changed = comparison.files.filter(
+    (file) => file.base.killed !== file.head.killed || file.base.total !== file.head.total,
+  );
+  const unchanged = comparison.files.length - changed.length;
+  const lines = [
+    `mutation-compare: base ${describe(comparison.base)} head ${describe(comparison.head)} delta ${deltaPoints(comparison.base, comparison.head)}`,
+  ];
+  for (const file of changed) {
+    lines.push(`  ${file.path}: ${describe(file.base)} -> ${describe(file.head)}`);
+  }
+  if (unchanged > 0) lines.push(`  ${unchanged} unchanged file(s)`);
+  if (comparison.regression) {
+    lines.push(`mutation-compare: REGRESSION (${deltaPoints(comparison.base, comparison.head)})${advisory ? " in advisory mode, exit 0" : ""}`);
+  } else {
+    lines.push(`mutation-compare: no regression${advisory ? " (advisory mode, exit 0)" : ""}`);
+  }
+  return lines.join("\n");
+}
+
+export function parseArgs(argv: readonly string[]): Options {
+  let advisory = false;
+  const paths: string[] = [];
+  for (const arg of argv) {
+    if (arg === "--advisory") advisory = true;
+    else if (arg.startsWith("--")) throw new ReportError(`${USAGE}: unknown flag ${arg}`);
+    else paths.push(arg);
+  }
+  if (paths.length !== 2 || paths[0] === undefined || paths[1] === undefined) throw new ReportError(USAGE);
+  return { basePath: paths[0], headPath: paths[1], advisory };
+}
+
+export function exitFor(comparison: Comparison, advisory: boolean): number {
+  return comparison.regression && !advisory ? 1 : 0;
+}
+
+async function load(path: string): Promise<string> {
+  try {
+    return await readFile(path, "utf8");
+  } catch {
+    throw new ReportError(`mutation-compare: cannot read ${path}`);
+  }
+}
+
+if (import.meta.main) {
+  try {
+    const options = parseArgs(process.argv.slice(2));
+    const comparison = compareReports(parseReport(await load(options.basePath)), parseReport(await load(options.headPath)));
+    console.log(formatComparison(comparison, options.advisory));
+    process.exit(exitFor(comparison, options.advisory));
+  } catch (error) {
+    console.error(error instanceof ReportError ? error.message : `mutation-compare: ${String(error)}`);
+    process.exit(2);
+  }
+}

package/stryker.preset.js

@@ -0,0 +1,13 @@
+export default {
+  packageManager: "npm",
+  plugins: ["@stryker-mutator/*", "@hughescr/stryker-bun-runner"],
+  testRunner: "bun",
+  bun: { timeout: 60000 },
+  inPlace: true,
+  tempDirName: "../.stryker-tmp",
+  coverageAnalysis: "perTest",
+  reporters: ["html", "json", "clear-text", "progress"],
+  timeoutMS: 60000,
+  concurrency: 8,
+  thresholds: { high: 85, low: 70, break: null },
+};