@avi2dg/checks 0.1.0 → 0.3.0

package/README.md

@@ -5,8 +5,9 @@ Deterministic checks shared across my TypeScript repos. One package,
 Effect language-service block, the shared commitlint config, the shared
 dependency-cruiser base, the test-layout check with its bunfig preset,
 the commit-identity check with its workflow, the comment gate with its
-workflow and backtest, and the Effect error-channel plugin compiled to
-JavaScript.
+workflow and backtest, the Stryker mutation-testing preset with its
+no-regression comparator, and the Effect error-channel plugin compiled
+to JavaScript.
 
 Published as `@avi2dg/checks` on the public npm registry.
 
@@ -50,7 +51,7 @@ cp node_modules/@avi2dg/checks/bunfig.toml bunfig.toml
 `package.json` gains three scripts:
 
 ```json
-"lint": "oxlint --type-aware && ./node_modules/@avi2dg/checks/scripts/lint-coverage.sh && bun ./node_modules/@avi2dg/checks/scripts/test-layout.ts && bun ./node_modules/@avi2dg/checks/scripts/commit-identity.ts HEAD",
+"lint": "oxlint --type-aware && checks-lint-coverage && checks-test-layout && checks-commit-identity HEAD",
 "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
 "test": "bun test --randomize"
 ```
@@ -64,14 +65,25 @@ cp node_modules/@avi2dg/checks/bunfig.toml bunfig.toml
 `commit-identity.ts` refuses a commit with an author other than the
 repository owner; see "Commit identity" below.
 
+A repo that runs mutation testing installs `@stryker-mutator/core` and
+`@hughescr/stryker-bun-runner`, then spreads the shipped preset in
+`stryker.conf.mjs`:
+
+```js
+import preset from "@avi2dg/checks/stryker.preset.js";
+
+export default {
+  ...preset,
+};
+```
+
 The registry version is pinned by the consumer's lockfile; bump
 `@avi2dg/checks` to adopt a new release.
 
 ## Test layout
 
-`bun ./node_modules/@avi2dg/checks/scripts/test-layout.ts` fails unless the
-repo holds this shape, and names the file and the path to move it to when
-it does not:
+`checks-test-layout` fails unless the repo holds this shape, and names the
+file and the path to move it to when it does not:
 
 - Every test file is `tests/**/*.test.ts` or `.tsx`. A `*.test.ts`, `*.spec.ts` or
   `*_test.ts` under `src/`, `test/`, `__tests__/` or the repo root fails.
@@ -192,8 +204,8 @@ commit-identity check below is the enforcement.
 one carries an identity other than the repository owner's:
 
 ```sh
-bun ./node_modules/@avi2dg/checks/scripts/commit-identity.ts <base-ref> <head-ref>
-bun ./node_modules/@avi2dg/checks/scripts/commit-identity.ts <ref>
+bun run checks-commit-identity <base-ref> <head-ref>
+bun run checks-commit-identity <ref>
 ```
 
 With one argument it checks that commit alone, which is the form the
@@ -235,8 +247,8 @@ leaves stale once the base branch advances after the pull request opens.
 when an added line carries a banned comment:
 
 ```sh
-bun ./node_modules/@avi2dg/checks/scripts/comment-gate.ts <base-ref> <head-ref>
-bun ./node_modules/@avi2dg/checks/scripts/comment-gate.ts <ref>
+bun run checks-comment-gate <base-ref> <head-ref>
+bun run checks-comment-gate <ref>
 ```
 
 With two arguments it diffs the base against the head. With one it diffs
@@ -268,7 +280,7 @@ jobs:
 at each recent commit, so a repository can measure its own history:
 
 ```sh
-bun ./node_modules/@avi2dg/checks/scripts/backtest.ts [commit-count]
+bun run checks-backtest [commit-count]
 ```
 
 It walks the last `commit-count` first-parent commits, 60 by default,
@@ -278,6 +290,52 @@ comment text as a share of added lines.
 `generated/`, `vendor/`, `repos/`, `node_modules/` and `dist/` are out
 of reach, so the figures are authored code.
 
+## Mutation compare
+
+`checks-mutation-compare` gates a pull request on no-regression rather
+than an absolute threshold: the head mutation score may not fall below
+the score at the merge-base.
+
+```sh
+checks-mutation-compare <base-report> <head-report> [--advisory]
+```
+
+Both reports are Stryker `mutation.json` files. It prints the overall
+score of each report and the per-file scores that differ, and exits 1
+when the head score is below the base score. The score is Stryker's:
+`Killed` and `Timeout` over those plus `Survived` and `NoCoverage`, so
+`CompileError`, `RuntimeError`, `Ignored` and `Pending` mutants leave
+it. Every file in a report counts toward that report's score, including
+files present in only one of the two.
+
+`--advisory` prints the same verdict and always exits 0. That is how
+consumers run it for the first month; after that they drop the flag and
+it blocks.
+
+Enforcement runs on pull requests, comparing the head report against a
+report built at the merge-base:
+
+```yaml
+jobs:
+  mutation-compare:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: bunx stryker run
+      - run: |
+          base="$(git merge-base HEAD origin/main)"
+          git worktree add /tmp/mutation-base "$base"
+          (cd /tmp/mutation-base && bun install --frozen-lockfile && bunx stryker run)
+      - run: bun run checks-mutation-compare --advisory /tmp/mutation-base/reports/mutation/mutation.json reports/mutation/mutation.json
+```
+
+The shared Stryker preset's `json` reporter writes
+`reports/mutation/mutation.json` in each worktree.
+
 ## Why it is shaped this way
 
 - `plugins` does not inherit through oxlint `extends`. `rules`,
@@ -308,9 +366,16 @@ of reach, so the figures are authored code.
   consumer's copy is compared key by key against the installed one
   instead. `[test] pathIgnorePatterns` is a real bunfig key, and an empty
   `--path-ignore-patterns` flag overrides the file's own list.
-- The layout check ships as `.ts` and is invoked with `bun`, which needs
-  no build step and no `dist/` entry, unlike the oxlint plugin that node
-  loads.
+- The Stryker preset is a JavaScript module, not JSON: Stryker 10 does
+  not resolve `extends` in a JSON config, but a `.mjs` config that
+  spreads an imported object consumes it. Keys the consumer sets after
+  the spread win.
+- Each runnable script ships a `checks-` bin entry, so consumer
+  `package.json` scripts call the short name, which the package manager
+  puts on `PATH` only there; a shell runs it through `bun run`, which
+  never falls back to the registry the way `bunx` does. The `.ts` checks
+  keep a `bun` shebang, which needs no build step and no `dist/`
+  entry, unlike the oxlint plugin that node loads.
 - `bun` counts as a built-in module. Nothing installed resolves it except
   `@types/bun`, which would otherwise make every runtime `bun` import look
   like a dev-only dependency.
@@ -344,7 +409,7 @@ a tag off `main` and reruns the build, `dist/` check, lint, typecheck
 and tests before it publishes:
 
 ```sh
-git tag v0.2.0 && git push origin v0.2.0
+git tag v0.3.0 && git push origin v0.3.0
 ```
 
 The `release` workflow publishes the tagged version through npm

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avi2dg/checks",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Deterministic checks shared across the captain's TypeScript repos",
   "license": "MIT",
   "repository": {
@@ -18,10 +18,12 @@
     "scripts/lint-coverage.sh",
     "scripts/test-layout.ts",
     "scripts/commit-identity.ts",
+    "scripts/mutation-compare.ts",
     "scripts/comments.ts",
     "scripts/comment-gate.ts",
     "scripts/backtest.ts",
     "oxlintrc.json",
+    "stryker.preset.js",
     "tsconfig.effect.json",
     "dist/index.js"
   ],
@@ -32,16 +34,26 @@
     "./scripts/lint-coverage.sh": "./scripts/lint-coverage.sh",
     "./scripts/test-layout.ts": "./scripts/test-layout.ts",
     "./scripts/commit-identity.ts": "./scripts/commit-identity.ts",
+    "./scripts/mutation-compare.ts": "./scripts/mutation-compare.ts",
     "./scripts/comments.ts": "./scripts/comments.ts",
     "./scripts/comment-gate.ts": "./scripts/comment-gate.ts",
     "./scripts/backtest.ts": "./scripts/backtest.ts",
     "./oxlintrc.json": "./oxlintrc.json",
+    "./stryker.preset.js": "./stryker.preset.js",
     "./tsconfig.effect.json": "./tsconfig.effect.json",
     "./dist/index.js": "./dist/index.js"
   },
+  "bin": {
+    "checks-lint-coverage": "scripts/lint-coverage.sh",
+    "checks-test-layout": "scripts/test-layout.ts",
+    "checks-commit-identity": "scripts/commit-identity.ts",
+    "checks-mutation-compare": "scripts/mutation-compare.ts",
+    "checks-comment-gate": "scripts/comment-gate.ts",
+    "checks-backtest": "scripts/backtest.ts"
+  },
   "scripts": {
     "build": "bun build effect-channel/index.ts --outdir dist --target node --format esm",
-    "lint": "oxlint --type-aware && ./scripts/lint-coverage.sh && bun scripts/test-layout.ts && bun scripts/commit-identity.ts HEAD && bun scripts/comment-gate.ts HEAD && depcruise --config .dependency-cruiser.cjs effect-channel scripts tests commitlint.config.js dependency-cruiser.config.js .dependency-cruiser.cjs",
+    "lint": "oxlint --type-aware && ./scripts/lint-coverage.sh && bun scripts/test-layout.ts && bun scripts/commit-identity.ts HEAD && bun scripts/comment-gate.ts HEAD && depcruise --config .dependency-cruiser.cjs effect-channel scripts tests commitlint.config.js dependency-cruiser.config.js stryker.preset.js .dependency-cruiser.cjs",
     "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
     "test": "bun test --randomize"
   },

package/scripts/mutation-compare.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env bun
+import { readFile } from "node:fs/promises";
+
+export type Tally = {
+  readonly killed: number;
+  readonly total: number;
+};
+
+export type FileComparison = {
+  readonly path: string;
+  readonly base: Tally;
+  readonly head: Tally;
+};
+
+export type Comparison = {
+  readonly base: Tally;
+  readonly head: Tally;
+  readonly files: readonly FileComparison[];
+  readonly regression: boolean;
+};
+
+export type Options = {
+  readonly basePath: string;
+  readonly headPath: string;
+  readonly advisory: boolean;
+};
+
+export class ReportError extends Error {}
+
+const DETECTED = new Set(["Killed", "Timeout"]);
+const UNDETECTED = new Set(["Survived", "NoCoverage"]);
+const USAGE = "usage: mutation-compare.ts [--advisory] <base-report> <head-report>";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+export function parseReport(text: string): Map<string, readonly string[]> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new ReportError("mutation-compare: report is not valid JSON");
+  }
+  if (!isRecord(parsed) || !isRecord(parsed["files"])) {
+    throw new ReportError("mutation-compare: report has no files table");
+  }
+  const files = new Map<string, readonly string[]>();
+  for (const [path, entry] of Object.entries(parsed["files"])) {
+    if (!isRecord(entry) || !Array.isArray(entry["mutants"])) {
+      throw new ReportError(`mutation-compare: ${path} has no mutants list`);
+    }
+    const statuses: string[] = [];
+    for (const mutant of entry["mutants"]) {
+      if (!isRecord(mutant) || typeof mutant["status"] !== "string") {
+        throw new ReportError(`mutation-compare: ${path} carries a mutant without a status`);
+      }
+      statuses.push(mutant["status"]);
+    }
+    files.set(path, statuses);
+  }
+  return files;
+}
+
+function tally(statuses: readonly string[]): Tally {
+  let killed = 0;
+  let total = 0;
+  for (const status of statuses) {
+    if (DETECTED.has(status)) killed += 1;
+    if (DETECTED.has(status) || UNDETECTED.has(status)) total += 1;
+  }
+  return { killed, total };
+}
+
+const EMPTY: Tally = { killed: 0, total: 0 };
+
+export function compareReports(
+  baseFiles: ReadonlyMap<string, readonly string[]>,
+  headFiles: ReadonlyMap<string, readonly string[]>,
+): Comparison {
+  const paths = [...new Set([...baseFiles.keys(), ...headFiles.keys()])].sort();
+  const files = paths.map((path) => {
+    const baseStatuses = baseFiles.get(path);
+    const headStatuses = headFiles.get(path);
+    return {
+      path,
+      base: baseStatuses === undefined ? EMPTY : tally(baseStatuses),
+      head: headStatuses === undefined ? EMPTY : tally(headStatuses),
+    };
+  });
+  const base = tally([...baseFiles.values()].flat());
+  const head = tally([...headFiles.values()].flat());
+  return { base, head, files, regression: regressed(base, head) };
+}
+
+export function regressed(base: Tally, head: Tally): boolean {
+  if (base.total === 0) return head.total > 0 && head.killed < head.total;
+  if (head.total === 0) return false;
+  return head.killed * base.total < base.killed * head.total;
+}
+
+function points(count: Tally): number {
+  return count.total === 0 ? 100 : (100 * count.killed) / count.total;
+}
+
+function percent(count: Tally): string {
+  return count.total === 0 ? "n/a" : `${points(count).toFixed(2)}%`;
+}
+
+function describe(count: Tally): string {
+  return `${percent(count)} (${count.killed}/${count.total})`;
+}
+
+function deltaPoints(base: Tally, head: Tally): string {
+  const delta = points(head) - points(base);
+  return `${delta < 0 ? "-" : "+"}${Math.abs(delta).toFixed(2)}pp`;
+}
+
+export function formatComparison(comparison: Comparison, advisory: boolean): string {
+  const changed = comparison.files.filter(
+    (file) => file.base.killed !== file.head.killed || file.base.total !== file.head.total,
+  );
+  const unchanged = comparison.files.length - changed.length;
+  const lines = [
+    `mutation-compare: base ${describe(comparison.base)} head ${describe(comparison.head)} delta ${deltaPoints(comparison.base, comparison.head)}`,
+  ];
+  for (const file of changed) {
+    lines.push(`  ${file.path}: ${describe(file.base)} -> ${describe(file.head)}`);
+  }
+  if (unchanged > 0) lines.push(`  ${unchanged} unchanged file(s)`);
+  if (comparison.regression) {
+    lines.push(`mutation-compare: REGRESSION (${deltaPoints(comparison.base, comparison.head)})${advisory ? " in advisory mode, exit 0" : ""}`);
+  } else {
+    lines.push(`mutation-compare: no regression${advisory ? " (advisory mode, exit 0)" : ""}`);
+  }
+  return lines.join("\n");
+}
+
+export function parseArgs(argv: readonly string[]): Options {
+  let advisory = false;
+  const paths: string[] = [];
+  for (const arg of argv) {
+    if (arg === "--advisory") advisory = true;
+    else if (arg.startsWith("--")) throw new ReportError(`${USAGE}: unknown flag ${arg}`);
+    else paths.push(arg);
+  }
+  if (paths.length !== 2 || paths[0] === undefined || paths[1] === undefined) throw new ReportError(USAGE);
+  return { basePath: paths[0], headPath: paths[1], advisory };
+}
+
+export function exitFor(comparison: Comparison, advisory: boolean): number {
+  return comparison.regression && !advisory ? 1 : 0;
+}
+
+async function load(path: string): Promise<string> {
+  try {
+    return await readFile(path, "utf8");
+  } catch {
+    throw new ReportError(`mutation-compare: cannot read ${path}`);
+  }
+}
+
+if (import.meta.main) {
+  try {
+    const options = parseArgs(process.argv.slice(2));
+    const comparison = compareReports(parseReport(await load(options.basePath)), parseReport(await load(options.headPath)));
+    console.log(formatComparison(comparison, options.advisory));
+    process.exit(exitFor(comparison, options.advisory));
+  } catch (error) {
+    console.error(error instanceof ReportError ? error.message : `mutation-compare: ${String(error)}`);
+    process.exit(2);
+  }
+}

package/scripts/test-layout.ts

@@ -40,6 +40,7 @@ const BANNED_GLOBAL_CALLS: readonly string[] = ["fetch"];
 
 export const REQUIRED_TEST_SCRIPT = "bun test --randomize";
 export const LAYOUT_CHECK_MARK = "scripts/test-layout.ts";
+export const LAYOUT_CHECK_BIN = "checks-test-layout";
 
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
@@ -215,11 +216,11 @@ export function scriptViolations(manifest: unknown): readonly Violation[] {
       message: `scripts.test must be exactly "${REQUIRED_TEST_SCRIPT}", found ${JSON.stringify(test ?? null)}`,
     });
   }
-  if (typeof lint !== "string" || !lint.includes(LAYOUT_CHECK_MARK)) {
+  if (typeof lint !== "string" || (!lint.includes(LAYOUT_CHECK_MARK) && !lint.includes(LAYOUT_CHECK_BIN))) {
     violations.push({
       file,
       line: undefined,
-      message: `scripts.lint must run the layout check: add "bun ./node_modules/@avi2dg/checks/${LAYOUT_CHECK_MARK}"`,
+      message: `scripts.lint must run the layout check: add "${LAYOUT_CHECK_BIN}"`,
     });
   }
   return violations;

package/stryker.preset.js

@@ -0,0 +1,13 @@
+export default {
+  packageManager: "npm",
+  plugins: ["@stryker-mutator/*", "@hughescr/stryker-bun-runner"],
+  testRunner: "bun",
+  bun: { timeout: 60000 },
+  inPlace: true,
+  tempDirName: "../.stryker-tmp",
+  coverageAnalysis: "perTest",
+  reporters: ["html", "json", "clear-text", "progress"],
+  timeoutMS: 60000,
+  concurrency: 8,
+  thresholds: { high: 85, low: 70, break: null },
+};