@avi2dg/checks 0.9.0 → 0.10.0

package/README.md

@@ -2,13 +2,16 @@
 
 Deterministic checks shared across my TypeScript repos. One package,
 `@avi2dg/checks`: the `checks-lint` entry point that runs every lint gate
-below over a range it resolves itself, the oxlint base config, the
-tsconfig fragment with the Effect language-service block, the shared
-commitlint config, the shared dependency-cruiser base, the test-layout
-check with its bunfig preset, the commit-identity check, the comment
-gate with its backtest, the oxlint suppressions ratchet, the Stryker
-mutation-testing preset with its no-regression comparator, the CI-wiring
-check, and the Effect error-channel plugin compiled to JavaScript.
+below over a range it resolves itself, the `checks-test` entry point
+that runs the suite and refuses an undeclared skip, the `checks-flake`
+run that records the seeds a failing test fails with, the oxlint base
+config, the tsconfig fragment with the Effect language-service block,
+the shared commitlint config, the shared dependency-cruiser base, the
+test-layout check with its bunfig preset, the commit-identity check, the
+comment gate with its backtest, the oxlint suppressions ratchet, the
+Stryker mutation-testing preset with its no-regression comparator, the
+CI-wiring check, and the Effect error-channel plugin compiled to
+JavaScript.
 
 Published as `@avi2dg/checks` on the public npm registry.
 
@@ -54,9 +57,12 @@ cp node_modules/@avi2dg/checks/bunfig.toml bunfig.toml
 ```json
 "lint": "oxlint --type-aware && checks-lint",
 "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
-"test": "bun test --randomize"
+"test": "checks-test"
 ```
 
+`checks-test` runs `bun test --randomize` and fails on a skip the
+repository has not declared; see "Test entry point" below.
+
 `checks-lint` runs every kit gate a lint needs; see "Lint entry point"
 below. Three of them:
 
@@ -285,8 +291,9 @@ file and the path to move it to when it does not:
   them; `tests/fixtures/**` is data and is not parsed. Detection parses
   with swc and reads import specifiers and identifier use, so a test that
   only carries `"node:child_process"` as a string is not a violation.
-- `scripts.test` is exactly `bun test --randomize` and `scripts.lint` runs
-  this check, itself or through `checks-lint` called by its bare bin name.
+- `scripts.test` is exactly `checks-test`, which runs `bun test --randomize`
+  (see "Test entry point"), and `scripts.lint` runs this check, itself or
+  through `checks-lint` called by its bare bin name.
 - `bunfig.toml` carries every `[test]` key of the shipped preset with the
   same value, and `[test].pathIgnorePatterns` is always
   `["**/tests/quarantine/**"]`: the check pins it itself, so this repo,
@@ -305,6 +312,109 @@ still run on demand:
 bun test --path-ignore-patterns='' tests/quarantine
 ```
 
+## Test entry point
+
+`checks-test` runs the whole suite with `bun test --randomize`, passes
+bun's output through, and then reads bun's JUnit report of the same run.
+bun exits 0 with tests skipped, so a green run says nothing about the
+tests that never ran. `checks-test` fails when a test was skipped, by
+`test.skip`, `test.skipIf`, `test.if`, `describe.skip` or `test.todo`,
+without a declaration in `package.json`:
+
+```json
+"testSkips": [
+  {
+    "file": "tests/e2e/docker.test.ts",
+    "test": "images > builds the release image",
+    "reason": "the runner has no docker daemon",
+    "when": "ci"
+  }
+]
+```
+
+`file` is the path bun reports, relative to the package root, and `test`
+is the name bun's console prints: the describe blocks and the test name
+joined by ` > `. `reason` is required. `when` is `ci` or `local` for a
+test skipped only there, and a declaration without it holds in both;
+`checks-test` counts a run as `ci` when `CI` is set true, as GitHub
+Actions sets it. A declaration that holds for the run but matches no
+skipped test fails a ci run too, so a fixed or renamed test takes its
+declaration with it. A local run only warns about it, because whether a
+test skips there can hang on the machine, such as a docker daemon being
+up:
+
+```
+checks-test: 1 skipped test(s) undeclared and 1 declaration(s) matching no skipped test in this ci run:
+  tests/pricing.test.ts:12 pricing > rounds half to even: skipped with no declaration; run it, or declare it in package.json testSkips with its reason
+  tests/e2e/docker.test.ts > images > builds the release image: declared, but no such test skipped; delete the declaration
+```
+
+It exits 1 when a test failed or a skip is undeclared or, in a ci run,
+a declaration stale, and 2 when `testSkips` does not parse or bun
+passed without writing its report. It takes no arguments: a `-t`
+filter reports every test it leaves out as skipped and a path filter
+drops files a declaration names, so a narrowed run is plain
+`bun test --randomize` with the arguments. Files under
+`tests/quarantine/` are never run and so never reported; see "Test
+layout".
+
+## Flake run
+
+A green run proves nothing failed in that run, not that no test is
+flaky. `checks-flake` runs the whole suite several times, each with its
+own `--seed`, and records per failing test the seeds it failed with:
+
+```sh
+checks-flake [--runs <count> | --seed <seed>...] [--report <file>]
+```
+
+`--runs` defaults to 10 runs on random seeds, and `--seed`, given once
+per run, replays chosen seeds, such as the ones a report recorded.
+`bun test --randomize --seed=<seed>` puts the suite in the same order,
+so a seed reproduces a failure that hangs on order. `--report` writes the
+record as JSON, every run's seed and failing tests and every failing
+test's seeds, and under GitHub Actions the summary below is appended to
+the job summary:
+
+```
+checks-flake: 3 of 10 run(s) failed, 1 test(s) failing in them
+
+| Test | Failed | Seeds |
+| --- | --- | --- |
+| tests/cache.test.ts:6 reads the cache | 3 of 10 runs | 2170533150, 4046124386, 180394251 |
+
+Reproduce a failing run with bun test --randomize --seed=<seed>.
+```
+
+A run that fails with no failing test, such as a test file that throws
+while loading, is listed with its seed on its own line. It exits 1 when
+any run failed and 2 when bun passed without writing its report.
+
+A consumer runs it on a schedule and keeps the record as an artifact:
+
+```yaml
+on:
+  schedule:
+    - cron: "17 5 * * *"
+  workflow_dispatch:
+jobs:
+  flake:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: bunx checks-flake --runs 10 --report flake-report.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: flake-report
+          path: flake-report.json
+```
+
+and declares the step in `ciWiring.scheduled`, so `checks-ci-wiring`
+fails once the schedule stops running it; see "CI wiring".
+
 ## Dependency rules
 
 `.dependency-cruiser.cjs` extends the shared base, which carries
@@ -555,9 +665,31 @@ ci-wiring: 1 of 8 gate(s) do not run on pull requests to main:
     .github/workflows/release.yml job publish step 7: .github/workflows/release.yml does not trigger on pull_request
 ```
 
-It exits 2 when `package.json` declares no gates, a gate is not one
-plain command, or a workflow does not parse. Whether a workflow is
-well formed is actionlint's question, not this one's.
+It exits 2 when `package.json` declares no gates, `scheduled` is not an
+array, a gate or scheduled command is not one plain command, or a
+workflow does not parse. Whether a workflow is well formed is
+actionlint's question, not this one's.
+
+A command a schedule must run, such as the flake run, goes in
+`"scheduled"` beside `"gates"`:
+
+```json
+"ciWiring": {
+  "gates": ["bun run lint", "bun run typecheck", "bun run test"],
+  "scheduled": ["bunx checks-flake --runs 10 --report flake-report.json"]
+}
+```
+
+Each counts only as a step of the same plain shape in a workflow whose
+`on` carries `schedule` with at least one `cron`, under the same
+`if: false`, `continue-on-error: true` and `needs` rules as a gate. It
+exits 1 naming each one no schedule runs:
+
+```
+ci-wiring: 1 of 1 scheduled command(s) do not run on a schedule:
+  bunx checks-flake --runs 10 --report flake-report.json
+    .github/workflows/ci.yml job checks step 5: .github/workflows/ci.yml does not trigger on a schedule
+```
 
 ### Gate selection
 
@@ -741,6 +873,11 @@ The shared Stryker preset's `json` reporter writes
   that check, which is why a selection without it, or without another
   gate that applies everywhere, is refused as `checks-lint` reads it:
   nothing would check the selection otherwise.
+- `checks-test` runs bun itself rather than reading a report some other
+  run left: a skip taken only on CI is visible only in CI's own run, and
+  an earlier run's report may be stale or narrowed. It reads the JUnit
+  report bun writes to a temporary directory, since bun has no other
+  per-test output meant for a program.
 - `checks-ci-wiring` runs inside `lint`, not in a workflow of its own:
   deleting the step that runs a check is the violation it catches, so the
   local `lint` is where it has to fail.
@@ -781,7 +918,8 @@ a tag off `main` and reruns the build, `dist/` check, lint, typecheck
 and tests before it publishes:
 
 ```sh
-git tag v0.9.0 && git push origin v0.9.0
+tag="v$(bun -p 'require("./package.json").version')"
+git tag "$tag" && git push origin "$tag"
 ```
 
 The `release` workflow publishes the tagged version through npm

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avi2dg/checks",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Deterministic checks shared across the captain's TypeScript repos",
   "license": "MIT",
   "repository": {
@@ -21,6 +21,9 @@
     "scripts/gates.ts",
     "scripts/lint.ts",
     "scripts/test-layout.ts",
+    "scripts/test.ts",
+    "scripts/test-report.ts",
+    "scripts/flake.ts",
     "scripts/commit-identity.ts",
     "scripts/mutation-compare.ts",
     "scripts/ci-wiring.ts",
@@ -41,6 +44,8 @@
     "./scripts/lint-coverage.sh": "./scripts/lint-coverage.sh",
     "./scripts/lint.ts": "./scripts/lint.ts",
     "./scripts/test-layout.ts": "./scripts/test-layout.ts",
+    "./scripts/test.ts": "./scripts/test.ts",
+    "./scripts/flake.ts": "./scripts/flake.ts",
     "./scripts/commit-identity.ts": "./scripts/commit-identity.ts",
     "./scripts/mutation-compare.ts": "./scripts/mutation-compare.ts",
     "./scripts/ci-wiring.ts": "./scripts/ci-wiring.ts",
@@ -58,6 +63,8 @@
     "checks-lint": "scripts/lint.ts",
     "checks-lint-coverage": "scripts/lint-coverage.sh",
     "checks-test-layout": "scripts/test-layout.ts",
+    "checks-test": "scripts/test.ts",
+    "checks-flake": "scripts/flake.ts",
     "checks-commit-identity": "scripts/commit-identity.ts",
     "checks-mutation-compare": "scripts/mutation-compare.ts",
     "checks-ci-wiring": "scripts/ci-wiring.ts",
@@ -69,7 +76,7 @@
     "build": "bun build effect-channel/index.ts --outdir dist --target node --format esm",
     "lint": "oxlint --type-aware && bun scripts/lint.ts && depcruise --config .dependency-cruiser.cjs .",
     "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
-    "test": "bun test --randomize"
+    "test": "bun scripts/test.ts"
   },
   "ciWiring": {
     "gates": [
@@ -79,6 +86,9 @@
       "bun run typecheck",
       "bun run test",
       "./node_modules/.bin/commitlint"
+    ],
+    "scheduled": [
+      "bun scripts/flake.ts --runs 10 --report flake-report.json"
     ]
   },
   "peerDependencies": {

package/scripts/ci-wiring.ts

@@ -21,6 +21,7 @@ export type Gate = {
 
 export type Declaration = {
   readonly gates: readonly Gate[];
+  readonly scheduled: readonly Gate[];
   readonly defaultBranch: string;
   readonly lintGates: readonly KitGate[];
 };
@@ -53,6 +54,8 @@ type RunStep = {
   readonly script: string;
 };
 
+type Trigger = (workflow: Workflow) => string | undefined;
+
 export class WiringError extends Schema.TaggedError<WiringError>()("WiringError", {
   message: Schema.String,
 }) {}
@@ -161,7 +164,7 @@ function selects(patterns: readonly string[], branch: string): boolean {
   return selected;
 }
 
-function triggerBlocker(workflow: Workflow, branch: string): string | undefined {
+const pullRequestTrigger = (branch: string): Trigger => (workflow) => {
   const on = isRecord(workflow.document) ? workflow.document["on"] : undefined;
   const events = isRecord(on) ? Object.keys(on) : (names(on) ?? []);
   if (!events.includes("pull_request")) return `${workflow.path} does not trigger on pull_request`;
@@ -183,7 +186,14 @@ function triggerBlocker(workflow: Workflow, branch: string): string | undefined
   const missing = types === undefined ? [] : GATING_TYPES.filter((type) => !types.includes(type));
   if (missing.length > 0) return `${workflow.path} limits pull_request to types without ${missing.join(", ")}`;
   return undefined;
-}
+};
+
+const scheduleTrigger: Trigger = (workflow) => {
+  const on = isRecord(workflow.document) ? workflow.document["on"] : undefined;
+  const schedule = isRecord(on) ? on["schedule"] : undefined;
+  const crons = Array.isArray(schedule) ? schedule.filter((entry) => isRecord(entry) && typeof entry["cron"] === "string") : [];
+  return crons.length > 0 ? undefined : `${workflow.path} does not trigger on a schedule`;
+};
 
 // GitHub prefixes any other if: with success(), so only a status function overrides a needed job's skip.
 function skippedBy(jobs: Readonly<Record<string, unknown>>, id: string, seen: readonly string[]): string | undefined {
@@ -202,7 +212,7 @@ function localCall(uses: unknown): string | undefined {
   return typeof uses === "string" && uses.startsWith("./") ? uses.slice(2) : undefined;
 }
 
-function runSteps(workflows: readonly Workflow[], branch: string): readonly RunStep[] {
+function runSteps(workflows: readonly Workflow[], trigger: Trigger): readonly RunStep[] {
   const documents = new Map(workflows.map((workflow) => [workflow.path, workflow.document]));
   const steps: RunStep[] = [];
 
@@ -240,7 +250,7 @@ function runSteps(workflows: readonly Workflow[], branch: string): readonly RunS
   };
 
   for (const workflow of workflows) {
-    visit(workflow.document, workflow.path, triggerBlocker(workflow, branch), [workflow.path]);
+    visit(workflow.document, workflow.path, trigger(workflow), [workflow.path]);
   }
   return steps;
 }
@@ -250,10 +260,9 @@ function entryPointCommand(gate: Command, lintGates: readonly KitGate[]): Comman
   return index === -1 ? undefined : [...gate.slice(0, index), ENTRY_POINT.bin];
 }
 
-export function findGaps(declaration: Declaration, workflows: readonly Workflow[]): readonly Gap[] {
-  const steps = runSteps(workflows, declaration.defaultBranch);
-  return declaration.gates.flatMap((gate) => {
-    const entryPoint = entryPointCommand(gate.words, declaration.lintGates);
+function gapsAmong(gates: readonly Gate[], steps: readonly RunStep[], lintGates: readonly KitGate[]): readonly Gap[] {
+  return gates.flatMap((gate) => {
+    const entryPoint = entryPointCommand(gate.words, lintGates);
     const commands = [
       { command: gate.command, words: gate.words },
       ...(entryPoint === undefined ? [] : [{ command: entryPoint.join(" "), words: entryPoint }]),
@@ -274,10 +283,16 @@ export function findGaps(declaration: Declaration, workflows: readonly Workflow[
   });
 }
 
-export function formatReport(declaration: Declaration, gaps: readonly Gap[]): string {
-  const target = `pull requests to ${declaration.defaultBranch}`;
-  if (gaps.length === 0) return `ci-wiring: ${declaration.gates.length} gate(s) run on ${target}`;
-  const lines = [`ci-wiring: ${gaps.length} of ${declaration.gates.length} gate(s) do not run on ${target}:`];
+export function findGaps(declaration: Declaration, workflows: readonly Workflow[]): readonly Gap[] {
+  return gapsAmong(declaration.gates, runSteps(workflows, pullRequestTrigger(declaration.defaultBranch)), declaration.lintGates);
+}
+
+export function findScheduledGaps(declaration: Declaration, workflows: readonly Workflow[]): readonly Gap[] {
+  return gapsAmong(declaration.scheduled, runSteps(workflows, scheduleTrigger), declaration.lintGates);
+}
+
+function gapLines(gaps: readonly Gap[]): readonly string[] {
+  const lines: string[] = [];
   for (const gap of gaps) {
     lines.push(`  ${gap.gate}`);
     if (gap.blocked.length === 0) {
@@ -285,11 +300,38 @@ export function formatReport(declaration: Declaration, gaps: readonly Gap[]): st
     }
     for (const { location, blocker } of gap.blocked) lines.push(`    ${location}: ${blocker}`);
   }
-  return lines.join("\n");
+  return lines;
 }
 
 const decodeWiring = Schema.decodeUnknownEffect(LintWiring);
 
+export function formatReport(declaration: Declaration, gaps: readonly Gap[]): string {
+  const target = `pull requests to ${declaration.defaultBranch}`;
+  if (gaps.length === 0) return `ci-wiring: ${declaration.gates.length} gate(s) run on ${target}`;
+  return [`ci-wiring: ${gaps.length} of ${declaration.gates.length} gate(s) do not run on ${target}:`, ...gapLines(gaps)].join("\n");
+}
+
+export function formatScheduledReport(declaration: Declaration, gaps: readonly Gap[]): string {
+  const total = declaration.scheduled.length;
+  if (gaps.length === 0) return `ci-wiring: ${total} scheduled command(s) run on a schedule`;
+  return [`ci-wiring: ${gaps.length} of ${total} scheduled command(s) do not run on a schedule:`, ...gapLines(gaps)].join("\n");
+}
+
+const plainCommands = Effect.fnUntraced(function* (
+  listed: readonly unknown[],
+  subject: string,
+): Effect.fn.Return<readonly Gate[], WiringError> {
+  const gates: Gate[] = [];
+  for (const command of listed) {
+    const words = typeof command === "string" ? plainCommand(command) : undefined;
+    if (typeof command !== "string" || words === undefined) {
+      return yield* new WiringError({ message: `${subject} ${JSON.stringify(command)} is not one plain command` });
+    }
+    gates.push({ command, words });
+  }
+  return gates;
+});
+
 export const parseDeclaration = Effect.fnUntraced(function* (
   manifest: unknown,
   source: string,
@@ -299,22 +341,18 @@ export const parseDeclaration = Effect.fnUntraced(function* (
   if (!Array.isArray(listed) || listed.length === 0) {
     return yield* new WiringError({ message: `${source} sets no ciWiring.gates, a non-empty array of commands` });
   }
-  const commands: readonly unknown[] = listed;
-  const gates: Gate[] = [];
-  for (const command of commands) {
-    const words = typeof command === "string" ? plainCommand(command) : undefined;
-    if (typeof command !== "string" || words === undefined) {
-      return yield* new WiringError({
-        message: `${source} ciWiring gate ${JSON.stringify(command)} is not one plain command`,
-      });
-    }
-    gates.push({ command, words });
+  const gates = yield* plainCommands(listed, `${source} ciWiring gate`);
+  const scheduledListed = configured["scheduled"] ?? [];
+  if (!Array.isArray(scheduledListed)) {
+    return yield* new WiringError({ message: `${source} ciWiring.scheduled is not an array of commands` });
   }
+  const scheduled = yield* plainCommands(scheduledListed, `${source} ciWiring scheduled command`);
   const { ciWiring } = yield* decodeWiring(manifest).pipe(
     Effect.mapError((cause) => new WiringError({ message: `${source}: ${cause.message}` })),
   );
   return {
     gates,
+    scheduled,
     defaultBranch: ciWiring?.defaultBranch ?? DEFAULT_BRANCH,
     lintGates: selectedGates(ciWiring?.lintGates),
   };
@@ -389,7 +427,8 @@ export const readWorkflows = Effect.fn("readWorkflows")(function* (root: string)
 const wiring = Effect.gen(function* () {
   const root = (yield* git(["rev-parse", "--show-toplevel"])).trim();
   const declaration = yield* readDeclaration(root);
-  const gaps = findGaps(declaration, yield* readWorkflows(root));
+  const workflows = yield* readWorkflows(root);
+  const gaps = findGaps(declaration, workflows);
   const omissions = yield* findOmissions(root, declaration.lintGates);
   const gapReport = formatReport(declaration, gaps);
   yield* gaps.length > 0 ? Console.error(gapReport) : Console.log(gapReport);
@@ -397,7 +436,11 @@ const wiring = Effect.gen(function* () {
   if (omissionReport !== undefined) {
     yield* omissions.length > 0 ? Console.error(omissionReport) : Console.log(omissionReport);
   }
-  return gaps.length === 0 && omissions.length === 0;
+  if (declaration.scheduled.length === 0) return gaps.length === 0 && omissions.length === 0;
+  const scheduledGaps = findScheduledGaps(declaration, workflows);
+  const scheduledReport = formatScheduledReport(declaration, scheduledGaps);
+  yield* scheduledGaps.length > 0 ? Console.error(scheduledReport) : Console.log(scheduledReport);
+  return gaps.length === 0 && omissions.length === 0 && scheduledGaps.length === 0;
 });
 
 if (import.meta.main) runMain("ci-wiring", wiring);

package/scripts/flake.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+import { Config, Console, Effect, FileSystem, Option, Path, Random, Schema } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { runMain, Usage } from "./main.ts";
+import { NAME_SEPARATOR, parseReport, ReportError, reporterArgs, type TestResult } from "./test-report.ts";
+
+export type Run = {
+  readonly seed: number;
+  readonly passed: boolean;
+  readonly tests: number;
+  readonly failed: readonly TestResult[];
+};
+
+export type Failure = {
+  readonly file: string;
+  readonly test: string;
+  readonly line: number;
+  readonly seeds: readonly number[];
+};
+
+export type RunRecord = Omit<Run, "failed"> & { readonly failed: readonly string[] };
+
+export type FlakeRecord = {
+  readonly runs: readonly RunRecord[];
+  readonly failures: readonly Failure[];
+  readonly outsideTests: readonly number[];
+};
+
+type Plan = { readonly seeds: readonly number[] } | { readonly runs: number };
+
+const NAME = "checks-flake";
+const USAGE = `usage: ${NAME} [--runs <count> | --seed <seed>...] [--report <file>]`;
+const DEFAULT_RUNS = 10;
+const MAX_SEED = 2 ** 32 - 1;
+
+const wholeBetween = (minimum: number, maximum: number) =>
+  Schema.decodeUnknownEffect(Schema.NumberFromString.check(Schema.isInt(), Schema.isBetween({ minimum, maximum })));
+const decodeRuns = wholeBetween(1, 1000);
+const decodeSeed = wholeBetween(0, MAX_SEED);
+
+const parseArgs = Effect.fnUntraced(function* (args: readonly string[]) {
+  const usage = (reason?: string) => new Usage({ message: reason === undefined ? USAGE : `${reason}; ${USAGE}` });
+  let runs: string | undefined;
+  let report: string | undefined;
+  const seeds: string[] = [];
+  for (let index = 0; index < args.length; index += 2) {
+    const [flag, value] = [args[index], args[index + 1]];
+    if (value === undefined) return yield* usage();
+    if (flag === "--runs" && runs === undefined) runs = value;
+    else if (flag === "--seed") seeds.push(value);
+    else if (flag === "--report" && report === undefined) report = value;
+    else return yield* usage();
+  }
+  if (runs !== undefined && seeds.length > 0) return yield* usage();
+  const invalid = (flag: string) => (cause: { readonly message: string }) => usage(`${flag}: ${cause.message}`);
+  const plan: Plan =
+    seeds.length > 0
+      ? { seeds: yield* Effect.forEach(seeds, (seed) => decodeSeed(seed).pipe(Effect.mapError(invalid("--seed")))) }
+      : { runs: runs === undefined ? DEFAULT_RUNS : yield* decodeRuns(runs).pipe(Effect.mapError(invalid("--runs"))) };
+  return { plan, report };
+});
+
+const seedsOf = (plan: Plan) =>
+  "seeds" in plan ? Effect.succeed(plan.seeds) : Effect.replicateEffect(Random.nextIntBetween(0, MAX_SEED), plan.runs);
+
+const runOnce = Effect.fn("runOnce")(function* (seed: number, outfile: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const args = ["test", "--randomize", `--seed=${seed}`, ...reporterArgs(outfile)];
+  const exitCode = yield* spawner.exitCode(
+    ChildProcess.make(process.execPath, args, { stdin: "ignore", stdout: "ignore", stderr: "ignore" }),
+  );
+  const passed = exitCode === ChildProcessSpawner.ExitCode(0);
+  if (!(yield* fs.exists(outfile))) {
+    if (passed) return yield* new ReportError({ message: `bun test passed with seed ${seed} but wrote no report` });
+    return { seed, passed, tests: 0, failed: [] } satisfies Run;
+  }
+  const results = yield* parseReport(yield* fs.readFileString(outfile));
+  return { seed, passed, tests: results.length, failed: results.filter((result) => result.outcome === "failed") } satisfies Run;
+});
+
+function outcomeOf(run: Run): string {
+  if (run.passed) return "passed";
+  return run.failed.length === 0 ? "failed outside any test" : `failed, ${run.failed.length} test(s) failing`;
+}
+
+function nameOf(result: TestResult): string {
+  return `${result.file}${NAME_SEPARATOR}${result.name}`;
+}
+
+export function recordOf(runs: readonly Run[]): FlakeRecord {
+  const failures = new Map<string, { file: string; test: string; line: number; seeds: number[] }>();
+  for (const run of runs) {
+    const failing = new Map(run.failed.map((result) => [nameOf(result), result]));
+    for (const [name, result] of failing) {
+      const failure = failures.get(name) ?? { file: result.file, test: result.name, line: result.line, seeds: [] };
+      failure.seeds.push(run.seed);
+      failures.set(name, failure);
+    }
+  }
+  return {
+    runs: runs.map(({ seed, passed, tests, failed }) => ({ seed, passed, tests, failed: failed.map(nameOf) })),
+    failures: [...failures.values()].sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line),
+    outsideTests: runs.filter((run) => !run.passed && run.failed.length === 0).map((run) => run.seed),
+  };
+}
+
+function cell(text: string): string {
+  return text.replaceAll("|", "\\|");
+}
+
+export function summary(record: FlakeRecord): string {
+  const total = record.runs.length;
+  const failedRuns = record.runs.filter((run) => !run.passed).length;
+  if (failedRuns === 0) return `${NAME}: ${total} run(s) passed, with seeds ${record.runs.map((run) => run.seed).join(", ")}`;
+  const lines = [`${NAME}: ${failedRuns} of ${total} run(s) failed, ${record.failures.length} test(s) failing in them`];
+  if (record.failures.length > 0) {
+    lines.push("", "| Test | Failed | Seeds |", "| --- | --- | --- |");
+    for (const failure of record.failures) {
+      const place = cell(`${failure.file}:${failure.line} ${failure.test}`);
+      lines.push(`| ${place} | ${failure.seeds.length} of ${total} runs | ${failure.seeds.join(", ")} |`);
+    }
+  }
+  if (record.outsideTests.length > 0) {
+    lines.push("", `${record.outsideTests.length} run(s) failed outside any test, with seeds ${record.outsideTests.join(", ")}`);
+  }
+  lines.push("", "Reproduce a failing run with bun test --randomize --seed=<seed>.");
+  return lines.join("\n");
+}
+
+const flake = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const { plan, report } = yield* parseArgs(process.argv.slice(2));
+  const seeds = yield* seedsOf(plan);
+  const dir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-flake-" });
+
+  const runs = yield* Effect.forEach(seeds, (seed, index) =>
+    runOnce(seed, path.join(dir, `${index}.xml`)).pipe(
+      Effect.tap((run) => Console.log(`${NAME}: run ${index + 1} of ${seeds.length}, seed ${seed}: ${outcomeOf(run)}`)),
+    ),
+  );
+  const record = recordOf(runs);
+  const text = summary(record);
+  yield* Console.log(text);
+
+  if (report !== undefined) yield* fs.writeFileString(report, `${JSON.stringify(record, null, 2)}\n`);
+  const stepSummary = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY"));
+  if (Option.isSome(stepSummary)) yield* fs.writeFileString(stepSummary.value, `${text}\n`, { flag: "a" });
+  return runs.every((run) => run.passed);
+}).pipe(Effect.scoped);
+
+if (import.meta.main) runMain(NAME, flake);

package/scripts/gates.ts

@@ -19,6 +19,8 @@ export type KitGate = Program & {
 
 export const ENTRY_POINT: Program = { bin: "checks-lint", script: "lint.ts" };
 
+export const TEST_ENTRY_POINT: Program = { bin: "checks-test", script: "test.ts" };
+
 export const DEFAULT_BRANCH = "main";
 
 const TYPESCRIPT_SOURCE: TrackedContent = { pathspecs: ["*.ts", "*.tsx"], content: "TypeScript source" };

package/scripts/test-layout.ts

@@ -3,7 +3,7 @@ import { parse } from "@swc/core";
 import { Console, Effect, FileSystem, Path, Schema } from "effect";
 import { git } from "./git.ts";
 import { runMain } from "./main.ts";
-import { ENTRY_POINT } from "./gates.ts";
+import { ENTRY_POINT, TEST_ENTRY_POINT } from "./gates.ts";
 
 export type Violation = {
   readonly file: string;
@@ -39,13 +39,13 @@ const BANNED_BUN_NAMES: readonly string[] = [
 ];
 const BANNED_GLOBAL_CALLS: readonly string[] = ["fetch"];
 
-export const REQUIRED_TEST_SCRIPT = "bun test --randomize";
 const REQUIRED_TEST_TABLE: Record<string, unknown> = {
   pathIgnorePatterns: ["**/tests/quarantine/**"],
 };
 export const LAYOUT_CHECK_MARK = "scripts/test-layout.ts";
 export const LAYOUT_CHECK_BIN = "checks-test-layout";
 const OWN_ENTRY_POINT = `scripts/${ENTRY_POINT.script}`;
+const TEST_SCRIPTS: readonly string[] = [TEST_ENTRY_POINT.bin, `bun scripts/${TEST_ENTRY_POINT.script}`];
 
 export class LayoutError extends Schema.TaggedError<LayoutError>()("LayoutError", {
   message: Schema.String,
@@ -226,11 +226,11 @@ export function scriptViolations(manifest: unknown): readonly Violation[] {
   const test = isRecord(scripts) ? scripts["test"] : undefined;
   const lint = isRecord(scripts) ? scripts["lint"] : undefined;
   const violations: Violation[] = [];
-  if (test !== REQUIRED_TEST_SCRIPT) {
+  if (typeof test !== "string" || !TEST_SCRIPTS.includes(test)) {
     violations.push({
       file,
       line: undefined,
-      message: `scripts.test must be exactly "${REQUIRED_TEST_SCRIPT}", found ${JSON.stringify(test ?? null)}`,
+      message: `scripts.test must be exactly "${TEST_ENTRY_POINT.bin}", which runs bun test --randomize and judges its skips, found ${JSON.stringify(test ?? null)}`,
     });
   }
   if (typeof lint !== "string" || !runsLayoutCheck(lint)) {

package/scripts/test-report.ts

@@ -0,0 +1,115 @@
+import { Effect, Schema } from "effect";
+
+export type Outcome = "passed" | "failed" | "skipped" | "todo";
+
+export type TestResult = {
+  readonly file: string;
+  readonly name: string;
+  readonly line: number;
+  readonly outcome: Outcome;
+};
+
+export class ReportError extends Schema.TaggedError<ReportError>()("ReportError", {
+  message: Schema.String,
+}) {}
+
+type Element = {
+  readonly tag: string;
+  readonly attributes: Readonly<Record<string, string>>;
+  readonly children: Element[];
+};
+
+// bun's console prints a test inside describe blocks as "outer > inner > name".
+export const NAME_SEPARATOR = " > ";
+
+export function reporterArgs(outfile: string): readonly string[] {
+  return ["--reporter=junit", `--reporter-outfile=${outfile}`];
+}
+
+const MARKUP =
+  /<(?:\?[\s\S]*?\?>|!--[\s\S]*?-->|!\[CDATA\[[\s\S]*?\]\]>|(\/?)([A-Za-z_][\w.:-]*)((?:\s+[A-Za-z_][\w.:-]*\s*=\s*"[^"<]*")*)\s*(\/?)>)/g;
+const ATTRIBUTE = /([A-Za-z_][\w.:-]*)\s*=\s*"([^"<]*)"/g;
+const ENTITY = /&(?:(amp|lt|gt|quot|apos)|#(\d+)|#x([0-9A-Fa-f]+));/g;
+const NAMED_ENTITIES: Readonly<Record<string, string>> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
+
+const decodeSuite = Schema.decodeUnknownEffect(Schema.Struct({ name: Schema.String }));
+const decodeCase = Schema.decodeUnknownEffect(
+  Schema.Struct({ name: Schema.String, file: Schema.String, line: Schema.NumberFromString.check(Schema.isInt()) }),
+);
+
+function unescape(text: string): string {
+  return text.replace(ENTITY, (_, named: string | undefined, decimal: string | undefined, hex: string | undefined) => {
+    if (named !== undefined) return NAMED_ENTITIES[named] ?? "";
+    return String.fromCodePoint(decimal === undefined ? Number.parseInt(hex ?? "", 16) : Number.parseInt(decimal, 10));
+  });
+}
+
+function attributesOf(source: string): Readonly<Record<string, string>> {
+  const attributes: Record<string, string> = {};
+  for (const [, name, value] of source.matchAll(ATTRIBUTE)) {
+    if (name !== undefined && value !== undefined) attributes[name] = unescape(value);
+  }
+  return attributes;
+}
+
+const parseElements = Effect.fnUntraced(function* (xml: string): Effect.fn.Return<Element, ReportError> {
+  const malformed = (reason: string) => new ReportError({ message: `the junit report is malformed: ${reason}` });
+  const document: Element = { tag: "", attributes: {}, children: [] };
+  const open: Element[] = [document];
+  let consumed = 0;
+  for (const match of xml.matchAll(MARKUP)) {
+    if (xml.slice(consumed, match.index).includes("<")) return yield* malformed(`stray < before offset ${match.index}`);
+    consumed = match.index + match[0].length;
+    const [, closing, tag, attributes, selfClosing] = match;
+    if (tag === undefined) continue;
+    const parent = open.at(-1);
+    if (parent === undefined) return yield* malformed(`<${tag}> after the root closed`);
+    if (closing === "/") {
+      if (parent.tag !== tag) return yield* malformed(`</${tag}> closes <${parent.tag}>`);
+      open.pop();
+      continue;
+    }
+    const element: Element = { tag, attributes: attributesOf(attributes ?? ""), children: [] };
+    parent.children.push(element);
+    if (selfClosing !== "/") open.push(element);
+  }
+  if (xml.slice(consumed).includes("<")) return yield* malformed("stray < after the last element");
+  if (open.length !== 1) return yield* malformed(`<${open.at(-1)?.tag}> never closes`);
+  const [root, ...extra] = document.children;
+  if (root?.tag !== "testsuites" || extra.length > 0) return yield* malformed("the root is not one <testsuites>");
+  return root;
+});
+
+function outcomeOf(testcase: Element): Outcome {
+  const skipped = testcase.children.find((child) => child.tag === "skipped");
+  if (skipped !== undefined) return skipped.attributes["message"] === "TODO" ? "todo" : "skipped";
+  return testcase.children.some((child) => child.tag === "failure" || child.tag === "error") ? "failed" : "passed";
+}
+
+const collect = Effect.fnUntraced(function* (
+  suite: Element,
+  describes: readonly string[],
+  results: TestResult[],
+): Effect.fn.Return<void, ReportError> {
+  const unreadable = (tag: string) => (cause: { readonly message: string }) =>
+    new ReportError({ message: `the junit report holds a <${tag}> bun never writes: ${cause.message}` });
+  for (const child of suite.children) {
+    if (child.tag === "testsuite") {
+      const { name } = yield* decodeSuite(child.attributes).pipe(Effect.mapError(unreadable(child.tag)));
+      yield* collect(child, [...describes, name], results);
+    } else if (child.tag === "testcase") {
+      const { name, file, line } = yield* decodeCase(child.attributes).pipe(Effect.mapError(unreadable(child.tag)));
+      results.push({ file, name: [...describes, name].join(NAME_SEPARATOR), line, outcome: outcomeOf(child) });
+    }
+  }
+});
+
+// A top-level <testsuite> is named for its file, so only the describe blocks nested in it join a test's name.
+export const parseReport = Effect.fn("parseReport")(function* (xml: string) {
+  const root = yield* parseElements(xml);
+  const results: TestResult[] = [];
+  for (const fileSuite of root.children) {
+    if (fileSuite.tag === "testsuite") yield* collect(fileSuite, [], results);
+  }
+  return results;
+});

package/scripts/test.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { TEST_ENTRY_POINT } from "./gates.ts";
+import { runMain, Usage } from "./main.ts";
+import { NAME_SEPARATOR, parseReport, ReportError, reporterArgs, type TestResult } from "./test-report.ts";
+
+const Environment = Schema.Literals(["ci", "local"]);
+
+export type Environment = typeof Environment.Type;
+
+const SkipDeclaration = Schema.Struct({
+  file: Schema.NonEmptyString,
+  test: Schema.NonEmptyString,
+  reason: Schema.NonEmptyString,
+  when: Schema.optionalKey(Environment),
+});
+
+export type SkipDeclaration = typeof SkipDeclaration.Type;
+
+export type Verdict = {
+  readonly environment: Environment;
+  readonly skipped: number;
+  readonly undeclared: readonly TestResult[];
+  readonly stale: readonly SkipDeclaration[];
+  readonly unjudged: number;
+};
+
+export class SkipGateError extends Schema.TaggedError<SkipGateError>()("SkipGateError", {
+  message: Schema.String,
+}) {}
+
+const NAME = TEST_ENTRY_POINT.bin;
+const DECLARATIONS = "package.json testSkips";
+const USAGE = `usage: ${NAME} takes no arguments, since a narrowed run skips every test it leaves out; run bun test --randomize <args> for one`;
+
+const decodeManifest = Schema.decodeUnknownEffect(
+  Schema.fromJsonString(Schema.Struct({ testSkips: Schema.optionalKey(Schema.Array(SkipDeclaration)) })),
+);
+
+function matches(declaration: SkipDeclaration, result: TestResult): boolean {
+  return declaration.file === result.file && declaration.test === result.name;
+}
+
+function byPlace(a: TestResult, b: TestResult): number {
+  return a.file.localeCompare(b.file) || a.line - b.line || a.name.localeCompare(b.name);
+}
+
+export function judgeSkips(
+  results: readonly TestResult[],
+  declarations: readonly SkipDeclaration[],
+  environment: Environment,
+): Verdict {
+  const skipped = results.filter((result) => result.outcome === "skipped" || result.outcome === "todo");
+  const applying = declarations.filter((declaration) => (declaration.when ?? environment) === environment);
+  return {
+    environment,
+    skipped: skipped.length,
+    undeclared: skipped.filter((result) => !applying.some((declaration) => matches(declaration, result))).sort(byPlace),
+    stale: applying.filter((declaration) => !skipped.some((result) => matches(declaration, result))),
+    unjudged: declarations.length - applying.length,
+  };
+}
+
+function refusedStale(verdict: Verdict): readonly SkipDeclaration[] {
+  return verdict.environment === "ci" ? verdict.stale : [];
+}
+
+export function passes(verdict: Verdict): boolean {
+  return verdict.undeclared.length === 0 && refusedStale(verdict).length === 0;
+}
+
+function staleLine(declaration: SkipDeclaration): string {
+  return `  ${declaration.file}${NAME_SEPARATOR}${declaration.test}: declared, but no such test skipped; delete the declaration`;
+}
+
+function verdictLines(verdict: Verdict): readonly string[] {
+  const { environment, skipped, undeclared, unjudged } = verdict;
+  const run = `this ${environment} run`;
+  if (passes(verdict)) {
+    const other = environment === "ci" ? "local" : "ci";
+    const aside = unjudged === 0 ? "" : `; ${unjudged} declaration(s) for ${other} not judged in ${run}`;
+    return [skipped === 0 ? `${NAME}: no test skipped${aside}` : `${NAME}: ${skipped} skipped test(s), each declared in ${DECLARATIONS}${aside}`];
+  }
+  const stale = refusedStale(verdict);
+  const counted = environment === "ci" ? ` and ${stale.length} declaration(s) matching no skipped test` : "";
+  return [
+    `${NAME}: ${undeclared.length} skipped test(s) undeclared${counted} in ${run}:`,
+    ...undeclared.map((result) => {
+      const kind = result.outcome === "todo" ? "a todo" : "skipped";
+      return `  ${result.file}:${result.line} ${result.name}: ${kind} with no declaration; run it, or declare it in ${DECLARATIONS} with its reason`;
+    }),
+    ...stale.map(staleLine),
+  ];
+}
+
+export function report(verdict: Verdict): string {
+  const warned = verdict.environment === "local" ? verdict.stale : [];
+  const warning =
+    warned.length === 0
+      ? []
+      : [
+          `${NAME}: warning: ${warned.length} declaration(s) matching no skipped test in this local run, refused only in a ci run:`,
+          ...warned.map(staleLine),
+        ];
+  return [...verdictLines(verdict), ...warning].join("\n");
+}
+
+const readDeclarations = Effect.fn("readDeclarations")(function* (root: string) {
+  const manifest = (yield* Path.Path).join(root, "package.json");
+  const { testSkips } = yield* (yield* FileSystem.FileSystem).readFileString(manifest).pipe(
+    Effect.flatMap(decodeManifest),
+    Effect.mapError((cause) => new SkipGateError({ message: `cannot read ${DECLARATIONS}: ${cause.message}` })),
+  );
+  return testSkips ?? [];
+});
+
+const environmentOf = Config.Boolean("CI").pipe(
+  Config.withDefault(false),
+  Config.map((ci): Environment => (ci ? "ci" : "local")),
+  Effect.mapError((cause) => new SkipGateError({ message: `cannot tell a ci run from a local one: ${cause.message}` })),
+);
+
+const runSuite = Effect.fn("runSuite")(function* (outfile: string) {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const args = ["test", "--randomize", ...reporterArgs(outfile)];
+  return yield* spawner.exitCode(
+    ChildProcess.make(process.execPath, args, { stdin: "ignore", stdout: "inherit", stderr: "inherit" }),
+  );
+});
+
+const testEntry = Effect.gen(function* () {
+  if (process.argv.length > 2) return yield* new Usage({ message: USAGE });
+  const fs = yield* FileSystem.FileSystem;
+  const declarations = yield* readDeclarations(process.cwd());
+  const environment = yield* environmentOf;
+
+  const outfile = (yield* Path.Path).join(yield* fs.makeTempDirectoryScoped({ prefix: "checks-test-" }), "junit.xml");
+  const suitePassed = (yield* runSuite(outfile)) === ChildProcessSpawner.ExitCode(0);
+  if (!(yield* fs.exists(outfile))) {
+    if (!suitePassed) {
+      yield* Console.error(`${NAME}: bun test failed before it wrote a report, so no skip was judged`);
+      return false;
+    }
+    return yield* new ReportError({ message: `bun test passed but wrote no report to ${outfile}` });
+  }
+
+  const verdict = judgeSkips(yield* parseReport(yield* fs.readFileString(outfile)), declarations, environment);
+  yield* passes(verdict) ? Console.log(report(verdict)) : Console.error(report(verdict));
+  return suitePassed && passes(verdict);
+}).pipe(Effect.scoped);
+
+if (import.meta.main) runMain(NAME, testEntry);