@avi2dg/checks 0.8.0 → 0.10.0

package/README.md

@@ -2,13 +2,16 @@
 
 Deterministic checks shared across my TypeScript repos. One package,
 `@avi2dg/checks`: the `checks-lint` entry point that runs every lint gate
-below over a range it resolves itself, the oxlint base config, the
-tsconfig fragment with the Effect language-service block, the shared
-commitlint config, the shared dependency-cruiser base, the test-layout
-check with its bunfig preset, the commit-identity check, the comment
-gate with its backtest, the oxlint suppressions ratchet, the Stryker
-mutation-testing preset with its no-regression comparator, the CI-wiring
-check, and the Effect error-channel plugin compiled to JavaScript.
+below over a range it resolves itself, the `checks-test` entry point
+that runs the suite and refuses an undeclared skip, the `checks-flake`
+run that records the seeds a failing test fails with, the oxlint base
+config, the tsconfig fragment with the Effect language-service block,
+the shared commitlint config, the shared dependency-cruiser base, the
+test-layout check with its bunfig preset, the commit-identity check, the
+comment gate with its backtest, the oxlint suppressions ratchet, the
+Stryker mutation-testing preset with its no-regression comparator, the
+CI-wiring check, and the Effect error-channel plugin compiled to
+JavaScript.
 
 Published as `@avi2dg/checks` on the public npm registry.
 
@@ -54,15 +57,20 @@ cp node_modules/@avi2dg/checks/bunfig.toml bunfig.toml
 ```json
 "lint": "oxlint --type-aware && checks-lint",
 "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
-"test": "bun test --randomize"
+"test": "checks-test"
 ```
 
+`checks-test` runs `bun test --randomize` and fails on a skip the
+repository has not declared; see "Test entry point" below.
+
 `checks-lint` runs every kit gate a lint needs; see "Lint entry point"
 below. Three of them:
 
 `lint-coverage.sh` fails when oxlint silently skips a tracked `.ts` or
 `.tsx` file, for example through a stray `.gitignore` entry. It compares
 `git ls-files` against oxlint's own file walk and names the missing files.
+It exits 2 when oxlint cannot walk the tree, as when it is not on `PATH`
+or its config does not parse.
 
 `test-layout.ts` decides the test layout described below.
 
@@ -87,7 +95,9 @@ The registry version is pinned by the consumer's lockfile; bump
 ## Lint entry point
 
 `checks-lint` runs each of the kit's lint gates in turn and names every
-one that fails, rather than stopping at the first:
+one that fails, rather than stopping at the first. A repository whose
+tracked files give a gate nothing to check can leave it out through
+`ciWiring.lintGates`; see "Gate selection" under "CI wiring".
 
 | Gate | Reads |
 | --- | --- |
@@ -122,13 +132,29 @@ tip: commits the base branch gained after the head branched off would
 otherwise count against the head. When the head is the merge base, as
 on a push to the default branch or a local run on it, the range would be
 empty, so each range gate is handed that tip commit alone and checks it
-against its parent:
+against its parent, or against the empty tree when it is a repository's
+first commit:
 
 ```
 checks-lint: tip 10ba7d8935b73ed72624120a1542e51bd21ca7c7 from HEAD against origin/main
 ```
 
-It prints the range, each gate's own report, then its verdict:
+A clone with no remote-tracking refs at all, such as a freshly
+initialised repository with no remote or one whose remote was never
+fetched, has no default branch to start from, so each range gate is
+handed `HEAD` alone the same way:
+
+```
+checks-lint: tip 10ba7d8935b73ed72624120a1542e51bd21ca7c7 from HEAD alone, as the clone has no remote-tracking refs
+```
+
+Once any ref sits under `refs/remotes/`, a missing
+`origin/<default branch>` exits 2 instead, since that is the shape of a
+shallow CI checkout, where `HEAD` alone would leave the commits before
+it unchecked.
+
+It prints the range, the declared selection if there is one, each
+gate's own report, then its verdict:
 
 ```
 checks-lint: range 2504acf098d120e73a8ece3c96f22b934f35c6a8..10ba7d8935b73ed72624120a1542e51bd21ca7c7 from HEAD against origin/main
@@ -136,10 +162,11 @@ checks-lint: range 2504acf098d120e73a8ece3c96f22b934f35c6a8..10ba7d8935b73ed7262
 checks-lint: 3 of 6 gate(s) failed: checks-commit-identity, checks-comment-gate, checks-suppressions-ratchet
 ```
 
-It exits 1 when any gate found a violation, and 2 when the range does
-not resolve or no failing gate could decide. Every gate runs, so a
-repository on `checks-lint` declares `ciWiring.gates` (see "CI wiring")
-and holds the test layout.
+It exits 1 when any gate found a violation, and 2 when the range or the
+selection does not resolve, or no failing gate could decide. ci-wiring
+always runs, so a repository on `checks-lint` declares `ciWiring.gates`
+(see "CI wiring"), and it holds the test layout unless its selection
+leaves out `checks-test-layout`.
 
 CI runs it through `lint`. The checkout fetches the whole history, which
 the merge base needs:
@@ -207,7 +234,9 @@ which need the `node`, `promise` and `unicorn` plugins in the override's
 `plugins`. `unicorn/no-process-exit` passes over any file that opens with
 a shebang, so a bin also needs `no-restricted-properties` on
 `process.exit`. Sites standing when the override lands go in oxlint's own
-baseline, `oxlint --suppress-all`, so their count can only fall.
+baseline, `oxlint --suppress-all`, so their count can only fall. A later
+override lifts `effect-channel/no-throw` from `scripts/comment-matchers.ts`,
+the one file there a host loads without `node_modules`.
 
 The language service holds the same paths to Effect-native IO through
 `overrides` in the plugin block of `tsconfig.json`. effect-tsgo keeps the
@@ -262,8 +291,9 @@ file and the path to move it to when it does not:
   them; `tests/fixtures/**` is data and is not parsed. Detection parses
   with swc and reads import specifiers and identifier use, so a test that
   only carries `"node:child_process"` as a string is not a violation.
-- `scripts.test` is exactly `bun test --randomize` and `scripts.lint` runs
-  this check, itself or through `checks-lint` called by its bare bin name.
+- `scripts.test` is exactly `checks-test`, which runs `bun test --randomize`
+  (see "Test entry point"), and `scripts.lint` runs this check, itself or
+  through `checks-lint` called by its bare bin name.
 - `bunfig.toml` carries every `[test]` key of the shipped preset with the
   same value, and `[test].pathIgnorePatterns` is always
   `["**/tests/quarantine/**"]`: the check pins it itself, so this repo,
@@ -282,6 +312,109 @@ still run on demand:
 bun test --path-ignore-patterns='' tests/quarantine
 ```
 
+## Test entry point
+
+`checks-test` runs the whole suite with `bun test --randomize`, passes
+bun's output through, and then reads bun's JUnit report of the same run.
+bun exits 0 with tests skipped, so a green run says nothing about the
+tests that never ran. `checks-test` fails when a test was skipped, by
+`test.skip`, `test.skipIf`, `test.if`, `describe.skip` or `test.todo`,
+without a declaration in `package.json`:
+
+```json
+"testSkips": [
+  {
+    "file": "tests/e2e/docker.test.ts",
+    "test": "images > builds the release image",
+    "reason": "the runner has no docker daemon",
+    "when": "ci"
+  }
+]
+```
+
+`file` is the path bun reports, relative to the package root, and `test`
+is the name bun's console prints: the describe blocks and the test name
+joined by ` > `. `reason` is required. `when` is `ci` or `local` for a
+test skipped only there, and a declaration without it holds in both;
+`checks-test` counts a run as `ci` when `CI` is set true, as GitHub
+Actions sets it. A declaration that holds for the run but matches no
+skipped test fails a ci run too, so a fixed or renamed test takes its
+declaration with it. A local run only warns about it, because whether a
+test skips there can hang on the machine, such as a docker daemon being
+up:
+
+```
+checks-test: 1 skipped test(s) undeclared and 1 declaration(s) matching no skipped test in this ci run:
+  tests/pricing.test.ts:12 pricing > rounds half to even: skipped with no declaration; run it, or declare it in package.json testSkips with its reason
+  tests/e2e/docker.test.ts > images > builds the release image: declared, but no such test skipped; delete the declaration
+```
+
+It exits 1 when a test failed or a skip is undeclared or, in a ci run,
+a declaration stale, and 2 when `testSkips` does not parse or bun
+passed without writing its report. It takes no arguments: a `-t`
+filter reports every test it leaves out as skipped and a path filter
+drops files a declaration names, so a narrowed run is plain
+`bun test --randomize` with the arguments. Files under
+`tests/quarantine/` are never run and so never reported; see "Test
+layout".
+
+## Flake run
+
+A green run proves nothing failed in that run, not that no test is
+flaky. `checks-flake` runs the whole suite several times, each with its
+own `--seed`, and records per failing test the seeds it failed with:
+
+```sh
+checks-flake [--runs <count> | --seed <seed>...] [--report <file>]
+```
+
+`--runs` defaults to 10 runs on random seeds, and `--seed`, given once
+per run, replays chosen seeds, such as the ones a report recorded.
+`bun test --randomize --seed=<seed>` puts the suite in the same order,
+so a seed reproduces a failure that hangs on order. `--report` writes the
+record as JSON, every run's seed and failing tests and every failing
+test's seeds, and under GitHub Actions the summary below is appended to
+the job summary:
+
+```
+checks-flake: 3 of 10 run(s) failed, 1 test(s) failing in them
+
+| Test | Failed | Seeds |
+| --- | --- | --- |
+| tests/cache.test.ts:6 reads the cache | 3 of 10 runs | 2170533150, 4046124386, 180394251 |
+
+Reproduce a failing run with bun test --randomize --seed=<seed>.
+```
+
+A run that fails with no failing test, such as a test file that throws
+while loading, is listed with its seed on its own line. It exits 1 when
+any run failed and 2 when bun passed without writing its report.
+
+A consumer runs it on a schedule and keeps the record as an artifact:
+
+```yaml
+on:
+  schedule:
+    - cron: "17 5 * * *"
+  workflow_dispatch:
+jobs:
+  flake:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: bunx checks-flake --runs 10 --report flake-report.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: flake-report
+          path: flake-report.json
+```
+
+and declares the step in `ciWiring.scheduled`, so `checks-ci-wiring`
+fails once the schedule stops running it; see "CI wiring".
+
 ## Dependency rules
 
 `.dependency-cruiser.cjs` extends the shared base, which carries
@@ -408,15 +541,23 @@ bun run checks-comment-gate <ref>
 ```
 
 With two arguments it diffs the base against the head. With one it diffs
-that commit against its parent, and exits 2 when that parent is not in
-the clone rather than widening to the whole tree, so a shallow checkout
-running the one-argument form needs `fetch-depth: 2`. Only added lines
+that commit against its parent, or against the empty tree for a
+repository's first commit, which has none. It exits 2 when the parent
+exists but is not in the clone rather than widening to the whole tree,
+so a shallow checkout running the one-argument form needs
+`fetch-depth: 2`. Only added lines
 are checked, so a violation in a file the diff never touches stays
 silent, and a refusal counts when any line of the comment carrying it was
 added. The check refuses a machine-read directive, a record or ticket
 pointer, a doc block, and a file opening with a rationale block over
-three lines, licence headers excepted; `scripts/comments.ts` holds the
-scanner the gate and the backtest share.
+three lines, licence headers excepted.
+
+`scripts/comment-matchers.ts` holds the scanner, the comment syntaxes
+and a synchronous `refused()`, and imports nothing, so a host such as a
+hook bundle can copy it alone into a directory with no `node_modules`
+and import it as `@avi2dg/checks/scripts/comment-matchers.ts`. The repo's
+cruise fails when it gains an import. `scripts/comments.ts` wraps the
+same matchers in Effect for the gate and the backtest.
 
 `checks-lint` runs it over each pull request's range; see "Lint entry point".
 
@@ -434,8 +575,9 @@ bun run checks-suppressions-ratchet <base-ref> <head-ref>
 bun run checks-suppressions-ratchet <ref>
 ```
 
-With one argument it compares that commit with its parent, and exits 2
-when the parent is not in the clone. With two it reads the base where
+With one argument it compares that commit with its parent, or with the
+empty tree for a repository's first commit, and exits 2 when the parent
+exists but is not in the clone. With two it reads the base where
 the head branched off, at their merge-base, so a count the base branch
 lowered since does not read as a rise at the head. A count that fell and
 an entry that went both pass. A commit without the file counts as empty,
@@ -523,9 +665,80 @@ ci-wiring: 1 of 8 gate(s) do not run on pull requests to main:
     .github/workflows/release.yml job publish step 7: .github/workflows/release.yml does not trigger on pull_request
 ```
 
-It exits 2 when `package.json` declares no gates, a gate is not one
-plain command, or a workflow does not parse. Whether a workflow is
-well formed is actionlint's question, not this one's.
+It exits 2 when `package.json` declares no gates, `scheduled` is not an
+array, a gate or scheduled command is not one plain command, or a
+workflow does not parse. Whether a workflow is well formed is
+actionlint's question, not this one's.
+
+A command a schedule must run, such as the flake run, goes in
+`"scheduled"` beside `"gates"`:
+
+```json
+"ciWiring": {
+  "gates": ["bun run lint", "bun run typecheck", "bun run test"],
+  "scheduled": ["bunx checks-flake --runs 10 --report flake-report.json"]
+}
+```
+
+Each counts only as a step of the same plain shape in a workflow whose
+`on` carries `schedule` with at least one `cron`, under the same
+`if: false`, `continue-on-error: true` and `needs` rules as a gate. It
+exits 1 naming each one no schedule runs:
+
+```
+ci-wiring: 1 of 1 scheduled command(s) do not run on a schedule:
+  bunx checks-flake --runs 10 --report flake-report.json
+    .github/workflows/ci.yml job checks step 5: .github/workflows/ci.yml does not trigger on a schedule
+```
+
+### Gate selection
+
+A repository with no TypeScript source gives `checks-lint-coverage` and
+`checks-test-layout` nothing to check, and test-layout still refuses its
+missing `bun test` script and `bunfig.toml`. It declares the gates
+`checks-lint` runs as `lintGates`, beside `gates`:
+
+```json
+"ciWiring": {
+  "gates": ["bun run lint"],
+  "lintGates": [
+    "checks-commit-identity",
+    "checks-comment-gate",
+    "checks-suppressions-ratchet",
+    "checks-ci-wiring"
+  ]
+}
+```
+
+`checks-lint` runs exactly those, in the "Lint entry point" table's
+order, and all six when `lintGates` is absent. A step running
+`checks-lint` then counts only for a declared gate that `lintGates`
+keeps.
+
+A selection may leave out only a gate that does not apply:
+
+| Gate | Applies when the repository |
+| --- | --- |
+| `checks-lint-coverage` | tracks a `.ts` or `.tsx` file |
+| `checks-test-layout` | tracks a `.ts` or `.tsx` file |
+| `checks-commit-identity` | always |
+| `checks-comment-gate` | always |
+| `checks-suppressions-ratchet` | always |
+| `checks-ci-wiring` | always |
+
+Both bins exit 2 on a `lintGates` that names an unknown gate or leaves
+out one that always applies. ci-wiring exits 1 when the
+selection leaves out a gate the repository's tracked files make
+applicable, and names the gate and the files:
+
+```
+ci-wiring: ciWiring.lintGates leaves out 2 gate(s) this repository's contents make applicable:
+  checks-lint-coverage: the repository tracks TypeScript source (src/widget.ts)
+  checks-test-layout: the repository tracks TypeScript source (src/widget.ts)
+```
+
+It reads the files tracked at the checkout, so the pull request that
+adds the first TypeScript file is the one refused.
 
 ### Limits
 
@@ -655,6 +868,16 @@ The shared Stryker preset's `json` reporter writes
   the entry point, and `lint-coverage.sh` stays a shell script. The
   gates run one at a time with their output passed straight through, so
   each report reads whole and in the table's order.
+- A gate selection is checked against the repository's contents rather
+  than trusted, so it cannot skip a gate that applies. ci-wiring does
+  that check, which is why a selection without it, or without another
+  gate that applies everywhere, is refused as `checks-lint` reads it:
+  nothing would check the selection otherwise.
+- `checks-test` runs bun itself rather than reading a report some other
+  run left: a skip taken only on CI is visible only in CI's own run, and
+  an earlier run's report may be stale or narrowed. It reads the JUnit
+  report bun writes to a temporary directory, since bun has no other
+  per-test output meant for a program.
 - `checks-ci-wiring` runs inside `lint`, not in a workflow of its own:
   deleting the step that runs a check is the violation it catches, so the
   local `lint` is where it has to fail.
@@ -695,7 +918,8 @@ a tag off `main` and reruns the build, `dist/` check, lint, typecheck
 and tests before it publishes:
 
 ```sh
-git tag v0.8.0 && git push origin v0.8.0
+tag="v$(bun -p 'require("./package.json").version')"
+git tag "$tag" && git push origin "$tag"
 ```
 
 The `release` workflow publishes the tagged version through npm

package/oxlintrc.json

@@ -14,8 +14,8 @@
     "typescript/no-unnecessary-type-parameters": "error",
     "typescript/use-unknown-in-catch-callback-variable": "error",
     "eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_", "ignoreRestSiblings": true }],
-    "typescript/no-unsafe-type-assertion": "off",
-    "typescript/no-non-null-assertion": "off",
+    "typescript/no-unsafe-type-assertion": "error",
+    "typescript/no-non-null-assertion": "error",
     "typescript/consistent-return": "off"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avi2dg/checks",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Deterministic checks shared across the captain's TypeScript repos",
   "license": "MIT",
   "repository": {
@@ -21,9 +21,13 @@
     "scripts/gates.ts",
     "scripts/lint.ts",
     "scripts/test-layout.ts",
+    "scripts/test.ts",
+    "scripts/test-report.ts",
+    "scripts/flake.ts",
     "scripts/commit-identity.ts",
     "scripts/mutation-compare.ts",
     "scripts/ci-wiring.ts",
+    "scripts/comment-matchers.ts",
     "scripts/comments.ts",
     "scripts/comment-gate.ts",
     "scripts/suppressions-ratchet.ts",
@@ -40,9 +44,12 @@
     "./scripts/lint-coverage.sh": "./scripts/lint-coverage.sh",
     "./scripts/lint.ts": "./scripts/lint.ts",
     "./scripts/test-layout.ts": "./scripts/test-layout.ts",
+    "./scripts/test.ts": "./scripts/test.ts",
+    "./scripts/flake.ts": "./scripts/flake.ts",
     "./scripts/commit-identity.ts": "./scripts/commit-identity.ts",
     "./scripts/mutation-compare.ts": "./scripts/mutation-compare.ts",
     "./scripts/ci-wiring.ts": "./scripts/ci-wiring.ts",
+    "./scripts/comment-matchers.ts": "./scripts/comment-matchers.ts",
     "./scripts/comments.ts": "./scripts/comments.ts",
     "./scripts/comment-gate.ts": "./scripts/comment-gate.ts",
     "./scripts/suppressions-ratchet.ts": "./scripts/suppressions-ratchet.ts",
@@ -56,6 +63,8 @@
     "checks-lint": "scripts/lint.ts",
     "checks-lint-coverage": "scripts/lint-coverage.sh",
     "checks-test-layout": "scripts/test-layout.ts",
+    "checks-test": "scripts/test.ts",
+    "checks-flake": "scripts/flake.ts",
     "checks-commit-identity": "scripts/commit-identity.ts",
     "checks-mutation-compare": "scripts/mutation-compare.ts",
     "checks-ci-wiring": "scripts/ci-wiring.ts",
@@ -65,9 +74,9 @@
   },
   "scripts": {
     "build": "bun build effect-channel/index.ts --outdir dist --target node --format esm",
-    "lint": "oxlint --type-aware && bun scripts/lint.ts && depcruise --config .dependency-cruiser.cjs effect-channel scripts tests commitlint.config.js dependency-cruiser.config.js stryker.preset.js .dependency-cruiser.cjs",
+    "lint": "oxlint --type-aware && bun scripts/lint.ts && depcruise --config .dependency-cruiser.cjs .",
     "typecheck": "tsc --noEmit && effect-tsgo diagnostics --project tsconfig.json --format text --strict",
-    "test": "bun test --randomize"
+    "test": "bun scripts/test.ts"
   },
   "ciWiring": {
     "gates": [
@@ -77,6 +86,9 @@
       "bun run typecheck",
       "bun run test",
       "./node_modules/.bin/commitlint"
+    ],
+    "scheduled": [
+      "bun scripts/flake.ts --runs 10 --report flake-report.json"
     ]
   },
   "peerDependencies": {

package/scripts/backtest.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { Console, Effect, Option } from "effect";
-import { comments, refused, SYNTAXES } from "./comments.ts";
+import { comments, refused, syntaxOf } from "./comments.ts";
 import { git } from "./git.ts";
 import { runMain, Usage } from "./main.ts";
 
@@ -18,8 +18,7 @@ const EXCLUDED = ["generated/", "vendor/", "repos/", "node_modules/", "dist/"];
 
 function readable(path: string): boolean {
   if (EXCLUDED.some((prefix) => path.startsWith(prefix))) return false;
-  const extension = path.slice(path.lastIndexOf(".") + 1);
-  return extension in SYNTAXES;
+  return syntaxOf(path) !== undefined;
 }
 
 const at = (sha: string, path: string) => git(["show", `${sha}:${path}`]).pipe(Effect.option);