@ic-reactor/mcp 0.0.0-stage → 0.0.0

package/scripts/verify-test-fails-rules.test.mjs

@@ -0,0 +1,215 @@
+/**
+ * Tests of what verify-test-fails decides: the base it compares with by
+ * default, how each test's two results sort, and the exit code.
+ *
+ * Run by `pnpm test:scripts`.
+ */
+import assert from "node:assert/strict"
+import { execFileSync } from "node:child_process"
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { after, describe, it } from "node:test"
+import {
+  classify,
+  defaultBase,
+  exitCodeFor,
+} from "./verify-test-fails-rules.mjs"
+
+const statuses = (entries) => new Map(Object.entries(entries))
+
+describe("classify", () => {
+  it("counts a test that fails on the base and passes now as catching the bug", () => {
+    const groups = classify(
+      statuses({ caught: "passed" }),
+      statuses({ caught: "failed" })
+    )
+    assert.deepEqual(groups.discriminating, ["caught"])
+  })
+
+  it("counts a test that passes on both as passing either way", () => {
+    const groups = classify(
+      statuses({ guard: "passed" }),
+      statuses({ guard: "passed" })
+    )
+    assert.deepEqual(groups.bothPass, ["guard"])
+  })
+
+  it("counts a test with no result on the base as no evidence, not as passing either way", () => {
+    // The base's sources could not load the file (v3, 3.x, under a 4.x
+    // test), so the base run reported nothing for it (#968).
+    const groups = classify(statuses({ unloaded: "passed" }), new Map())
+    assert.deepEqual(groups.noBaseResult, ["unloaded"])
+    assert.deepEqual(groups.bothPass, [])
+  })
+
+  it("counts a test skipped on the base as no evidence", () => {
+    const groups = classify(
+      statuses({
+        "skipped there": "passed",
+        "todo there": "passed",
+        "pending there": "passed",
+      }),
+      statuses({
+        "skipped there": "skipped",
+        "todo there": "todo",
+        "pending there": "pending",
+      })
+    )
+    assert.deepEqual(groups.noBaseResult, [
+      "skipped there",
+      "todo there",
+      "pending there",
+    ])
+  })
+
+  it("keeps a test that fails or is skipped now apart, whatever the base says", () => {
+    const groups = classify(
+      statuses({ broken: "failed", later: "skipped" }),
+      statuses({ broken: "failed", later: "failed" })
+    )
+    assert.deepEqual(groups.failingNow, ["broken"])
+    assert.deepEqual(groups.skipped, ["later"])
+    assert.deepEqual(groups.discriminating, [])
+  })
+})
+
+describe("exitCodeFor", () => {
+  const groups = (counts) => ({
+    discriminating: [],
+    bothPass: [],
+    noBaseResult: [],
+    failingNow: [],
+    skipped: [],
+    ...Object.fromEntries(
+      Object.entries(counts).map(([group, n]) => [
+        group,
+        Array.from({ length: n }, (_, i) => `${group} ${i}`),
+      ])
+    ),
+  })
+
+  it("is 0 when a test caught the bug and every other one has a result on the base", () => {
+    assert.equal(exitCodeFor(groups({ discriminating: 1, bothPass: 2 })), 0)
+  })
+
+  it("is 2 when a test has no result on the base, even beside one that caught the bug", () => {
+    assert.equal(exitCodeFor(groups({ noBaseResult: 12 })), 2)
+    assert.equal(exitCodeFor(groups({ discriminating: 1, noBaseResult: 1 })), 2)
+  })
+
+  it("is 2 when no test distinguishes the fix", () => {
+    assert.equal(exitCodeFor(groups({ bothPass: 3 })), 2)
+    assert.equal(exitCodeFor(groups({ skipped: 3 })), 2)
+  })
+
+  it("is 1 when a test fails now", () => {
+    assert.equal(
+      exitCodeFor(
+        groups({ failingNow: 1, discriminating: 1, noBaseResult: 1 })
+      ),
+      1
+    )
+  })
+})
+
+describe("defaultBase", () => {
+  const dirs = []
+  after(() => {
+    for (const dir of dirs) rmSync(dir, { recursive: true, force: true })
+  })
+
+  /**
+   * A repository whose `main` has A then C, and a branch `fix` that left it at A
+   * and added B, checked out. Each test gets its own.
+   */
+  function repository() {
+    const dir = mkdtempSync(join(tmpdir(), "vtf-base-"))
+    dirs.push(dir)
+    const git = (...args) =>
+      execFileSync(
+        "git",
+        [
+          "-c",
+          "user.name=test",
+          "-c",
+          "user.email=test@example.invalid",
+          "-c",
+          "commit.gpgsign=false",
+          ...args,
+        ],
+        { cwd: dir, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }
+      ).trim()
+    const commit = (message) => {
+      writeFileSync(join(dir, "file.txt"), message)
+      git("add", "file.txt")
+      git("commit", "-q", "--no-verify", "-m", message)
+      return git("rev-parse", "HEAD")
+    }
+    git("init", "-q", "-b", "main")
+    const left = commit("A")
+    git("checkout", "-q", "-b", "fix")
+    const fix = commit("B")
+    git("checkout", "-q", "main")
+    const moved = commit("C")
+    git("checkout", "-q", "fix")
+    return { git, commit, left, fix, moved }
+  }
+
+  it("is where the branch left the local main, not main's tip", () => {
+    const { git, left } = repository()
+    assert.deepEqual(defaultBase(git), { line: "main", commit: left })
+  })
+
+  it("is where the branch left origin/main in a clone with no local main, as a fresh clone of another branch has", () => {
+    const { git, left, moved } = repository()
+    git("update-ref", "refs/remotes/origin/main", moved)
+    git("branch", "-q", "-D", "main")
+    assert.deepEqual(defaultBase(git), { line: "origin/main", commit: left })
+  })
+
+  it("is the newer branch point when the local main is behind origin/main", () => {
+    // The branch left origin/main at C, which the stale local main never got.
+    // Reverting to A would also undo C, and a test could fail there for C.
+    const { git, commit, left, moved } = repository()
+    git("update-ref", "refs/remotes/origin/main", moved)
+    git("checkout", "-q", "-B", "fix", moved)
+    commit("B'")
+    git("branch", "-q", "-f", "main", left)
+    assert.deepEqual(defaultBase(git), { line: "origin/main", commit: moved })
+  })
+
+  it("is the newer branch point when origin/main is behind the local main", () => {
+    const { git, left, moved } = repository()
+    git("update-ref", "refs/remotes/origin/main", left)
+    git("checkout", "-q", "-B", "fix", "main")
+    assert.deepEqual(defaultBase(git), { line: "main", commit: moved })
+  })
+
+  it("is HEAD on main itself", () => {
+    const { git, moved } = repository()
+    git("checkout", "-q", "main")
+    assert.deepEqual(defaultBase(git), { line: "main", commit: moved })
+  })
+
+  it("reads the branch main, not a tag of that name", () => {
+    const { git, left, fix } = repository()
+    git("tag", "main", fix)
+    assert.deepEqual(defaultBase(git), { line: "main", commit: left })
+  })
+
+  it("is undefined with neither, so the script asks for --base instead of guessing v3", () => {
+    const { git, left } = repository()
+    git("branch", "-q", "v3", left)
+    git("checkout", "-q", "v3")
+    git("branch", "-q", "-D", "main")
+    assert.equal(defaultBase(git), undefined)
+  })
+
+  it("throws when HEAD shares no history with the line", () => {
+    const { git } = repository()
+    git("checkout", "-q", "--orphan", "unrelated")
+    git("commit", "-q", "--no-verify", "-m", "unrelated")
+    assert.throws(() => defaultBase(git), /HEAD shares no history with main/)
+  })
+})

package/scripts/verify-test-fails.js

@@ -0,0 +1,371 @@
+#!/usr/bin/env node
+/**
+ * Does this test actually catch the bug it was written for?
+ *
+ * Reverts package sources to a base revision, re-runs the given tests against
+ * that older code, and reports which ones flipped. A test that passes both with
+ * and without the fix proves nothing about the fix — it is either an invariant
+ * guard (fine, and worth having) or vacuous (not fine, and easy to write by
+ * accident: an inline closure that changes a memo dependency every render, or a
+ * fixture that never reaches the state the defect lives in).
+ *
+ * Usage
+ *   node scripts/verify-test-fails.js <vitest-file-pattern> [options]
+ *
+ *   --package <name>   workspace package to run in     (default: auto-detected)
+ *   --base <ref>       revision to revert sources to   (default: where
+ *                      HEAD left main, `git merge-base HEAD main origin/main`
+ *                      over those that exist; with none, it stops)
+ *   --keep             leave the reverted sources in place (debugging)
+ *
+ * Example
+ *   node scripts/verify-test-fails.js tests/retry-classification.test.ts --package core
+ *
+ * It exits 0 when a test caught the bug and every test has a result on the
+ * base, 1 when a test fails now or the run cannot start, and 2 otherwise: no
+ * test distinguishes the fix, or a test has no result on the base (its file
+ * did not load there, a hook failed there, or it was skipped there), which is
+ * no evidence either way. The rules are in `verify-test-fails-rules.mjs`.
+ *
+ * Sources are copied aside and restored from that copy rather than via git
+ * stash, so an interrupted run cannot leave conflict markers behind.
+ *
+ * CI does not run this: it compares a test against a base revision, which
+ * wants a person to pick the test and the base. What CI runs is
+ * `scripts/verify-faults.mjs` (`pnpm verify:faults`), which holds each test
+ * listed in `scripts/faults.json` to a fault written out in that file, applied
+ * to a copy of the package. When a test added with a fix should stay proven,
+ * add its entry there.
+ */
+import { execFileSync, spawnSync } from "node:child_process"
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  rmSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+} from "node:fs"
+import { dirname, join, relative } from "node:path"
+import { tmpdir } from "node:os"
+import { fileURLToPath } from "node:url"
+import {
+  BASE_LINES,
+  classify,
+  defaultBase,
+  exitCodeFor,
+} from "./verify-test-fails-rules.mjs"
+
+const rootDir = join(dirname(fileURLToPath(import.meta.url)), "..")
+
+const args = process.argv.slice(2)
+const VALUE_FLAGS = new Set(["--package", "--base"])
+const BOOL_FLAGS = new Set(["--keep"])
+
+const options = {}
+const positional = []
+for (let i = 0; i < args.length; i++) {
+  const arg = args[i]
+  if (VALUE_FLAGS.has(arg)) options[arg.slice(2)] = args[++i]
+  else if (BOOL_FLAGS.has(arg)) options[arg.slice(2)] = true
+  else if (arg.startsWith("--")) {
+    console.error(`Unknown option: ${arg}`)
+    process.exit(1)
+  } else positional.push(arg)
+}
+
+const flag = (name, fallback) => options[name] ?? fallback
+const pattern = positional[0]
+const keep = Boolean(flag("keep", false))
+
+if (!pattern) {
+  console.error(
+    "Usage: node scripts/verify-test-fails.js <test-file> [--package <name>] [--base <ref>]"
+  )
+  process.exit(1)
+}
+
+const packagesDir = join(rootDir, "packages")
+const allPackages = readdirSync(packagesDir).filter((p) =>
+  existsSync(join(packagesDir, p, "package.json"))
+)
+
+/** Which package owns the test file, if the caller did not say. */
+function detectPackage() {
+  const named = flag("package")
+  if (named) return named
+  for (const p of allPackages) {
+    if (existsSync(join(packagesDir, p, pattern))) return p
+  }
+  console.error(
+    `Could not tell which package "${pattern}" belongs to. Pass --package <name>.`
+  )
+  process.exit(1)
+}
+
+const pkg = detectPackage()
+const pkgDir = join(packagesDir, pkg)
+
+if (pkg === "parser") {
+  console.error(
+    "parser is not supported: its tests import the built `dist/nodejs` WASM\n" +
+      "artifact, so reverting `src/lib.rs` changes nothing that the tests load.\n" +
+      "Both passes would run the same binary and every test would look\n" +
+      "non-discriminating. Rebuild and compare by hand instead."
+  )
+  process.exit(1)
+}
+
+function git(...a) {
+  return execFileSync("git", a, { cwd: rootDir, encoding: "utf8" }).trim()
+}
+
+const named = flag("base")
+let defaulted
+if (named === undefined) {
+  try {
+    defaulted = defaultBase(git)
+  } catch (error) {
+    console.error(
+      `✖ ${error.message} (a shallow clone?): git fetch --unshallow, or pass --base <ref>.`
+    )
+    process.exit(1)
+  }
+  if (defaulted === undefined) {
+    console.error(
+      `✖ No ${BASE_LINES.map(({ name }) => name).join(" or ")} to compare with: git fetch origin main:refs/remotes/origin/main (deepen a shallow clone), or pass --base <ref>.`
+    )
+    process.exit(1)
+  }
+}
+/** What the sources are reverted to, and how the report names it. */
+const base = named ?? defaulted.commit
+const baseName = named ?? git("rev-parse", "--short", base)
+const baseHeader = named ?? `${baseName} (where HEAD left ${defaulted.line})`
+
+/**
+ * Run the tests and return a map of test name -> status, and the first line
+ * of the error of each test file that failed with no test result (it did not
+ * load), the way `verify-faults.mjs` tells a broken file.
+ *
+ * A non-zero exit is expected on the "before" run, so the exit code is ignored
+ * in favour of the JSON report.
+ */
+function runTests(label) {
+  const reportPath = join(mkdtempSync(join(tmpdir(), "vtf-")), "report.json")
+  spawnSync(
+    "./node_modules/.bin/vitest",
+    ["run", pattern, "--reporter=json", "--outputFile", reportPath],
+    { cwd: pkgDir, encoding: "utf8", stdio: "ignore" }
+  )
+  if (!existsSync(reportPath)) {
+    console.error(
+      `\n✖ ${label}: vitest produced no report — is the path right?`
+    )
+    process.exit(1)
+  }
+  const report = JSON.parse(readFileSync(reportPath, "utf8"))
+  const results = new Map()
+  const fileErrors = []
+  for (const file of report.testResults ?? []) {
+    for (const t of file.assertionResults ?? []) {
+      // Keep the status verbatim. Collapsing it to a boolean made every
+      // skipped or todo test look like a failure, and this repo has both.
+      results.set(t.fullName ?? t.title, t.status)
+    }
+    if (file.status === "failed" && !file.assertionResults?.length) {
+      const where = file.name ? relative(pkgDir, file.name) : pattern
+      const first = String(file.message ?? "").split("\n")[0]
+      fileErrors.push(`${where}: ${first || "failed with no test result"}`)
+    }
+  }
+  return { results, fileErrors }
+}
+
+// Sources are what we revert; tests stay exactly as written.
+const sourcePaths = allPackages
+  .map((p) => join("packages", p, "src"))
+  .filter((rel) => existsSync(join(rootDir, rel)))
+
+const backupDir = mkdtempSync(join(tmpdir(), "verify-test-fails-"))
+let restored = false
+
+/**
+ * The caller's staged changes under the source trees, as a patch.
+ *
+ * `git checkout <base> -- <paths>` overwrites those index entries, and the
+ * `git reset` that undoes it resets them to HEAD — silently discarding whatever
+ * the caller had staged, including a partial-staging selection. Captured here
+ * and re-applied on the way out.
+ */
+const stagedPatch = (() => {
+  try {
+    const diff = execFileSync(
+      "git",
+      ["diff", "--cached", "--binary", "--", ...sourcePaths],
+      { cwd: rootDir, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }
+    )
+    return diff.trim() ? diff : undefined
+  } catch {
+    return undefined
+  }
+})()
+
+function restore() {
+  if (restored || keep) return
+  restored = true
+  for (const rel of sourcePaths) {
+    const saved = join(backupDir, rel.replaceAll("/", "__"))
+    if (!existsSync(saved)) continue
+    rmSync(join(rootDir, rel), { recursive: true, force: true })
+    cpSync(saved, join(rootDir, rel), { recursive: true })
+  }
+  // `git checkout <base> -- <paths>` writes the base content to the index as
+  // well as the worktree. Putting the files back leaves the index still holding
+  // the base version, which shows up as staged changes and could be committed
+  // by accident — so unstage those paths too.
+  try {
+    execFileSync("git", ["reset", "-q", "--", ...sourcePaths], { cwd: rootDir })
+    if (stagedPatch) {
+      const patchFile = join(backupDir, "staged.patch")
+      writeFileSync(patchFile, stagedPatch)
+      execFileSync("git", ["apply", "--cached", patchFile], { cwd: rootDir })
+    }
+  } catch {
+    console.error(
+      "\n⚠ Could not fully restore the git index for the source paths.\n" +
+        `    git reset -- ${sourcePaths.join(" ")}\n` +
+        (stagedPatch ? "    (your staged changes may need re-staging)\n" : "")
+    )
+  }
+  rmSync(backupDir, { recursive: true, force: true })
+}
+
+// Restore on any exit path, including Ctrl-C.
+process.on("exit", restore)
+for (const sig of ["SIGINT", "SIGTERM"]) process.on(sig, () => process.exit(1))
+
+console.log(`\nverify-test-fails`)
+console.log(`  package : ${pkg}`)
+console.log(`  tests   : ${pattern}`)
+console.log(`  base    : ${baseHeader}\n`)
+
+console.log("→ running against the current sources…")
+const { results: after, fileErrors: errorsNow } = runTests("current")
+if (after.size === 0) {
+  for (const error of errorsNow) console.error(`      ! ${error}`)
+  console.error("✖ No tests matched — check the path.")
+  process.exit(1)
+}
+
+for (const rel of sourcePaths) {
+  cpSync(join(rootDir, rel), join(backupDir, rel.replaceAll("/", "__")), {
+    recursive: true,
+  })
+}
+const colocatedTestsBefore = colocatedTests()
+
+/** Every colocated test file under the reverted source trees. */
+function colocatedTests() {
+  const found = []
+  const walk = (dir) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name)
+      if (entry.isDirectory()) walk(full)
+      else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(entry.name)) found.push(full)
+    }
+  }
+  for (const rel of sourcePaths) walk(join(rootDir, rel))
+  return found
+}
+
+console.log(`→ reverting sources to ${baseName} and re-running…`)
+try {
+  git("checkout", base, "--", ...sourcePaths)
+  // Some packages colocate tests under `src` (codegen does). Checking out the
+  // whole tree would swap those for their base versions — or delete ones the
+  // branch adds — so the tests being judged would not be the ones written.
+  // Put them back from the copy taken a moment ago.
+  for (const rel of sourcePaths) {
+    const srcRoot = join(rootDir, rel)
+    const savedRoot = join(backupDir, rel.replaceAll("/", "__"))
+    for (const abs of colocatedTestsBefore) {
+      if (!abs.startsWith(`${srcRoot}/`)) continue
+      const saved = join(savedRoot, relative(srcRoot, abs))
+      if (!existsSync(saved)) continue
+      mkdirSync(dirname(abs), { recursive: true })
+      cpSync(saved, abs)
+    }
+  }
+} catch (error) {
+  console.error(
+    `✖ Could not check out sources from ${baseName}: ${error.message}`
+  )
+  process.exit(1)
+}
+const { results: before, fileErrors } = runTests("base")
+restore()
+
+// ── report ────────────────────────────────────────────────────────────────
+const groups = classify(after, before)
+// A matched file that does not load now fails now, as verify-faults counts it
+// broken, rather than dropping out of the report.
+groups.failingNow.unshift(...errorsNow)
+const { discriminating, bothPass, noBaseResult, failingNow, skipped } = groups
+
+const pad = (n) => String(n).padStart(3)
+console.log(
+  `\n${pad(discriminating.length)} caught the bug   (fail on ${baseName}, pass now)`
+)
+for (const n of discriminating) console.log(`      ✓ ${n}`)
+
+if (bothPass.length) {
+  console.log(
+    `\n${pad(bothPass.length)} passed either way  (invariant guards, or vacuous — check each)`
+  )
+  for (const n of bothPass) console.log(`      • ${n}`)
+}
+
+if (noBaseResult.length) {
+  console.log(
+    `\n${pad(noBaseResult.length)} no result on the base  (not evidence either way)`
+  )
+  for (const error of fileErrors) console.log(`      ! ${error}`)
+  for (const n of noBaseResult) console.log(`      ? ${n}`)
+}
+
+if (failingNow.length) {
+  console.log(`\n${pad(failingNow.length)} failing now       (fix these first)`)
+  for (const n of failingNow) console.log(`      ✗ ${n}`)
+}
+
+if (skipped.length) {
+  console.log(
+    `\n${pad(skipped.length)} skipped            (not evidence either way)`
+  )
+  for (const n of skipped) console.log(`      – ${n}`)
+}
+
+console.log("")
+const code = exitCodeFor(groups)
+if (code !== 2) process.exit(code)
+if (noBaseResult.length) {
+  console.log(
+    `${noBaseResult.length} test(s) did not run on the base (the file did not load there, or\n` +
+      "a hook failed or the test was skipped there), so this run says nothing\n" +
+      "about them. Pick a base they run on (--base <ref>). If they need the\n" +
+      "fix itself to load or set up, no base will run them: hold them to a\n" +
+      "fault in scripts/faults.json instead.\n"
+  )
+}
+if (discriminating.length === 0 && bothPass.length) {
+  console.log(
+    "No test in this file distinguishes the fix. That is correct for a pure\n" +
+      "refactor or a file of invariant guards — otherwise the fix is untested.\n"
+  )
+} else if (discriminating.length === 0 && !noBaseResult.length) {
+  console.log("Every test here is skipped, so this run proves nothing.\n")
+}
+process.exit(2)