flagrm 0.1.1 → 0.2.0

package/dist/core/report.js

@@ -1,6 +1,5 @@
 import pc from "picocolors";
 import { plural, relativePath } from "./util.js";
-import { shortTestNames } from "./verify/test-results.js";
 /** A path relative to the working directory, with `/` separators on every platform. */
 function rel(file) {
     return relativePath(process.cwd(), file) || file;
@@ -28,12 +27,21 @@ export function printInventory(report) {
                 (refs.tests ? ` (${refs.tests} in tests)` : "") +
                 pc.dim(` · ${byProject}`);
         console.log(`${pc.bold(entry.flag)}  ${pc.dim(`${entry.type} · ${stateLabel(entry.state)} ·`)} ${usage}`);
-        const project = pad([...entry.definitions, ...entry.config].map((x) => x.project));
+        const project = pad([...entry.definitions, ...entry.config, ...entry.flow].map((x) => x.project));
         const kind = pad(entry.definitions.map((d) => d.kind));
         const name = pad(entry.definitions.map((d) => d.name));
         for (const d of entry.definitions) {
             console.log(`  ${pc.dim("defined")}  ${pc.dim(project(d.project))}  ${pc.dim(kind(d.kind))}  ${name(d.name)}  ${pc.cyan(`${rel(d.file)}:${d.line}`)}`);
         }
+        const flowKind = pad(entry.flow.map((n) => n.kind));
+        const flowName = pad(entry.flow.map((n) => n.name));
+        for (const n of entry.flow) {
+            console.log(`  ${pc.dim("flows  ")}  ${pc.dim(project(n.project))}  ${pc.dim(flowKind(n.kind))}  ${flowName(n.name)}  ${pc.cyan(`${rel(n.file)}:${n.line}`)}` +
+                (n.record ? "" : pc.dim("  (suggested)")));
+        }
+        if (entry.parameterizedTests.length) {
+            console.log(`  ${pc.dim("tests  ")}  ${plural(entry.parameterizedTests.length, "test call")} pass the flag's value as a literal`);
+        }
         const env = pad(entry.config.map((c) => c.environment));
         const state = pad(entry.config.map((c) => configLabel(c.state, c.filters)));
         for (const c of entry.config) {
@@ -41,7 +49,8 @@ export function printInventory(report) {
         }
         console.log("");
     }
-    console.log(pc.dim("Reference counts are direct uses of the flag's literal and definitions, not reads through wrappers."));
+    console.log(pc.dim("Reference counts are direct uses of the flag's literal and definitions, not reads through wrappers. " +
+        "`flows` are the names its value travels through; `baseline` records the ones not marked suggested."));
 }
 function stateLabel(state) {
     return state === "unconfigured" ? "not configured" : state;
@@ -58,11 +67,21 @@ export function printBaseline(baseline, file) {
     for (const c of checks ?? []) {
         const status = c.exitCode === 0 ? pc.green("pass") : pc.red(`fail (exit ${c.exitCode})`);
         console.log(`  ${c.project} ${c.check}: ${status} ${pc.dim(`${c.command} — ${Math.round(c.durationMs / 100) / 10}s`)}`);
+        if (c.failure)
+            console.log(`    ${pc.yellow("setup problem:")} ${c.failure.message}`);
     }
     if (baseline.names.length) {
-        const names = baseline.names.map((n) => (n.kind === "literal" ? n.name : `${n.name} (${n.kind})`));
+        const names = baseline.names.map((n) => n.kind === "literal" ? n.name : `${n.name} (${n.kind}${n.file ? ` in ${n.file}` : ""})`);
         console.log(`  names: ${pc.dim(names.join(", "))}`);
     }
+    for (const n of baseline.suggestedNames ?? []) {
+        console.log(`  ${pc.yellow("suggested")} ${n.name} ${pc.dim(`(${n.kind})`)} ${pc.cyan(`${rel(n.file)}:${n.line}`)} ` +
+            pc.dim(`if it only carries this flag: --name ${n.name} --kind wrapper` +
+                (n.kind === "method" || n.kind === "property" ? "" : ` --file ${rel(n.file)}`)));
+    }
+    for (const t of baseline.parameterizedTests ?? []) {
+        console.log(`  ${pc.dim("test passes")} ${t.parameter}: ${t.value} ${pc.dim(`to ${t.method}`)} ${pc.cyan(`${rel(t.file)}:${t.line}`)}`);
+    }
 }
 const CHECK_STYLE = {
     pass: { icon: "✓", color: pc.green },
@@ -94,13 +113,6 @@ export function printVerifyReport(report, file) {
         if (check.findings.length > shown.length) {
             console.log(pc.dim(`      … ${check.findings.length - shown.length} more (see --json)`));
         }
-        if (check.id === "tests") {
-            const removed = shortTestNames(report.tests.flatMap((t) => t.removed));
-            for (const name of removed.slice(0, MAX_FINDINGS))
-                console.log(pc.dim(`      no longer runs: ${name}`));
-            if (removed.length > MAX_FINDINGS)
-                console.log(pc.dim(`      … ${removed.length - MAX_FINDINGS} more (see --md)`));
-        }
     }
     if (report.strict && warned)
         console.log(pc.dim("\n  --strict: warnings count as failures."));
@@ -133,3 +145,34 @@ export function printDoctor(report) {
     }
     console.log(report.status === "ok" ? pc.green("\nflagrm doctor: ok") : pc.red("\nflagrm doctor: problems found"));
 }
+/** `flagrm scope`: each .NET project's setup problems and what flagrm.slnf leaves out. */
+export function printScope(setups, root, written) {
+    if (setups.length === 0) {
+        console.log(pc.dim("No .NET projects in the config."));
+        return;
+    }
+    for (const s of setups) {
+        const solution = s.setup ? relativePath(root, s.setup.solution) : s.note;
+        console.log(`${pc.bold(s.project)} ${pc.dim(solution ?? "")}`);
+        if (s.sdk)
+            console.log(`  ${pc.red("✗")} ${s.sdk}`);
+        for (const p of s.setup?.problems ?? [])
+            console.log(`  ${pc.yellow("!")} ${p}`);
+        const out = s.setup?.leaveOut ?? [];
+        if (!s.sdk && !s.setup?.problems.length)
+            console.log(`  ${pc.green("✓")} this machine has what it needs`);
+        if (out.length) {
+            const total = s.setup?.projects.length ?? 0;
+            console.log(`  ${written ? "flagrm.slnf leaves out" : "flagrm.slnf would leave out"} ${out.length} of ${total} projects:`);
+            for (const l of out.slice(0, 30))
+                console.log(`    ${l.entry} ${pc.dim(`— ${l.reason}`)}`);
+            if (out.length > 30)
+                console.log(pc.dim(`    … ${out.length - 30} more`));
+            console.log(pc.dim(written
+                ? "  Their builds and tests won't run: verify can't check edits to them, so say so in the PR."
+                : "  Run `flagrm scope --write` to create it and point the build and test at it."));
+        }
+        if (s.sdk)
+            console.log(pc.dim("  A solution filter can't help here: install the SDK or change global.json."));
+    }
+}

package/dist/core/scope.js

@@ -0,0 +1,192 @@
+/**
+ * `flagrm scope`: check that the machine can build and test each .NET
+ * project, and when it lacks something only some projects need (`cargo`,
+ * Docker), narrow the build and test to a solution filter, `flagrm.slnf`,
+ * that leaves those projects out. A smaller scope that runs beats a full one
+ * that can't: the baseline's results are then real.
+ */
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import fg from "fast-glob";
+import YAML from "yaml";
+import { sdkProblem, solutionSetup } from "../adapters/dotnet/setup.js";
+import { projectCommands } from "./config-snapshot.js";
+import { relativePath } from "./util.js";
+export const SLNF = "flagrm.slnf";
+/** Check each .NET project of the workspace. */
+export function setupReport(projects, root, probes) {
+    const out = [];
+    for (const { adapter, ctx } of projects) {
+        if (adapter.id !== "dotnet")
+            continue;
+        const entry = { project: ctx.name };
+        const sdk = sdkProblem(ctx.root, probes, root);
+        if (sdk)
+            entry.sdk = sdk;
+        const solution = solutionFor(ctx, projectCommands(adapter, ctx));
+        if ("note" in solution)
+            entry.note = solution.note;
+        else {
+            try {
+                entry.setup = solutionSetup(solution.solution, probes, root, solution.only);
+            }
+            catch (err) {
+                entry.note = `can't read ${relativePath(root, solution.solution)}: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        out.push(entry);
+    }
+    return out;
+}
+/** A `.sln`, `.slnx` or `.slnf` named by a command. */
+const SOLUTION_ARG = /(?<=^|\s)(?:(["'])([^"']+\.sln[xf]?)\1|([^\s"']+\.sln[xf]?))(?=\s|$)/;
+/** The solution path in a SOLUTION_ARG match, quoted or not. */
+const solutionArg = (m) => m?.[2] ?? m?.[3];
+/** A project file (`.csproj`, `.fsproj`, ...) named by a command. */
+const PROJECT_ARG = /(?<=^|\s)["']?[^\s"']+\.\w*proj["']?(?=\s|$)/;
+/**
+ * The solution a project builds: the one its build or test command names, or
+ * the only one in its directory. When the command names a solution filter
+ * (a user's own or flagrm.slnf), the solution it filters, with `only` its
+ * projects: that is the scope, which flagrm only narrows further.
+ */
+function solutionFor(ctx, commands) {
+    for (const command of [commands.build, commands.test]) {
+        if (command && PROJECT_ARG.test(command) && !SOLUTION_ARG.test(command)) {
+            return { note: `\`${command}\` builds a project, not a solution: nothing to narrow` };
+        }
+    }
+    for (const command of [commands.build, commands.test]) {
+        const arg = command && solutionArg(SOLUTION_ARG.exec(command));
+        if (!arg)
+            continue;
+        const file = path.resolve(ctx.root, arg);
+        if (!fs.existsSync(file))
+            return { note: `${arg} (from \`${command}\`) not found` };
+        if (!file.endsWith(".slnf"))
+            return { solution: file };
+        try {
+            const { path: inner, projects } = JSON.parse(fs.readFileSync(file, "utf8"))?.solution ?? {};
+            if (typeof inner === "string") {
+                const solution = path.resolve(path.dirname(file), inner.replace(/\\/g, "/"));
+                const dir = path.dirname(solution);
+                const only = Array.isArray(projects)
+                    ? projects.filter((p) => typeof p === "string").map((p) => path.resolve(dir, p.replace(/\\/g, "/")))
+                    : undefined;
+                return { solution, only };
+            }
+        }
+        catch {
+            return { note: `can't read ${arg}` };
+        }
+    }
+    const found = fg.sync("*.{sln,slnx}", { cwd: ctx.root, absolute: true });
+    if (found.length === 1)
+        return { solution: found[0] };
+    return { note: found.length ? "several solutions: name one in build/test" : "no solution file found" };
+}
+/**
+ * Create each solution's flagrm.slnf without the projects to leave out, keep
+ * it out of git through `.git/info/exclude` (it describes this machine, not
+ * the repository), and point the project's build and test at it in the config.
+ */
+export function writeScope(setups, projects, root, configFile) {
+    const steps = [];
+    const commandUpdates = [];
+    for (const s of setups) {
+        if (!s.setup?.leaveOut.length)
+            continue;
+        const { solution, leaveOut, projects: all } = s.setup;
+        const slnf = path.join(path.dirname(solution), SLNF);
+        const out = new Set(leaveOut.map((l) => l.entry));
+        const content = `${JSON.stringify({ solution: { path: path.basename(solution), projects: all.map((p) => p.entry).filter((e) => !out.has(e)) } }, null, 2)}\n`;
+        const before = fs.existsSync(slnf) ? fs.readFileSync(slnf, "utf8") : undefined;
+        if (before !== content)
+            fs.writeFileSync(slnf, content, "utf8");
+        steps.push({
+            path: relativePath(root, slnf),
+            status: before === undefined ? "created" : before === content ? "unchanged" : "updated",
+            note: `${all.length - out.size} of ${all.length} projects`,
+        });
+        steps.push(excludeFromGit(root, slnf));
+        const target = projects.find((p) => p.ctx.name === s.project);
+        if (target) {
+            const commands = projectCommands(target.adapter, target.ctx);
+            const relative = relativePath(target.ctx.root, slnf) || SLNF;
+            const rel = /\s/.test(relative) ? `"${relative}"` : relative;
+            commandUpdates.push({
+                project: s.project,
+                // The resolved commands: the configured ones, or the adapter's defaults when none are.
+                build: commands.build && pointAt(commands.build, rel),
+                test: commands.test && pointAt(commands.test, rel),
+            });
+        }
+    }
+    if (commandUpdates.length)
+        steps.push(updateConfig(configFile, root, commandUpdates));
+    return steps;
+}
+/**
+ * `command` building `slnf` instead of the solution it names, or the solution
+ * `dotnet` would find. A command naming a project file is left as it is:
+ * `dotnet` takes one project or solution, not both.
+ */
+export function pointAt(command, slnf) {
+    if (SOLUTION_ARG.test(command))
+        return command.replace(SOLUTION_ARG, slnf);
+    if (PROJECT_ARG.test(command))
+        return command;
+    return command.replace(/\bdotnet\s+(build|test)\b/, `dotnet $1 ${slnf}`);
+}
+function excludeFromGit(root, file) {
+    const git = (...args) => spawnSync("git", args, { cwd: root, encoding: "utf8" });
+    const top = git("rev-parse", "--show-toplevel");
+    const exclude = git("rev-parse", "--path-format=absolute", "--git-path", "info/exclude");
+    if (top.status !== 0 || exclude.status !== 0)
+        return { path: ".git/info/exclude", status: "skipped", note: "not a git repository" };
+    const excludeFile = exclude.stdout.trim();
+    const line = `/${relativePath(fs.realpathSync(top.stdout.trim()), fs.realpathSync(file))}`;
+    const text = fs.existsSync(excludeFile) ? fs.readFileSync(excludeFile, "utf8") : "";
+    const shown = relativePath(root, excludeFile);
+    if (text.split(/\r?\n/).includes(line))
+        return { path: shown, status: "unchanged" };
+    fs.mkdirSync(path.dirname(excludeFile), { recursive: true });
+    fs.appendFileSync(excludeFile, `${text && !text.endsWith("\n") ? "\n" : ""}${line}\n`, "utf8");
+    return { path: shown, status: "updated", note: line };
+}
+/** Set the build and test commands of `projects:` entries in place, keeping the file's comments and layout. */
+function updateConfig(configFile, root, updates) {
+    const paste = updates.map((u) => `${u.project}: build: ${u.build}; test: ${u.test}`).join("; ");
+    if (!configFile || configFile.endsWith(".json")) {
+        return {
+            path: configFile ? relativePath(root, configFile) : "flagrm.config.yaml",
+            status: "skipped",
+            note: `set ${paste}`,
+        };
+    }
+    const text = fs.readFileSync(configFile, "utf8");
+    const doc = YAML.parseDocument(text);
+    const list = doc.get("projects");
+    const missing = [];
+    for (const u of updates) {
+        const index = YAML.isSeq(list)
+            ? list.items.findIndex((item) => YAML.isMap(item) && item.get("name") === u.project)
+            : -1;
+        if (index === -1) {
+            missing.push(u.project);
+            continue;
+        }
+        if (u.build)
+            doc.setIn(["projects", index, "build"], u.build);
+        if (u.test)
+            doc.setIn(["projects", index, "test"], u.test);
+    }
+    const next = doc.toString();
+    if (next !== text)
+        fs.writeFileSync(configFile, next, "utf8");
+    const note = missing.length
+        ? `not in projects: (${missing.join(", ")}): set ${paste}`
+        : updates.map((u) => `${u.project}: \`${u.build}\`, \`${u.test}\``).join("; ");
+    return { path: relativePath(root, configFile), status: next === text ? "unchanged" : "updated", note };
+}

package/dist/core/tools.js

@@ -0,0 +1,108 @@
+/**
+ * Which AI coding tools `flagrm init` sets up, and how the choice is made:
+ * `--tool`, the `tools:` saved in flagrm.config.yaml, or a question with the
+ * tools the repository already uses pre-selected.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import readline from "node:readline/promises";
+import YAML from "yaml";
+import { findDefaultConfigFile, parseTools, TOOL_IDS } from "./config.js";
+import { relativePath } from "./util.js";
+export const TOOL_LABELS = {
+    "claude-code": "Claude Code",
+    copilot: "GitHub Copilot",
+    codex: "Codex",
+};
+/** The tools a repository already uses: `.claude/`, Copilot instructions or skills, AGENTS.md or `.agents/`. */
+export function detectTools(root) {
+    const has = (rel) => fs.existsSync(path.join(root, rel));
+    const found = {
+        "claude-code": has(".claude"),
+        copilot: has(".github/copilot-instructions.md") || has(".github/skills") || has(".github/prompts"),
+        codex: has("AGENTS.md") || has(".agents"),
+    };
+    return TOOL_IDS.filter((t) => found[t]);
+}
+/** The `tools:` saved in the config file in `root`, if any. Throws on an invalid value. */
+export function savedTools(root) {
+    const file = findDefaultConfigFile(root);
+    if (!file)
+        return undefined;
+    const text = fs.readFileSync(file, "utf8");
+    const raw = file.endsWith(".json") ? JSON.parse(text) : YAML.parse(text);
+    return parseTools(raw?.tools, `${path.basename(file)}: tools`);
+}
+/** Save `tools` in the config file in `root`, in place (comments and layout kept). */
+export function saveTools(root, tools) {
+    const file = findDefaultConfigFile(root);
+    if (!file)
+        return { path: "flagrm.config.yaml", status: "skipped" };
+    const rel = relativePath(root, file);
+    const text = fs.readFileSync(file, "utf8");
+    let current;
+    try {
+        current = savedTools(root)?.join();
+    }
+    catch {
+        // An invalid `tools:` (say `[cursor]`) is what `init --tool` replaces.
+    }
+    if (current === tools.join())
+        return { path: rel, status: "unchanged" };
+    let next;
+    if (file.endsWith(".json")) {
+        next = `${JSON.stringify({ ...JSON.parse(text), tools }, null, 2)}\n`;
+    }
+    else {
+        const doc = YAML.parseDocument(text);
+        const seq = doc.createNode(tools);
+        seq.flow = true;
+        if (doc.has("tools"))
+            doc.set("tools", seq);
+        // First, above `projects:`, as the template has it.
+        else if (YAML.isMap(doc.contents))
+            doc.contents.items.unshift(doc.createPair("tools", seq));
+        else
+            return { path: rel, status: "skipped", note: `add \`tools: [${tools.join(", ")}]\`` };
+        next = doc.toString();
+    }
+    if (next === text)
+        return { path: rel, status: "unchanged" };
+    fs.writeFileSync(file, next, "utf8");
+    return { path: rel, status: "updated", note: `tools: [${tools.join(", ")}]` };
+}
+/**
+ * Ask which tools to set up, with `preselected` checked: the user types the
+ * numbers to set up (`1,3`), or presses Enter to keep the pre-selection.
+ */
+export async function askTools(preselected, input = process.stdin, output = process.stdout) {
+    const rl = readline.createInterface({ input, output });
+    try {
+        output.write("Which AI coding tools should flagrm set up?\n");
+        TOOL_IDS.forEach((t, i) => {
+            output.write(`  ${i + 1}. [${preselected.includes(t) ? "x" : " "}] ${TOOL_LABELS[t]}\n`);
+        });
+        const fallback = preselected.length ? preselected : [...TOOL_IDS];
+        const defaults = fallback.map((t) => TOOL_IDS.indexOf(t) + 1).join(",");
+        for (;;) {
+            let answer;
+            try {
+                answer = (await rl.question(`Numbers, comma-separated [${defaults}]: `)).trim();
+            }
+            catch {
+                // Ctrl+D or Ctrl+C: nothing was chosen.
+                throw new Error(`Cancelled: no tools chosen. Run flagrm init again, or pass --tool ${TOOL_IDS.join(",")}.`);
+            }
+            if (!answer)
+                return fallback;
+            const picks = answer.split(/[\s,]+/).map(Number);
+            if (picks.every((n) => Number.isInteger(n) && n >= 1 && n <= TOOL_IDS.length)) {
+                return TOOL_IDS.filter((_, i) => picks.includes(i + 1));
+            }
+            output.write(`Enter numbers from 1 to ${TOOL_IDS.length}, e.g. 1,3.\n`);
+        }
+    }
+    finally {
+        rl.close();
+    }
+}

package/dist/core/types.js

@@ -14,8 +14,14 @@
  *   `failedNames`: the comparison is in `tests`, failed tests in the findings),
  *   and one `tests` warning for all tests that no longer run.
  * - 6: `verify` reports `skipped`, the checks `--skip` left out.
+ * - 7: `baseline --json` prints a summary of `baseline.json`: test results as
+ *   counts and at most 20 short failed test names (`failedTests`), no
+ *   `config`. `baseline.json` itself is unchanged, so version 6 files are
+ *   still read.
  */
-export const JSON_SCHEMA_VERSION = 6;
+export const JSON_SCHEMA_VERSION = 7;
+/** `baseline.json` versions this flagrm reads: 7 only added optional fields to 6. */
+export const READABLE_BASELINE_VERSIONS = [6, 7];
 // ---------------------------------------------------------------------------
 // verify
 // ---------------------------------------------------------------------------

package/dist/core/util.js

@@ -16,3 +16,13 @@ export function readJson(file) {
         return undefined;
     }
 }
+/** Whether `exe` can be run from `cwd`: a path that exists, or a program on PATH (with PATHEXT on Windows). */
+export function onPath(exe, cwd) {
+    if (exe.includes("/") || exe.includes("\\"))
+        return fs.existsSync(path.resolve(cwd, exe));
+    const exts = process.platform === "win32" ? ["", ...(process.env.PATHEXT ?? ".EXE;.CMD;.BAT").split(";")] : [""];
+    return (process.env.PATH ?? "")
+        .split(path.delimiter)
+        .filter(Boolean)
+        .some((dir) => exts.some((ext) => fs.existsSync(path.join(dir, exe + ext))));
+}

package/dist/core/verify/commands.js

@@ -10,8 +10,9 @@ import path from "node:path";
 import { errorExcerpt, templateCheckGap } from "../baseline.js";
 import { escapeRegExp, isTestPath } from "../scan.js";
 import { relativePath } from "../util.js";
-import { checkResult, plural, skippedResult } from "./context.js";
-import { diffTestNames } from "./test-results.js";
+import { checkResult, configFindings, plural, skippedResult } from "./context.js";
+import { classifyRemovedTests } from "./removed-tests.js";
+import { diffTestNames, shortTestNames, splitFailures } from "./test-results.js";
 function failedRunFindings(run) {
     const findings = [
         {
@@ -21,15 +22,29 @@ function failedRunFindings(run) {
             file: run.log,
         },
     ];
-    for (const line of errorExcerpt(run.log))
-        findings.push({ severity: "info", message: line, project: run.project });
+    if (run.failure)
+        findings.push(setupFinding(run));
+    else
+        for (const line of errorExcerpt(run.log))
+            findings.push({ severity: "info", message: line, project: run.project });
     return findings;
 }
+/** A failed run's setup problem: the machine, not the edit, is what needs fixing. */
+function setupFinding(run) {
+    return { severity: "info", message: `setup problem: ${run.failure?.message}`, project: run.project };
+}
+/** No command to run: skipped, unless the baseline had one, which is worth a warning. */
+function noCommandResult(v, id) {
+    const summary = `no ${id === "build" ? "build" : "test"} command configured`;
+    const findings = configFindings(v, id);
+    return findings.length ? checkResult(id, summary, findings) : skippedResult(id, summary);
+}
 export function buildCheck(v) {
     const runs = v.runs.filter((r) => r.check === "build");
     if (runs.length === 0)
-        return skippedResult("build", "no build command configured");
+        return noCommandResult(v, "build");
     const findings = runs.flatMap((r) => (r.exitCode === 0 ? [] : failedRunFindings(r)));
+    findings.push(...configFindings(v, "build"));
     for (const run of runs) {
         const project = v.projects.find(({ ctx }) => ctx.name === run.project);
         const gap = project && templateCheckGap(project.adapter, run.command);
@@ -58,24 +73,104 @@ export function compareTests(v) {
             after: after?.total,
             removed: [],
             added: [],
+            deleted: [],
+            renamed: [],
+            excludedByConfig: [],
+            unexplained: [],
+            newFailures: [],
+            knownFailures: [],
         };
-        if (before && after)
-            Object.assign(comparison, diffTestNames(before.names, after.names));
+        if (after)
+            comparison.failed = after.failed;
+        const failures = splitBaselineFailures(v, run);
+        if (failures) {
+            comparison.newFailures = shortTestNames(failures.new);
+            comparison.knownFailures = shortTestNames(failures.known);
+        }
+        if (before && after) {
+            const { removed, added } = diffTestNames(before.names, after.names);
+            Object.assign(comparison, { removed, added });
+            Object.assign(comparison, classifyRemovedTests(removed, added, after.names, {
+                root: v.root,
+                sha: v.baseline.git.sha,
+                changedFiles: v.changedFiles,
+                names: v.baseline.names,
+                testConfigChanged: v.configChanges.some((c) => c.check === "tests" && c.project === run.project),
+            }));
+        }
         out.push(comparison);
     }
     return out;
 }
+/**
+ * For a test run that exited non-zero: its failed tests split into the ones
+ * that already failed at the baseline and new ones. Undefined when that can't
+ * be told (no result files now or at the baseline, or none of them failed, so
+ * the command broke some other way).
+ */
+function splitBaselineFailures(v, run) {
+    const before = v.baseline.checks?.find((c) => c.project === run.project && c.check === "test")?.results;
+    if (!before || !run.results?.failed)
+        return undefined;
+    return splitFailures(before.failedNames, run.results.failedNames);
+}
+/**
+ * A failed test run fails the check, unless every failed test already failed
+ * at the baseline: those are known failures, and only warn.
+ */
+function failedTestFindings(run, failures) {
+    const list = (names, prefix) => shortTestNames(names)
+        .slice(0, 20)
+        .map((name) => ({ severity: "info", message: `${prefix}: ${name}`, project: run.project }));
+    if (!failures) {
+        return [...failedRunFindings(run).slice(0, 1), ...list(run.results?.failedNames ?? [], "failed")];
+    }
+    if (failures.new.length === 0) {
+        return [
+            {
+                severity: "warn",
+                message: `${plural(failures.known.length, "test fails", "tests fail")}, as at the baseline (known failures; log: ${run.log})`,
+                project: run.project,
+                file: run.log,
+            },
+            ...list(failures.known, "known failure"),
+        ];
+    }
+    return [
+        {
+            severity: "fail",
+            message: `${plural(failures.new.length, "test fails that", "tests fail that")} passed or didn't run at the baseline (log: ${run.log})`,
+            project: run.project,
+            file: run.log,
+        },
+        ...list(failures.new, "new failure"),
+        ...(failures.known.length
+            ? [
+                {
+                    severity: "info",
+                    message: `plus ${plural(failures.known.length, "known failure")} from the baseline`,
+                    project: run.project,
+                },
+            ]
+            : []),
+    ];
+}
 export function testsCheck(v, comparisons) {
     const runs = v.runs.filter((r) => r.check === "test");
     if (runs.length === 0)
-        return skippedResult("tests", "no test command configured");
-    const findings = [];
+        return noCommandResult(v, "tests");
+    const findings = configFindings(v, "tests");
+    let failed = 0;
+    let known = 0;
     for (const run of runs) {
         if (run.exitCode !== 0) {
-            findings.push(...failedRunFindings(run).slice(0, 1));
-            for (const name of run.results?.failedNames.slice(0, 20) ?? []) {
-                findings.push({ severity: "info", message: `failed: ${name}`, project: run.project });
-            }
+            const failures = splitBaselineFailures(v, run);
+            if (failures?.new.length !== 0)
+                failed++;
+            known += failures?.known.length ?? 0;
+            findings.push(...failedTestFindings(run, failures));
+            if (run.failure)
+                findings.push(setupFinding(run));
         }
         const ctx = v.projects.find(({ ctx }) => ctx.name === run.project)?.ctx;
         const project = ctx?.project;
@@ -94,13 +189,16 @@ export function testsCheck(v, comparisons) {
                 project: run.project,
             });
         }
-        const removed = comparison?.removed.length ?? 0;
-        if (removed) {
+        const unexplained = comparison?.unexplained ?? [];
+        if (unexplained.length) {
             findings.push({
                 severity: "warn",
-                message: `${plural(removed, "test no longer runs", "tests no longer run")} (fine if they only covered the OFF path or were renamed; say so in the PR)`,
+                message: `${plural(unexplained.length, "test no longer runs", "tests no longer run")} though the diff doesn't delete or rename it and the test config is unchanged: account for ${unexplained.length === 1 ? "it" : "them"} in the PR`,
                 project: run.project,
             });
+            for (const name of shortTestNames(unexplained).slice(0, 20)) {
+                findings.push({ severity: "info", message: `no longer runs: ${name}`, project: run.project });
+            }
         }
         if (run.results && ctx) {
             for (const file of changedTestFiles(v, ctx.root)) {
@@ -117,11 +215,21 @@ export function testsCheck(v, comparisons) {
     }
     const counts = comparisons
         .filter((c) => c.after !== undefined)
-        .map((c) => (c.before === undefined ? `${c.project} ${c.after}` : `${c.project} ${c.before} → ${c.after}`));
-    const failed = runs.filter((r) => r.exitCode !== 0).length;
+        .map((c) => {
+        if (c.before === undefined)
+            return `${c.project} ${c.after}`;
+        const why = [
+            c.deleted.length && `${c.deleted.length} deleted`,
+            c.renamed.length && `${c.renamed.length} renamed`,
+            c.excludedByConfig.length && `${c.excludedByConfig.length} excluded by config`,
+            c.unexplained.length && `${c.unexplained.length} unexplained`,
+        ].filter(Boolean);
+        return `${c.project} ${c.before} → ${c.after}${why.length ? `: ${why.join(", ")}` : ""}`;
+    });
     const summary = [
         failed ? `${plural(failed, "test run")} failed` : "tests passed",
-        counts.length ? `(${counts.join(", ")})` : undefined,
+        known ? `apart from ${plural(known, "known failure")}` : undefined,
+        counts.length ? `(${counts.join("; ")})` : undefined,
     ]
         .filter(Boolean)
         .join(" ");

package/dist/core/verify/context.js

@@ -1,3 +1,12 @@
+/** Warnings about the configuration changes that affect check `id`. */
+export function configFindings(v, id) {
+    return v.configChanges
+        .filter((c) => c.check === id)
+        .flatMap((c) => [
+        { severity: "warn", message: c.message, project: c.project },
+        ...(c.details ?? []).map((line) => ({ severity: "info", message: line, project: c.project })),
+    ]);
+}
 /** A check's status is the worst severity among its findings. */
 export function checkResult(id, summary, findings) {
     const status = findings.some((f) => f.severity === "fail")