@agjs/tsforge 0.2.0 → 0.2.2

package/src/detect-gate.ts

@@ -2,6 +2,7 @@ import { join, dirname } from "node:path";
 import { existsSync } from "node:fs";
 import { ESLint } from "eslint";
 import { WEB_TEMPLATES, type WebFramework } from "./web-templates";
+import { isRecord } from "./lib/guards";
 
 /**
  * Build the gate that confirms "done" — and makes tsforge a TypeScript-SPECIALIZED
@@ -106,10 +107,16 @@ const STRICT_TSCONFIG = `{
 /** Strict overlay for a project that ALREADY has a tsconfig: extend it (so the
  *  project's paths/jsx/module/lib still resolve — a bare strict config would
  *  mis-compile a real app) but FORCE every strictness flag on top, so a loosely-
- *  configured repo still gets tsforge's strict-TS floor. Written as a sibling
- *  `tsforge.tsconfig.json` and gated with `tsc -p`. */
-const STRICT_TSCONFIG_OVERRIDE = `{
-  "extends": "./tsconfig.json",
+ *  configured repo still gets tsforge's strict-TS floor.
+ *
+ *  PERSISTENCE POLICY: written under `.tsforge/` (tsforge's cache namespace), NOT
+ *  as a sibling in the project root — so the gate never litters the user's repo
+ *  with a `tsforge.tsconfig.json`. `extends` points one level up to the project's
+ *  own config, and `include`/`exclude` are re-stated relative to the subdir
+ *  because `extends` does not inherit them (they default to the config's own
+ *  directory otherwise — which under `.tsforge/` would compile nothing). */
+const STRICT_TSCONFIG_OVERLAY = `{
+  "extends": "../tsconfig.json",
   "compilerOptions": {
     "strict": true,
     "noUncheckedIndexedAccess": true,
@@ -119,10 +126,16 @@ const STRICT_TSCONFIG_OVERRIDE = `{
     "erasableSyntaxOnly": true,
     "skipLibCheck": true,
     "noEmit": true
-  }
+  },
+  "include": ["../**/*.ts", "../**/*.tsx"],
+  "exclude": ["../node_modules", "../dist", "../build", "../scratch", "../.tsforge"]
 }
 `;
 
+/** The gate overlay's home: tsforge's cache dir + the overlay filename. */
+const GATE_TSCONFIG_DIR = ".tsforge";
+const GATE_TSCONFIG_FILE = "tsconfig.gate.json";
+
 // The web-stack scaffolds (Vite + React full-kit, or Vite vanilla) live in the
 // registry; this module just lays them down and builds their gate. shadcn/TanStack
 // boilerplate is held to a web-tailored strict config (no `I`-prefix — React names
@@ -373,7 +386,12 @@ export function buildWebGate(framework: WebFramework): IGate {
   // HARNESS-authored and app-agnostic: we deliberately do NOT run a model-authored
   // checks.json — the 27b writes over-strict interaction assertions (exact
   // placeholders/fill flows) it then can't satisfy and spirals on (iter3/4).
-  const render = `bun "${BROWSER_CHECK}" dist/index.html --smoke --crawl`;
+  // OPT-IN quality oracles (default OFF so existing web runs are unchanged):
+  // TSFORGE_A11Y=1 adds axe (serious/critical fail), TSFORGE_SCREENSHOTS=1 writes
+  // per-route PNGs. A "frontend"/"strict" profile can set these.
+  const a11y = process.env.TSFORGE_A11Y === "1" ? " --a11y" : "";
+  const shots = process.env.TSFORGE_SCREENSHOTS === "1" ? " --screenshots" : "";
+  const render = `bun "${BROWSER_CHECK}" dist/index.html --smoke --crawl${a11y}${shots}`;
   // Prettier enforces formatting (the fix step runs `prettier --write` first, so
   // this passes without the model ever hand-formatting). Respects .prettierignore
   // (vendored ui/ + lib/ skipped). Runs after lint so a parse error fails there.
@@ -444,6 +462,22 @@ export function buildWebFix(framework: WebFramework): string {
   return `${lintFix} ; ${format}`;
 }
 
+/**
+ * The core (non-web) auto-fix command — same janitor as buildWebFix but uses the
+ * bundled strict.eslint.config.mjs. Run BEFORE the gate each cycle so padding-line,
+ * prefer-const, curly, etc. are squashed without model turns.
+ */
+export function buildCoreFix(): string {
+  const lintFix =
+    `"${ESLINT_BIN}" --no-config-lookup -c "${STRICT_CONFIG}" --fix .`.replace(
+      /\s+/g,
+      " "
+    );
+  const format = `"${PRETTIER_BIN}" --write .`;
+
+  return `${lintFix} ; ${format}`;
+}
+
 async function ensureFile(
   cwd: string,
   name: string,
@@ -468,7 +502,7 @@ export async function buildGate(
   cwd: string,
   packs?: readonly string[],
   ruleOverrides?: Readonly<Record<string, "error" | "warn" | "off">>,
-  options?: { enableTypeAware?: boolean }
+  options?: { enableTypeAware?: boolean; includeTests?: boolean }
 ): Promise<IGate> {
   const parts: string[] = [];
   const labels: string[] = [];
@@ -494,29 +528,95 @@ export async function buildGate(
     }
   }
 
+  // Tests run LAST (after the cheap static floor) so a type/lint error fails
+  // fast without paying for a test run. Only appended when the project actually
+  // has tests to run — a strict-floor-only run, or a project with none, skips it.
+  if (options?.includeTests === true) {
+    const test = await discoverTestCommand(cwd);
+
+    if (test !== null) {
+      parts.push(test);
+      labels.push("tests");
+    }
+  }
+
   return { command: parts.join(" && "), label: labels.join(" + ") };
 }
 
+/** The npm-init placeholder test script — running it always fails, so it must
+ *  NOT count as "the project has tests". */
+const PLACEHOLDER_TEST = /no test specified/i;
+
+/**
+ * The project's test command for the gate, or null when there's nothing to run.
+ * Prefers an explicit, real package.json `test` script (run via `bun run test`);
+ * else falls back to `bun test` when the project has test files; else null — so
+ * a greenfield app with no tests yet stays at the strict floor instead of
+ * failing a gate that runs a placeholder/absent test command.
+ */
+export async function discoverTestCommand(cwd: string): Promise<string | null> {
+  const pkgFile = Bun.file(join(cwd, "package.json"));
+
+  if (await pkgFile.exists()) {
+    try {
+      const pkg: unknown = await pkgFile.json();
+      const scripts = isRecord(pkg) ? pkg.scripts : undefined;
+      const script = isRecord(scripts) ? scripts.test : undefined;
+
+      if (
+        typeof script === "string" &&
+        script.trim().length > 0 &&
+        !PLACEHOLDER_TEST.test(script)
+      ) {
+        return "bun run test";
+      }
+    } catch {
+      // Malformed package.json — fall through to file detection.
+    }
+  }
+
+  return (await hasTestFiles(cwd)) ? "bun test" : null;
+}
+
+/** True when the project has at least one *.test.* / *.spec.* file (outside
+ *  node_modules) — the signal that a bare `bun test` has something to run. */
+async function hasTestFiles(cwd: string): Promise<boolean> {
+  const glob = new Bun.Glob("**/*.{test,spec}.{ts,tsx,js,jsx}");
+
+  for await (const path of glob.scan({ cwd, onlyFiles: true })) {
+    if (!path.includes("node_modules")) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 /**
  * The type-aware floor — ALWAYS tsforge-strict (user policy: a repo's own config
- * is never trusted to be strict enough). With a project tsconfig, extend it but
- * force the strict flags; greenfield, bring the full strict one. null when not a
- * TS project. (The strict override / bundled config win over whatever the repo set.)
+ * is never trusted to be strict enough). With a project tsconfig, extend it under
+ * `.tsforge/` but force the strict flags; greenfield, bring the full strict one.
+ * null when not a TS project. (The strict overlay / bundled config win over
+ * whatever the repo set.)
  */
 async function tscPart(cwd: string): Promise<string | null> {
   const hasTsconfig = await Bun.file(join(cwd, "tsconfig.json")).exists();
 
   if (hasTsconfig) {
+    // EPHEMERAL gate artifact: lives in .tsforge/ (Bun.write makes the dir), so
+    // we never drop a tsforge.tsconfig.json in the user's project root.
     await Bun.write(
-      join(cwd, "tsforge.tsconfig.json"),
-      STRICT_TSCONFIG_OVERRIDE
+      join(cwd, GATE_TSCONFIG_DIR, GATE_TSCONFIG_FILE),
+      STRICT_TSCONFIG_OVERLAY
     );
+    await ignoreGateArtifact(cwd);
 
-    return `"${TSC_BIN}" --noEmit -p tsforge.tsconfig.json`;
+    return `"${TSC_BIN}" --noEmit -p ${GATE_TSCONFIG_DIR}/${GATE_TSCONFIG_FILE}`;
   }
 
   // Greenfield: bring a strict tsconfig so tsc can gate — but only when this is
   // actually a TS project (has a package.json), so we never litter a random dir.
+  // Unlike the overlay, a greenfield tsconfig.json is a DURABLE project file.
   if (await Bun.file(join(cwd, "package.json")).exists()) {
     await Bun.write(join(cwd, "tsconfig.json"), STRICT_TSCONFIG);
 
@@ -526,6 +626,20 @@ async function tscPart(cwd: string): Promise<string | null> {
   return null;
 }
 
+/** Keep the ephemeral gate overlay out of git WITHOUT touching the user's root
+ *  .gitignore: drop a scoped `.tsforge/.gitignore` ignoring just the overlay.
+ *  Created only when absent, so a user-authored `.tsforge/.gitignore` (e.g. one
+ *  that intentionally tracks rules.json) is never clobbered. */
+async function ignoreGateArtifact(cwd: string): Promise<void> {
+  const ignore = join(cwd, GATE_TSCONFIG_DIR, ".gitignore");
+
+  if (await Bun.file(ignore).exists()) {
+    return;
+  }
+
+  await Bun.write(ignore, `${GATE_TSCONFIG_FILE}\n`);
+}
+
 /** The syntactic idiom layer — ALWAYS tsforge's bundled strict eslint config
  *  (user policy). We deliberately do NOT defer to the project's own `lint`
  *  script: that's exactly how a weak repo would dodge the strict-TS floor. The

package/src/eval/eval.types.ts

@@ -1,3 +1,5 @@
+import type { FailureClass } from "./failure-class";
+
 export interface IJudgeInput {
   goal: string;
   criteria: string;
@@ -21,6 +23,9 @@ export interface IRunRecord {
   ms: number;
   /** LLM-judge quality score (1–5), when available. */
   quality?: number;
+  /** Structured reason a failed run failed (from classifyRun); omitted/`none`
+   *  for a passing run. The substrate for turning failures into interventions. */
+  failureClass?: FailureClass;
 }
 
 /** Aggregated metrics for a variant across its runs. */
@@ -33,4 +38,8 @@ export interface IVariantSummary {
   avgMs: number;
   /** Average quality across runs that were scored (0 if none). */
   avgQuality: number;
+  /** Count of failed runs by failure class (e.g. {"type-error": 2}); empty when
+   *  no run carried a class. Lets a sweep show WHY a variant failed, not just how
+   *  often. */
+  failureClasses: Record<string, number>;
 }

package/src/eval/failure-class.ts

@@ -0,0 +1,263 @@
+import type { ILoopEvent } from "../loop/loop.types";
+import type { ErrorSet } from "../validate/validate.types";
+
+/**
+ * Why a run failed — a structured reason, so every failed run maps to a possible
+ * harness intervention (the self-improving north-star). Derived purely from the
+ * event stream (+ an optional final gate error set), so the same classifier
+ * serves the live loop, the eval sweep, and the offline log analyzer.
+ */
+export const FAILURE_CLASS = {
+  /** The run reached a green gate — no failure. */
+  none: "none",
+  /** Model emitted tool calls the parser couldn't read (repair L3 / salvage). */
+  toolMalformed: "tool-malformed",
+  /** Edits kept missing their target (missing-file / not-found / ambiguous). */
+  editReject: "edit-reject",
+  /** Hit the turn cap or the gate stalled with no decisive error class. */
+  noProgress: "no-progress",
+  /** Final gate red dominated by tsc type errors. */
+  typeError: "type-error",
+  /** Final gate red dominated by ESLint rule violations. */
+  lintRule: "lint-rule",
+  /** Imported a module that doesn't exist (TS2307 / "Cannot find module"). */
+  hallucinatedImport: "hallucinated-import",
+  /** Output degenerated into a repetition loop (StreamGuard fired). */
+  degeneration: "degeneration",
+  /** A per-call/timeout backstop tripped. */
+  timeout: "timeout",
+  /** A route rendered as an empty/phantom page. */
+  routePhantom: "route-phantom",
+  /** The built app failed to render / threw in the browser oracle. */
+  browserFail: "browser-fail",
+  /** The bundler/build step (vite) failed. */
+  buildFail: "build-fail",
+  /** Failed, but no signal was decisive. */
+  unknown: "unknown",
+} as const;
+
+export type FailureClass = (typeof FAILURE_CLASS)[keyof typeof FAILURE_CLASS];
+
+/** Per-signal tallies behind a classification — kept for debugging/telemetry. */
+export interface IFailureSignals {
+  repairs: number;
+  salvages: number;
+  editRejects: number;
+  degenerated: boolean;
+  tsErrors: number;
+  lintErrors: number;
+  missingModule: number;
+  browser: number;
+  build: number;
+}
+
+export interface IFailureSummary {
+  failureClass: FailureClass;
+  /** The dominant rule/code for type-error|lint-rule (e.g. "TS18048", "no-as"). */
+  detail?: string;
+  signals: IFailureSignals;
+}
+
+const TS_CODE = /^TS\d+$/;
+const MISSING_MODULE = /cannot find module/i;
+const DEGENERATE = /degenerat/i;
+const TOOL_MALFORMED = /salvage|recovered|malformed|re-ask/i;
+const REJECTED = /reject/i;
+const BROWSER = /blank|did not render|did not mount|page error|uncaught/i;
+const ROUTE = /route|phantom|stub/i;
+const BUILD = /vite|esbuild|build failed|bundl/i;
+
+/** The most frequently occurring string, or undefined for an empty list. */
+function mostCommon(values: readonly string[]): string | undefined {
+  const counts = new Map<string, number>();
+  let best: string | undefined;
+  let bestN = 0;
+
+  for (const value of values) {
+    const n = (counts.get(value) ?? 0) + 1;
+
+    counts.set(value, n);
+
+    if (n > bestN) {
+      bestN = n;
+      best = value;
+    }
+  }
+
+  return best;
+}
+
+/** The final red gate's rules: prefer the explicit error set, else the rules
+ *  carried on the last failing `validated` event. */
+function finalRules(
+  events: readonly ILoopEvent[],
+  finalErrors?: ErrorSet
+): string[] {
+  if (finalErrors !== undefined) {
+    return finalErrors.flatMap((e) => (e.rule === undefined ? [] : [e.rule]));
+  }
+
+  let last: readonly string[] = [];
+
+  for (const event of events) {
+    if (event.kind === "validated" && event.passed === false && event.rules) {
+      last = event.rules;
+    }
+  }
+
+  return [...last];
+}
+
+/** Concatenated message/output text across the run — for keyword signals that
+ *  aren't structured into a dedicated field (missing module, browser, build). */
+function runText(
+  events: readonly ILoopEvent[],
+  finalErrors?: ErrorSet
+): string {
+  const parts: string[] = [];
+
+  for (const event of events) {
+    parts.push(event.message);
+
+    if (event.output !== undefined) {
+      parts.push(event.output);
+    }
+  }
+
+  for (const e of finalErrors ?? []) {
+    parts.push(e.message);
+  }
+
+  return parts.join("\n");
+}
+
+function gatherSignals(
+  events: readonly ILoopEvent[],
+  finalErrors?: ErrorSet
+): IFailureSignals {
+  const rules = finalRules(events, finalErrors);
+  const text = runText(events, finalErrors);
+  const missingModule =
+    rules.filter((r) => r === "TS2307").length +
+    (MISSING_MODULE.test(text) ? 1 : 0);
+
+  return {
+    repairs: events.filter((e) => e.kind === "repair").length,
+    salvages: events.filter(
+      (e) => e.kind === "tool" && TOOL_MALFORMED.test(e.message)
+    ).length,
+    editRejects: events.filter(
+      (e) => e.kind === "edit" && REJECTED.test(e.message)
+    ).length,
+    degenerated: events.some((e) => DEGENERATE.test(e.message)),
+    tsErrors: rules.filter((r) => TS_CODE.test(r) && r !== "TS2307").length,
+    lintErrors: rules.filter((r) => !TS_CODE.test(r)).length,
+    missingModule,
+    browser: BROWSER.test(text) ? 1 : 0,
+    build: BUILD.test(text) ? 1 : 0,
+  };
+}
+
+function finalStatusOf(
+  events: readonly ILoopEvent[]
+): "done" | "stuck" | "none" {
+  let status: "done" | "stuck" | "none" = "none";
+
+  for (const event of events) {
+    if (event.kind === "done") {
+      status = "done";
+    } else if (event.kind === "stuck") {
+      status = "stuck";
+    }
+  }
+
+  return status;
+}
+
+/** Pick the dominant gate-error class (type vs lint), with its commonest code. */
+function classifyGateErrors(
+  events: readonly ILoopEvent[],
+  finalErrors: ErrorSet | undefined,
+  signals: IFailureSignals
+): IFailureSummary | undefined {
+  const rules = finalRules(events, finalErrors);
+
+  if (signals.tsErrors > 0 && signals.tsErrors >= signals.lintErrors) {
+    return {
+      failureClass: FAILURE_CLASS.typeError,
+      detail: mostCommon(rules.filter((r) => TS_CODE.test(r))),
+      signals,
+    };
+  }
+
+  if (signals.lintErrors > 0) {
+    return {
+      failureClass: FAILURE_CLASS.lintRule,
+      detail: mostCommon(rules.filter((r) => !TS_CODE.test(r))),
+      signals,
+    };
+  }
+
+  return undefined;
+}
+
+/** Behavioral fallback when no gate-error class dominates. */
+function classifyBehavior(signals: IFailureSignals): FailureClass {
+  if (signals.degenerated) {
+    return FAILURE_CLASS.degeneration;
+  }
+
+  if (signals.salvages > 0 || signals.repairs > 0) {
+    return FAILURE_CLASS.toolMalformed;
+  }
+
+  if (signals.editRejects > 0) {
+    return FAILURE_CLASS.editReject;
+  }
+
+  return FAILURE_CLASS.noProgress;
+}
+
+/**
+ * Classify a run from its event stream. Pass the final gate `ErrorSet` when the
+ * caller has it (authoritative); otherwise the classifier falls back to the
+ * `rules` carried on the last failing `validated` event and keyword signals.
+ * A run that reached a green gate classifies as `none`.
+ */
+export function classifyRun(
+  events: readonly ILoopEvent[],
+  finalErrors?: ErrorSet
+): IFailureSummary {
+  const signals = gatherSignals(events, finalErrors);
+
+  if (finalStatusOf(events) === "done") {
+    return { failureClass: FAILURE_CLASS.none, signals };
+  }
+
+  if (signals.missingModule > 0) {
+    return { failureClass: FAILURE_CLASS.hallucinatedImport, signals };
+  }
+
+  if (signals.browser > 0) {
+    const text = runText(events, finalErrors);
+
+    return {
+      failureClass: ROUTE.test(text)
+        ? FAILURE_CLASS.routePhantom
+        : FAILURE_CLASS.browserFail,
+      signals,
+    };
+  }
+
+  if (signals.build > 0 && signals.tsErrors === 0 && signals.lintErrors === 0) {
+    return { failureClass: FAILURE_CLASS.buildFail, signals };
+  }
+
+  const gate = classifyGateErrors(events, finalErrors, signals);
+
+  if (gate !== undefined) {
+    return gate;
+  }
+
+  return { failureClass: classifyBehavior(signals), signals };
+}

package/src/eval/index.ts

@@ -2,6 +2,14 @@ export * from "./eval.types";
 export { judge } from "./judge";
 export { summarize } from "./score";
 export { analyzeEvents, type IRunMetrics } from "./metrics";
+export {
+  classifyRun,
+  FAILURE_CLASS,
+  type FailureClass,
+  type IFailureSummary,
+  type IFailureSignals,
+} from "./failure-class";
+export { parseEventLog } from "./parse-log";
 export {
   buildSweepReport,
   renderSweepReportMarkdown,

package/src/eval/metrics.ts

@@ -1,4 +1,5 @@
 import type { ILoopEvent } from "../loop/loop.types";
+import { classifyRun, type FailureClass } from "./failure-class";
 
 /** Behavioral metrics distilled from a run's event stream — the signals the
  *  local-model literature says predict outcomes (tokens-to-solution, repair
@@ -6,6 +7,10 @@ import type { ILoopEvent } from "../loop/loop.types";
  *  the cli-metrics script. */
 export interface IRunMetrics {
   finalStatus: "done" | "stuck" | "none";
+  /** Structured reason the run failed (`none` when it reached green). The single
+   *  source of truth for failure classification — the cli-metrics analyzer and
+   *  the eval sweep both read this rather than re-deriving it. */
+  failureClass: FailureClass;
   /** Model turns (one per `cycle` event). */
   turns: number;
   /** Model calls (one per `usage` event). */
@@ -29,6 +34,7 @@ export interface IRunMetrics {
 function emptyMetrics(): IRunMetrics {
   return {
     finalStatus: "none",
+    failureClass: "none",
     turns: 0,
     modelCalls: 0,
     tokensOut: 0,
@@ -82,6 +88,7 @@ export function analyzeEvents(events: readonly ILoopEvent[]): IRunMetrics {
 
   m.filesCreated = created.size;
   m.avgTokensPerSecond = tpsCount > 0 ? Math.round(tpsSum / tpsCount) : 0;
+  m.failureClass = classifyRun(events).failureClass;
 
   return m;
 }

package/src/eval/parse-log.ts

@@ -0,0 +1,105 @@
+import type { ILoopEvent } from "../loop/loop.types";
+import { isRecord } from "../lib/guards";
+
+/** The known event kinds, as a runtime set, so a JSONL line can be validated
+ *  into a typed ILoopEvent without an `as` cast. Keep in sync with ILoopEvent. */
+const KNOWN_KINDS = new Set<string>([
+  "start",
+  "red",
+  "cycle",
+  "token",
+  "message",
+  "fix",
+  "edit",
+  "create",
+  "validated",
+  "done",
+  "stuck",
+  "run",
+  "tool",
+  "repair",
+  "timing",
+  "usage",
+  "ttsr",
+]);
+
+function isKind(value: string): value is ILoopEvent["kind"] {
+  return KNOWN_KINDS.has(value);
+}
+
+function optionalString(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function stringArray(value: unknown): string[] | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+
+  return value.filter((v): v is string => typeof v === "string");
+}
+
+/** Coerce one parsed JSONL record into an ILoopEvent, or null when it isn't one.
+ *  Reads only the fields the failure classifier + metrics consume — enough to
+ *  reconstruct a typed event stream from a `--log` file. */
+function coerceEvent(record: unknown): ILoopEvent | null {
+  if (!isRecord(record)) {
+    return null;
+  }
+
+  const kind = record.kind;
+
+  if (typeof kind !== "string" || !isKind(kind)) {
+    return null;
+  }
+
+  const event: ILoopEvent = {
+    kind,
+    task: optionalString(record.task) ?? "",
+    message: optionalString(record.message) ?? "",
+  };
+  const output = optionalString(record.output);
+  const rules = stringArray(record.rules);
+
+  if (output !== undefined) {
+    event.output = output;
+  }
+
+  if (typeof record.passed === "boolean") {
+    event.passed = record.passed;
+  }
+
+  if (rules !== undefined) {
+    event.rules = rules;
+  }
+
+  return event;
+}
+
+/** Parse a `--log` JSONL transcript (one serialized event per line) into a typed
+ *  event stream. Malformed lines and non-event records are skipped. */
+export function parseEventLog(jsonl: string): ILoopEvent[] {
+  const events: ILoopEvent[] = [];
+
+  for (const line of jsonl.split("\n")) {
+    if (line.trim().length === 0) {
+      continue;
+    }
+
+    let parsed: unknown;
+
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    const event = coerceEvent(parsed);
+
+    if (event !== null) {
+      events.push(event);
+    }
+  }
+
+  return events;
+}

package/src/eval/report.ts

@@ -164,5 +164,24 @@ export function renderSweepReportMarkdown(report: ISweepReport): string {
     ...rows,
     "",
     "`*` = significant at p < 0.05 (two-proportion z-test vs baseline).",
+    ...failureSection(report),
   ].join("\n");
 }
+
+/** Format a variant's failure-class tally, e.g. "type-error×2, no-progress×1". */
+function formatFailureClasses(classes: Record<string, number>): string {
+  return Object.entries(classes)
+    .sort(([, a], [, b]) => b - a)
+    .map(([cls, n]) => `${cls}×${String(n)}`)
+    .join(", ");
+}
+
+/** A "why failures happened" section — per-variant failure-class breakdown.
+ *  Empty (no lines) when every run passed, so a clean sweep stays terse. */
+function failureSection(report: ISweepReport): string[] {
+  const lines = report.variants
+    .filter((v) => Object.keys(v.failureClasses).length > 0)
+    .map((v) => `- **${v.label}**: ${formatFailureClasses(v.failureClasses)}`);
+
+  return lines.length === 0 ? [] : ["", "### Failure breakdown", ...lines];
+}