@agjs/tsforge 0.2.0 → 0.2.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agjs/tsforge",
   "type": "module",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "license": "MIT",
   "description": "TypeScript coding harness with a deterministic gate, stack-aware guardrails, and stream-level correction.",
   "repository": {
@@ -19,7 +19,8 @@
     "src",
     "scripts",
     "strict.eslint.config.mjs",
-    "strict.web.eslint.config.mjs"
+    "strict.web.eslint.config.mjs",
+    "strict.type-aware.eslint.config.mjs"
   ],
   "engines": {
     "bun": ">=1.3.14"

package/scripts/browser-check.ts

@@ -5,27 +5,59 @@
 //
 //   bun browser-check.ts <htmlFile>                 # render-only (no errors)
 //   bun browser-check.ts <htmlFile> --smoke         # render + generic behaviour smoke
+//   bun browser-check.ts <htmlFile> --a11y          # + axe accessibility (serious/critical fail)
+//   bun browser-check.ts <htmlFile> --screenshots[=dir]  # + per-route PNGs (artifact)
+//   bun browser-check.ts <htmlFile> --perf          # + a basic DOM-size/mount-time budget
 //   bun browser-check.ts <htmlFile> <checks.json>   # render + interaction checks
 //   bun browser-check.ts <htmlFile> <selector> [text]
 import { readdir } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { renderCheck, parseChecks, type IRenderOptions } from "../src/browser";
+import {
+  renderCheck,
+  parseChecks,
+  type IRenderOptions,
+  type IPerfBudget,
+} from "../src/browser";
 import { crawlableRoutePaths } from "../src/web-routes";
 
 const rawArgs = process.argv.slice(2);
 const smoke = rawArgs.includes("--smoke");
 const crawl = rawArgs.includes("--crawl");
-const [file, arg2, arg3] = rawArgs.filter(
-  (a) => a !== "--smoke" && a !== "--crawl"
-);
+const a11y = rawArgs.includes("--a11y");
+const perf = rawArgs.includes("--perf");
+const screenshotsArg = rawArgs.find((a) => a.startsWith("--screenshots"));
+// Positionals are anything that isn't a recognized `--flag`.
+const [file, arg2, arg3] = rawArgs.filter((a) => !a.startsWith("--"));
 
 if (file === undefined) {
   process.stderr.write(
-    "usage: browser-check.ts <htmlFile> [--smoke] [--crawl] [checks.json | selector [text]]\n"
+    "usage: browser-check.ts <htmlFile> [--smoke] [--crawl] [--a11y] " +
+      "[--screenshots[=dir]] [--perf] [checks.json | selector [text]]\n"
   );
   process.exit(2);
 }
 
+/** A conservative default budget — a tripwire for runaway render trees / slow
+ *  mounts, not a tuned Lighthouse target. */
+const DEFAULT_PERF_BUDGET: IPerfBudget = {
+  maxDomNodes: 5000,
+  maxMountMs: 6000,
+};
+
+/** The screenshot dir: `--screenshots=<dir>`, else a `screenshots/` folder next
+ *  to the HTML file. undefined when `--screenshots` wasn't passed. */
+function screenshotDir(): string | undefined {
+  if (screenshotsArg === undefined) {
+    return undefined;
+  }
+
+  const eq = screenshotsArg.indexOf("=");
+
+  return eq === -1
+    ? join(dirname(file ?? "."), "screenshots")
+    : screenshotsArg.slice(eq + 1);
+}
+
 /** With --crawl, enumerate the app's static routes from `<buildDir>/src/routes/`
  *  (the build dir is the parent of dist/) so every page — not just the home —
  *  is render-checked. Dynamic ($param) routes are skipped. */
@@ -66,10 +98,14 @@ async function checksFor(): Promise<Partial<IRenderOptions>> {
   };
 }
 
+const shots = screenshotDir();
 const result = await renderCheck({
   file,
   smoke,
+  a11y,
   routes: await routesFor(),
+  ...(perf ? { perfBudget: DEFAULT_PERF_BUDGET } : {}),
+  ...(shots !== undefined ? { screenshotDir: shots } : {}),
   ...(await checksFor()),
 });
 

package/scripts/cli-metrics.ts

@@ -10,6 +10,7 @@ import { readdir } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { isRecord } from "../src/lib/guards";
+import { classifyRun, parseEventLog } from "../src/eval";
 
 function num(value: unknown): number {
   return typeof value === "number" ? value : 0;
@@ -168,6 +169,9 @@ async function main(): Promise<void> {
   const text = await Bun.file(path).text();
   const lines = text.split("\n").filter((l) => l.trim().length > 0);
   const m = analyze(lines);
+  // Single source of truth for WHY a run failed — the same classifier the eval
+  // sweep and the reusable analyzeEvents() use, fed the typed event stream.
+  const failure = classifyRun(parseEventLog(text));
   const pct =
     m.contextWindow > 0
       ? Math.round((m.peakContext / m.contextWindow) * 100)
@@ -182,6 +186,12 @@ async function main(): Promise<void> {
     ["model", m.model],
     ["context window", String(m.contextWindow)],
     ["final status", m.finalStatus],
+    [
+      "failure class",
+      failure.detail === undefined
+        ? failure.failureClass
+        : `${failure.failureClass} (${failure.detail})`,
+    ],
     ["turns (repair iterations)", String(m.turns)],
     ["model calls", String(m.modelCalls)],
     ["tokens out (→ solution)", String(m.tokensOut)],

package/scripts/sweep.ts

@@ -6,13 +6,19 @@
 import { mkdir, readdir, rm, stat } from "node:fs/promises";
 import { join } from "node:path";
 import { parseSpec } from "../src/spec";
-import { buildGate, prettierWriteCommand } from "../src/detect-gate";
+import { buildGate, buildCoreFix } from "../src/detect-gate";
 import { runSpec, qualityRepair } from "../src/loop";
 import { modelAgent } from "../src/agent";
 import { OpenAICompatibleProvider } from "../src/inference";
 import { resolveActiveModel, resolveApiKey } from "../src/models-config";
 import { providerConfig } from "../src/cli";
-import { summarize, type IRunRecord } from "../src/eval";
+import {
+  summarize,
+  classifyRun,
+  renderSweepReportMarkdown,
+  buildSweepReport,
+  type IRunRecord,
+} from "../src/eval";
 import { renderEvent } from "../src/render";
 import type { ILoopEvent } from "../src/loop";
 
@@ -252,24 +258,33 @@ async function runOne(
     // (an unguarded index access, an `as any`) slipped through as GREEN. Now
     // every task and the whole-spec verify must clear the strict floor BEFORE
     // its functional tests count.
-    // prettier --write FIRST (auto-format), then tsc-strict + eslint. The model
-    // never hand-formats, but the gate still enforces type-safety + idioms.
-    const strictGate = `${prettierWriteCommand()} && ${(await buildGate(runDir)).command}`;
+    // buildCoreFix (eslint --fix + prettier) runs as task.fix before each gate
+    // check — same janitor as the interactive CLI — so padding-line, prefer-const,
+    // etc. are squashed without model turns.
+    const gateCommand = (await buildGate(runDir)).command;
+    const fixCommand = buildCoreFix();
     const gatedSpec = {
       ...spec,
       tasks: spec.tasks.map((t) => ({
         ...t,
-        accept: `${strictGate} && ${t.accept}`,
+        fix: fixCommand,
+        accept: `${gateCommand} && ${t.accept}`,
       })),
       verify:
-        spec.verify.length > 0 ? `${strictGate} && ${spec.verify}` : strictGate,
+        spec.verify.length > 0
+          ? `${gateCommand} && ${spec.verify}`
+          : gateCommand,
     };
 
     // Every run gets a full transcript at <runDir>/run.log; stream to the
     // terminal too when TSFORGE_STREAM=1.
     const log = Bun.file(join(runDir, "run.log")).writer();
+    // Keep the structured events so a failed run can be classified (WHY it
+    // failed), not just counted — fed to classifyRun below.
+    const runEvents: ILoopEvent[] = [];
 
     const onEvent = (e: ILoopEvent): void => {
+      runEvents.push(e);
       void log.write(renderEvent(e, { color: false }));
       // Flush per event — otherwise Bun's FileSink buffers and `tail -f` shows
       // nothing until the run ends. The log must be live.
@@ -359,6 +374,9 @@ async function runOne(
     );
 
     const vLabel = variantLabel(variantEnv);
+    const failureClass = passed
+      ? undefined
+      : classifyRun(runEvents).failureClass;
 
     records.push({
       label: `${vLabel} temp=${temp}`,
@@ -366,9 +384,10 @@ async function runOne(
       cycles,
       ms,
       quality,
+      ...(failureClass === undefined ? {} : { failureClass }),
     });
     process.stdout.write(
-      `  ${seed} ${vLabel} temp=${temp} #${i + 1}: ${passed ? "done" : "blocked"} (${cycles} cyc, ${edits} edits, ${regressions} regress, ${ms}ms${quality === undefined ? "" : `, Q${quality}/5`}) → ${runId}\n`
+      `  ${seed} ${vLabel} temp=${temp} #${i + 1}: ${passed ? "done" : `blocked[${failureClass ?? "unknown"}]`} (${cycles} cyc, ${edits} edits, ${regressions} regress, ${ms}ms${quality === undefined ? "" : `, Q${quality}/5`}) → ${runId}\n`
     );
   } finally {
     restore();
@@ -380,11 +399,22 @@ const summaries = summarize(records);
 process.stdout.write(`\n=== sweep: ${seed} (${repeats} runs/variant) ===\n`);
 
 for (const s of summaries) {
+  const failures = Object.entries(s.failureClasses)
+    .sort(([, a], [, b]) => b - a)
+    .map(([cls, n]) => `${cls}×${String(n)}`)
+    .join(", ");
+
   process.stdout.write(
-    `${s.label.padEnd(10)}  pass ${Math.round(s.passRate * 100)}% (${s.passed}/${s.runs})  Q ${s.avgQuality.toFixed(1)}/5  avg ${s.avgCycles.toFixed(1)} cyc  ${Math.round(s.avgMs)}ms\n`
+    `${s.label.padEnd(10)}  pass ${Math.round(s.passRate * 100)}% (${s.passed}/${s.runs})  Q ${s.avgQuality.toFixed(1)}/5  avg ${s.avgCycles.toFixed(1)} cyc  ${Math.round(s.avgMs)}ms${failures.length > 0 ? `  [${failures}]` : ""}\n`
   );
 }
 
+// The statistical report (Wilson CI + z-test vs baseline) now also tabulates a
+// per-variant failure-class breakdown — WHY runs failed, not just how often.
+process.stdout.write(
+  `\n${renderSweepReportMarkdown(buildSweepReport(records))}\n`
+);
+
 const outPath = join(evalsRoot, "runs", `sweep-${seed}-${stamp()}.json`);
 
 await Bun.write(

package/src/browser/index.ts

@@ -1,8 +1,11 @@
 export {
   renderCheck,
+  summarizeAxeViolations,
+  checkPerfBudget,
   type IRenderOptions,
   type IRenderExpect,
   type IRenderResult,
   type IStep,
+  type IPerfBudget,
 } from "./oracle";
 export { parseChecks } from "./checks";

package/src/browser/oracle.ts

@@ -1,4 +1,5 @@
 import { resolve, dirname, basename, join } from "node:path";
+import { isRecord } from "../lib/guards";
 // `playwright` is an OPTIONAL peer: bundling it (+ a browser binary) into every
 // install is too heavy, so the import is dynamic and the render-check skips when
 // it's absent. The type-only import is erased at runtime, so it can't crash a
@@ -14,6 +15,20 @@ async function loadChromium(): Promise<typeof Chromium | null> {
   }
 }
 
+/** Run axe against a page and return its raw result; null when @axe-core/
+ *  playwright isn't installed (a11y is an optional enhancement, like the browser
+ *  itself). Kept untyped at the boundary — extractAxeViolations narrows it. */
+async function runAxe(page: Page): Promise<unknown> {
+  try {
+    const mod = await import("@axe-core/playwright");
+    const builder = new mod.AxeBuilder({ page });
+
+    return await builder.analyze();
+  } catch {
+    return null;
+  }
+}
+
 /**
  * The browser oracle — renders a built web page in headless chromium and reports
  * whether it actually WORKS, beyond what tsc/eslint can see: it fails on uncaught
@@ -55,22 +70,121 @@ export interface IRenderOptions {
    *  single-page smoke misses them. Served with SPA fallback so the client
    *  router handles the path. Empty/undefined → no crawl (unchanged behavior). */
   routes?: string[];
+  /** Run axe accessibility checks on the page (and each crawled route). Serious
+   *  and critical violations become gate errors; minor/moderate are skipped.
+   *  Skipped gracefully when @axe-core/playwright isn't installed. */
+  a11y?: boolean;
+  /** Directory to write a screenshot per page/route into (desktop + mobile
+   *  viewports). An artifact for human/visual review — never a pass/fail signal. */
+  screenshotDir?: string;
+  /** A perf budget (DOM node count + mount time) checked on the initial page. */
+  perfBudget?: IPerfBudget;
   /** Navigation timeout (default 15s). */
   timeoutMs?: number;
 }
 
+/** Screenshot viewports — a desktop and a mobile pass per page. */
+const VIEWPORTS = [
+  { name: "desktop", width: 1280, height: 800 },
+  { name: "mobile", width: 390, height: 844 },
+] as const;
+
 export interface IRenderResult {
   ok: boolean;
   /** Human-readable failures (console errors, page errors, missing content). */
   errors: string[];
   /** True when the check was skipped because playwright isn't installed. */
   skipped?: boolean;
+  /** Paths of screenshots captured (when `screenshotDir` was set). */
+  screenshots?: string[];
+}
+
+/** A simple performance budget: fail the render when the built app blows past
+ *  these. Intentionally minimal (no full Lighthouse) — a tripwire, not a profiler. */
+export interface IPerfBudget {
+  /** Max total DOM nodes after load (a proxy for over-heavy render trees). */
+  maxDomNodes?: number;
+  /** Max time from navigation start to DOMContentLoaded, in ms. */
+  maxMountMs?: number;
+}
+
+/** axe impact levels that FAIL the a11y check — minor/moderate are reported by
+ *  axe but don't gate (too noisy to block a build on). */
+const AXE_FAIL_IMPACTS = new Set(["serious", "critical"]);
+
+/** The subset of an axe violation the oracle reports on. */
+interface IAxeViolation {
+  id: string;
+  impact: string | undefined;
+  nodeCount: number;
+}
+
+/** Extract the reportable violations from axe's (untyped, dynamically-imported)
+ *  result — narrowed with guards, no casts. */
+function extractAxeViolations(result: unknown): IAxeViolation[] {
+  if (!isRecord(result) || !Array.isArray(result.violations)) {
+    return [];
+  }
+
+  const out: IAxeViolation[] = [];
+
+  for (const v of result.violations) {
+    if (!isRecord(v) || typeof v.id !== "string") {
+      continue;
+    }
+
+    out.push({
+      id: v.id,
+      impact: typeof v.impact === "string" ? v.impact : undefined,
+      nodeCount: Array.isArray(v.nodes) ? v.nodes.length : 0,
+    });
+  }
+
+  return out;
+}
+
+/** Turn axe violations into gate errors — only serious/critical fail. Pure. */
+export function summarizeAxeViolations(
+  violations: readonly IAxeViolation[],
+  where: string
+): string[] {
+  return violations
+    .filter((v) => v.impact !== undefined && AXE_FAIL_IMPACTS.has(v.impact))
+    .map(
+      (v) =>
+        `a11y ${v.impact ?? "?"} at ${where}: ${v.id} (${String(v.nodeCount)} node(s))`
+    );
+}
+
+/** Evaluate a perf budget against measured values → gate errors. Pure. */
+export function checkPerfBudget(
+  domNodes: number,
+  mountMs: number,
+  budget: IPerfBudget,
+  where: string
+): string[] {
+  const errors: string[] = [];
+
+  if (budget.maxDomNodes !== undefined && domNodes > budget.maxDomNodes) {
+    errors.push(
+      `perf at ${where}: ${String(domNodes)} DOM nodes > budget ${String(budget.maxDomNodes)}`
+    );
+  }
+
+  if (budget.maxMountMs !== undefined && mountMs > budget.maxMountMs) {
+    errors.push(
+      `perf at ${where}: mount ${String(Math.round(mountMs))}ms > budget ${String(budget.maxMountMs)}ms`
+    );
+  }
+
+  return errors;
 }
 
 export async function renderCheck(
   opts: IRenderOptions
 ): Promise<IRenderResult> {
   const errors: string[] = [];
+  const screenshots: string[] = [];
   const chromium = await loadChromium();
 
   // No playwright → skip the render check rather than fail the gate. The build
@@ -87,7 +201,10 @@ export async function renderCheck(
   const browser = await chromium.launch({ args: ["--no-sandbox"] });
 
   try {
-    const page = await browser.newPage();
+    // Page via an explicit context (not browser.newPage()) — axe-core/playwright
+    // requires a context-owned page; browser.close() tears the context down too.
+    const context = await browser.newContext();
+    const page = await context.newPage();
     const timeout = opts.timeoutMs ?? 15_000;
 
     page.on("console", (message) => {
@@ -113,30 +230,39 @@ export async function renderCheck(
           waitUntil: "load",
           timeout,
         });
-        await runChecks(page, opts, errors);
+        await runChecks(page, opts, errors, screenshots);
 
         if (opts.routes !== undefined && opts.routes.length > 0) {
-          await crawlRoutes(page, base, opts.routes, errors, timeout);
+          await crawlRoutes(page, base, opts.routes, errors, timeout, {
+            opts,
+            screenshots,
+          });
         }
       } finally {
         await server.stop(true);
       }
     } else {
       await page.setContent(opts.html ?? "", { waitUntil: "load", timeout });
-      await runChecks(page, opts, errors);
+      await runChecks(page, opts, errors, screenshots);
     }
 
-    return { ok: errors.length === 0, errors };
+    return {
+      ok: errors.length === 0,
+      errors,
+      ...(screenshots.length > 0 ? { screenshots } : {}),
+    };
   } finally {
     await browser.close();
   }
 }
 
-/** The expectation + step + smoke checks that run against the loaded page. */
+/** The expectation + step + smoke checks that run against the loaded page, then
+ *  the optional quality oracles (a11y, perf budget, screenshots). */
 async function runChecks(
   page: Page,
   opts: IRenderOptions,
-  errors: string[]
+  errors: string[],
+  screenshots: string[]
 ): Promise<void> {
   await checkExpectations(page, opts.expect, errors);
 
@@ -147,6 +273,76 @@ async function runChecks(
   if (opts.smoke === true) {
     await runSmoke(page, errors);
   }
+
+  await runQualityOracles(page, opts, "index", errors, screenshots);
+}
+
+/** The opt-in quality layer: accessibility (axe), a perf budget, and screenshots.
+ *  Each is independent and skips cleanly when not requested / dep absent. */
+async function runQualityOracles(
+  page: Page,
+  opts: IRenderOptions,
+  where: string,
+  errors: string[],
+  screenshots: string[]
+): Promise<void> {
+  if (opts.a11y === true) {
+    const violations = extractAxeViolations(await runAxe(page));
+
+    errors.push(...summarizeAxeViolations(violations, where));
+  }
+
+  if (opts.perfBudget !== undefined) {
+    const { domNodes, mountMs } = await measurePage(page);
+
+    errors.push(...checkPerfBudget(domNodes, mountMs, opts.perfBudget, where));
+  }
+
+  if (opts.screenshotDir !== undefined) {
+    await capturePage(page, opts.screenshotDir, where, screenshots);
+  }
+}
+
+/** Measure DOM size + mount time for the perf budget. */
+async function measurePage(
+  page: Page
+): Promise<{ domNodes: number; mountMs: number }> {
+  return page.evaluate(() => {
+    const nav = performance.getEntriesByType("navigation")[0];
+    const mountMs =
+      nav instanceof PerformanceNavigationTiming
+        ? nav.domContentLoadedEventEnd - nav.startTime
+        : 0;
+
+    return { domNodes: document.querySelectorAll("*").length, mountMs };
+  });
+}
+
+/** Filesystem-safe label for a route (e.g. "/a/b" → "a-b", "/" → "index"). */
+function routeLabel(route: string): string {
+  const cleaned = route.replace(/^\/+|\/+$/g, "").replace(/\//g, "-");
+
+  return cleaned.length === 0 ? "index" : cleaned;
+}
+
+/** Capture a desktop + mobile screenshot of the current page into `dir`. */
+async function capturePage(
+  page: Page,
+  dir: string,
+  label: string,
+  screenshots: string[]
+): Promise<void> {
+  for (const vp of VIEWPORTS) {
+    const path = join(dir, `${label}-${vp.name}.png`);
+
+    try {
+      await page.setViewportSize({ width: vp.width, height: vp.height });
+      await page.screenshot({ path, fullPage: true });
+      screenshots.push(path);
+    } catch {
+      // A screenshot is a best-effort artifact, never a gate failure.
+    }
+  }
 }
 
 /** Serve a directory on an ephemeral localhost port. SPA FALLBACK: an
@@ -187,7 +383,8 @@ async function crawlRoutes(
   base: string,
   routes: readonly string[],
   errors: string[],
-  timeout: number
+  timeout: number,
+  quality: { opts: IRenderOptions; screenshots: string[] }
 ): Promise<void> {
   for (const route of routes) {
     try {
@@ -207,7 +404,17 @@ async function crawlRoutes(
 
       if (blank) {
         errors.push(`route ${route} rendered blank`);
+        continue;
       }
+
+      // a11y + screenshots per route (perf budget stays an initial-page check).
+      await runQualityOracles(
+        page,
+        { ...quality.opts, perfBudget: undefined },
+        routeLabel(route),
+        errors,
+        quality.screenshots
+      );
     } catch (error) {
       errors.push(
         `route ${route} failed to load: ${error instanceof Error ? error.message : String(error)}`

package/src/cli.ts

@@ -38,6 +38,7 @@ import {
   buildGate,
   buildWebGate,
   buildWebFix,
+  buildCoreFix,
   buildWebTypeGate,
   buildWebTscCheck,
   scaffoldWeb,
@@ -102,11 +103,15 @@ export interface ICliArgs {
   /** Plan mode: a from-scratch build pauses after the design phase to show its
    *  plan for review/edit before implementing (`--plan`; also toggled by /plan). */
   plan: boolean;
+  /** Keep the auto-gate at the strict TS floor only — do NOT append the
+   *  project's discovered tests (`--strict-floor-only`). By default the auto-gate
+   *  also runs the project's tests, so "green" means floor + tests pass. */
+  strictFloorOnly: boolean;
 }
 
 const BOOL_FLAGS: Record<
   string,
-  "continue" | "noGate" | "web" | "log" | "plan"
+  "continue" | "noGate" | "web" | "log" | "plan" | "strictFloorOnly"
 > = {
   "--continue": "continue",
   "-c": "continue",
@@ -114,6 +119,7 @@ const BOOL_FLAGS: Record<
   "--web": "web",
   "--log": "log",
   "--plan": "plan",
+  "--strict-floor-only": "strictFloorOnly",
 };
 
 const VALUE_FLAGS = new Set([
@@ -140,6 +146,7 @@ export function parseArgs(argv: readonly string[]): ICliArgs {
     web: false,
     log: false,
     plan: false,
+    strictFloorOnly: false,
   };
 
   for (let i = 0; i < argv.length; i += 1) {
@@ -812,7 +819,13 @@ async function baseGate(
     args.dir,
     activePacks,
     Object.keys(ruleOverrides).length > 0 ? ruleOverrides : undefined,
-    { enableTypeAware: profile === "strict" }
+    {
+      enableTypeAware: profile === "strict",
+      // "Green" should mean the strict floor AND the project's own tests pass —
+      // not just that it type-checks and lints. discoverTestCommand appends them
+      // only when the project actually has tests; --strict-floor-only opts out.
+      includeTests: !args.strictFloorOnly,
+    }
   );
 
   return { accept: auto.command, gateLabel: auto.label };
@@ -891,7 +904,7 @@ async function repl(args: ICliArgs): Promise<number> {
           fix: buildWebFix("react"),
           incrementalCheck: buildWebTscCheck(),
         }
-      : { scaffoldWeb: true }),
+      : { scaffoldWeb: true, fix: buildCoreFix() }),
     ...(thinkingTokenBudget === undefined ? {} : { thinkingTokenBudget }),
     ...(autoCompactAt === undefined ? {} : { autoCompactAt }),
     // Thinking OFF for interactive replies so they STREAM immediately instead of