@agjs/tsforge 0.1.19 → 0.2.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agjs/tsforge",
   "type": "module",
-  "version": "0.1.19",
+  "version": "0.2.1",
   "license": "MIT",
   "description": "TypeScript coding harness with a deterministic gate, stack-aware guardrails, and stream-level correction.",
   "repository": {
@@ -19,7 +19,8 @@
     "src",
     "scripts",
     "strict.eslint.config.mjs",
-    "strict.web.eslint.config.mjs"
+    "strict.web.eslint.config.mjs",
+    "strict.type-aware.eslint.config.mjs"
   ],
   "engines": {
     "bun": ">=1.3.14"
@@ -34,6 +35,9 @@
     "@stylistic/eslint-plugin": "^5.10.0",
     "@typescript-eslint/utils": "8.60.0",
     "cli-highlight": "2.1.11",
+    "eslint-plugin-react": "^7.37.5",
+    "eslint-plugin-react-hooks": "^7.1.1",
+    "eslint-plugin-jsx-a11y": "^6.10.2",
     "eslint": "10.4.0",
     "prettier": "3.8.3",
     "typescript": "6.0.3",

package/scripts/browser-check.ts

@@ -5,27 +5,59 @@
 //
 //   bun browser-check.ts <htmlFile>                 # render-only (no errors)
 //   bun browser-check.ts <htmlFile> --smoke         # render + generic behaviour smoke
+//   bun browser-check.ts <htmlFile> --a11y          # + axe accessibility (serious/critical fail)
+//   bun browser-check.ts <htmlFile> --screenshots[=dir]  # + per-route PNGs (artifact)
+//   bun browser-check.ts <htmlFile> --perf          # + a basic DOM-size/mount-time budget
 //   bun browser-check.ts <htmlFile> <checks.json>   # render + interaction checks
 //   bun browser-check.ts <htmlFile> <selector> [text]
 import { readdir } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { renderCheck, parseChecks, type IRenderOptions } from "../src/browser";
+import {
+  renderCheck,
+  parseChecks,
+  type IRenderOptions,
+  type IPerfBudget,
+} from "../src/browser";
 import { crawlableRoutePaths } from "../src/web-routes";
 
 const rawArgs = process.argv.slice(2);
 const smoke = rawArgs.includes("--smoke");
 const crawl = rawArgs.includes("--crawl");
-const [file, arg2, arg3] = rawArgs.filter(
-  (a) => a !== "--smoke" && a !== "--crawl"
-);
+const a11y = rawArgs.includes("--a11y");
+const perf = rawArgs.includes("--perf");
+const screenshotsArg = rawArgs.find((a) => a.startsWith("--screenshots"));
+// Positionals are anything that isn't a recognized `--flag`.
+const [file, arg2, arg3] = rawArgs.filter((a) => !a.startsWith("--"));
 
 if (file === undefined) {
   process.stderr.write(
-    "usage: browser-check.ts <htmlFile> [--smoke] [--crawl] [checks.json | selector [text]]\n"
+    "usage: browser-check.ts <htmlFile> [--smoke] [--crawl] [--a11y] " +
+      "[--screenshots[=dir]] [--perf] [checks.json | selector [text]]\n"
   );
   process.exit(2);
 }
 
+/** A conservative default budget — a tripwire for runaway render trees / slow
+ *  mounts, not a tuned Lighthouse target. */
+const DEFAULT_PERF_BUDGET: IPerfBudget = {
+  maxDomNodes: 5000,
+  maxMountMs: 6000,
+};
+
+/** The screenshot dir: `--screenshots=<dir>`, else a `screenshots/` folder next
+ *  to the HTML file. undefined when `--screenshots` wasn't passed. */
+function screenshotDir(): string | undefined {
+  if (screenshotsArg === undefined) {
+    return undefined;
+  }
+
+  const eq = screenshotsArg.indexOf("=");
+
+  return eq === -1
+    ? join(dirname(file ?? "."), "screenshots")
+    : screenshotsArg.slice(eq + 1);
+}
+
 /** With --crawl, enumerate the app's static routes from `<buildDir>/src/routes/`
  *  (the build dir is the parent of dist/) so every page — not just the home —
  *  is render-checked. Dynamic ($param) routes are skipped. */
@@ -66,10 +98,14 @@ async function checksFor(): Promise<Partial<IRenderOptions>> {
   };
 }
 
+const shots = screenshotDir();
 const result = await renderCheck({
   file,
   smoke,
+  a11y,
   routes: await routesFor(),
+  ...(perf ? { perfBudget: DEFAULT_PERF_BUDGET } : {}),
+  ...(shots !== undefined ? { screenshotDir: shots } : {}),
   ...(await checksFor()),
 });
 

package/scripts/build-rules-md.ts

@@ -1,9 +1,10 @@
-// Generate RULES.md: a catalog of all rule packs and meta-rules.
-// This produces a deterministic, human-readable reference of what gets enforced.
-//   bun run packages/core/scripts/build-rules-md.ts
+// Generate RULES.md grouped by adoption tier, then pack.
 import { join } from "node:path";
 import { RULE_PACKS } from "../src/rule-packs";
 import { META_RULES } from "../src/meta-rules";
+import { getRuleCatalogEntry } from "../src/rule-packs/rule-metadata";
+import type { RuleTier } from "../src/rule-packs/rule-catalog.types";
+import { PROFILE_DEFINITIONS } from "../src/config/profiles";
 
 function getRuleDescription(obj: unknown): string | undefined {
   const isObject = (val: unknown): val is Record<string, unknown> =>
@@ -30,15 +31,28 @@ function getRuleDescription(obj: unknown): string | undefined {
   return typeof description === "string" ? description : undefined;
 }
 
+const TIER_ORDER: readonly RuleTier[] = [
+  "safety",
+  "framework",
+  "architecture",
+  "experimental",
+];
+
 const out: string[] = [
   "# Rules and Meta-Rules Catalog",
   "",
-  "This document lists all rules enforced by tsforge across rule packs and meta-rules.",
+  "Rules are grouped by **adoption tier**. Use `profile` in `tsforge.config.json` to control which tiers are active by default.",
+  "",
+  "## Profiles",
   "",
 ];
 
-// Section: Rule Packs
-out.push("## Rule Packs");
+for (const profile of Object.values(PROFILE_DEFINITIONS)) {
+  out.push(`- **${profile.id}**: ${profile.description}`);
+}
+
+out.push("");
+out.push("## Rule Packs by Tier");
 out.push("");
 
 type PackId = keyof typeof RULE_PACKS;
@@ -47,36 +61,62 @@ function isPackId(id: string): id is PackId {
   return id in RULE_PACKS;
 }
 
-const packIds = Object.keys(RULE_PACKS).sort();
+const entriesByTier = new Map<
+  RuleTier,
+  { packId: string; ruleName: string; severity: string; description: string }[]
+>();
 
-for (const packId of packIds) {
+for (const packId of Object.keys(RULE_PACKS).sort()) {
   if (!isPackId(packId)) {
     continue;
   }
 
   const pack = RULE_PACKS[packId];
 
-  out.push(`### ${packId}`);
-  out.push("");
-  out.push(pack.description);
-  out.push("");
-
-  const ruleNames = Object.keys(pack.rules).sort();
-
-  for (const ruleName of ruleNames) {
+  for (const ruleName of Object.keys(pack.rules).sort()) {
     const rule = pack.rules[ruleName];
     const severity = pack.rulesConfig[ruleName] ?? "warn";
     const description = getRuleDescription(rule) ?? ruleName;
-    const severityUpper = severity.toUpperCase();
-    const line = `- **${ruleName}** [${severityUpper}]: ${description}`;
+    const tier = getRuleCatalogEntry(ruleName, packId).tier;
+    const list = entriesByTier.get(tier) ?? [];
+
+    list.push({
+      packId,
+      ruleName,
+      severity: severity.toUpperCase(),
+      description,
+    });
+    entriesByTier.set(tier, list);
+  }
+}
+
+for (const tier of TIER_ORDER) {
+  const entries = entriesByTier.get(tier) ?? [];
+
+  if (entries.length === 0) {
+    continue;
+  }
 
-    out.push(line);
+  out.push(`### Tier: ${tier}`);
+  out.push("");
+
+  for (const entry of entries.sort((a, b) => {
+    const byPack = a.packId.localeCompare(b.packId);
+
+    if (byPack !== 0) {
+      return byPack;
+    }
+
+    return a.ruleName.localeCompare(b.ruleName);
+  })) {
+    out.push(
+      `- **${entry.packId}/${entry.ruleName}** [${entry.severity}]: ${entry.description}`
+    );
   }
 
   out.push("");
 }
 
-// Section: Meta-Rules
 out.push("## Meta-Rules");
 out.push("");
 out.push(
@@ -103,7 +143,6 @@ for (const rule of META_RULES) {
   rulesByCategory.set(cat, rules);
 }
 
-// Render meta-rules by category.
 for (const category of categoryOrder) {
   const rules = rulesByCategory.get(category) ?? [];
 
@@ -123,6 +162,24 @@ for (const category of categoryOrder) {
   out.push("");
 }
 
+out.push("## Out of scope");
+out.push("");
+out.push(
+  "The following are intentionally deferred — wrong tool for the syntactic ESLint gate, or require cross-file analysis:"
+);
+out.push("");
+out.push(
+  "- GraphQL/WebSocket/OpenAPI contract rules (until OpenAPI dep + parser)"
+);
+out.push(
+  "- Container/Kubernetes YAML hardening (future meta-rules when Dockerfile/k8s detected)"
+);
+out.push("- LLM/MCP security packs (opt-in when AI SDK deps detected)");
+out.push("- FSD layer DAG / full authorization taint tracking");
+out.push("- Lighthouse / bundle-analyzer CI gates");
+out.push("- Violation ratcheting / baseline snapshots (Phase 5)");
+out.push("");
+
 const path = join(import.meta.dir, "..", "RULES.md");
 
 await Bun.write(path, out.join("\n"));

package/scripts/cli-metrics.ts

@@ -10,6 +10,7 @@ import { readdir } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { isRecord } from "../src/lib/guards";
+import { classifyRun, parseEventLog } from "../src/eval";
 
 function num(value: unknown): number {
   return typeof value === "number" ? value : 0;
@@ -168,6 +169,9 @@ async function main(): Promise<void> {
   const text = await Bun.file(path).text();
   const lines = text.split("\n").filter((l) => l.trim().length > 0);
   const m = analyze(lines);
+  // Single source of truth for WHY a run failed — the same classifier the eval
+  // sweep and the reusable analyzeEvents() use, fed the typed event stream.
+  const failure = classifyRun(parseEventLog(text));
   const pct =
     m.contextWindow > 0
       ? Math.round((m.peakContext / m.contextWindow) * 100)
@@ -182,6 +186,12 @@ async function main(): Promise<void> {
     ["model", m.model],
     ["context window", String(m.contextWindow)],
     ["final status", m.finalStatus],
+    [
+      "failure class",
+      failure.detail === undefined
+        ? failure.failureClass
+        : `${failure.failureClass} (${failure.detail})`,
+    ],
     ["turns (repair iterations)", String(m.turns)],
     ["model calls", String(m.modelCalls)],
     ["tokens out (→ solution)", String(m.tokensOut)],

package/scripts/sweep.ts

@@ -11,7 +11,14 @@ import { runSpec, qualityRepair } from "../src/loop";
 import { modelAgent } from "../src/agent";
 import { OpenAICompatibleProvider } from "../src/inference";
 import { resolveActiveModel, resolveApiKey } from "../src/models-config";
-import { summarize, type IRunRecord } from "../src/eval";
+import { providerConfig } from "../src/cli";
+import {
+  summarize,
+  classifyRun,
+  renderSweepReportMarkdown,
+  buildSweepReport,
+  type IRunRecord,
+} from "../src/eval";
 import { renderEvent } from "../src/render";
 import type { ILoopEvent } from "../src/loop";
 
@@ -109,28 +116,32 @@ const seedFiles = await readdir(seedDir, { recursive: true });
 // unreachable endpoint and hung with an empty run.log.)
 const { entry: activeModel } = await resolveActiveModel();
 
-const provider = new OpenAICompatibleProvider({
-  baseUrl: activeModel.baseUrl,
-  model: activeModel.model,
-  apiKey: resolveApiKey(activeModel),
-  // Thinking tokens count against the limit, so give reasoning + code room.
-  maxTokens: Number(process.env.TSFORGE_MAX_TOKENS ?? "16384"),
-  // Opt-in only: a repetition penalty breaks rare temp-0 loops but DEGRADES
-  // algorithmic code (it made `money` write unsafe/any code that failed the
-  // strict gate). Default off; enable via env if a target genuinely loops.
-  repetitionPenalty:
-    process.env.TSFORGE_REPETITION_PENALTY === undefined
-      ? undefined
-      : Number(process.env.TSFORGE_REPETITION_PENALTY),
-});
+// Build the wire config the SAME way the CLI does (`providerConfig`), so the
+// sweep inherits the active entry's provider dialect — `reasoning`,
+// `reasoningEffort`, `extraBody`, `extraHeaders`. Hand-rolling the config here
+// dropped those fields, so a DeepSeek sweep sent qwen-only params and hit the
+// 400s the interactive path already handles. maxTokens still defaults to
+// PROVIDER_LIMITS (16384) — thinking tokens count against it, so reasoning +
+// code get room. Repetition penalty stays opt-in via TSFORGE_REPETITION_PENALTY.
+const provider = new OpenAICompatibleProvider(providerConfig(activeModel));
 
 // The judge scores quality. Point it at a flagship via TSFORGE_JUDGE_URL/MODEL
-// (+ TSFORGE_JUDGE_KEY) to measure the gap; defaults to the active model judging itself.
-const judgeProvider = new OpenAICompatibleProvider({
-  baseUrl: process.env.TSFORGE_JUDGE_URL ?? activeModel.baseUrl,
-  model: process.env.TSFORGE_JUDGE_MODEL ?? activeModel.model,
-  apiKey: process.env.TSFORGE_JUDGE_KEY ?? resolveApiKey(activeModel),
-});
+// (+ TSFORGE_JUDGE_KEY) to measure the gap. When NOT overridden, the active
+// model judges itself — reuse its full dialect via providerConfig so a
+// self-judge against DeepSeek speaks DeepSeek too. An explicit external judge
+// is a plain generic call (its own endpoint, no inherited reasoning dialect).
+const judgeOverridden =
+  process.env.TSFORGE_JUDGE_URL !== undefined ||
+  process.env.TSFORGE_JUDGE_MODEL !== undefined;
+const judgeProvider = new OpenAICompatibleProvider(
+  judgeOverridden
+    ? {
+        baseUrl: process.env.TSFORGE_JUDGE_URL ?? activeModel.baseUrl,
+        model: process.env.TSFORGE_JUDGE_MODEL ?? activeModel.model,
+        apiKey: process.env.TSFORGE_JUDGE_KEY ?? resolveApiKey(activeModel),
+      }
+    : providerConfig(activeModel)
+);
 
 /** Sortable timestamp `YYYYMMDD-HHMMSS` so run dirs sort newest-last by name. */
 function stamp(): string {
@@ -263,8 +274,12 @@ async function runOne(
     // Every run gets a full transcript at <runDir>/run.log; stream to the
     // terminal too when TSFORGE_STREAM=1.
     const log = Bun.file(join(runDir, "run.log")).writer();
+    // Keep the structured events so a failed run can be classified (WHY it
+    // failed), not just counted — fed to classifyRun below.
+    const runEvents: ILoopEvent[] = [];
 
     const onEvent = (e: ILoopEvent): void => {
+      runEvents.push(e);
       void log.write(renderEvent(e, { color: false }));
       // Flush per event — otherwise Bun's FileSink buffers and `tail -f` shows
       // nothing until the run ends. The log must be live.
@@ -354,6 +369,9 @@ async function runOne(
     );
 
     const vLabel = variantLabel(variantEnv);
+    const failureClass = passed
+      ? undefined
+      : classifyRun(runEvents).failureClass;
 
     records.push({
       label: `${vLabel} temp=${temp}`,
@@ -361,9 +379,10 @@ async function runOne(
       cycles,
       ms,
       quality,
+      ...(failureClass === undefined ? {} : { failureClass }),
     });
     process.stdout.write(
-      `  ${seed} ${vLabel} temp=${temp} #${i + 1}: ${passed ? "done" : "blocked"} (${cycles} cyc, ${edits} edits, ${regressions} regress, ${ms}ms${quality === undefined ? "" : `, Q${quality}/5`}) → ${runId}\n`
+      `  ${seed} ${vLabel} temp=${temp} #${i + 1}: ${passed ? "done" : `blocked[${failureClass ?? "unknown"}]`} (${cycles} cyc, ${edits} edits, ${regressions} regress, ${ms}ms${quality === undefined ? "" : `, Q${quality}/5`}) → ${runId}\n`
     );
   } finally {
     restore();
@@ -375,11 +394,22 @@ const summaries = summarize(records);
 process.stdout.write(`\n=== sweep: ${seed} (${repeats} runs/variant) ===\n`);
 
 for (const s of summaries) {
+  const failures = Object.entries(s.failureClasses)
+    .sort(([, a], [, b]) => b - a)
+    .map(([cls, n]) => `${cls}×${String(n)}`)
+    .join(", ");
+
   process.stdout.write(
-    `${s.label.padEnd(10)}  pass ${Math.round(s.passRate * 100)}% (${s.passed}/${s.runs})  Q ${s.avgQuality.toFixed(1)}/5  avg ${s.avgCycles.toFixed(1)} cyc  ${Math.round(s.avgMs)}ms\n`
+    `${s.label.padEnd(10)}  pass ${Math.round(s.passRate * 100)}% (${s.passed}/${s.runs})  Q ${s.avgQuality.toFixed(1)}/5  avg ${s.avgCycles.toFixed(1)} cyc  ${Math.round(s.avgMs)}ms${failures.length > 0 ? `  [${failures}]` : ""}\n`
   );
 }
 
+// The statistical report (Wilson CI + z-test vs baseline) now also tabulates a
+// per-variant failure-class breakdown — WHY runs failed, not just how often.
+process.stdout.write(
+  `\n${renderSweepReportMarkdown(buildSweepReport(records))}\n`
+);
+
 const outPath = join(evalsRoot, "runs", `sweep-${seed}-${stamp()}.json`);
 
 await Bun.write(