@agjs/tsforge 0.1.13 → 0.1.15

package/scripts/build-rules-md.ts

@@ -0,0 +1,129 @@
+// Generate RULES.md: a catalog of all rule packs and meta-rules.
+// This produces a deterministic, human-readable reference of what gets enforced.
+//   bun run packages/core/scripts/build-rules-md.ts
+import { join } from "node:path";
+import { RULE_PACKS } from "../src/rule-packs";
+import { META_RULES } from "../src/meta-rules";
+
+function getRuleDescription(obj: unknown): string | undefined {
+  const isObject = (val: unknown): val is Record<string, unknown> =>
+    val !== null && typeof val === "object";
+
+  if (!isObject(obj)) {
+    return undefined;
+  }
+
+  const meta = obj.meta;
+
+  if (!isObject(meta)) {
+    return undefined;
+  }
+
+  const docs = meta.docs;
+
+  if (!isObject(docs)) {
+    return undefined;
+  }
+
+  const description = docs.description;
+
+  return typeof description === "string" ? description : undefined;
+}
+
+const out: string[] = [
+  "# Rules and Meta-Rules Catalog",
+  "",
+  "This document lists all rules enforced by tsforge across rule packs and meta-rules.",
+  "",
+];
+
+// Section: Rule Packs
+out.push("## Rule Packs");
+out.push("");
+
+type PackId = keyof typeof RULE_PACKS;
+
+function isPackId(id: string): id is PackId {
+  return id in RULE_PACKS;
+}
+
+const packIds = Object.keys(RULE_PACKS).sort();
+
+for (const packId of packIds) {
+  if (!isPackId(packId)) {
+    continue;
+  }
+
+  const pack = RULE_PACKS[packId];
+
+  out.push(`### ${packId}`);
+  out.push("");
+  out.push(pack.description);
+  out.push("");
+
+  const ruleNames = Object.keys(pack.rules).sort();
+
+  for (const ruleName of ruleNames) {
+    const rule = pack.rules[ruleName];
+    const severity = pack.rulesConfig[ruleName] ?? "warn";
+    const description = getRuleDescription(rule) ?? ruleName;
+    const severityUpper = severity.toUpperCase();
+    const line = `- **${ruleName}** [${severityUpper}]: ${description}`;
+
+    out.push(line);
+  }
+
+  out.push("");
+}
+
+// Section: Meta-Rules
+out.push("## Meta-Rules");
+out.push("");
+out.push(
+  "Meta-rules enforce project structure and configuration invariants that ESLint cannot express."
+);
+out.push("");
+
+const categoryOrder = [
+  "supply-chain",
+  "config",
+  "source-text",
+  "testing",
+  "stack-layout",
+  "ci",
+] as const;
+
+const rulesByCategory = new Map<string, (typeof META_RULES)[number][]>();
+
+for (const rule of META_RULES) {
+  const cat = rule.category;
+  const rules = rulesByCategory.get(cat) ?? [];
+
+  rules.push(rule);
+  rulesByCategory.set(cat, rules);
+}
+
+// Render meta-rules by category.
+for (const category of categoryOrder) {
+  const rules = rulesByCategory.get(category) ?? [];
+
+  if (rules.length === 0) {
+    continue;
+  }
+
+  out.push(`### ${category}`);
+  out.push("");
+
+  for (const rule of rules.sort((a, b) => a.id.localeCompare(b.id))) {
+    out.push(
+      `- **${rule.id}** [${rule.severity.toUpperCase()}]: ${rule.description}`
+    );
+  }
+
+  out.push("");
+}
+
+const path = join(import.meta.dir, "..", "RULES.md");
+
+await Bun.write(path, out.join("\n"));
+process.stdout.write(`\nwrote rules catalog → ${path}\n`);

package/scripts/cli-metrics.ts

@@ -0,0 +1,203 @@
+// Turn a tsforge CLI `--log` (JSONL) into the metrics the local-Qwen literature
+// says actually predict behavior — tokens-to-solution, repair iterations,
+// hallucinated imports, peak context — measured against the model/window the log
+// records. "Measure, don't assume": many apparent model failures are really
+// quant/config/harness failures, so judge a run by these, not by vibes.
+//
+// Run:  bun run packages/core/scripts/cli-metrics.ts [logfile]
+//   (no arg → the newest log under ~/.tsforge/logs)
+import { readdir } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { isRecord } from "../src/lib/guards";
+
+function num(value: unknown): number {
+  return typeof value === "number" ? value : 0;
+}
+
+function str(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+async function newestLog(): Promise<string> {
+  const dir = join(process.env.TSFORGE_HOME ?? homedir(), ".tsforge", "logs");
+  // Filenames are ISO-timestamp-prefixed, so lexicographic sort = chronological.
+  const names = (await readdir(dir)).filter((n) => n.endsWith(".jsonl")).sort();
+  const latest = names.at(-1);
+
+  if (latest === undefined) {
+    throw new Error(`no .jsonl logs in ${dir}`);
+  }
+
+  return join(dir, latest);
+}
+
+interface IMetrics {
+  model: string;
+  contextWindow: number;
+  finalStatus: string;
+  turns: number;
+  modelCalls: number;
+  tokensOut: number;
+  peakContext: number;
+  edits: number;
+  filesCreated: number;
+  gateRuns: number;
+  compactions: number;
+  buildNudges: number;
+  salvaged: number;
+  hallucinatedImports: string[];
+  wallClockSeconds: number;
+}
+
+type EventRecord = Record<string, unknown>;
+
+/** One counter per event kind — a flat registry keeps `analyze` branch-free. */
+const HANDLERS: Record<
+  string,
+  (m: IMetrics, event: EventRecord, created: Set<string>) => void
+> = {
+  start: (m, e) => {
+    if (str(e.model).length > 0) {
+      m.model = str(e.model);
+    }
+
+    if (num(e.contextWindow) > 0) {
+      m.contextWindow = num(e.contextWindow);
+    }
+  },
+  cycle: (m) => {
+    m.turns += 1;
+  },
+  usage: (m, e) => {
+    m.modelCalls += 1;
+    m.tokensOut += num(e.completionTokens);
+    m.peakContext = Math.max(m.peakContext, num(e.promptTokens));
+  },
+  create: (m, e, created) => {
+    m.edits += 1;
+
+    if (str(e.file).length > 0) {
+      created.add(str(e.file));
+    }
+  },
+  edit: (m) => {
+    m.edits += 1;
+  },
+  timing: (m, e) => {
+    m.wallClockSeconds += Math.round(num(e.ms) / 1000);
+  },
+  validated: (m) => {
+    m.gateRuns += 1;
+  },
+  done: (m) => {
+    m.finalStatus = "done";
+  },
+  stuck: (m) => {
+    m.finalStatus = "stuck";
+  },
+  tool: (m, e) => {
+    const msg = str(e.message);
+
+    m.compactions += msg.includes("compacted") ? 1 : 0;
+    m.buildNudges += msg.includes("nudging") ? 1 : 0;
+    m.salvaged += msg.includes("recovered") ? 1 : 0;
+  },
+};
+
+function emptyMetrics(): IMetrics {
+  return {
+    model: "?",
+    contextWindow: 0,
+    finalStatus: "(none)",
+    turns: 0,
+    modelCalls: 0,
+    tokensOut: 0,
+    peakContext: 0,
+    edits: 0,
+    filesCreated: 0,
+    gateRuns: 0,
+    compactions: 0,
+    buildNudges: 0,
+    salvaged: 0,
+    hallucinatedImports: [],
+    wallClockSeconds: 0,
+  };
+}
+
+function analyze(lines: string[]): IMetrics {
+  const created = new Set<string>();
+  const m = emptyMetrics();
+  let allText = "";
+
+  for (const line of lines) {
+    let event: unknown;
+
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    if (!isRecord(event)) {
+      continue;
+    }
+
+    // Concatenate ALL message/output text, then scan once — a "Cannot find
+    // module" can be split across streamed token chunks.
+    allText += `${str(event.message)}\n${str(event.output)}\n`;
+    HANDLERS[str(event.kind)]?.(m, event, created);
+  }
+
+  const hallucinated = new Set<string>();
+
+  for (const match of allText.matchAll(
+    /Cannot find module ['"]([^'"]+)['"]/g
+  )) {
+    hallucinated.add(match[1] ?? "");
+  }
+
+  m.filesCreated = created.size;
+  m.hallucinatedImports = [...hallucinated];
+
+  return m;
+}
+
+async function main(): Promise<void> {
+  const path = process.argv[2] ?? (await newestLog());
+  const text = await Bun.file(path).text();
+  const lines = text.split("\n").filter((l) => l.trim().length > 0);
+  const m = analyze(lines);
+  const pct =
+    m.contextWindow > 0
+      ? Math.round((m.peakContext / m.contextWindow) * 100)
+      : 0;
+  const imports =
+    m.hallucinatedImports.length > 0
+      ? `${m.hallucinatedImports.length} → ${m.hallucinatedImports.join(", ")}`
+      : "0";
+
+  const rows: [string, string][] = [
+    ["log", path],
+    ["model", m.model],
+    ["context window", String(m.contextWindow)],
+    ["final status", m.finalStatus],
+    ["turns (repair iterations)", String(m.turns)],
+    ["model calls", String(m.modelCalls)],
+    ["tokens out (→ solution)", String(m.tokensOut)],
+    ["peak context", `${m.peakContext} (${pct}% of window)`],
+    ["edits/creates", `${m.edits} (${m.filesCreated} files created)`],
+    ["gate runs", String(m.gateRuns)],
+    ["auto-compactions", String(m.compactions)],
+    ["build nudges", String(m.buildNudges)],
+    ["salvaged tool calls", String(m.salvaged)],
+    ["hallucinated imports", imports],
+    ["wall clock", `${m.wallClockSeconds}s`],
+  ];
+
+  for (const [label, value] of rows) {
+    console.log(`${label.padEnd(26)} ${value}`);
+  }
+}
+
+await main();

package/scripts/coverage-check.ts

@@ -0,0 +1,33 @@
+#!/usr/bin/env bun
+// Gate step (catalog builds): fail unless every declared entity has real UI — a
+// feature folder with a component, not just a `.types.ts`. Wired into the web gate
+// by headless-build so an app can't green with half its entities unbuilt (a run
+// greened with 4 of 8 entities as types-only). Usage:
+//   bun coverage-check.ts <buildDir> "<Entity1>" "<Entity2>" ...
+import { uncoveredEntities } from "../src/web-coverage";
+
+const [, , dir, ...entities] = process.argv;
+
+if (dir === undefined || entities.length === 0) {
+  // No entity list → nothing to enforce (ad-hoc build). Pass.
+  process.exit(0);
+}
+
+const missing = await uncoveredEntities(dir, entities);
+
+if (missing.length > 0) {
+  const noun = missing.length === 1 ? "entity has" : "entities have";
+
+  process.stdout.write(
+    `coverage: ${String(missing.length)} declared ${noun} NO UI (types only) — ` +
+      `build each one's list + create + detail routes and wire a reachable "New" ` +
+      `button; the app is NOT done until every entity is reachable. Missing: ` +
+      `${missing.join(", ")}\n`
+  );
+  process.exit(1);
+}
+
+process.stdout.write(
+  `coverage: all ${String(entities.length)} declared entities have UI\n`
+);
+process.exit(0);

package/scripts/edit-benchmark.ts

@@ -0,0 +1,314 @@
+// Compare edit mechanisms (edit vs edit_lines) across completed run artifacts.
+// Parses run.log + result.json from sweep runs to measure: tool calls, success rates,
+// stale-anchor recovery, token cost (args size), gate failures, turns to green.
+//
+// Run:  bun run packages/core/scripts/edit-benchmark.ts <dir> <dir> ...
+//   (analyze run dirs produced with hashline on vs off)
+// Or:   bun run packages/core/scripts/edit-benchmark.ts --json <output.json> <dir> <dir> ...
+import { readFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { isRecord } from "../src/lib/guards";
+
+interface IEditMetrics {
+  runId: string;
+  hashlineEnabled: boolean;
+  editToolCalls: number;
+  editLinesToolCalls: number;
+  editRejections: number;
+  editLinesRejections: number;
+  staleAnchorRecoveries: number;
+  /** Mean tool-args bytes per successful edit (token-cost proxy). */
+  meanToolArgBytes: number;
+  gateFails: number;
+  /** Turns from start to first GREEN gate result. */
+  turnsToGreen: number;
+  totalTurns: number;
+  passed: boolean;
+  quality?: number;
+}
+
+interface IEditSummary {
+  label: string;
+  runs: number;
+  avgEditToolCalls: number;
+  avgEditLinesToolCalls: number;
+  editSuccessRate: number;
+  editLinesSuccessRate: number;
+  avgStaleAnchorRecoveries: number;
+  avgMeanToolArgBytes: number;
+  avgGateFails: number;
+  avgTurnsToGreen: number;
+  passRate: number;
+  avgQuality: number;
+}
+
+function countPattern(lines: string[], pattern: RegExp): number {
+  return lines.filter((line) => pattern.test(line)).length;
+}
+
+async function parseRunLog(
+  logPath: string
+): Promise<Omit<IEditMetrics, "runId" | "hashlineEnabled">> {
+  const logText = await readFile(logPath, "utf-8");
+  const lines = logText.split("\n");
+
+  const editToolCalls = countPattern(lines, /✎ edit .*/);
+  const editLinesToolCalls = countPattern(lines, /edit_lines /);
+  const editRejections = countPattern(lines, /edit .*REJECTED/);
+  const editLinesRejections = countPattern(lines, /edit_lines .*REJECTED/);
+  const staleAnchorRecoveries = countPattern(
+    lines,
+    /snapshot.*merge|anchor.*stale|recovery.*suggest/
+  );
+  const successfulEdits = countPattern(
+    lines,
+    /edited .*\(new hash #|✎ edit .* \)$/
+  );
+  const gateFails = countPattern(lines, /turn \d+: red \(\d+ error/);
+  const isGreen = lines.some((line) => /spec ".*": done/.test(line));
+
+  // Calculate total args bytes from successful edits
+  const successLines = lines.filter((line) =>
+    /edited .*\(new hash #|✎ edit .* \)$/.test(line)
+  );
+  const totalArgBytes = successLines.reduce(
+    (sum, line) => sum + line.length,
+    0
+  );
+
+  // Find turn counts
+  let totalTurns = 0;
+  let turnsToGreen = -1;
+
+  for (const line of lines) {
+    const askMatch = /turn (\d+): asking model/.exec(line);
+
+    if (askMatch !== null) {
+      totalTurns = Math.max(totalTurns, Number(askMatch[1]));
+    }
+
+    if (/· turn \d+: GREEN/.test(line) && turnsToGreen < 0) {
+      turnsToGreen = totalTurns;
+    }
+  }
+
+  return {
+    editToolCalls,
+    editLinesToolCalls,
+    editRejections,
+    editLinesRejections,
+    staleAnchorRecoveries,
+    meanToolArgBytes:
+      successfulEdits > 0 ? Math.round(totalArgBytes / successfulEdits) : 0,
+    gateFails,
+    turnsToGreen: turnsToGreen >= 0 ? turnsToGreen : totalTurns,
+    totalTurns,
+    passed: isGreen,
+  };
+}
+
+async function parseResultJson(
+  jsonPath: string
+): Promise<{ quality?: number; hashlineEnabled: boolean }> {
+  try {
+    const text = await readFile(jsonPath, "utf-8");
+    const parsed: unknown = JSON.parse(text);
+
+    if (!isRecord(parsed)) {
+      return { hashlineEnabled: false };
+    }
+
+    const quality =
+      typeof parsed.quality === "number" ? parsed.quality : undefined;
+    const features = isRecord(parsed.features) ? parsed.features : {};
+
+    return {
+      quality,
+      hashlineEnabled:
+        features.TSFORGE_HASHLINE === "1" ||
+        process.env.TSFORGE_HASHLINE === "1",
+    };
+  } catch {
+    return { hashlineEnabled: false };
+  }
+}
+
+async function analyzeRunDir(dir: string): Promise<IEditMetrics | null> {
+  const logPath = join(dir, "run.log");
+  const jsonPath = join(dir, "result.json");
+  const runId = basename(dir);
+
+  try {
+    const logMetrics = await parseRunLog(logPath);
+    const jsonData = await parseResultJson(jsonPath);
+
+    return {
+      runId,
+      hashlineEnabled: jsonData.hashlineEnabled,
+      ...logMetrics,
+      quality: jsonData.quality,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function summarizeMetrics(metrics: IEditMetrics[]): IEditSummary {
+  const m0 = metrics[0];
+  const label =
+    m0 !== undefined
+      ? m0.hashlineEnabled
+        ? "hashline=on"
+        : "hashline=off"
+      : "unknown";
+
+  const passed = metrics.filter((m) => m.passed).length;
+  const scored = metrics.filter((m) => m.quality !== undefined);
+
+  const sum = (sel: (m: IEditMetrics) => number): number =>
+    metrics.reduce((a, m) => a + sel(m), 0);
+  const avg = (sel: (m: IEditMetrics) => number): number =>
+    metrics.length > 0 ? sum(sel) / metrics.length : 0;
+
+  const totalEditCalls = sum((m) => m.editToolCalls);
+  const totalEditLines = sum((m) => m.editLinesToolCalls);
+
+  return {
+    label,
+    runs: metrics.length,
+    avgEditToolCalls: avg((m) => m.editToolCalls),
+    avgEditLinesToolCalls: avg((m) => m.editLinesToolCalls),
+    editSuccessRate:
+      totalEditCalls > 0
+        ? 1 - sum((m) => m.editRejections) / totalEditCalls
+        : 1,
+    editLinesSuccessRate:
+      totalEditLines > 0
+        ? 1 - sum((m) => m.editLinesRejections) / totalEditLines
+        : 1,
+    avgStaleAnchorRecoveries: avg((m) => m.staleAnchorRecoveries),
+    avgMeanToolArgBytes: avg((m) => m.meanToolArgBytes),
+    avgGateFails: avg((m) => m.gateFails),
+    avgTurnsToGreen: avg((m) => m.turnsToGreen),
+    passRate: metrics.length > 0 ? passed / metrics.length : 0,
+    avgQuality:
+      scored.length > 0
+        ? scored.reduce((a, m) => a + (m.quality ?? 0), 0) / scored.length
+        : 0,
+  };
+}
+
+function renderSummaryTable(summaries: IEditSummary[]): string {
+  const headers = [
+    "condition",
+    "runs",
+    "edit calls",
+    "edit_lines calls",
+    "edit % success",
+    "edit_lines % success",
+    "stale recovery",
+    "mean args (bytes)",
+    "gate fails",
+    "turns to green",
+    "pass rate",
+    "avg quality",
+  ];
+
+  const rows = summaries.map((s) => [
+    s.label,
+    String(s.runs),
+    s.avgEditToolCalls.toFixed(1),
+    s.avgEditLinesToolCalls.toFixed(1),
+    (s.editSuccessRate * 100).toFixed(0),
+    (s.editLinesSuccessRate * 100).toFixed(0),
+    s.avgStaleAnchorRecoveries.toFixed(2),
+    String(Math.round(s.avgMeanToolArgBytes)),
+    s.avgGateFails.toFixed(1),
+    s.avgTurnsToGreen.toFixed(1),
+    (s.passRate * 100).toFixed(0),
+    s.avgQuality.toFixed(1),
+  ]);
+
+  // Simple ASCII table
+  const colWidths = headers.map((h, i) => {
+    return Math.max(
+      h.length,
+      ...rows.map((r) => {
+        const cell = r[i];
+
+        return cell !== undefined ? cell.length : 0;
+      })
+    );
+  });
+
+  const headerLine = headers
+    .map((h, i) => h.padEnd(colWidths[i] ?? 0))
+    .join(" | ");
+  const separator = colWidths.map((w) => "-".repeat(w)).join("-+-");
+  const dataLines = rows
+    .map((r) => r.map((c, i) => c.padEnd(colWidths[i] ?? 0)).join(" | "))
+    .join("\n");
+
+  return `${headerLine}\n${separator}\n${dataLines}`;
+}
+
+const args = process.argv.slice(2);
+const jsonOutput = args[0] === "--json" ? args[1] : undefined;
+const dirs = jsonOutput !== undefined ? args.slice(2) : args;
+
+if (dirs.length === 0) {
+  process.stdout.write(
+    "Usage: edit-benchmark [--json <output.json>] <dir> [<dir> ...]\n"
+  );
+  process.exit(1);
+}
+
+const allMetrics: IEditMetrics[] = [];
+
+for (const dir of dirs) {
+  const metrics = await analyzeRunDir(dir);
+
+  if (metrics !== null) {
+    allMetrics.push(metrics);
+  }
+}
+
+// Group by hashline on/off
+const byHashline = new Map<boolean, IEditMetrics[]>();
+
+for (const m of allMetrics) {
+  const list = byHashline.get(m.hashlineEnabled) ?? [];
+
+  list.push(m);
+  byHashline.set(m.hashlineEnabled, list);
+}
+
+const summaries: IEditSummary[] = [];
+
+for (const [, metrics] of byHashline) {
+  summaries.push(summarizeMetrics(metrics));
+}
+
+summaries.sort((a, b) => a.label.localeCompare(b.label));
+
+const table = renderSummaryTable(summaries);
+
+process.stdout.write("\n=== edit-benchmark comparison ===\n\n");
+process.stdout.write(table);
+process.stdout.write("\n");
+
+if (jsonOutput !== undefined) {
+  await Bun.write(
+    jsonOutput,
+    JSON.stringify(
+      {
+        analyzed: allMetrics.length,
+        summaries,
+        table,
+      },
+      null,
+      2
+    )
+  );
+  process.stdout.write(`\nsaved JSON report to ${jsonOutput}\n`);
+}