@agjs/tsforge 0.1.14 → 0.1.15

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agjs/tsforge",
   "type": "module",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "license": "MIT",
   "description": "TypeScript coding harness with a deterministic gate, stack-aware guardrails, and stream-level correction.",
   "repository": {
@@ -17,6 +17,7 @@
   "files": [
     "bin",
     "src",
+    "scripts",
     "strict.eslint.config.mjs",
     "strict.web.eslint.config.mjs"
   ],

package/scripts/analyze-malformed.ts

@@ -0,0 +1,264 @@
+#!/usr/bin/env bun
+// Correlate tool-call repair incidents with the per-call thinking mode,
+// across every JSONL event log we have (~/.tsforge/logs + evals/runs). Tracks:
+//   - Per-rule repair rates (L0: drop-null, unwrap-autolink; L1: coerce:*; etc.)
+//   - L3 re-ask frequency (when repair gave up)
+//   - Correlation with thinking mode (hypothesis: thinking-off has higher failure)
+//
+//   bun packages/core/scripts/analyze-malformed.ts
+//
+// Old logs predate the per-call `thinking` field — those calls land in the
+// "unknown" bucket; rates firm up as new logs accumulate.
+import { readdirSync, readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { isRecord } from "../src/lib/guards";
+
+interface IBucket {
+  calls: number;
+  salvaged: number;
+  malformedNudges: number;
+  repairs: Map<string, number>; // rule name → count
+  reasks: number; // L3 re-ask count
+}
+
+type ThinkingMode = "on" | "off" | "unknown";
+
+const buckets: Record<ThinkingMode, IBucket> = {
+  on: {
+    calls: 0,
+    salvaged: 0,
+    malformedNudges: 0,
+    repairs: new Map(),
+    reasks: 0,
+  },
+  off: {
+    calls: 0,
+    salvaged: 0,
+    malformedNudges: 0,
+    repairs: new Map(),
+    reasks: 0,
+  },
+  unknown: {
+    calls: 0,
+    salvaged: 0,
+    malformedNudges: 0,
+    repairs: new Map(),
+    reasks: 0,
+  },
+};
+
+function modeOf(event: Record<string, unknown>): ThinkingMode {
+  if (event.thinking === true) {
+    return "on";
+  }
+
+  return event.thinking === false ? "off" : "unknown";
+}
+
+/** The thinking mode of the most recent usage event — repair events
+ *  carry no flag of their own (they fire after the call), so they inherit it. */
+let lastCallMode: ThinkingMode = "unknown";
+
+function ingestTool(event: Record<string, unknown>): void {
+  const msg = event.message;
+
+  if (typeof msg !== "string") {
+    return;
+  }
+
+  if (msg.includes("recovered") && msg.includes("malformed")) {
+    // Salvage warnings carry their own per-call flag (new logs).
+    buckets[
+      event.thinking === undefined ? lastCallMode : modeOf(event)
+    ].salvaged += 1;
+  }
+
+  if (msg.includes("malformed tool-call text")) {
+    buckets[lastCallMode].malformedNudges += 1;
+  }
+}
+
+function ingestRepair(event: Record<string, unknown>): void {
+  const mode = event.thinking === undefined ? lastCallMode : modeOf(event);
+  const bucket = buckets[mode];
+  const msg = event.message;
+
+  if (typeof msg !== "string") {
+    return;
+  }
+
+  // Format: "tool:L0:drop-null:field" or "tool:L1:coerce:files" or "tool:L3-re-ask"
+  const parts = msg.split(":");
+
+  if (parts[parts.length - 1] === "L3-re-ask") {
+    bucket.reasks += 1;
+  } else {
+    // Extract the rule name (e.g. "drop-null:field" → "drop-null")
+    const rule = parts.slice(1).join(":");
+
+    bucket.repairs.set(rule, (bucket.repairs.get(rule) ?? 0) + 1);
+  }
+}
+
+function ingestLine(line: string): void {
+  let event: unknown;
+
+  try {
+    event = JSON.parse(line);
+  } catch {
+    return;
+  }
+
+  if (!isRecord(event) || typeof event.message !== "string") {
+    return;
+  }
+
+  if (event.kind === "usage") {
+    lastCallMode = modeOf(event);
+    buckets[lastCallMode].calls += 1;
+
+    return;
+  }
+
+  if (event.kind === "tool") {
+    ingestTool(event);
+
+    return;
+  }
+
+  if (event.kind === "repair") {
+    ingestRepair(event);
+  }
+}
+
+function ingestFile(path: string): void {
+  lastCallMode = "unknown";
+
+  for (const line of readFileSync(path, "utf8").split("\n")) {
+    if (line.length > 0) {
+      ingestLine(line);
+    }
+  }
+}
+
+function collectLogs(): string[] {
+  const files: string[] = [];
+  const home = join(homedir(), ".tsforge", "logs");
+
+  if (existsSync(home)) {
+    for (const f of readdirSync(home)) {
+      if (f.endsWith(".jsonl")) {
+        files.push(join(home, f));
+      }
+    }
+  }
+
+  const runs = join("evals", "runs");
+
+  if (existsSync(runs)) {
+    for (const dir of readdirSync(runs)) {
+      for (const name of ["events.jsonl", "run.jsonl", "log.jsonl"]) {
+        const candidate = join(runs, dir, name);
+
+        if (existsSync(candidate)) {
+          files.push(candidate);
+        }
+      }
+    }
+  }
+
+  return files;
+}
+
+const logs = collectLogs();
+
+for (const f of logs) {
+  ingestFile(f);
+}
+
+function rate(b: IBucket): string {
+  if (b.calls === 0) {
+    return "—";
+  }
+
+  return `${(((b.salvaged + b.malformedNudges) / b.calls) * 100).toFixed(2)}%`;
+}
+
+function repairRate(b: IBucket): string {
+  const totalRepairs = Array.from(b.repairs.values()).reduce(
+    (a, c) => a + c,
+    0
+  );
+
+  if (b.calls === 0) {
+    return "—";
+  }
+
+  return `${((totalRepairs / b.calls) * 100).toFixed(2)}%`;
+}
+
+process.stdout.write(`scanned ${String(logs.length)} log file(s)\n\n`);
+
+// Salvage & malformed incidents by thinking mode
+process.stdout.write(
+  "thinking  calls  salvaged  malformed-nudges  incident-rate\n"
+);
+
+for (const mode of ["on", "off", "unknown"] as const) {
+  const b = buckets[mode];
+
+  process.stdout.write(
+    `${mode.padEnd(9)} ${String(b.calls).padStart(5)}  ${String(b.salvaged).padStart(8)}  ${String(b.malformedNudges).padStart(16)}  ${rate(b)}\n`
+  );
+}
+
+process.stdout.write(
+  "\n" +
+    "incident-rate = (salvaged + malformed-nudges) / model calls. 'unknown' =\n" +
+    "logs predating the per-call thinking flag.\n"
+);
+
+// Repair ladder statistics
+process.stdout.write("\n\nREPAIR LADDER STATISTICS:\n\n");
+process.stdout.write("thinking  calls  repairs  reasks  repair-rate\n");
+
+for (const mode of ["on", "off", "unknown"] as const) {
+  const b = buckets[mode];
+  const totalRepairs = Array.from(b.repairs.values()).reduce(
+    (a, c) => a + c,
+    0
+  );
+
+  process.stdout.write(
+    `${mode.padEnd(9)} ${String(b.calls).padStart(5)}  ${String(totalRepairs).padStart(7)}  ${String(b.reasks).padStart(5)}  ${repairRate(b)}\n`
+  );
+}
+
+// Top repair rules across all modes
+const allRules = new Map<string, number>();
+
+for (const b of Object.values(buckets)) {
+  for (const [rule, count] of b.repairs) {
+    allRules.set(rule, (allRules.get(rule) ?? 0) + count);
+  }
+}
+
+if (allRules.size > 0) {
+  process.stdout.write("\n\nTOP REPAIR RULES (across all modes):\n");
+  const sorted = Array.from(allRules.entries())
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 20);
+
+  for (const [rule, count] of sorted) {
+    process.stdout.write(`  ${String(count).padStart(5)}  ${rule}\n`);
+  }
+}
+
+process.stdout.write(
+  "\n" +
+    "repair-rate = total repairs / model calls. Re-run as new logs accumulate;\n" +
+    "track per-rule rates to identify systemic model failures worth adding L2\n" +
+    "safe-defaults for. L3 re-ask rate should trend to near-zero (recoverable\n" +
+    "repairs succeed; unrecoverable args are infrequent and addressed in prompting).\n"
+);

package/scripts/analyze-runs.ts

@@ -0,0 +1,279 @@
+// Extract luck-INDEPENDENT mechanism signals from eval run logs, so harness
+// changes can be judged by what they actually did — not by a single noisy
+// turn-count. Reads each run dir's plain-text run.log (+ result.json) and
+// tabulates, then summarizes the spread across runs.
+//
+// Run:  bun run packages/core/scripts/analyze-runs.ts money 5
+//   (analyze the latest 5 `money-*` run dirs)
+// Or:   bun run packages/core/scripts/analyze-runs.ts <dir> <dir> ...
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { isRecord } from "../src/lib/guards";
+
+const evalsRoot = join(import.meta.dir, "..", "..", "..", "evals");
+
+interface IRunMetrics {
+  runId: string;
+  passed: boolean;
+  turns: number;
+  totalSeconds: number;
+  /** Highest per-turn gate error count seen — >1 proves the combined parser
+   *  surfaced structured, per-error feedback (not one opaque blob). */
+  maxErrorsSurfaced: number;
+  /** Times the model enumerated source lines by hand to locate an error —
+   *  should be 0 since gate feedback shows the offending line. */
+  handCountingLines: number;
+  /** Most file mutations applied in a SINGLE turn — >1 means it fixed several
+   *  sites at once instead of one-per-turn. */
+  maxEditsPerTurn: number;
+  totalEdits: number;
+  /** Longest single turn (s) — usually one heavy reasoning turn; shows that
+   *  wall-time variance is the model thinking, not harness churn. */
+  slowestTurnSeconds: number;
+  /** Char volume of the heaviest turn (reasoning+content). Shows whether a
+   *  thinking_token_budget binds (drops it) and flags spirals (huge value). */
+  maxTurnChars: number;
+  /** Tool calls the harness rejected (bad input / scope / match failure) — the
+   *  open-model tool-calling friction; 0 = clean. Repaired calls excluded. */
+  toolRejects: number;
+  regressions: number;
+  quality: number | undefined;
+}
+
+const TIMING = /⏱ turn (\d+) took ([\d.]+)(s|ms) \(total ([\d.]+)(s|ms)\)/;
+const RED = /turn \d+: red \((\d+) error/;
+const ASKING = /turn (\d+): asking model/;
+// Hand-counting = the model re-typing the file with SEQUENTIAL line numbers
+// (`1: …`, `2: …`) to LOCATE an error it can't see — the costly pattern the
+// located-feedback fix removes. Deliberately excludes `Line 37:`-style citations
+// of feedback-provided lines, which are the model USING the located errors.
+const HAND_COUNT = /^\s*\d+:\s+(?:export|const|function|return|if|for|\}|\/\/)/;
+
+interface ITurnTiming {
+  turn: number;
+  tookSeconds: number | null;
+  totalSeconds: number | null;
+}
+
+/** Parse a `turn N took Xs (total Ys)` line into its seconds (ms normalized). */
+function parseTiming(line: string): ITurnTiming | null {
+  const m = TIMING.exec(line);
+
+  if (m?.[1] === undefined) {
+    return null;
+  }
+
+  const took =
+    m[2] === undefined
+      ? null
+      : m[3] === "ms"
+        ? Number(m[2]) / 1000
+        : Number(m[2]);
+  const total =
+    m[4] === undefined
+      ? null
+      : m[5] === "ms"
+        ? Number(m[4]) / 1000
+        : Number(m[4]);
+
+  return { turn: Number(m[1]), tookSeconds: took, totalSeconds: total };
+}
+
+function parseLog(
+  runId: string,
+  log: string
+): Omit<IRunMetrics, "regressions" | "quality"> {
+  const lines = log.split("\n");
+
+  let turns = 0;
+  let totalSeconds = 0;
+  let slowestTurnSeconds = 0;
+  let maxErrorsSurfaced = 0;
+  let handCountingLines = 0;
+  let totalEdits = 0;
+  let maxEditsPerTurn = 0;
+  let editsThisTurn = 0;
+  let charsThisTurn = 0;
+  let maxTurnChars = 0;
+  let toolRejects = 0;
+  const passed = /spec ".*": done/.test(log) || /· turn \d+: GREEN/.test(log);
+
+  for (const line of lines) {
+    const asking = ASKING.exec(line);
+
+    if (asking !== null) {
+      maxEditsPerTurn = Math.max(maxEditsPerTurn, editsThisTurn);
+      editsThisTurn = 0;
+      charsThisTurn = 0;
+    }
+
+    // Reasoning+content volume of the turn — how "does a thinking_token_budget
+    // bind?" shows up here (and a spiral is a huge maxTurnChars).
+    charsThisTurn += line.length;
+
+    if (line.includes("✎ edit") || line.includes("✚ create")) {
+      totalEdits += 1;
+      editsThisTurn += 1;
+    }
+
+    const timing = parseTiming(line);
+
+    if (timing !== null) {
+      turns = Math.max(turns, timing.turn);
+      maxTurnChars = Math.max(maxTurnChars, charsThisTurn);
+
+      if (timing.tookSeconds !== null) {
+        slowestTurnSeconds = Math.max(slowestTurnSeconds, timing.tookSeconds);
+      }
+
+      if (timing.totalSeconds !== null) {
+        totalSeconds = timing.totalSeconds;
+      }
+    }
+
+    const red = RED.exec(line);
+
+    if (red?.[1] !== undefined) {
+      maxErrorsSurfaced = Math.max(maxErrorsSurfaced, Number(red[1]));
+    }
+
+    if (HAND_COUNT.test(line)) {
+      handCountingLines += 1;
+    }
+
+    if (/tool_input_rejected:|tool_rejected:/.test(line)) {
+      toolRejects += 1;
+    }
+  }
+
+  maxEditsPerTurn = Math.max(maxEditsPerTurn, editsThisTurn);
+
+  return {
+    runId,
+    passed,
+    turns,
+    totalSeconds,
+    maxErrorsSurfaced,
+    handCountingLines,
+    maxEditsPerTurn,
+    totalEdits,
+    slowestTurnSeconds,
+    maxTurnChars,
+    toolRejects,
+  };
+}
+
+async function readResult(
+  dir: string
+): Promise<{ regressions: number; quality: number | undefined }> {
+  const file = Bun.file(join(dir, "result.json"));
+
+  if (!(await file.exists())) {
+    return { regressions: 0, quality: undefined };
+  }
+
+  const data: unknown = JSON.parse(await file.text());
+
+  if (!isRecord(data)) {
+    return { regressions: 0, quality: undefined };
+  }
+
+  const quality = typeof data.quality === "number" ? data.quality : undefined;
+  let regressions = 0;
+
+  if (Array.isArray(data.tasks)) {
+    for (const t of data.tasks) {
+      if (isRecord(t) && typeof t.regressions === "number") {
+        regressions += t.regressions;
+      }
+    }
+  }
+
+  return { regressions, quality };
+}
+
+async function resolveDirs(): Promise<string[]> {
+  const args = process.argv.slice(2);
+
+  // `<seed> <count>` form: latest N run dirs whose name starts with the seed.
+  if (args.length === 2 && /^\d+$/.test(args[1] ?? "")) {
+    const prefix = args[0] ?? "";
+    const count = Number(args[1]);
+    const all = await readdir(evalsRoot, { withFileTypes: true });
+    const dirs = all
+      .filter((d) => d.isDirectory() && d.name.startsWith(prefix))
+      .map((d) => d.name)
+      .sort();
+
+    return dirs.slice(-count).map((name) => join(evalsRoot, name));
+  }
+
+  return args.map((a) => (a.startsWith("/") ? a : join(evalsRoot, a)));
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+
+  if (sorted.length === 0) {
+    return 0;
+  }
+
+  return sorted.length % 2 === 0
+    ? ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2
+    : (sorted[mid] ?? 0);
+}
+
+const dirs = await resolveDirs();
+const metrics: IRunMetrics[] = [];
+
+for (const dir of dirs) {
+  const log = Bun.file(join(dir, "run.log"));
+
+  if (!(await log.exists())) {
+    continue;
+  }
+
+  const runId = dir.split("/").slice(-1)[0] ?? dir;
+  const base = parseLog(runId, await log.text());
+  const extra = await readResult(dir);
+
+  metrics.push({ ...base, ...extra });
+}
+
+process.stdout.write(`\n=== run analysis (${metrics.length} runs) ===\n\n`);
+process.stdout.write(
+  "pass  turns  time(s)  slowTurn(s)  maxTurnChars  maxErr  handCount  toolRej  maxEdits/turn  edits  regress  Q\n"
+);
+
+for (const m of metrics) {
+  process.stdout.write(
+    [
+      m.passed ? " ✓  " : " ✗  ",
+      String(m.turns).padStart(5),
+      m.totalSeconds.toFixed(0).padStart(8),
+      m.slowestTurnSeconds.toFixed(0).padStart(12),
+      String(m.maxTurnChars).padStart(13),
+      String(m.maxErrorsSurfaced).padStart(7),
+      String(m.handCountingLines).padStart(10),
+      String(m.toolRejects).padStart(8),
+      String(m.maxEditsPerTurn).padStart(14),
+      String(m.totalEdits).padStart(7),
+      String(m.regressions).padStart(8),
+      (m.quality === undefined ? "-" : String(m.quality)).padStart(3),
+      `  ${m.runId}`,
+    ].join("") + "\n"
+  );
+}
+
+const turns = metrics.map((m) => m.turns);
+const times = metrics.map((m) => m.totalSeconds);
+const passRate = metrics.filter((m) => m.passed).length;
+
+process.stdout.write(
+  `\nturns:  min ${Math.min(...turns)}  median ${median(turns)}  max ${Math.max(...turns)}  (spread ${Math.max(...turns) - Math.min(...turns)})\n`
+);
+process.stdout.write(
+  `time:   min ${Math.min(...times).toFixed(0)}s  median ${median(times).toFixed(0)}s  max ${Math.max(...times).toFixed(0)}s\n`
+);
+process.stdout.write(`pass:   ${passRate}/${metrics.length}\n`);