agent-gauntlet 1.1.0 → 1.2.0

package/dist/index.js

@@ -299,7 +299,7 @@ import { Command } from "commander";
 // package.json
 var package_default = {
   name: "agent-gauntlet",
-  version: "1.1.0",
+  version: "1.2.0",
   description: "A CLI tool for testing AI coding agents",
   license: "MIT",
   author: "Paul Caplan",
@@ -8454,14 +8454,10 @@ var SKILLS_SOURCE_DIR = (() => {
     throw err;
   }
 })();
-var SKILL_ACTIONS = ["run", "check", "status", "help", "setup"];
-var SKILL_DESCRIPTIONS = {
-  run: "Run the verification suite",
-  check: "Run checks only (no reviews)",
-  status: "Show gauntlet status",
-  help: "Diagnose and explain gauntlet behavior",
-  setup: "Configure checks and reviews interactively"
-};
+async function getSkillDirNames() {
+  const entries = await fs32.readdir(SKILLS_SOURCE_DIR, { withFileTypes: true });
+  return entries.filter((e) => e.isDirectory()).map((e) => e.name).sort();
+}
 var CLI_PREFERENCE_ORDER = [
   "codex",
   "claude",
@@ -8520,7 +8516,7 @@ async function runInit(options) {
   }
   await installExternalFiles(projectRoot, hookAdapters, skipPrompts);
   await addToGitignore(projectRoot, "gauntlet_logs");
-  printPostInitInstructions(instructionCLINames);
+  await printPostInitInstructions(instructionCLINames);
 }
 function printNoCLIsMessage() {
   console.log();
@@ -8559,8 +8555,7 @@ async function copyDirRecursive(opts) {
 }
 async function installSkillsWithChecksums(projectRoot, skipPrompts) {
   const skillsDir = path30.join(projectRoot, ".claude", "skills");
-  for (const action of SKILL_ACTIONS) {
-    const dirName = `gauntlet-${action}`;
+  for (const dirName of await getSkillDirNames()) {
     const sourceDir = path30.join(SKILLS_SOURCE_DIR, dirName);
     const targetDir = path30.join(skillsDir, dirName);
     const relativeDir = `${path30.relative(projectRoot, targetDir)}/`;
@@ -8585,7 +8580,7 @@ async function installExternalFiles(projectRoot, devAdapters, skipPrompts) {
   await installSkillsWithChecksums(projectRoot, skipPrompts);
   await installHooksForAdapters(projectRoot, devAdapters, skipPrompts);
 }
-function printPostInitInstructions(devCLINames) {
+async function printPostInitInstructions(devCLINames) {
   const hasNative = devCLINames.some((name) => NATIVE_CLIS.has(name));
   const nonNativeNames = devCLINames.filter((name) => !NATIVE_CLIS.has(name));
   const hasNonNative = nonNativeNames.length > 0;
@@ -8597,8 +8592,8 @@ function printPostInitInstructions(devCLINames) {
     console.log(chalk11.bold("To complete setup, reference the setup skill in your CLI: @.claude/skills/gauntlet-setup/SKILL.md. This will guide you through configuring the static checks (unit tests, linters, etc.) that Agent Gauntlet will run."));
     console.log();
     console.log("Available skills:");
-    for (const action of SKILL_ACTIONS) {
-      console.log(`  @.claude/skills/gauntlet-${action}/SKILL.md — ${SKILL_DESCRIPTIONS[action]}`);
+    for (const dirName of await getSkillDirNames()) {
+      console.log(`  @.claude/skills/${dirName}/SKILL.md`);
     }
   }
 }
@@ -8775,9 +8770,361 @@ function registerReviewCommand(program) {
     enableReviews: new Set(options.enableReview ?? [])
   }));
 }
-// src/core/run-executor-lock.ts
-import fs33 from "node:fs/promises";
+// src/scripts/review-audit.ts
+import fs33 from "node:fs";
 import path31 from "node:path";
+import readline from "node:readline";
+function parseKeyValue2(text) {
+  const result = {};
+  for (const [, key, value] of text.matchAll(/(\w+)=(\S+)/g))
+    if (key && value)
+      result[key] = value;
+  return result;
+}
+var parseTimestamp2 = (line) => line.match(/^\[([^\]]+)\]/)?.[1] ?? "";
+var parseEventType2 = (line) => line.match(/^\[[^\]]+\]\s+(\S+)/)?.[1] ?? "";
+var parseEventBody2 = (line) => line.match(/^\[[^\]]+\]\s+\S+\s*(.*)/)?.[1] ?? "";
+var safeNum = (v) => {
+  const n = Number(v ?? 0);
+  return Number.isNaN(n) ? 0 : n;
+};
+var parseDuration = (d) => {
+  const m = d.match(/^([\d.]+)(ms|s|m)?$/);
+  const val = safeNum(m?.[1]);
+  if (m?.[2] === "ms")
+    return val / 1000;
+  if (m?.[2] === "m")
+    return val * 60;
+  return val;
+};
+function getLogDir3(cwd) {
+  const configPath = path31.join(cwd, ".gauntlet", "config.yml");
+  try {
+    const content = fs33.readFileSync(configPath, "utf-8");
+    const match = content.match(/^log_dir:\s*(.+)$/m);
+    if (match?.[1])
+      return match[1].trim();
+  } catch {}
+  return "gauntlet_logs";
+}
+function handleRunStart(ts, body) {
+  const kv = parseKeyValue2(body);
+  return {
+    timestamp: ts,
+    mode: kv.mode ?? "unknown",
+    linesAdded: safeNum(kv.lines_added),
+    linesRemoved: safeNum(kv.lines_removed),
+    reviewGates: [],
+    priorPassSkips: 0,
+    telemetry: []
+  };
+}
+function handleGateResult(current, body) {
+  const gateIdMatch = body.match(/^(\S+)/);
+  const gateId = gateIdMatch?.[1] ?? "";
+  if (!gateId.startsWith("review:"))
+    return;
+  const kv = parseKeyValue2(body);
+  if (kv.cli) {
+    current.reviewGates.push({
+      reviewType: gateId.split(":").at(-1) ?? "other",
+      cli: kv.cli,
+      durationS: parseDuration(kv.duration ?? "0s"),
+      violations: safeNum(kv.violations)
+    });
+  } else {
+    current.priorPassSkips++;
+  }
+}
+function handleTelemetry(current, body) {
+  const kv = parseKeyValue2(body);
+  if (!kv.adapter)
+    return;
+  current.telemetry.push({
+    adapter: kv.adapter,
+    inTokens: safeNum(kv.in),
+    cacheTokens: safeNum(kv.cache),
+    outTokens: safeNum(kv.out),
+    thoughtTokens: safeNum(kv.thought),
+    toolTokens: safeNum(kv.tool),
+    apiRequests: safeNum(kv.api_requests),
+    cacheRead: safeNum(kv.cacheRead),
+    cacheWrite: safeNum(kv.cacheWrite)
+  });
+}
+function handleRunEnd(current, body) {
+  const kv = parseKeyValue2(body);
+  current.end = {
+    status: kv.status ?? "unknown",
+    fixed: safeNum(kv.fixed),
+    skipped: safeNum(kv.skipped),
+    failed: safeNum(kv.failed)
+  };
+}
+var emptyStat = () => ({
+  count: 0,
+  totalDuration: 0,
+  totalViolations: 0
+});
+function addGate(s, g) {
+  s.count++;
+  s.totalDuration += g.durationS;
+  s.totalViolations += g.violations;
+}
+function getOrCreate(map, key, init) {
+  if (!map.has(key))
+    map.set(key, init());
+  return map.get(key);
+}
+var REVIEW_TYPES = ["code-quality", "task-compliance", "artifact-review"];
+function accumulateBlock(block, a) {
+  for (const g of block.reviewGates) {
+    const inner = getOrCreate(a.cells, g.reviewType, () => new Map);
+    addGate(getOrCreate(inner, g.cli, emptyStat), g);
+    addGate(getOrCreate(a.cliTotals, g.cli, emptyStat), g);
+    addGate(getOrCreate(a.typeTotals, g.reviewType, emptyStat), g);
+    addGate(a.grandTotal, g);
+  }
+  const diff = block.linesAdded + block.linesRemoved;
+  if (diff <= 0)
+    return;
+  for (const cli of new Set(block.reviewGates.map((g) => g.cli))) {
+    const dur = block.reviewGates.filter((g) => g.cli === cli).reduce((s, g) => s + g.durationS, 0);
+    const p = getOrCreate(a.per100, cli, () => ({ dur: 0, diff: 0 }));
+    p.dur += dur;
+    p.diff += diff;
+  }
+}
+function buildCrossTab(blocks) {
+  const a = {
+    cells: new Map,
+    cliTotals: new Map,
+    typeTotals: new Map,
+    per100: new Map,
+    grandTotal: emptyStat()
+  };
+  for (const block of blocks)
+    accumulateBlock(block, a);
+  const allTypes = [
+    ...REVIEW_TYPES.filter((t) => a.typeTotals.has(t)),
+    ...[...a.typeTotals.keys()].filter((t) => !REVIEW_TYPES.includes(t))
+  ];
+  return { ...a, allTypes, allClis: [...a.cliTotals.keys()] };
+}
+var emptyTokenStats = (adapter) => ({
+  adapter,
+  inTokens: 0,
+  cacheTokens: 0,
+  outTokens: 0,
+  thoughtTokens: 0,
+  toolTokens: 0,
+  apiRequests: 0,
+  cacheRead: 0,
+  cacheWrite: 0,
+  runsWithTelemetry: 0
+});
+function accumulateTelemetryEntry(t, statsMap) {
+  const s = statsMap.get(t.adapter) ?? emptyTokenStats(t.adapter);
+  statsMap.set(t.adapter, s);
+  s.inTokens += t.inTokens;
+  s.cacheTokens += t.cacheTokens;
+  s.outTokens += t.outTokens;
+  s.thoughtTokens += t.thoughtTokens;
+  s.toolTokens += t.toolTokens;
+  s.apiRequests += t.apiRequests;
+  s.cacheRead += t.cacheRead;
+  s.cacheWrite += t.cacheWrite;
+}
+function aggregateTokenStats(blocks) {
+  const statsMap = new Map;
+  for (const block of blocks) {
+    const adaptersInBlock = new Set(block.telemetry.map((t) => t.adapter));
+    for (const t of block.telemetry)
+      accumulateTelemetryEntry(t, statsMap);
+    for (const adapter of adaptersInBlock) {
+      const s = statsMap.get(adapter);
+      if (s)
+        s.runsWithTelemetry++;
+    }
+  }
+  return Array.from(statsMap.values());
+}
+var formatNum = (n) => n.toLocaleString("en-US");
+var padRight = (s, w) => s + " ".repeat(Math.max(0, w - s.length));
+var padLeft = (s, w) => " ".repeat(Math.max(0, w - s.length)) + s;
+var capitalize = (s) => s.charAt(0).toUpperCase() + s.slice(1);
+var fmtType = (t) => t.split("-").map(capitalize).join("-");
+function formatCrossTable(title, rowLabels, colLabels, cell, rowTotal, colTotal, grandTotal) {
+  const rlW = Math.max(17, ...rowLabels.map((r) => fmtType(r).length)) + 2;
+  const cW = 12;
+  const hdr = padRight("", rlW) + colLabels.map((c) => padRight(capitalize(c), cW)).join("") + "Total";
+  const rows = rowLabels.map((r) => padRight(fmtType(r), rlW) + colLabels.map((c) => padRight(cell(r, c), cW)).join("") + rowTotal(r));
+  const totalRow = padRight("Total", rlW) + colLabels.map((c) => padRight(colTotal(c), cW)).join("") + grandTotal;
+  return [title, hdr, ...rows, totalRow, ""];
+}
+function formatRunCounts(ct) {
+  return formatCrossTable("=== Run Counts ===", ct.allTypes, ct.allClis, (t, c) => String(ct.cells.get(t)?.get(c)?.count ?? 0), (t) => String(ct.typeTotals.get(t)?.count ?? 0), (c) => String(ct.cliTotals.get(c)?.count ?? 0), String(ct.grandTotal.count));
+}
+function formatTiming(ct) {
+  const avg = (s) => s && s.count > 0 ? `${(s.totalDuration / s.count).toFixed(1)}s` : "n/a";
+  const lines = formatCrossTable("=== Timing ===", ct.allTypes, ct.allClis, (t, c) => avg(ct.cells.get(t)?.get(c)), (t) => avg(ct.typeTotals.get(t)), (c) => avg(ct.cliTotals.get(c)), avg(ct.grandTotal));
+  const p100parts = ct.allClis.map((c) => {
+    const p = ct.per100.get(c);
+    return p && p.diff > 0 ? `${c}=${(p.dur / p.diff * 100).toFixed(1)}s` : "";
+  }).filter(Boolean);
+  if (p100parts.length > 0)
+    lines.splice(lines.length - 1, 0, `Per 100 diff lines (excl. zero-diff): ${p100parts.join("  ")}`);
+  return lines;
+}
+function formatViolations(ct) {
+  const avg = (s) => s && s.count > 0 ? (s.totalViolations / s.count).toFixed(2) : "n/a";
+  return formatCrossTable("=== Violations (avg per run) ===", ct.allTypes, ct.allClis, (t, c) => avg(ct.cells.get(t)?.get(c)), (t) => avg(ct.typeTotals.get(t)), (c) => avg(ct.cliTotals.get(c)), avg(ct.grandTotal));
+}
+function formatTokenEntry(t, totalRuns) {
+  const out = [
+    `${capitalize(t.adapter)} (${t.runsWithTelemetry} of ${totalRuns} runs had telemetry):`
+  ];
+  if (t.inTokens > 0 || t.cacheTokens > 0) {
+    const total = t.inTokens + t.cacheTokens;
+    out.push(`  Input:        ${padLeft(formatNum(total), 12)}   (non-cached: ${formatNum(t.inTokens)} | cached: ${formatNum(t.cacheTokens)})`);
+  }
+  if (t.outTokens > 0)
+    out.push(`  Output:       ${padLeft(formatNum(t.outTokens), 12)}`);
+  if (t.thoughtTokens > 0)
+    out.push(`  Thinking:     ${padLeft(formatNum(t.thoughtTokens), 12)}`);
+  if (t.toolTokens > 0)
+    out.push(`  Tool tokens:  ${padLeft(formatNum(t.toolTokens), 12)}`);
+  if (t.cacheRead > 0 || t.cacheWrite > 0) {
+    out.push(`  Cache reads:  ${padLeft(formatNum(t.cacheRead), 12)}`);
+    out.push(`  Cache writes: ${padLeft(formatNum(t.cacheWrite), 12)}`);
+  }
+  if (t.apiRequests > 0) {
+    const avg = t.runsWithTelemetry > 0 ? (t.apiRequests / t.runsWithTelemetry).toFixed(1) : "?";
+    out.push(`  API requests: ${padLeft(formatNum(t.apiRequests), 12)}    (avg ${avg}/run)`);
+  }
+  out.push("");
+  return out;
+}
+function formatTokenUsage(ts, m) {
+  if (ts.length === 0)
+    return ["=== Token Usage ===", "No telemetry data found.", ""];
+  return [
+    "=== Token Usage ===",
+    ...ts.flatMap((t) => formatTokenEntry(t, m.get(t.adapter) ?? t.runsWithTelemetry))
+  ];
+}
+function formatFixSkip(blocks) {
+  const withEnd = blocks.filter((b) => b.end);
+  const total = withEnd.length;
+  const passed = withEnd.filter((b) => b.end?.status === "pass").length;
+  const fixed = withEnd.reduce((s, b) => s + (b.end?.fixed ?? 0), 0);
+  const skipped = withEnd.reduce((s, b) => s + (b.end?.skipped ?? 0), 0);
+  const failed = withEnd.reduce((s, b) => s + (b.end?.failed ?? 0), 0);
+  const priorPass = blocks.reduce((s, b) => s + b.priorPassSkips, 0);
+  const lines = [
+    "=== Fix / Skip ===",
+    `Gauntlet runs: ${total} total  (${passed} passed, ${total - passed} failed)`,
+    `  Violations fixed:   ${fixed}`,
+    `  Violations skipped: ${skipped}`,
+    `  Gates failed:       ${failed}`,
+    `  Review gates skipped (prior pass): ${priorPass}`
+  ];
+  const totalFixedSkipped = fixed + skipped;
+  if (totalFixedSkipped > 0) {
+    const fp = (fixed / totalFixedSkipped * 100).toFixed(1);
+    const sp = (skipped / totalFixedSkipped * 100).toFixed(1);
+    lines.push(`  (fixed: ${fp}% | skipped: ${sp}% of fixed+skipped)`);
+  }
+  return lines;
+}
+function formatAuditReport(blocks, date) {
+  if (blocks.length === 0)
+    return `Review Execution Audit — ${date}
+
+No gauntlet runs found for this date.`;
+  const ct = buildCrossTab(blocks);
+  const tokenStats = aggregateTokenStats(blocks);
+  const cliBlockCount = new Map;
+  for (const block of blocks) {
+    for (const cli of new Set(block.reviewGates.map((g) => g.cli)))
+      cliBlockCount.set(cli, (cliBlockCount.get(cli) ?? 0) + 1);
+  }
+  return [
+    `Review Execution Audit — ${date}`,
+    "",
+    ...formatRunCounts(ct),
+    ...formatTiming(ct),
+    ...formatViolations(ct),
+    ...formatTokenUsage(tokenStats, cliBlockCount),
+    ...formatFixSkip(blocks)
+  ].join(`
+`);
+}
+function todayLocalDate() {
+  const now = new Date;
+  const y = now.getFullYear();
+  const mo = String(now.getMonth() + 1).padStart(2, "0");
+  const d = String(now.getDate()).padStart(2, "0");
+  return `${y}-${mo}-${d}`;
+}
+async function readBlocks(filePath, date) {
+  const rl = readline.createInterface({
+    input: fs33.createReadStream(filePath)
+  });
+  const blocks = [];
+  let current = null;
+  for await (const line of rl) {
+    if (!line.trim())
+      continue;
+    const ts = parseTimestamp2(line);
+    if (!ts.startsWith(date))
+      continue;
+    const event = parseEventType2(line);
+    const body = parseEventBody2(line);
+    if (event === "RUN_START") {
+      current = handleRunStart(ts, body);
+      blocks.push(current);
+      continue;
+    }
+    if (!current)
+      continue;
+    if (event === "GATE_RESULT")
+      handleGateResult(current, body);
+    else if (event === "TELEMETRY")
+      handleTelemetry(current, body);
+    else if (event === "RUN_END")
+      handleRunEnd(current, body);
+  }
+  return blocks;
+}
+async function main2(date) {
+  const cwd = process.cwd();
+  if (date && !/^\d{4}-\d{2}-\d{2}$/.test(date)) {
+    console.error("Invalid --date. Expected YYYY-MM-DD");
+    process.exit(1);
+  }
+  const targetDate = date ?? todayLocalDate();
+  const debugLogPath = path31.join(cwd, getLogDir3(cwd), ".debug.log");
+  if (!fs33.existsSync(debugLogPath)) {
+    console.log(`No debug log found. (looked in ${getLogDir3(cwd)}/.debug.log)`);
+    process.exit(0);
+  }
+  const blocks = await readBlocks(debugLogPath, targetDate);
+  console.log(formatAuditReport(blocks, targetDate));
+}
+var isDirectRun2 = (import.meta.url === `file://${process.argv[1]}` || typeof Bun !== "undefined" && import.meta.url === `file://${Bun.main}`) && (process.argv[1]?.endsWith("review-audit.ts") || process.argv[1]?.endsWith("review-audit.js"));
+if (isDirectRun2)
+  main2();
+
+// src/commands/review-audit.ts
+function registerReviewAuditCommand(program) {
+  program.command("review-audit").description("Audit review execution for a given date from the debug log").option("--date <YYYY-MM-DD>", "Date to filter (default: today)").action(async (opts) => {
+    await main2(opts.date);
+  });
+}
+// src/core/run-executor-lock.ts
+import fs34 from "node:fs/promises";
+import path32 from "node:path";
 var LOCK_FILENAME2 = ".gauntlet-run.lock";
 var STALE_LOCK_MS = 10 * 60 * 1000;
 function isProcessAlive(pid) {
@@ -8793,9 +9140,9 @@ function isProcessAlive(pid) {
 }
 async function isLockStale(lockPath) {
   try {
-    const lockContent = await fs33.readFile(lockPath, "utf-8");
+    const lockContent = await fs34.readFile(lockPath, "utf-8");
     const lockPid = Number.parseInt(lockContent.trim(), 10);
-    const lockStat = await fs33.stat(lockPath);
+    const lockStat = await fs34.stat(lockPath);
     const lockAgeMs = Date.now() - lockStat.mtimeMs;
     const pidValid = !Number.isNaN(lockPid);
     if (pidValid && !isProcessAlive(lockPid)) {
@@ -8810,10 +9157,10 @@ async function isLockStale(lockPath) {
   }
 }
 async function tryAcquireLock(logDir) {
-  await fs33.mkdir(logDir, { recursive: true });
-  const lockPath = path31.resolve(logDir, LOCK_FILENAME2);
+  await fs34.mkdir(logDir, { recursive: true });
+  const lockPath = path32.resolve(logDir, LOCK_FILENAME2);
   try {
-    await fs33.writeFile(lockPath, String(process.pid), { flag: "wx" });
+    await fs34.writeFile(lockPath, String(process.pid), { flag: "wx" });
     return true;
   } catch (err) {
     const isExist = typeof err === "object" && err !== null && "code" in err && err.code === "EEXIST";
@@ -8824,9 +9171,9 @@ async function tryAcquireLock(logDir) {
     if (!stale) {
       return false;
     }
-    await fs33.rm(lockPath, { force: true });
+    await fs34.rm(lockPath, { force: true });
     try {
-      await fs33.writeFile(lockPath, String(process.pid), { flag: "wx" });
+      await fs34.writeFile(lockPath, String(process.pid), { flag: "wx" });
       return true;
     } catch {
       return false;
@@ -8835,7 +9182,7 @@ async function tryAcquireLock(logDir) {
 }
 async function findLatestConsoleLog(logDir) {
   try {
-    const files = await fs33.readdir(logDir);
+    const files = await fs34.readdir(logDir);
     let maxNum = -1;
     let latestFile = null;
     for (const file of files) {
@@ -8851,7 +9198,7 @@ async function findLatestConsoleLog(logDir) {
         }
       }
     }
-    return latestFile ? path31.join(logDir, latestFile) : null;
+    return latestFile ? path32.join(logDir, latestFile) : null;
   } catch {
     return null;
   }
@@ -9253,8 +9600,8 @@ function registerSkipCommand(program) {
   });
 }
 // src/commands/start-hook.ts
-import fs34 from "node:fs/promises";
-import path32 from "node:path";
+import fs35 from "node:fs/promises";
+import path33 from "node:path";
 import YAML8 from "yaml";
 var START_HOOK_MESSAGE = `<IMPORTANT>
 This project uses Agent Gauntlet for automated quality verification.
@@ -9302,9 +9649,9 @@ function isValidConfig(content) {
 }
 function registerStartHookCommand(program) {
   program.command("start-hook").description("Session start hook - primes agent with gauntlet verification instructions").option("--adapter <adapter>", "Output format: claude or cursor", "claude").action(async (options) => {
-    const configPath = path32.join(process.cwd(), ".gauntlet", "config.yml");
+    const configPath = path33.join(process.cwd(), ".gauntlet", "config.yml");
     try {
-      const content = await fs34.readFile(configPath, "utf-8");
+      const content = await fs35.readFile(configPath, "utf-8");
       if (!isValidConfig(content)) {
         return;
       }
@@ -9314,7 +9661,7 @@ function registerStartHookCommand(program) {
     const adapter = options.adapter;
     try {
       const cwd = process.cwd();
-      const logDir = path32.join(cwd, await getLogDir2(cwd));
+      const logDir = path33.join(cwd, await getLogDir2(cwd));
       const globalConfig = await loadGlobalConfig();
       const projectDebugLogConfig = await getDebugLogConfig(cwd);
       const debugLogConfig = mergeDebugLogConfig(projectDebugLogConfig, globalConfig.debug_log);
@@ -9353,6 +9700,7 @@ registerRunCommand(program);
 registerCheckCommand(program);
 registerCICommand(program);
 registerCleanCommand(program);
+registerReviewAuditCommand(program);
 registerReviewCommand(program);
 registerDetectCommand(program);
 registerListCommand(program);
@@ -9369,4 +9717,4 @@ if (process.argv.length < 3) {
 }
 program.parse(process.argv);
 
-//# debugId=9CAFDAA7414B382764756E2164756E21
+//# debugId=CAFA132ED75F0B9564756E2164756E21