@mmerterden/multi-agent-pipeline 8.6.0

package/pipeline/scripts/test-gap-scan.mjs

@@ -0,0 +1,343 @@
+#!/usr/bin/env node
+
+/**
+ * @file test-gap-scan.mjs — v8.3.0
+ *
+ * Static test-gap detector for Phase 5. Walks the diff between base and head
+ * to find newly added/changed public symbols, then checks whether the diff
+ * (or the repo tree, when --scan-tree is set) contains a paired test file
+ * with a test method covering the symbol.
+ *
+ * Heuristic, deterministic, no LLM. Advisory by default; consumers can flip
+ * to blocking via prefs.testGap.blockingThreshold.
+ *
+ * Stack rules live in `pipeline/scripts/test-gap-rules/<stack>.json`.
+ *
+ * Inputs:
+ *   --base <ref>       Base ref. Default: origin/main, fallback: main
+ *   --head <ref>       Head ref. Default: HEAD
+ *   --diff <path>      Pre-computed diff file (skip git invocation)
+ *   --root <path>      Repo root for git -C and tree scan. Default: cwd
+ *   --stack <id>       ios | android | python | node. Required.
+ *   --scan-tree        When set, also scan repo for existing tests by symbol
+ *                      name (slower; default off — diff-only is sufficient
+ *                      for the common case).
+ *   --json             Pretty-print
+ *   --severity-promote Make every gap "important" instead of "suggestion".
+ *                      Used by Phase 5 when audit-mode is on.
+ *
+ * Output:
+ *   stdout — JSON conforming to pipeline/schemas/test-gap.schema.json
+ *
+ * Exit codes:
+ *   0 — success (gap report rendered, possibly empty)
+ *   1 — git invocation or rule-load error
+ *   2 — empty diff (no changed sources for this stack)
+ *  64 — usage error
+ *
+ * @module pipeline/scripts/test-gap-scan
+ */
+
+import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join, dirname, extname, basename } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const argv = process.argv.slice(2);
+const flags = {};
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i];
+  if (a.startsWith("--")) {
+    const k = a.slice(2);
+    const v = argv[i + 1];
+    if (v !== undefined && !v.startsWith("--")) {
+      flags[k] = v;
+      i++;
+    } else {
+      flags[k] = true;
+    }
+  }
+}
+
+const ROOT = flags.root || process.cwd();
+const HEAD = flags.head || "HEAD";
+let BASE = flags.base || null;
+const DIFF_FILE = flags.diff || null;
+const STACK = flags.stack;
+const SCAN_TREE = flags["scan-tree"] === true;
+const PRETTY = flags.json === true;
+const PROMOTE = flags["severity-promote"] === true;
+
+if (!STACK) {
+  process.stderr.write("test-gap-scan: --stack <ios|android|python|node> required\n");
+  process.exit(64);
+}
+
+const RULES_PATH = join(__dirname, "test-gap-rules", `${STACK}.json`);
+let rules;
+try {
+  rules = JSON.parse(readFileSync(RULES_PATH, "utf8"));
+} catch (e) {
+  process.stderr.write(`test-gap-scan: cannot load rules ${RULES_PATH}: ${e.message}\n`);
+  process.exit(1);
+}
+
+function git(args) {
+  return execFileSync("git", ["-C", ROOT, ...args], { encoding: "utf8" }).trim();
+}
+
+function resolveBase() {
+  if (BASE) return BASE;
+  for (const ref of ["origin/main", "main", "origin/master", "master", "develop", "origin/develop"]) {
+    try {
+      execFileSync("git", ["-C", ROOT, "rev-parse", "--verify", ref], { stdio: "ignore" });
+      return ref;
+    } catch { /* continue */ }
+  }
+  throw new Error("no base ref found");
+}
+
+function loadDiff() {
+  if (DIFF_FILE) {
+    if (!existsSync(DIFF_FILE)) throw new Error(`diff file not found: ${DIFF_FILE}`);
+    return readFileSync(DIFF_FILE, "utf8");
+  }
+  BASE = resolveBase();
+  return git(["diff", "--unified=0", `${BASE}...${HEAD}`]);
+}
+
+function isExcluded(path) {
+  if (!Array.isArray(rules.excludePathGlobs)) return false;
+  for (const pat of rules.excludePathGlobs) {
+    if (pat.endsWith("/")) {
+      if (path.includes(pat) || path.startsWith(pat)) return true;
+    } else if (pat.includes("**")) {
+      const re = new RegExp("^" + pat.replace(/\./g, "\\.").replace(/\*\*/g, ".*").replace(/\*/g, "[^/]*") + "$");
+      if (re.test(path)) return true;
+    } else if (path.endsWith(pat)) return true;
+  }
+  return false;
+}
+
+function isTestPath(path) {
+  if (Array.isArray(rules.testPathSuffixes) && rules.testPathSuffixes.some((s) => path.endsWith(s))) return true;
+  if (Array.isArray(rules.testPathPrefixes) && rules.testPathPrefixes.some((p) => basename(path).startsWith(p))) return true;
+  if (Array.isArray(rules.testPathContains) && rules.testPathContains.some((c) => path.includes(c))) return true;
+  return false;
+}
+
+function isSourcePath(path) {
+  if (isTestPath(path)) return false;
+  if (isExcluded(path)) return false;
+  if (!Array.isArray(rules.sourceExtensions)) return false;
+  return rules.sourceExtensions.some((ext) => path.endsWith(ext));
+}
+
+function expectedTestPathsFor(srcPath) {
+  // Generic: append common test suffixes alongside the source.
+  const dir = dirname(srcPath);
+  const ext = extname(srcPath);
+  const base = basename(srcPath, ext);
+  const candidates = [];
+  if (Array.isArray(rules.testPathSuffixes)) {
+    for (const suf of rules.testPathSuffixes) {
+      if (suf.startsWith(".")) {
+        candidates.push(`${dir}/${base}${suf}`);
+      } else {
+        // e.g. "Tests.swift" → ButtonTests.swift, ButtonTest.swift
+        candidates.push(`${dir}/${base.replace(ext, "")}${suf}`);
+      }
+    }
+  }
+  if (Array.isArray(rules.testPathPrefixes)) {
+    for (const pre of rules.testPathPrefixes) {
+      candidates.push(`${dir}/${pre}${base}${ext}`);
+    }
+  }
+  // De-dupe
+  return Array.from(new Set(candidates));
+}
+
+function parseDiffAddedLines(diffText) {
+  const perFile = new Map();
+  let cur = null;
+  let isAdditionFile = false;
+  for (const line of diffText.split("\n")) {
+    const m = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+    if (m) {
+      cur = m[2];
+      isAdditionFile = false;
+      if (!perFile.has(cur)) perFile.set(cur, { lines: [], isNew: false });
+      continue;
+    }
+    if (!cur) continue;
+    if (line.startsWith("new file mode")) {
+      perFile.get(cur).isNew = true;
+      continue;
+    }
+    if (line.startsWith("+++") || line.startsWith("---") || line.startsWith("@@")) continue;
+    if (line.startsWith("+")) perFile.get(cur).lines.push(line.slice(1));
+  }
+  return perFile;
+}
+
+function extractSymbols(addedLines) {
+  const out = [];
+  const allPats = [
+    ...(rules.publicApiPatterns || []),
+    ...(rules.swiftUiViewPatterns || []),
+    ...(rules.composablePatterns || []),
+  ];
+  for (const line of addedLines) {
+    for (const p of allPats) {
+      const re = new RegExp(p.regex);
+      const m = line.match(re);
+      if (m) {
+        const sym = (m[1] || p.id).trim();
+        out.push({ symbol: sym, kind: p.id });
+      }
+    }
+  }
+  // De-dup by (kind, symbol)
+  const seen = new Set();
+  return out.filter((s) => {
+    const key = s.kind + "::" + s.symbol;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function scanTreeForTests() {
+  // Returns Map<symbolLower, paths[]> for tests already present in the tree.
+  // Limited to top 8 dirs to avoid wandering into node_modules; we rely on
+  // rules.testPathContains / suffixes to narrow.
+  const hits = new Map();
+  if (!SCAN_TREE) return hits;
+  const visit = (dir) => {
+    let entries;
+    try { entries = readdirSync(dir, { withFileTypes: true }); }
+    catch { return; }
+    for (const ent of entries) {
+      const full = join(dir, ent.name);
+      const rel = full.replace(ROOT + "/", "");
+      if (isExcluded(rel)) continue;
+      if (ent.isDirectory()) {
+        if (ent.name.startsWith(".") || ent.name === "node_modules") continue;
+        visit(full);
+      } else if (ent.isFile() && isTestPath(rel)) {
+        let body = "";
+        try { body = readFileSync(full, "utf8"); } catch { continue; }
+        // crude symbol mention: any Identifier-ish token. Cap body size.
+        const slice = body.slice(0, 400_000);
+        const tokens = slice.match(/[A-Za-z_][A-Za-z0-9_]{2,}/g) || [];
+        for (const t of new Set(tokens.map((x) => x.toLowerCase()))) {
+          if (!hits.has(t)) hits.set(t, []);
+          hits.get(t).push(rel);
+        }
+      }
+    }
+  };
+  visit(ROOT);
+  return hits;
+}
+
+function severityFor(symbol, kind) {
+  if (PROMOTE) return "important";
+  // Public API additions without tests are important; pure boolean exports
+  // (e.g. const FLAG = true) tend to be suggestion-tier.
+  if (kind === "objc_export" || kind === "interface" || kind === "public_proto") return "important";
+  if (kind.startsWith("public_") || kind === "open_class" || kind === "open_fun" || kind === "named_export" || kind === "default_export") return "suggestion";
+  if (kind === "view_struct" || kind === "config_struct") return "important";
+  if (kind === "composable_fun") return "important";
+  if (kind === "top_def" || kind === "top_class" || kind === "async_def") return "suggestion";
+  return "suggestion";
+}
+
+function symbolMentionedInTest(diffText, symbol, srcPath) {
+  // Was the symbol mentioned in any *added* test line (whole-word)?
+  const re = new RegExp("\\b" + symbol.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&") + "\\b");
+  const lines = diffText.split("\n");
+  let inTestFile = false;
+  for (const line of lines) {
+    const fm = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+    if (fm) { inTestFile = isTestPath(fm[2]); continue; }
+    if (!inTestFile) continue;
+    if (!line.startsWith("+")) continue;
+    if (re.test(line)) return true;
+  }
+  return false;
+}
+
+function main() {
+  const diffText = loadDiff();
+  const perFile = parseDiffAddedLines(diffText);
+  if (perFile.size === 0) {
+    process.stderr.write("test-gap-scan: empty diff\n");
+    process.exit(2);
+  }
+
+  const sources = [];
+  const tests = [];
+  for (const [path, meta] of perFile.entries()) {
+    if (isTestPath(path)) tests.push({ path, ...meta });
+    else if (isSourcePath(path)) sources.push({ path, ...meta });
+  }
+  if (sources.length === 0) {
+    process.stderr.write(`test-gap-scan: no ${STACK} sources changed in this diff\n`);
+    process.exit(2);
+  }
+
+  const treeIndex = scanTreeForTests();
+
+  const gaps = [];
+  for (const src of sources) {
+    const symbols = extractSymbols(src.lines);
+    if (symbols.length === 0) continue;
+    for (const { symbol, kind } of symbols) {
+      // Mentioned in any added test line?
+      if (symbolMentionedInTest(diffText, symbol, src.path)) continue;
+      // Tree scan fallback (when --scan-tree)
+      const lower = symbol.toLowerCase();
+      if (SCAN_TREE && treeIndex.has(lower)) continue;
+      gaps.push({
+        sourcePath: src.path,
+        symbol,
+        kind,
+        severity: severityFor(symbol, kind),
+        expectedTestPaths: expectedTestPathsFor(src.path),
+        hint: rules.hint || `Add a test that exercises ${symbol}.`,
+      });
+    }
+  }
+
+  const totals = {
+    sourcesScanned: sources.length,
+    testsScanned: tests.length,
+    gapCount: gaps.length,
+    gapBySeverity: {
+      blocking:   gaps.filter((g) => g.severity === "blocking").length,
+      important:  gaps.filter((g) => g.severity === "important").length,
+      suggestion: gaps.filter((g) => g.severity === "suggestion").length,
+    },
+  };
+
+  const out = {
+    schemaVersion: "1.0.0",
+    stack: STACK,
+    totals,
+    gaps,
+  };
+
+  process.stdout.write(PRETTY ? JSON.stringify(out, null, 2) + "\n" : JSON.stringify(out) + "\n");
+  process.exit(0);
+}
+
+try {
+  main();
+} catch (e) {
+  process.stderr.write(`test-gap-scan: ${e.message}\n`);
+  process.exit(1);
+}

package/pipeline/scripts/token-budget-report.mjs

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+// token-budget-report.mjs — aggregates token + cost metrics from metrics.jsonl,
+// alerts on budget breach per token-budget.json.
+//
+// Usage:
+//   node token-budget-report.mjs                              # last 24h
+//   node token-budget-report.mjs --period weekly              # last 7d
+//   node token-budget-report.mjs --period monthly             # last 30d
+//   node token-budget-report.mjs --task PROJ-1234             # one task
+//   node token-budget-report.mjs --json                       # machine-readable
+//
+// Reads:
+//   $METRICS_FILE  (default $HOME/.claude/logs/multi-agent/metrics.jsonl)
+//   $BUDGET_FILE   (default $HOME/.claude/configs/token-budget.json — optional)
+//
+// Writes:
+//   stdout (text or json depending on --json)
+//   appends a snapshot row to $HOME/.claude/logs/multi-agent/metrics-summary.md
+//   when invoked with --append
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const args = parseArgs(process.argv.slice(2));
+const METRICS = process.env.METRICS_FILE
+  || path.join(os.homedir(), ".claude/logs/multi-agent/metrics.jsonl");
+const BUDGET = process.env.BUDGET_FILE
+  || path.join(os.homedir(), ".claude/configs/token-budget.json");
+const SUMMARY = path.join(os.homedir(), ".claude/logs/multi-agent/metrics-summary.md");
+
+const PERIOD_DAYS = { daily: 1, weekly: 7, monthly: 30 };
+const periodDays = PERIOD_DAYS[args.period] || 1;
+const cutoffMs = Date.now() - periodDays * 24 * 3600 * 1000;
+
+if (!fs.existsSync(METRICS)) {
+  console.error(`metrics file not found: ${METRICS}`);
+  console.error("(no telemetry has been emitted yet — run a task first)");
+  process.exit(args.json ? 0 : 2);
+}
+
+const lines = fs.readFileSync(METRICS, "utf8").split("\n").filter(Boolean);
+const events = lines
+  .map(l => { try { return JSON.parse(l); } catch { return null; } })
+  .filter(e => e && e.ts)
+  .filter(e => Date.parse(e.ts) >= cutoffMs)
+  .filter(e => !args.task || e.task_id === args.task);
+
+// Per-task aggregation
+const byTask = new Map();
+for (const e of events) {
+  if (!byTask.has(e.task_id)) byTask.set(e.task_id, {
+    tokens_in: 0, tokens_out: 0, premium_calls: 0, phases: new Set(), models: new Set()
+  });
+  const t = byTask.get(e.task_id);
+  t.phases.add(e.phase);
+  const d = e.details || {};
+  if (typeof d.tokens_in === "number") t.tokens_in += d.tokens_in;
+  if (typeof d.tokens_out === "number") t.tokens_out += d.tokens_out;
+  if (d.model) t.models.add(d.model);
+  if (d.premium === true || /opus|gpt-/i.test(String(d.model || ""))) t.premium_calls += 1;
+}
+
+// Budget (best-effort)
+let budget = {};
+if (fs.existsSync(BUDGET)) {
+  try { budget = JSON.parse(fs.readFileSync(BUDGET, "utf8")); }
+  catch { /* tolerate malformed */ }
+}
+const limits = {
+  perTaskTokens: budget.perTaskTokens || 250000,
+  perTaskTokensMultiRepo: budget.perTaskTokensMultiRepo || 500000,
+  perTaskUsd: budget.perTaskUsd || 2.50,
+  perTaskUsdMultiRepo: budget.perTaskUsdMultiRepo || 5.00,
+};
+// Crude USD estimate: opus ~$15/M in, $75/M out; sonnet ~$3/M, $15/M out.
+function estimateUsd(t) {
+  const inK = t.tokens_in / 1e6;
+  const outK = t.tokens_out / 1e6;
+  const opusShare = t.models.has("opus") ? 0.5 : 0; // rough
+  return (inK * (15 * opusShare + 3 * (1 - opusShare))) + (outK * (75 * opusShare + 15 * (1 - opusShare)));
+}
+
+const summary = {
+  period: args.period || "daily",
+  windowStart: new Date(cutoffMs).toISOString(),
+  windowEnd: new Date().toISOString(),
+  taskCount: byTask.size,
+  totalTokensIn: 0,
+  totalTokensOut: 0,
+  totalUsd: 0,
+  breaches: [],
+};
+for (const [taskId, t] of byTask) {
+  const usd = estimateUsd(t);
+  summary.totalTokensIn += t.tokens_in;
+  summary.totalTokensOut += t.tokens_out;
+  summary.totalUsd += usd;
+  const total = t.tokens_in + t.tokens_out;
+  const isMulti = t.phases.has("4") && t.premium_calls > 5; // heuristic
+  const tokenLimit = isMulti ? limits.perTaskTokensMultiRepo : limits.perTaskTokens;
+  const usdLimit = isMulti ? limits.perTaskUsdMultiRepo : limits.perTaskUsd;
+  if (total > tokenLimit) summary.breaches.push({ taskId, kind: "tokens", actual: total, limit: tokenLimit });
+  if (usd > usdLimit)    summary.breaches.push({ taskId, kind: "usd",    actual: usd.toFixed(2), limit: usdLimit });
+}
+
+if (args.json) {
+  console.log(JSON.stringify(summary, null, 2));
+} else {
+  console.log(`Token & cost report — ${summary.period} (${summary.windowStart} → ${summary.windowEnd})`);
+  console.log(`  tasks:        ${summary.taskCount}`);
+  console.log(`  tokens in:    ${summary.totalTokensIn.toLocaleString()}`);
+  console.log(`  tokens out:   ${summary.totalTokensOut.toLocaleString()}`);
+  console.log(`  est. cost:    $${summary.totalUsd.toFixed(2)} (rough)`);
+  if (summary.breaches.length) {
+    console.log(`  ⚠ breaches:`);
+    for (const b of summary.breaches) {
+      console.log(`    - ${b.taskId} ${b.kind} ${b.actual} > ${b.limit}`);
+    }
+  } else {
+    console.log(`  ✓ no budget breaches`);
+  }
+}
+
+if (args.append) {
+  const row = `| ${summary.windowEnd.slice(0,10)} | ${summary.period} | ${summary.taskCount} | ${summary.totalTokensIn} | ${summary.totalTokensOut} | $${summary.totalUsd.toFixed(2)} | ${summary.breaches.length} |`;
+  if (!fs.existsSync(SUMMARY)) {
+    fs.mkdirSync(path.dirname(SUMMARY), { recursive: true });
+    fs.writeFileSync(SUMMARY, "# Multi-Agent Pipeline Metrics Summary\n\n| date | period | tasks | tokens_in | tokens_out | est_usd | breaches |\n|---|---|---|---|---|---|---|\n");
+  }
+  fs.appendFileSync(SUMMARY, row + "\n");
+}
+
+process.exit(summary.breaches.length > 0 ? 1 : 0);
+
+function parseArgs(argv) {
+  const out = { period: "daily", json: false, append: false, task: null };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === "--period") out.period = argv[++i];
+    else if (argv[i] === "--json") out.json = true;
+    else if (argv[i] === "--append") out.append = true;
+    else if (argv[i] === "--task") out.task = argv[++i];
+  }
+  return out;
+}