@mmerterden/multi-agent-pipeline 8.6.0

package/pipeline/scripts/diff-risk-score.mjs

@@ -0,0 +1,328 @@
+#!/usr/bin/env node
+
+/**
+ * @file diff-risk-score.mjs — v8.3.0
+ *
+ * Heuristic risk score per file in a diff. Deterministic, zero LLM.
+ * Output sorted by score descending so reviewer prompts can attach a
+ * "priority files" list. Advisory only — never blocks the pipeline.
+ *
+ * Signals (all sourced from `git diff --numstat` + targeted hunk grep):
+ *   loc_changed      — base sensitivity to size of change         w=1.0
+ *   security_path    — touches Auth/Keychain/Network/Crypto/...   w=3.0
+ *   public_api       — adds/changes public func/class/Composable  w=2.0
+ *   no_test_change   — source touched, test file untouched        w=2.5
+ *   complexity_delta — added if/guard/case/switch/while count     w=1.5
+ *   ui_critical      — *View.swift / *Screen.kt / Configuration   w=1.5
+ *   migration        — DB schema / migration path                 w=4.0
+ *
+ * Inputs:
+ *   --base <ref>     Base ref. Default: origin/main, fallback: main
+ *   --head <ref>     Head ref. Default: HEAD
+ *   --diff <path>    Pre-computed diff file (skip git invocation)
+ *   --root <path>    Repo root for git -C. Default: cwd
+ *   --task-id <id>   Tag the output with a task id (no functional effect)
+ *   --top <N>        Truncate `files` to top N. Default: keep all,
+ *                    consumers (reviewer prompt template) cap separately
+ *   --json           Pretty print (default: compact JSON)
+ *
+ * Output:
+ *   stdout — JSON conforming to pipeline/schemas/diff-risk.schema.json
+ *
+ * Exit codes:
+ *   0 — success
+ *   1 — git/diff invocation failed or unparseable
+ *   2 — empty diff (no files changed) — caller can choose to skip
+ *
+ * @module pipeline/scripts/diff-risk-score
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+
+const argv = process.argv.slice(2);
+const flags = {};
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i];
+  if (a.startsWith("--")) {
+    const k = a.slice(2);
+    const v = argv[i + 1];
+    if (v !== undefined && !v.startsWith("--")) {
+      flags[k] = v;
+      i++;
+    } else {
+      flags[k] = true;
+    }
+  }
+}
+
+const ROOT = flags.root || process.cwd();
+const HEAD = flags.head || "HEAD";
+let BASE = flags.base || null;
+const DIFF_FILE = flags.diff || null;
+const TASK_ID = flags["task-id"] || null;
+const TOP = flags.top ? Number(flags.top) : null;
+const PRETTY = flags.json === true;
+
+function git(args) {
+  return execFileSync("git", ["-C", ROOT, ...args], { encoding: "utf8" }).trim();
+}
+
+function resolveBase() {
+  if (BASE) return BASE;
+  for (const ref of ["origin/main", "main", "origin/master", "master", "develop", "origin/develop"]) {
+    try {
+      execFileSync("git", ["-C", ROOT, "rev-parse", "--verify", ref], { stdio: "ignore" });
+      return ref;
+    } catch { /* try next */ }
+  }
+  throw new Error("no base ref found (tried origin/main, main, master, develop)");
+}
+
+function loadDiff() {
+  if (DIFF_FILE) {
+    if (!existsSync(DIFF_FILE)) throw new Error(`diff file not found: ${DIFF_FILE}`);
+    return readFileSync(DIFF_FILE, "utf8");
+  }
+  BASE = resolveBase();
+  return git(["diff", "--unified=0", `${BASE}...${HEAD}`]);
+}
+
+function loadNumstat() {
+  if (DIFF_FILE) {
+    // Approximate from the diff text — count + and - lines per file.
+    const text = readFileSync(DIFF_FILE, "utf8");
+    const map = new Map();
+    let cur = null;
+    for (const line of text.split("\n")) {
+      const m = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+      if (m) {
+        cur = m[2];
+        if (!map.has(cur)) map.set(cur, { added: 0, removed: 0 });
+        continue;
+      }
+      if (!cur) continue;
+      if (line.startsWith("+++") || line.startsWith("---") || line.startsWith("@@")) continue;
+      if (line.startsWith("+")) map.get(cur).added++;
+      else if (line.startsWith("-")) map.get(cur).removed++;
+    }
+    return Array.from(map.entries()).map(([path, v]) => ({ path, ...v }));
+  }
+  const out = git(["diff", "--numstat", `${BASE}...${HEAD}`]);
+  if (!out) return [];
+  return out.split("\n").map((line) => {
+    const [a, r, ...rest] = line.split("\t");
+    return {
+      path: rest.join("\t"),
+      added: a === "-" ? 0 : Number(a) || 0,
+      removed: r === "-" ? 0 : Number(r) || 0,
+    };
+  });
+}
+
+const SECURITY_GLOBS = [
+  /(^|\/)(Auth|Authentication|Keychain|Credential|Secret|Token|Crypto|Cipher|Password|Login|OAuth|JWT|Session)/i,
+  /(^|\/)Networking?(?:Layer|Manager)?\.(swift|kt|ts|js|mjs|py)$/i,
+  /(^|\/)(api|http)Client\.(swift|kt|ts|js|mjs|py)$/i,
+  /\.env(\..+)?$/,
+  /SecurityHeaders|SSL|TLS|Pinning|Certificate/i,
+];
+
+const UI_CRITICAL_GLOBS = [
+  /\/[A-Z][A-Za-z0-9]*View\.swift$/,
+  /\/[A-Z][A-Za-z0-9]*Screen\.swift$/,
+  /\/[A-Z][A-Za-z0-9]*Configuration\.swift$/,
+  /\/[A-Z][A-Za-z0-9]*Screen\.kt$/,
+  /\/[A-Z][A-Za-z0-9]*Content\.kt$/,
+  /\/[A-Z][A-Za-z0-9]*Composable\.kt$/,
+];
+
+const MIGRATION_GLOBS = [
+  /\/[Mm]igrations?\//,
+  /\/migration_/,
+  /\.(sql|ddl)$/,
+  /alembic\//,
+  /db\/migrate\//,
+  /knex|prisma\/migrations/,
+];
+
+const TEST_PATH_GLOBS = [
+  /Tests?\.(swift|kt|ts|tsx|js|mjs|py)$/i,
+  /(^|\/)(__tests__|tests|test)\//i,
+  /\.test\.(ts|tsx|js|mjs|py)$/i,
+  /\.spec\.(ts|tsx|js|mjs|py)$/i,
+];
+
+function matchesAny(path, regexes) {
+  return regexes.some((r) => r.test(path));
+}
+
+function isTestPath(path) {
+  return matchesAny(path, TEST_PATH_GLOBS);
+}
+
+function pairedTestPaths(srcPath) {
+  // Return likely test path candidates for a source file.
+  const parts = srcPath.split("/");
+  const file = parts[parts.length - 1];
+  const base = file.replace(/\.(swift|kt|ts|tsx|js|mjs|py)$/, "");
+  const ext = (file.match(/\.(swift|kt|ts|tsx|js|mjs|py)$/) || ["", ""])[1];
+  return [
+    `${base}Tests.${ext}`,
+    `${base}Test.${ext}`,
+    `${base}.test.${ext}`,
+    `${base}.spec.${ext}`,
+  ];
+}
+
+function parseHunks(diffText) {
+  // Group lines per file, but ONLY the added lines (we score what's new).
+  const perFile = new Map();
+  let cur = null;
+  for (const line of diffText.split("\n")) {
+    const m = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+    if (m) {
+      cur = m[2];
+      if (!perFile.has(cur)) perFile.set(cur, []);
+      continue;
+    }
+    if (!cur) continue;
+    if (line.startsWith("+++") || line.startsWith("---") || line.startsWith("@@")) continue;
+    if (line.startsWith("+")) perFile.get(cur).push(line.slice(1));
+  }
+  return perFile;
+}
+
+function publicApiSignal(addedLines) {
+  let count = 0;
+  for (const l of addedLines) {
+    if (/\bpublic\s+(func|class|struct|protocol|enum|init|static|let|var)\b/.test(l)) count++;
+    else if (/\b@objc\b/.test(l)) count++;
+    else if (/\b@JvmStatic\b/.test(l)) count++;
+    else if (/\bopen\s+(fun|class)\b/.test(l)) count++;
+    else if (/^export\s+(function|class|const|let|interface|type|default)\b/.test(l)) count++;
+    else if (/\b@Composable\b/.test(l)) count++;
+  }
+  return count;
+}
+
+function complexitySignal(addedLines) {
+  let count = 0;
+  for (const l of addedLines) {
+    if (/^\s*(if|guard|switch|case\s|while|for|do|try)\b/.test(l)) count++;
+    else if (/\b\?\s*:/.test(l)) count++; // ternary
+    else if (/&&|\|\|/.test(l)) count++;
+  }
+  return count;
+}
+
+function buildRow(stat, addedLines, allChangedPaths) {
+  const path = stat.path;
+  const signals = [];
+  let score = 0;
+
+  const loc = stat.added + stat.removed;
+  if (loc > 0) {
+    const w = 1.0;
+    signals.push({ name: "loc_changed", weight: w, value: loc });
+    score += loc * w;
+  }
+
+  if (matchesAny(path, SECURITY_GLOBS)) {
+    const w = 3.0;
+    signals.push({ name: "security_path", weight: w, value: 1 });
+    score += 30 * w; // boolean signal — fixed contribution
+  }
+
+  if (matchesAny(path, MIGRATION_GLOBS)) {
+    const w = 4.0;
+    signals.push({ name: "migration", weight: w, value: 1 });
+    score += 40 * w;
+  }
+
+  if (matchesAny(path, UI_CRITICAL_GLOBS)) {
+    const w = 1.5;
+    signals.push({ name: "ui_critical", weight: w, value: 1 });
+    score += 15 * w;
+  }
+
+  const apiCount = publicApiSignal(addedLines);
+  if (apiCount > 0) {
+    const w = 2.0;
+    signals.push({ name: "public_api", weight: w, value: apiCount });
+    score += apiCount * 5 * w;
+  }
+
+  const cplx = complexitySignal(addedLines);
+  if (cplx > 0) {
+    const w = 1.5;
+    signals.push({ name: "complexity_delta", weight: w, value: cplx });
+    score += cplx * 2 * w;
+  }
+
+  // No-test-change: source touched + no matching test file in the changed set.
+  if (!isTestPath(path)) {
+    const candidates = pairedTestPaths(path).map((c) => c.toLowerCase());
+    const anyTest = allChangedPaths.some((p) => {
+      const low = p.toLowerCase();
+      return isTestPath(p) && candidates.some((c) => low.includes(c));
+    });
+    if (!anyTest) {
+      const w = 2.5;
+      signals.push({ name: "no_test_change", weight: w, value: 1 });
+      score += 10 * w;
+    }
+  }
+
+  return {
+    path,
+    score: Math.round(score * 100) / 100,
+    loc_added: stat.added,
+    loc_removed: stat.removed,
+    signals,
+  };
+}
+
+function main() {
+  const numstat = loadNumstat();
+  if (numstat.length === 0) {
+    process.stderr.write("diff-risk-score: empty diff\n");
+    process.exit(2);
+  }
+  const diffText = loadDiff();
+  const perFileAdded = parseHunks(diffText);
+  const allChangedPaths = numstat.map((n) => n.path);
+  const rows = numstat.map((stat) =>
+    buildRow(stat, perFileAdded.get(stat.path) || [], allChangedPaths),
+  );
+  rows.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+
+  const totals = {
+    files: rows.length,
+    loc_added: rows.reduce((s, r) => s + r.loc_added, 0),
+    loc_removed: rows.reduce((s, r) => s + r.loc_removed, 0),
+    max_score: rows.length ? rows[0].score : 0,
+  };
+
+  const out = {
+    schemaVersion: "1.0.0",
+    task: {
+      id: TASK_ID,
+      base: BASE || "(diff-file)",
+      head: HEAD,
+      ts: new Date().toISOString(),
+    },
+    totals,
+    files: TOP ? rows.slice(0, Number(TOP)) : rows,
+  };
+
+  process.stdout.write(PRETTY ? JSON.stringify(out, null, 2) + "\n" : JSON.stringify(out) + "\n");
+  process.exit(0);
+}
+
+try {
+  main();
+} catch (e) {
+  process.stderr.write(`diff-risk-score: ${e.message}\n`);
+  process.exit(1);
+}

package/pipeline/scripts/eval-golden-tasks-live.mjs

@@ -0,0 +1,294 @@
+#!/usr/bin/env node
+
+/**
+ * @file eval-golden-tasks-live.mjs — v7.8.0 Paket A
+ *
+ * Opt-in live evaluation harness for golden-task fixtures. Unlike the
+ * always-on contract harness (`eval-golden-tasks.mjs`) which only validates
+ * schema shape, this harness optionally invokes a real model via the local
+ * `claude` CLI and compares the actual output against the fixture's expected
+ * scope (stack, blocker count, deferral count).
+ *
+ * **Default behavior is dry-run** — no model is invoked unless `--live` is
+ * passed AND `MULTI_AGENT_LIVE_EVAL=1` is set in the environment. This
+ * double-gate prevents accidental cost.
+ *
+ * Cost guard:
+ *   - Per-case budget cap (default $1 USD) — case skipped if exceeded
+ *   - Run-wide max-cases cap (default 1) — even with --live, only N cases run
+ *   - Run-wide hard ceiling: budget * max_cases (default $1 total)
+ *
+ * Zero-dep: shells out to the user's installed `claude` CLI rather than the
+ * Anthropic SDK. Honors ADR-4 and uses the user's existing CLI auth.
+ *
+ * Usage:
+ *   node eval-golden-tasks-live.mjs                      # dry-run (default)
+ *   node eval-golden-tasks-live.mjs --live               # blocked unless env set
+ *   MULTI_AGENT_LIVE_EVAL=1 node eval-golden-tasks-live.mjs --live --case 01-...
+ *   node eval-golden-tasks-live.mjs --budget=0.50 --max-cases=2
+ *   node eval-golden-tasks-live.mjs --json               # machine-readable summary
+ *
+ * Exit codes:
+ *   0  all run cases produced output matching expected scope
+ *   1  one or more cases mismatched expected
+ *   2  setup/usage error
+ *   3  budget exhausted before run completed
+ *
+ * @module pipeline/scripts/eval-golden-tasks-live
+ */
+
+import { readdirSync, readFileSync, existsSync } from "node:fs";
+import { join, dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const root = resolve(here, "..", "..");
+const evalDir = join(root, "pipeline", "eval", "golden-tasks");
+
+const argv = process.argv.slice(2);
+const flags = {};
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i];
+  if (a.startsWith("--")) {
+    const [key, ...rest] = a.slice(2).split("=");
+    if (rest.length) flags[key] = rest.join("=");
+    else if (argv[i + 1] && !argv[i + 1].startsWith("--")) {
+      flags[key] = argv[i + 1];
+      i++;
+    } else flags[key] = true;
+  }
+}
+
+if (flags.help || flags.h) {
+  console.log(`Usage: eval-golden-tasks-live.mjs [options]
+
+Modes:
+  (default)             dry-run: list what would run, no model invocation
+  --live                live: call \`claude -p\` for real (requires
+                        MULTI_AGENT_LIVE_EVAL=1 in env)
+
+Selection:
+  --case <slug>         run only this case (e.g. 01-ios-bugfix-darkmode)
+  --max-cases <N>       cap total cases to run (default: 1)
+
+Cost guard:
+  --budget <usd>        per-case budget cap (default: 1.00 USD)
+  --total-budget <usd>  run-wide ceiling (default: budget * max-cases)
+
+Output:
+  --json                machine-readable JSON summary
+`);
+  process.exit(0);
+}
+
+const isLive = !!flags.live;
+const liveEnvOK = process.env.MULTI_AGENT_LIVE_EVAL === "1";
+const budget = parseFloat(flags.budget || "1.00");
+const maxCases = parseInt(flags["max-cases"] || "1", 10);
+const totalBudget = parseFloat(flags["total-budget"] || String(budget * maxCases));
+const asJson = !!flags.json;
+
+if (isLive && !liveEnvOK) {
+  process.stderr.write(
+    "eval-golden-tasks-live: --live requires MULTI_AGENT_LIVE_EVAL=1 in env (cost gate).\n",
+  );
+  process.exit(2);
+}
+
+if (!Number.isFinite(budget) || budget <= 0) {
+  process.stderr.write(`eval-golden-tasks-live: invalid --budget ${flags.budget}\n`);
+  process.exit(2);
+}
+
+function listCases() {
+  if (!existsSync(evalDir)) return [];
+  return readdirSync(evalDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory() && /^\d{2}-/.test(e.name))
+    .map((e) => e.name)
+    .sort();
+}
+
+function loadCase(slug) {
+  const caseDir = join(evalDir, slug);
+  const taskFile = join(caseDir, "task.json");
+  if (!existsSync(taskFile)) return null;
+  return {
+    slug,
+    dir: caseDir,
+    task: JSON.parse(readFileSync(taskFile, "utf-8")),
+  };
+}
+
+function hasClaudeCLI() {
+  const r = spawnSync("which", ["claude"], { encoding: "utf-8" });
+  return r.status === 0 && (r.stdout || "").trim().length > 0;
+}
+
+/**
+ * Live invocation — wraps the task as a prompt for `claude -p`. Returns the
+ * raw stdout. The current iteration captures only the analysis-phase output
+ * by asking for a JSON-only response per the analysis schema. Future work
+ * extends this to full Phase 1/2/4 multi-call evaluation.
+ */
+function invokeLive(taskJson) {
+  const prompt = [
+    "Perform Phase 1 (Analysis) of the multi-agent-pipeline for the task below.",
+    "Return ONLY the analysis JSON (matching pipeline/schemas/analysis-output.schema.json).",
+    "Do not include explanatory prose, markdown fences, or commentary.",
+    "",
+    "Task:",
+    JSON.stringify(taskJson, null, 2),
+  ].join("\n");
+
+  const r = spawnSync("claude", ["-p", prompt], {
+    encoding: "utf-8",
+    timeout: 5 * 60 * 1000,
+  });
+  if (r.status !== 0) {
+    return { ok: false, error: `claude exit ${r.status}: ${r.stderr || ""}` };
+  }
+  return { ok: true, raw: r.stdout || "" };
+}
+
+function tryParseJson(s) {
+  try {
+    return { ok: true, value: JSON.parse(s) };
+  } catch (e) {
+    // Try to extract first {...} block
+    const m = /\{[\s\S]*\}/.exec(s);
+    if (m) {
+      try {
+        return { ok: true, value: JSON.parse(m[0]) };
+      } catch {
+        /* fallthrough */
+      }
+    }
+    return { ok: false, error: e.message };
+  }
+}
+
+function evaluateOne(c) {
+  const result = {
+    case: c.slug,
+    mode: isLive ? "live" : "dry-run",
+    expected: {
+      stack: c.task.expectedStack,
+      blockers: c.task.expectedBlockers,
+      deferrals: c.task.expectedDeferrals,
+    },
+    actual: null,
+    pass: null,
+    notes: [],
+  };
+
+  if (!isLive) {
+    result.pass = "skipped-dry-run";
+    result.notes.push("would invoke claude -p with task prompt");
+    return result;
+  }
+
+  if (!hasClaudeCLI()) {
+    result.pass = "skipped-no-cli";
+    result.notes.push("claude CLI not on PATH");
+    return result;
+  }
+
+  const live = invokeLive(c.task);
+  if (!live.ok) {
+    result.pass = false;
+    result.notes.push(`live invocation failed: ${live.error}`);
+    return result;
+  }
+
+  const parsed = tryParseJson(live.raw);
+  if (!parsed.ok) {
+    result.pass = false;
+    result.notes.push(`could not parse model output as JSON: ${parsed.error}`);
+    return result;
+  }
+
+  const actual = parsed.value;
+  result.actual = {
+    stack: actual?.stack?.primary ?? actual?.stack ?? null,
+  };
+  if (actual?.stack?.primary === c.task.expectedStack) {
+    result.pass = true;
+    result.notes.push(`stack match: ${actual.stack.primary}`);
+  } else {
+    result.pass = false;
+    result.notes.push(
+      `stack mismatch: expected ${c.task.expectedStack}, got ${result.actual.stack}`,
+    );
+  }
+  return result;
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+const allCases = listCases();
+let selected = allCases;
+if (flags.case) {
+  selected = selected.filter((s) => s === flags.case);
+  if (selected.length === 0) {
+    process.stderr.write(`eval-golden-tasks-live: case not found: ${flags.case}\n`);
+    process.exit(2);
+  }
+}
+selected = selected.slice(0, maxCases);
+
+const results = [];
+let projectedSpend = 0;
+
+for (const slug of selected) {
+  const c = loadCase(slug);
+  if (!c) {
+    results.push({ case: slug, pass: false, notes: ["fixture missing"] });
+    continue;
+  }
+  if (projectedSpend + budget > totalBudget) {
+    results.push({
+      case: slug,
+      pass: "skipped-budget-exhausted",
+      notes: [`would exceed total-budget ${totalBudget} (already projected ${projectedSpend})`],
+    });
+    continue;
+  }
+  const r = evaluateOne(c);
+  if (r.pass !== "skipped-dry-run" && r.pass !== "skipped-no-cli") {
+    projectedSpend += budget;
+  }
+  results.push(r);
+}
+
+const summary = {
+  mode: isLive ? "live" : "dry-run",
+  cases_total: allCases.length,
+  cases_run: results.length,
+  cases_passed: results.filter((r) => r.pass === true).length,
+  cases_failed: results.filter((r) => r.pass === false).length,
+  cases_skipped: results.filter((r) => typeof r.pass === "string" && r.pass.startsWith("skipped")).length,
+  budget_per_case: budget,
+  total_budget: totalBudget,
+  projected_spend: projectedSpend,
+};
+
+if (asJson) {
+  console.log(JSON.stringify({ summary, results }, null, 2));
+} else {
+  console.log("");
+  console.log(`eval-golden-tasks-live (${summary.mode})`);
+  console.log(`  cases: ${summary.cases_run}/${summary.cases_total} run, ${summary.cases_passed} passed, ${summary.cases_failed} failed, ${summary.cases_skipped} skipped`);
+  console.log(`  budget: $${budget.toFixed(2)}/case · total cap $${totalBudget.toFixed(2)} · projected spend $${projectedSpend.toFixed(2)}`);
+  console.log("");
+  for (const r of results) {
+    const icon = r.pass === true ? "✓" : r.pass === false ? "✗" : "·";
+    console.log(`  ${icon} ${r.case} [${r.pass}]`);
+    for (const n of r.notes) console.log(`      ${n}`);
+  }
+  console.log("");
+}
+
+const hasFailure = results.some((r) => r.pass === false);
+const budgetExhausted = results.some((r) => r.pass === "skipped-budget-exhausted");
+
+process.exit(hasFailure ? 1 : budgetExhausted ? 3 : 0);