@mmerterden/multi-agent-pipeline 12.7.0 → 12.9.0

package/pipeline/scripts/eval-golden-tasks-live.mjs

@@ -1,302 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @file eval-golden-tasks-live.mjs  -  v7.8.0 Paket A
- *
- * Opt-in live evaluation harness for golden-task fixtures. Unlike the
- * always-on contract harness (`eval-golden-tasks.mjs`) which only validates
- * schema shape, this harness optionally invokes a real model via the local
- * `claude` CLI and compares the actual output against the fixture's expected
- * scope (stack, blocker count, deferral count).
- *
- * **Default behavior is dry-run**  -  no model is invoked unless `--live` is
- * passed AND `MULTI_AGENT_LIVE_EVAL=1` is set in the environment. This
- * double-gate prevents accidental cost.
- *
- * Cost guard:
- *   - Per-case budget cap (default $1 USD)  -  case skipped if exceeded
- *   - Run-wide max-cases cap (default 1)  -  even with --live, only N cases run
- *   - Run-wide hard ceiling: budget * max_cases (default $1 total)
- *
- * Zero-dep: shells out to the user's installed `claude` CLI rather than the
- * Anthropic SDK. Honors ADR-4 and uses the user's existing CLI auth.
- *
- * Usage:
- *   node eval-golden-tasks-live.mjs                      # dry-run (default)
- *   node eval-golden-tasks-live.mjs --live               # blocked unless env set
- *   MULTI_AGENT_LIVE_EVAL=1 node eval-golden-tasks-live.mjs --live --case 01-...
- *   node eval-golden-tasks-live.mjs --budget=0.50 --max-cases=2
- *   node eval-golden-tasks-live.mjs --json               # machine-readable summary
- *
- * Exit codes:
- *   0  all run cases produced output matching expected scope
- *   1  one or more cases mismatched expected
- *   2  setup/usage error
- *   3  budget exhausted before run completed
- *
- * @module pipeline/scripts/eval-golden-tasks-live
- */
-
-import { readdirSync, readFileSync, existsSync } from "node:fs";
-import { join, dirname, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { spawnSync } from "node:child_process";
-
-const here = dirname(fileURLToPath(import.meta.url));
-const root = resolve(here, "..", "..");
-const evalDir = join(root, "pipeline", "eval", "golden-tasks");
-
-const argv = process.argv.slice(2);
-const flags = {};
-for (let i = 0; i < argv.length; i++) {
-  const a = argv[i];
-  if (a.startsWith("--")) {
-    const [key, ...rest] = a.slice(2).split("=");
-    if (rest.length) flags[key] = rest.join("=");
-    else if (argv[i + 1] && !argv[i + 1].startsWith("--")) {
-      flags[key] = argv[i + 1];
-      i++;
-    } else flags[key] = true;
-  }
-}
-
-if (flags.help || flags.h) {
-  console.log(`Usage: eval-golden-tasks-live.mjs [options]
-
-Modes:
-  (default)             dry-run: list what would run, no model invocation
-  --live                live: call \`claude -p\` for real (requires
-                        MULTI_AGENT_LIVE_EVAL=1 in env)
-
-Selection:
-  --case <slug>         run only this case (e.g. 01-ios-bugfix-darkmode)
-  --max-cases <N>       cap total cases to run (default: 1)
-
-Cost guard:
-  --budget <usd>        per-case budget cap (default: 1.00 USD)
-  --total-budget <usd>  run-wide ceiling (default: budget * max-cases)
-
-Output:
-  --json                machine-readable JSON summary
-`);
-  process.exit(0);
-}
-
-const isLive = !!flags.live;
-const liveEnvOK = process.env.MULTI_AGENT_LIVE_EVAL === "1";
-const budget = parseFloat(flags.budget || "1.00");
-const maxCases = parseInt(flags["max-cases"] || "1", 10);
-const totalBudget = parseFloat(flags["total-budget"] || String(budget * maxCases));
-const asJson = !!flags.json;
-
-if (isLive && !liveEnvOK) {
-  process.stderr.write(
-    "eval-golden-tasks-live: --live requires MULTI_AGENT_LIVE_EVAL=1 in env (cost gate).\n",
-  );
-  process.exit(2);
-}
-
-if (!Number.isFinite(budget) || budget <= 0) {
-  process.stderr.write(`eval-golden-tasks-live: invalid --budget ${flags.budget}\n`);
-  process.exit(2);
-}
-
-function listCases() {
-  if (!existsSync(evalDir)) return [];
-  return readdirSync(evalDir, { withFileTypes: true })
-    .filter((e) => e.isDirectory() && /^\d{2}-/.test(e.name))
-    .map((e) => e.name)
-    .sort();
-}
-
-function loadCase(slug) {
-  const caseDir = join(evalDir, slug);
-  const taskFile = join(caseDir, "task.json");
-  if (!existsSync(taskFile)) return null;
-  let task;
-  try {
-    task = JSON.parse(readFileSync(taskFile, "utf-8"));
-  } catch (e) {
-    process.stderr.write(`[eval-live] malformed task.json for "${slug}": ${e.message}\n`);
-    return null;
-  }
-  return { slug, dir: caseDir, task };
-}
-
-function hasClaudeCLI() {
-  const r = spawnSync("which", ["claude"], { encoding: "utf-8" });
-  return r.status === 0 && (r.stdout || "").trim().length > 0;
-}
-
-/**
- * Live invocation  -  wraps the task as a prompt for `claude -p`. Returns the
- * raw stdout. The current iteration captures only the analysis-phase output
- * by asking for a JSON-only response per the analysis schema. Future work
- * extends this to full Phase 1/2/4 multi-call evaluation.
- */
-function invokeLive(taskJson) {
-  const prompt = [
-    "Perform Phase 1 (Analysis) of the multi-agent-pipeline for the task below.",
-    "Return ONLY the analysis JSON (matching pipeline/schemas/analysis-output.schema.json).",
-    "Do not include explanatory prose, markdown fences, or commentary.",
-    "",
-    "Task:",
-    JSON.stringify(taskJson, null, 2),
-  ].join("\n");
-
-  const r = spawnSync("claude", ["-p", prompt], {
-    encoding: "utf-8",
-    timeout: 5 * 60 * 1000,
-  });
-  if (r.status !== 0) {
-    return { ok: false, error: `claude exit ${r.status}: ${r.stderr || ""}` };
-  }
-  return { ok: true, raw: r.stdout || "" };
-}
-
-function tryParseJson(s) {
-  try {
-    return { ok: true, value: JSON.parse(s) };
-  } catch (e) {
-    // Try to extract first {...} block
-    const m = /\{[\s\S]*\}/.exec(s);
-    if (m) {
-      try {
-        return { ok: true, value: JSON.parse(m[0]) };
-      } catch {
-        /* fallthrough */
-      }
-    }
-    return { ok: false, error: e.message };
-  }
-}
-
-function evaluateOne(c) {
-  const result = {
-    case: c.slug,
-    mode: isLive ? "live" : "dry-run",
-    expected: {
-      stack: c.task.expectedStack,
-      blockers: c.task.expectedBlockers,
-      deferrals: c.task.expectedDeferrals,
-    },
-    actual: null,
-    pass: null,
-    notes: [],
-  };
-
-  if (!isLive) {
-    result.pass = "skipped-dry-run";
-    result.notes.push("would invoke claude -p with task prompt");
-    return result;
-  }
-
-  if (!hasClaudeCLI()) {
-    result.pass = "skipped-no-cli";
-    result.notes.push("claude CLI not on PATH");
-    return result;
-  }
-
-  const live = invokeLive(c.task);
-  if (!live.ok) {
-    result.pass = false;
-    result.notes.push(`live invocation failed: ${live.error}`);
-    return result;
-  }
-
-  const parsed = tryParseJson(live.raw);
-  if (!parsed.ok) {
-    result.pass = false;
-    result.notes.push(`could not parse model output as JSON: ${parsed.error}`);
-    return result;
-  }
-
-  const actual = parsed.value;
-  result.actual = {
-    stack: actual?.stack?.primary ?? actual?.stack ?? null,
-  };
-  if (actual?.stack?.primary === c.task.expectedStack) {
-    result.pass = true;
-    result.notes.push(`stack match: ${actual.stack.primary}`);
-  } else {
-    result.pass = false;
-    result.notes.push(
-      `stack mismatch: expected ${c.task.expectedStack}, got ${result.actual.stack}`,
-    );
-  }
-  return result;
-}
-
-// ─────────────────────────────────────────────────────────────────────────
-const allCases = listCases();
-let selected = allCases;
-if (flags.case) {
-  selected = selected.filter((s) => s === flags.case);
-  if (selected.length === 0) {
-    process.stderr.write(`eval-golden-tasks-live: case not found: ${flags.case}\n`);
-    process.exit(2);
-  }
-}
-selected = selected.slice(0, maxCases);
-
-const results = [];
-let projectedSpend = 0;
-
-for (const slug of selected) {
-  const c = loadCase(slug);
-  if (!c) {
-    results.push({ case: slug, pass: false, notes: ["fixture missing"] });
-    continue;
-  }
-  if (projectedSpend + budget > totalBudget) {
-    results.push({
-      case: slug,
-      pass: "skipped-budget-exhausted",
-      notes: [`would exceed total-budget ${totalBudget} (already projected ${projectedSpend})`],
-    });
-    continue;
-  }
-  const r = evaluateOne(c);
-  if (r.pass !== "skipped-dry-run" && r.pass !== "skipped-no-cli") {
-    projectedSpend += budget;
-  }
-  results.push(r);
-}
-
-const summary = {
-  mode: isLive ? "live" : "dry-run",
-  cases_total: allCases.length,
-  cases_run: results.length,
-  cases_passed: results.filter((r) => r.pass === true).length,
-  cases_failed: results.filter((r) => r.pass === false).length,
-  cases_skipped: results.filter((r) => typeof r.pass === "string" && r.pass.startsWith("skipped"))
-    .length,
-  budget_per_case: budget,
-  total_budget: totalBudget,
-  projected_spend: projectedSpend,
-};
-
-if (asJson) {
-  console.log(JSON.stringify({ summary, results }, null, 2));
-} else {
-  console.log("");
-  console.log(`eval-golden-tasks-live (${summary.mode})`);
-  console.log(
-    `  cases: ${summary.cases_run}/${summary.cases_total} run, ${summary.cases_passed} passed, ${summary.cases_failed} failed, ${summary.cases_skipped} skipped`,
-  );
-  console.log(
-    `  budget: $${budget.toFixed(2)}/case · total cap $${totalBudget.toFixed(2)} · projected spend $${projectedSpend.toFixed(2)}`,
-  );
-  console.log("");
-  for (const r of results) {
-    const icon = r.pass === true ? "✓" : r.pass === false ? "✗" : "·";
-    console.log(`  ${icon} ${r.case} [${r.pass}]`);
-    for (const n of r.notes) console.log(`      ${n}`);
-  }
-  console.log("");
-}
-
-const hasFailure = results.some((r) => r.pass === false);
-const budgetExhausted = results.some((r) => r.pass === "skipped-budget-exhausted");
-
-process.exit(hasFailure ? 1 : budgetExhausted ? 3 : 0);

package/pipeline/scripts/eval-golden-tasks.mjs

@@ -1,224 +0,0 @@
-#!/usr/bin/env node
-// eval-golden-tasks.mjs  -  contract regression check for whole-pipeline fixtures.
-//
-// Walks pipeline/eval/golden-tasks/<NN>-<name>/ subdirectories. For each case:
-//   1. Reads task.json (input) + expected/phase-{1,2,4-review,4-triage}.json.
-//   2. Validates each expected/ file against its paired validator script
-//      (validate-analysis.mjs / validate-planning.mjs / validate-reviewer.mjs /
-//      validate-triage.mjs).
-//   3. Cross-file consistency: Phase 2 tasks[].files[] ⊆ Phase 1
-//      touchedAreas[].path. Phase 4 review findings[].file referenced in
-//      Phase 2 planned files (or flagged as deferred in triage).
-//   4. Stack match: phase-1.stack.primary === task.expectedStack.
-//   5. Blocker count match: triage.accepted[].filter(severity=blocking).length
-//      === task.expectedBlockers. Same for deferred bucket.
-//
-// What this catches:
-//   - Schema drift (any of the 4 phase schemas changes in a breaking way).
-//   - Fixture-vs-spec divergence (someone updates a schema without re-syncing
-//     the golden fixtures).
-//   - Coverage gap (a triage that loses findings vs. reviewer input).
-//
-// What this does NOT catch:
-//   - Real model output quality. This is a CONTRACT test, not a model eval.
-//     A separate multi-model harness (with API keys, non-deterministic) is
-//     future work. See pipeline/eval/golden-tasks/README.md.
-//
-// Usage:
-//   node pipeline/scripts/eval-golden-tasks.mjs                # all cases
-//   node pipeline/scripts/eval-golden-tasks.mjs --case 01-...  # one case
-//   node pipeline/scripts/eval-golden-tasks.mjs --json         # CI-friendly
-//
-// Exit codes: 0 all pass, 1 one or more fail, 2 usage/setup error.
-
-import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { spawnSync } from "node:child_process";
-
-const here = dirname(fileURLToPath(import.meta.url));
-const root = resolve(here, "..", "..");
-const evalDir = join(root, "pipeline", "eval", "golden-tasks");
-
-const validators = {
-  analysis: join(root, "pipeline", "scripts", "validate-analysis.mjs"),
-  planning: join(root, "pipeline", "scripts", "validate-planning.mjs"),
-  reviewer: join(root, "pipeline", "scripts", "validate-reviewer.mjs"),
-  triage: join(root, "pipeline", "scripts", "validate-triage.mjs"),
-};
-
-const args = process.argv.slice(2);
-const opts = { json: false, case: null };
-for (let i = 0; i < args.length; i++) {
-  if (args[i] === "--json") opts.json = true;
-  else if (args[i] === "--case") opts.case = args[++i];
-  else if (args[i] === "-h" || args[i] === "--help") {
-    console.log("Usage: eval-golden-tasks.mjs [--case <name>] [--json]");
-    process.exit(0);
-  }
-}
-
-function listCases() {
-  if (!existsSync(evalDir)) return [];
-  return readdirSync(evalDir)
-    .filter((f) => f !== "README.md" && statSync(join(evalDir, f)).isDirectory())
-    .sort();
-}
-
-function runValidator(scriptPath, payloadPath) {
-  // validate-reviewer.mjs reads stdin with `-`; others take a path.
-  // For simplicity, all validators accept either stdin or file path. Pass path.
-  const r = spawnSync("node", [scriptPath, payloadPath], { encoding: "utf-8" });
-  return { code: r.status, stdout: r.stdout || "", stderr: r.stderr || "" };
-}
-
-function runCase(name) {
-  const dir = join(evalDir, name);
-  const errors = [];
-
-  const taskPath = join(dir, "task.json");
-  const analysisPath = join(dir, "expected", "phase-1-analysis.json");
-  const planPath = join(dir, "expected", "phase-2-plan.json");
-  const reviewPath = join(dir, "expected", "phase-4-review.json");
-  const triagePath = join(dir, "expected", "phase-4-triage.json");
-
-  for (const [label, p] of [
-    ["task.json", taskPath],
-    ["phase-1-analysis.json", analysisPath],
-    ["phase-2-plan.json", planPath],
-    ["phase-4-review.json", reviewPath],
-    ["phase-4-triage.json", triagePath],
-  ]) {
-    if (!existsSync(p)) {
-      errors.push(`missing ${label}`);
-    }
-  }
-  if (errors.length) return { name, ok: false, errors };
-
-  let task, analysis, plan, review, triage;
-  try {
-    task = JSON.parse(readFileSync(taskPath, "utf-8"));
-    analysis = JSON.parse(readFileSync(analysisPath, "utf-8"));
-    plan = JSON.parse(readFileSync(planPath, "utf-8"));
-    review = JSON.parse(readFileSync(reviewPath, "utf-8"));
-    triage = JSON.parse(readFileSync(triagePath, "utf-8"));
-  } catch (e) {
-    return { name, ok: false, errors: [`JSON parse: ${e.message}`] };
-  }
-
-  // --- schema checks ---
-  const a = runValidator(validators.analysis, analysisPath);
-  if (a.code !== 0)
-    errors.push(`phase-1-analysis fails validate-analysis (exit ${a.code}): ${a.stderr.trim()}`);
-
-  const p = runValidator(validators.planning, planPath);
-  if (p.code !== 0)
-    errors.push(`phase-2-plan fails validate-planning (exit ${p.code}): ${p.stderr.trim()}`);
-
-  // phase-4-review is an array (one entry per reviewer)  -  validate each
-  if (!Array.isArray(review)) {
-    errors.push("phase-4-review.json must be a JSON array (one entry per reviewer)");
-  } else {
-    review.forEach((entry, idx) => {
-      // write temp file for each reviewer entry
-      const tmpPayload = JSON.stringify(entry);
-      const r = spawnSync("node", [validators.reviewer, "-"], {
-        input: tmpPayload,
-        encoding: "utf-8",
-      });
-      if (r.status !== 0)
-        errors.push(
-          `phase-4-review[${idx}] fails validate-reviewer (exit ${r.status}): ${(r.stderr || "").trim()}`,
-        );
-    });
-  }
-
-  const t = runValidator(validators.triage, triagePath);
-  // validate-triage exit codes: 0 valid+clean, 2 contradiction, 3 correction.
-  // For fixtures we accept 0 only.
-  if (t.code !== 0)
-    errors.push(`phase-4-triage fails validate-triage (exit ${t.code}): ${t.stderr.trim()}`);
-
-  // --- cross-file consistency ---
-
-  // 1. Stack match
-  if (analysis.stack?.primary !== task.expectedStack) {
-    errors.push(
-      `stack mismatch: task.expectedStack=${task.expectedStack}, phase-1.stack.primary=${analysis.stack?.primary}`,
-    );
-  }
-
-  // 2. plan files ⊆ analysis touchedAreas paths
-  const touched = new Set((analysis.touchedAreas || []).map((a) => a.path));
-  for (const task of plan.tasks || []) {
-    for (const f of task.files || []) {
-      if (!touched.has(f)) {
-        errors.push(`plan task ${task.id} references file not in Phase 1 touchedAreas: ${f}`);
-      }
-    }
-  }
-
-  // 3. every reviewer finding traces to a planned file OR appears in triage
-  //    (as accepted / deferred / rejected).
-  const plannedFiles = new Set();
-  for (const task of plan.tasks || []) for (const f of task.files || []) plannedFiles.add(f);
-
-  const triageAll = [
-    ...(triage.accepted || []).map((x) => x),
-    ...(triage.deferred || []).map((x) => x.finding),
-    ...(triage.rejected || []).map((x) => x.finding),
-  ].filter((f) => f && typeof f === "object");
-  const triageKeys = new Set(triageAll.map((f) => `${f.file}::${f.line}::${f.issue}`));
-
-  for (const rev of Array.isArray(review) ? review : []) {
-    for (const f of rev.findings || []) {
-      const key = `${f.file}::${f.line}::${f.issue}`;
-      const inPlan = plannedFiles.has(f.file);
-      const inTriage = triageKeys.has(key);
-      if (!inPlan && !inTriage) {
-        errors.push(`review finding not in plan+triage: ${key}`);
-      }
-    }
-  }
-
-  // 4. blocker + deferral counts match task.expected*
-  const acceptedBlockers = (triage.accepted || []).filter((x) => x.severity === "blocking").length;
-  if (typeof task.expectedBlockers === "number" && acceptedBlockers !== task.expectedBlockers) {
-    errors.push(
-      `expectedBlockers=${task.expectedBlockers}, triage.accepted blocking count=${acceptedBlockers}`,
-    );
-  }
-  const deferredCount = (triage.deferred || []).length;
-  if (typeof task.expectedDeferrals === "number" && deferredCount !== task.expectedDeferrals) {
-    errors.push(
-      `expectedDeferrals=${task.expectedDeferrals}, triage.deferred count=${deferredCount}`,
-    );
-  }
-
-  return { name, ok: errors.length === 0, errors };
-}
-
-// --- main ---
-
-const cases = opts.case ? [opts.case] : listCases();
-if (cases.length === 0) {
-  console.error("No golden-task fixtures found under pipeline/eval/golden-tasks/");
-  process.exit(2);
-}
-
-const results = cases.map(runCase);
-const passed = results.filter((r) => r.ok).length;
-const failed = results.length - passed;
-
-if (opts.json) {
-  console.log(JSON.stringify({ passed, failed, results }, null, 2));
-} else {
-  for (const r of results) {
-    const mark = r.ok ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m";
-    console.log(`${mark} ${r.name.padEnd(38)} ${r.ok ? "" : "FAIL"}`);
-    if (!r.ok) for (const e of r.errors) console.log(`    - ${e}`);
-  }
-  console.log(`\n══ eval-golden-tasks: ${passed}/${results.length} passed ══`);
-}
-
-process.exit(failed > 0 ? 1 : 0);

package/pipeline/scripts/eval-intent.mjs

@@ -1,107 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @file eval-intent.mjs  -  measured accuracy for the Phase 0 intent guard.
- *
- * Runs every labeled case in pipeline/eval/intent-cases.json through
- * pipeline/lib/classify-intent.sh and reports accuracy. This turns the
- * classifier from "a heuristic we hope works" into a number with a regression
- * gate: a dangerous miss is a clear task read as `question` (work skipped) or a
- * clear question read as `task` (a worktree spun up for nothing). `ambiguous`
- * is the safe middle and is only correct when the case labels it so.
- *
- * Exit 0 if accuracy >= threshold (default 0.95), else 1. Override with
- * --min <0..1>. Prints a JSON summary + the mismatches.
- *
- * @module pipeline/scripts/eval-intent
- */
-
-import { readFileSync } from "node:fs";
-import { execFileSync } from "node:child_process";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const HERE = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(HERE, "..", "..");
-const CLASSIFY = join(ROOT, "pipeline", "lib", "classify-intent.sh");
-const CASES = join(ROOT, "pipeline", "eval", "intent-cases.json");
-
-const argv = process.argv.slice(2);
-let minAccuracy = 0.95;
-for (let i = 0; i < argv.length; i++) {
-  if (argv[i] === "--min") {
-    minAccuracy = Number(argv[i + 1]);
-    // A typo'd `--min` with no/NaN value would set the threshold to NaN, and
-    // `accuracy < NaN` is always false - silently turning the gate into a
-    // no-op. Fail loudly instead.
-    if (!Number.isFinite(minAccuracy) || minAccuracy < 0 || minAccuracy > 1) {
-      console.error(`eval-intent: --min needs a number in [0,1], got: ${argv[i + 1]}`);
-      process.exit(2);
-    }
-  }
-}
-
-function classify(input) {
-  try {
-    return execFileSync("bash", [CLASSIFY, input], { encoding: "utf8" }).trim();
-  } catch {
-    return "ERROR";
-  }
-}
-
-const { cases } = JSON.parse(readFileSync(CASES, "utf8"));
-if (!Array.isArray(cases) || cases.length === 0) {
-  console.error("eval-intent: no cases found");
-  process.exit(2);
-}
-
-// Operationally-safe match: only TWO outcomes are dangerous, and they are what
-// the gate measures -
-//   - a `task` read as `question`  -> real work silently skipped
-//   - a `question` read as `task`  -> a worktree spun up for nothing
-// `ambiguous` proceeds as a task in Phase 0, so it is SAFE for a task/ambiguous
-// case but UNSAFE for a question case (ambiguous still avoids the worktree, so
-// it is acceptable for a question too - only `task` is the dangerous read).
-function isSafe(expected, got) {
-  if (got === expected) return true;
-  if (expected === "task") return got === "ambiguous"; // proceeds as task anyway
-  if (expected === "ambiguous") return got === "task"; // both proceed to dev
-  if (expected === "question") return got === "ambiguous"; // still no worktree spun up
-  return false;
-}
-
-const mismatches = []; // operationally unsafe (gate-failing)
-const exactMisses = []; // not exact, but operationally safe (informational)
-let safe = 0;
-let exact = 0;
-for (const c of cases) {
-  const got = classify(c.input);
-  if (got === c.expected) exact++;
-  else exactMisses.push({ input: c.input, expected: c.expected, got });
-  if (isSafe(c.expected, got)) safe++;
-  else mismatches.push({ input: c.input, expected: c.expected, got, danger: true });
-}
-
-const accuracy = safe / cases.length;
-const summary = {
-  total: cases.length,
-  safeMatches: safe,
-  safeAccuracy: Math.round(accuracy * 1000) / 1000,
-  exactMatches: exact,
-  exactAccuracy: Math.round((exact / cases.length) * 1000) / 1000,
-  minAccuracy,
-  dangerousMisclassifications: mismatches,
-  safeButInexact: exactMisses,
-};
-console.log(JSON.stringify(summary, null, 2));
-
-if (accuracy < minAccuracy) {
-  console.error(
-    `\neval-intent: safe accuracy ${(accuracy * 100).toFixed(1)}% < required ${(minAccuracy * 100).toFixed(0)}% (${mismatches.length} dangerous misclassification(s))`,
-  );
-  process.exit(1);
-}
-console.error(
-  `\neval-intent: safe ${(accuracy * 100).toFixed(1)}% (${safe}/${cases.length}), exact ${((exact / cases.length) * 100).toFixed(1)}% - pass`,
-);
-process.exit(0);