@mmerterden/multi-agent-pipeline 20.16.0 → 20.17.0

package/pipeline/scripts/bug-bash.mjs

@@ -0,0 +1,379 @@
+#!/usr/bin/env node
+/**
+ * bug-bash.mjs  -  charters, merge and the proof rule of /multi-agent:bug-bash.
+ *
+ * Exploratory sessions produce claims. A claim becomes a bug in the report only
+ * when a reproducing test fails with ASSERTION_FAILED on the assertion that
+ * encodes it; every other claim is listed as rejected, with the reason. The
+ * model plans charters, drives the sessions and writes the repro tests; this
+ * script holds the parts that must not be judgement:
+ *
+ *   charters --file FILE
+ *       Validates `[{slug, persona, area, start, goal, budget?, stance?}]`:
+ *       at most MAX_CHARTERS, unique kebab-case slugs, a known persona (or
+ *       `custom` with its stance written out), a one-sentence goal, a start
+ *       screen or route. Prints the charters with each budget resolved.
+ *       Exit 1 when any charter is invalid.
+ *
+ *   merge --sessions DIR
+ *       Reads DIR/<slug>.json, one exploratory run each (explore-run schema),
+ *       and merges issues that describe one defect (same screen, same
+ *       expectation) into candidates BB-1..n, keeping the clearest repro and
+ *       every charter that hit it. Warnings and contract failures are kept
+ *       apart; each charter's verdict and blocked causes are summarised.
+ *
+ *   report --merged FILE --verifications FILE [--json]
+ *       Applies `[{id, status, repro?, reason?, note?, rootCause?, productionRisk?}]`
+ *       to the candidates and renders the report: confirmed bugs, production
+ *       risks, rejected by reason, warnings, charters.
+ *
+ * Exit: 0 printed, 1 invalid charters, 2 usage or unreadable input.
+ */
+
+import { readFileSync, readdirSync, realpathSync } from "node:fs";
+import { join, basename } from "node:path";
+import { pathToFileURL } from "node:url";
+import { runMain } from "../lib/fatal.mjs";
+import {
+  resolveBudget,
+  validateFinding,
+  evaluateRun,
+  SEVERITY_WORDS,
+} from "./explore-findings.mjs";
+
+export const MAX_CHARTERS = 10;
+export const MAX_PARALLEL = 4;
+
+export const PERSONAS = Object.freeze({
+  "first-time-user":
+    "A first-time user: follows the screen's own cues, reads every label, and reports anything confusing, broken or inconsistent.",
+  skeptic:
+    "Distrusts every number, date, count and claim on screen and cross-checks each against every other place it appears.",
+  fuzzer:
+    "At every input runs the charter's input matrix (empty, very long, unicode, leading spaces, special characters) before anything else, judging each entry before the next.",
+  "state-keeper":
+    "Creates, edits and deletes, then backgrounds, relaunches, reloads or goes back after each change, and reports state that is lost or stale.",
+  "error-paths":
+    "Takes the unhappy path on purpose: wrong credentials, offline, denied permissions, expired sessions; reports errors that are missing, misleading or leak detail.",
+});
+
+export const REJECT_REASONS = [
+  "artifact",
+  "environment",
+  "design",
+  "fixture",
+  "did-not-reproduce",
+  "duplicate",
+  "unproven",
+  "unverified",
+];
+
+const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const nonEmpty = (v) => typeof v === "string" && v.trim().length > 0;
+
+/**
+ * @param {object[]} list
+ * @returns {{charters: object[], errors: string[]}}
+ */
+export function validateCharters(list) {
+  const errors = [];
+  const charters = [];
+  if (!Array.isArray(list) || !list.length) {
+    return { charters, errors: ["charters must be a non-empty array"] };
+  }
+  if (list.length > MAX_CHARTERS) {
+    errors.push(`at most ${MAX_CHARTERS} charters; got ${list.length}`);
+  }
+  const seen = new Set();
+  list.forEach((c, i) => {
+    const at = `charter ${i + 1}${c?.slug ? ` (${c.slug})` : ""}`;
+    if (!SLUG.test(c?.slug || "")) errors.push(`${at}: slug must be kebab-case`);
+    else if (seen.has(c.slug)) errors.push(`${at}: duplicate slug`);
+    seen.add(c?.slug);
+    if (c?.persona === "custom") {
+      if (!nonEmpty(c.stance)) errors.push(`${at}: a custom persona needs its stance written out`);
+    } else if (!PERSONAS[c?.persona]) {
+      errors.push(`${at}: persona must be one of ${Object.keys(PERSONAS).join(", ")} or custom`);
+    }
+    if (!nonEmpty(c?.area)) errors.push(`${at}: area is required`);
+    if (!nonEmpty(c?.start))
+      errors.push(`${at}: start (the screen or route it begins at) is required`);
+    const goal = (c?.goal || "").trim();
+    if (!goal) errors.push(`${at}: goal is required`);
+    else if (/\n/.test(goal) || /[.!?]\s+\S/.test(goal)) {
+      errors.push(`${at}: goal must be one sentence: one area, one posture`);
+    }
+    charters.push({
+      ...c,
+      stance: c?.persona === "custom" ? c.stance : PERSONAS[c?.persona],
+      budget: resolveBudget(c?.budget),
+    });
+  });
+  return { charters, errors };
+}
+
+const norm = (s) =>
+  String(s || "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim();
+
+/**
+ * @param {{slug:string, persona?:string, run:object}[]} sessions
+ */
+export function mergeSessions(sessions) {
+  const byKey = new Map();
+  const warnings = [];
+  const invalid = [];
+  const charters = [];
+  for (const s of sessions) {
+    const e = evaluateRun(s.run);
+    charters.push({
+      slug: s.slug,
+      persona: s.persona || s.run?.persona || null,
+      goal: s.run?.goal || null,
+      verdict: e.verdict,
+      ended: e.ended,
+      usage: e.usage,
+      blocked: Object.fromEntries(
+        Object.entries(e.blocked)
+          .filter(([, l]) => l.length)
+          .map(([k, l]) => [k, l.length]),
+      ),
+    });
+    for (const f of s.run?.findings || []) {
+      const errors = validateFinding(f);
+      if (errors.length) {
+        invalid.push({ charter: s.slug, title: f?.title ?? null, errors });
+        continue;
+      }
+      if (f.kind === "warning") {
+        warnings.push({ ...f, charters: [s.slug] });
+        continue;
+      }
+      const key = `${norm(f.screen)}|${norm(f.expected)}`;
+      const prev = byKey.get(key);
+      if (!prev) {
+        byKey.set(key, { ...f, charters: [s.slug] });
+        continue;
+      }
+      if (!prev.charters.includes(s.slug)) prev.charters.push(s.slug);
+      if (f.repro.length > prev.repro.length) {
+        Object.assign(prev, {
+          ...f,
+          charters: prev.charters,
+          severity: Math.max(prev.severity, f.severity),
+        });
+      } else {
+        prev.severity = Math.max(prev.severity, f.severity);
+      }
+    }
+  }
+  const candidates = [...byKey.values()]
+    .sort((a, b) => b.severity - a.severity)
+    .map((f, i) => ({ id: `BB-${i + 1}`, ...f }));
+  warnings.sort((a, b) => b.severity - a.severity);
+  return { candidates, warnings, invalid, charters };
+}
+
+function proofProblem(repro) {
+  if (!repro || typeof repro !== "object") return "no repro test was recorded";
+  if (!nonEmpty(repro.path)) return "repro.path (the test file) is required";
+  if (!nonEmpty(repro.command)) return "repro.command (how it was run) is required";
+  if (!nonEmpty(repro.assertion))
+    return "repro.assertion (the check that encodes the finding) is required";
+  if (repro.code === "PASSED" || repro.code === null || repro.code === undefined) {
+    return "did-not-reproduce";
+  }
+  if (repro.code !== "ASSERTION_FAILED") {
+    return `the repro failed with ${repro.code}, not ASSERTION_FAILED: the test is wrong, fix it and rerun`;
+  }
+  if (!nonEmpty(repro.failedAt) || repro.failedAt.trim() !== repro.assertion.trim()) {
+    return `the repro failed on a different assertion (${repro.failedAt || "none recorded"}) than the one that encodes the finding`;
+  }
+  return null;
+}
+
+/**
+ * @param {ReturnType<typeof mergeSessions>} merged
+ * @param {object[]} verifications
+ */
+export function buildReport(merged, verifications) {
+  const byId = new Map((verifications || []).map((v) => [v?.id, v]));
+  const confirmed = [];
+  const rejected = [];
+  const productionRisks = [];
+  for (const c of merged.candidates) {
+    const v = byId.get(c.id);
+    if (!v) {
+      rejected.push({ ...c, reason: "unverified", note: "no verification was recorded" });
+      continue;
+    }
+    if (v.status === "confirmed") {
+      const problem = proofProblem(v.repro);
+      if (!problem) {
+        confirmed.push({
+          ...c,
+          proof: v.repro,
+          rootCause: v.rootCause || null,
+          note: v.note || null,
+        });
+      } else if (problem === "did-not-reproduce") {
+        rejected.push({
+          ...c,
+          reason: "did-not-reproduce",
+          note: v.note || "the repro test passed",
+          proof: v.repro,
+        });
+      } else {
+        rejected.push({ ...c, reason: "unproven", note: problem, proof: v.repro });
+      }
+      continue;
+    }
+    if (v.status !== "rejected" || !REJECT_REASONS.includes(v.reason)) {
+      rejected.push({
+        ...c,
+        reason: "unverified",
+        note: `verification needs status confirmed, or rejected with a reason from: ${REJECT_REASONS.join(", ")}`,
+      });
+      continue;
+    }
+    const row = { ...c, reason: v.reason, note: v.note || null };
+    rejected.push(row);
+    if (v.reason === "environment" && v.productionRisk === true) productionRisks.push(row);
+  }
+  return {
+    confirmed,
+    productionRisks,
+    rejected,
+    warnings: merged.warnings,
+    invalid: merged.invalid,
+    charters: merged.charters,
+  };
+}
+
+/** @param {ReturnType<typeof buildReport>} r */
+export function renderBugBash(r) {
+  const out = [
+    "# Bug bash report",
+    "",
+    `Confirmed: ${r.confirmed.length} · production risks (unverified): ${r.productionRisks.length} · rejected: ${r.rejected.length} · warnings: ${r.warnings.length} · charters: ${r.charters.length}`,
+    "",
+    `## Confirmed (${r.confirmed.length})`,
+    "",
+  ];
+  r.confirmed.forEach((b) => {
+    out.push(
+      `### ${b.id} [${SEVERITY_WORDS[b.severity]}] ${b.title}`,
+      "",
+      `- **Screen**: ${b.screen || "-"}`,
+      `- **Expected / observed**: ${b.expected} / ${b.observed}`,
+      `- **Repro steps**: ${b.repro.map((s, i) => `${i + 1}. ${s}`).join(" ")}`,
+      `- **Repro test**: \`${b.proof.path}\` (\`${b.proof.command}\`) fails with ASSERTION_FAILED at \`${b.proof.assertion}\``,
+      `- **Root cause**: ${b.rootCause || "not located"}`,
+      `- **Evidence**: ${b.evidence.map((e) => `${e.kind}=${e.path}`).join(" ")}`,
+      `- **Found by**: ${b.charters.join(", ")}`,
+      "",
+    );
+  });
+  if (!r.confirmed.length) out.push("None.", "");
+  out.push(`## Production risks (${r.productionRisks.length}) - unverified`, "");
+  for (const p of r.productionRisks) out.push(`- ${p.id} ${p.title}: ${p.note || "-"}`);
+  if (!r.productionRisks.length) out.push("None.");
+  out.push("", `## Rejected (${r.rejected.length})`, "");
+  for (const reason of REJECT_REASONS) {
+    const rows = r.rejected.filter((x) => x.reason === reason);
+    if (!rows.length) continue;
+    out.push(`### ${reason} (${rows.length})`, "");
+    for (const x of rows) out.push(`- ${x.id} ${x.title}: ${x.note || "-"}`);
+    out.push("");
+  }
+  if (!r.rejected.length) out.push("None.", "");
+  out.push(`## Warnings (${r.warnings.length})`, "");
+  for (const w of r.warnings)
+    out.push(`- [${SEVERITY_WORDS[w.severity]}] ${w.title} (${w.charters.join(", ")})`);
+  if (!r.warnings.length) out.push("None.");
+  if (r.invalid.length) {
+    out.push("", `## Findings that failed the contract (${r.invalid.length})`, "");
+    for (const v of r.invalid)
+      out.push(`- ${v.charter}: ${v.title ?? "(untitled)"}: ${v.errors.join("; ")}`);
+  }
+  out.push(
+    "",
+    "## Charters",
+    "",
+    "| Charter | Persona | Verdict | Ended | Blocked |",
+    "|---|---|---|---|---|",
+  );
+  for (const c of r.charters) {
+    const blocked = Object.entries(c.blocked)
+      .map(([k, n]) => `${k} ${n}`)
+      .join(", ");
+    out.push(`| ${c.slug} | ${c.persona || "-"} | ${c.verdict} | ${c.ended} | ${blocked || "-"} |`);
+  }
+  return `${out.join("\n")}\n`;
+}
+
+function arg(argv, flag) {
+  const i = argv.indexOf(flag);
+  return i >= 0 ? argv[i + 1] : undefined;
+}
+
+const readJson = (p) => JSON.parse(readFileSync(p, "utf8"));
+
+function readSessions(dir) {
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".json"))
+    .sort()
+    .map((f) => {
+      const run = readJson(join(dir, f));
+      return { slug: basename(f, ".json"), persona: run.persona || null, run };
+    });
+}
+
+function main() {
+  const argv = process.argv.slice(2);
+  const mode = argv[0];
+  const usage = () => {
+    process.stderr.write(
+      "usage: bug-bash.mjs charters --file FILE\n" +
+        "       bug-bash.mjs merge --sessions DIR\n" +
+        "       bug-bash.mjs report --merged FILE --verifications FILE [--json]\n",
+    );
+    process.exitCode = 2;
+  };
+  try {
+    if (mode === "charters" && arg(argv, "--file")) {
+      const r = validateCharters(readJson(arg(argv, "--file")));
+      process.stdout.write(`${JSON.stringify(r, null, 2)}\n`);
+      if (r.errors.length) process.exitCode = 1;
+    } else if (mode === "merge" && arg(argv, "--sessions")) {
+      const merged = mergeSessions(readSessions(arg(argv, "--sessions")));
+      process.stdout.write(`${JSON.stringify(merged, null, 2)}\n`);
+    } else if (mode === "report" && arg(argv, "--merged") && arg(argv, "--verifications")) {
+      const r = buildReport(
+        readJson(arg(argv, "--merged")),
+        readJson(arg(argv, "--verifications")),
+      );
+      process.stdout.write(
+        argv.includes("--json") ? `${JSON.stringify(r, null, 2)}\n` : renderBugBash(r),
+      );
+    } else {
+      usage();
+    }
+  } catch (err) {
+    process.stderr.write(`bug-bash: ${err.message}\n`);
+    process.exitCode = 2;
+  }
+}
+
+function invokedDirectly() {
+  if (!process.argv[1]) return false;
+  try {
+    return import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href;
+  } catch {
+    return false;
+  }
+}
+
+if (invokedDirectly()) runMain("bug-bash", main);