@mmerterden/multi-agent-pipeline 20.2.0 → 20.3.0

package/pipeline/scripts/plan-critique-gate.mjs

@@ -0,0 +1,591 @@
+#!/usr/bin/env node
+/**
+ * plan-critique-gate.mjs  -  one round of "why will this plan fail?", one
+ * answer per objection, and a verdict decided by constitution rule id.
+ *
+ * At the end of Phase 1 the plan-critic persona writes objections to the plan
+ * under four lenses (scope, feasibility, security, a cheaper alternative), and
+ * the planner answers each one exactly once: accept and revise the plan, or
+ * rebut with evidence. Both halves live in one file,
+ * `<worktree>/.pipeline/plan-critique.json` (schemas/plan-critique.schema.json).
+ * This script is the judge. It makes no model call and reads no prose:
+ *
+ *   1. Shape. The file matches the schema. Checked here without a dependency,
+ *      because the installed scripts carry none.
+ *   2. One round. `round` is 1; every objection has exactly one reply; no reply
+ *      answers an objection nobody raised; and once a run's critique has been
+ *      judged (`state.planCritique`), a different critique or a different set
+ *      of replies is refused as a second round, never scored.
+ *   3. Anchors. Every `file` anchor resolves in the working tree the way
+ *      verify-citations.mjs resolves a citation (line in range, quote on the
+ *      line); a `plan` anchor names a task of the plan; an `analysis` anchor is
+ *      a requirement id or the whole text of a heading of an analysis document.
+ *      An objection none of whose anchors resolves is labelled `inference`.
+ *   4. The constitution, by id. An objection citing a `binding` rule
+ *      (constitution.mjs) blocks unless its reply resolves it: an accept with a
+ *      revision and an anchor that resolves to a plan task, or a rebuttal with
+ *      an anchor that resolves, where a file anchor counts only as `path:line`
+ *      with a quote of that line. Every other objection, including one citing a
+ *      `proposed` or undefined rule, and all of them when there is no
+ *      constitution, is advisory: returned, and carried into the plan and the
+ *      PR summary through `state.planCritique.advisory`.
+ *
+ * Quality gates active (lib/unattended.mjs, gatesActive): the verdict is
+ * recorded as `plan-critique` through gate-ledger.mjs, the judgement is written
+ * to `state.planCritique`, and a failing or unreadable verdict parks the run
+ * as verification-failed: there is no second round to fix it in. Otherwise the
+ * same report is printed on stdout marked attended, the exit is 0 and nothing
+ * is written.
+ *
+ * Usage:
+ *   plan-critique-gate.mjs [--state <agent-state.json>] [--critique <plan-critique.json>]
+ *                          [--plan <plan.json>] [--analysis <doc.md>]...
+ *                          [--constitution <constitution.json>] [--repo <dir>] [--json]
+ *
+ * Exit (gates active): 0 pass · 1 fail · 2 not-applicable (no plan) · 3 usage or unreadable input.
+ * Attended: 0, and 3 on a usage error.
+ *
+ * @module pipeline/scripts/plan-critique-gate
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { RULE_ID, resolveConstitution } from "./constitution.mjs";
+import { gateContext, park, recordQuietly, updateState } from "./gate-ledger.mjs";
+import { requirementIds } from "./spec-consistency-gate.mjs";
+import { resolveOne } from "./verify-citations.mjs";
+
+export const GATE = "plan-critique";
+export const LENSES = ["scope", "feasibility", "security", "alternative"];
+export const ANCHOR_KINDS = ["file", "plan", "analysis", "inference"];
+export const ACTIONS = ["accept", "rebut"];
+const OBJ_ID = /^OBJ-\d{2,3}$/;
+const TOP_KEYS = new Set(["version", "round", "critic", "objections", "replies"]);
+const OBJECTION_KEYS = new Set(["id", "lens", "claim", "rule", "evidence"]);
+const REPLY_KEYS = new Set(["objection", "action", "revision", "note", "evidence"]);
+const ANCHOR_KEYS = new Set(["kind", "ref", "quote"]);
+
+const isObject = (v) => Boolean(v) && typeof v === "object" && !Array.isArray(v);
+const filled = (v) => typeof v === "string" && v.trim() !== "";
+
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (isObject(value)) {
+    return Object.fromEntries(
+      Object.keys(value)
+        .sort()
+        .map((k) => [k, canonical(value[k])]),
+    );
+  }
+  return value;
+}
+
+/** sha256 of a value, independent of key order and formatting. */
+export function digest(value) {
+  return createHash("sha256")
+    .update(JSON.stringify(canonical(value ?? null)))
+    .digest("hex");
+}
+
+function extraKeys(obj, allowed, where, errors) {
+  for (const k of Object.keys(obj))
+    if (!allowed.has(k)) errors.push(`${where}: unknown field ${k}`);
+}
+
+function checkAnchors(list, where, errors, { required }) {
+  if (list === undefined && !required) return;
+  if (!Array.isArray(list)) {
+    errors.push(`${where}: evidence must be an array`);
+    return;
+  }
+  if (required && list.length === 0) errors.push(`${where}: evidence needs at least one anchor`);
+  list.forEach((a, i) => {
+    const at = `${where} evidence[${i}]`;
+    if (!isObject(a)) {
+      errors.push(`${at}: not an object`);
+      return;
+    }
+    extraKeys(a, ANCHOR_KEYS, at, errors);
+    if (!ANCHOR_KINDS.includes(a.kind))
+      errors.push(`${at}: kind must be one of ${ANCHOR_KINDS.join(", ")}`);
+    if (!filled(a.ref)) errors.push(`${at}: ref is empty`);
+    if (a.quote !== undefined && !filled(a.quote)) errors.push(`${at}: quote is empty`);
+  });
+}
+
+/**
+ * The shape the schema describes, and the one-round protocol.
+ *
+ * @param {object} doc
+ * @returns {{errors: string[], protocol: {kind: string, objection: string|null, detail: string}[]}}
+ */
+export function checkCritique(doc) {
+  const errors = [];
+  const protocol = [];
+  if (!isObject(doc)) return { errors: ["the critique is not a JSON object"], protocol };
+  extraKeys(doc, TOP_KEYS, "critique", errors);
+  if (doc.version !== 1) errors.push(`version must be 1, got ${JSON.stringify(doc.version)}`);
+  if (!Number.isInteger(doc.round)) {
+    errors.push("round must be an integer");
+  } else if (doc.round !== 1) {
+    protocol.push({
+      kind: "second-round",
+      objection: null,
+      detail: `the critique is round ${doc.round}; a plan gets one round`,
+    });
+  }
+  if (doc.critic !== undefined && !isObject(doc.critic)) errors.push("critic must be an object");
+  if (!Array.isArray(doc.objections)) errors.push("objections must be an array");
+  if (!Array.isArray(doc.replies)) errors.push("replies must be an array");
+  if (errors.length) return { errors, protocol };
+
+  const ids = new Set();
+  doc.objections.forEach((o, i) => {
+    const where = typeof o?.id === "string" ? o.id : `objections[${i}]`;
+    if (!isObject(o)) {
+      errors.push(`${where}: not an object`);
+      return;
+    }
+    extraKeys(o, OBJECTION_KEYS, where, errors);
+    if (!OBJ_ID.test(o.id ?? "")) errors.push(`${where}: id must match OBJ-NN`);
+    else if (ids.has(o.id)) {
+      protocol.push({
+        kind: "duplicate-objection",
+        objection: o.id,
+        detail: `${o.id} is raised twice`,
+      });
+    }
+    ids.add(o.id);
+    if (!LENSES.includes(o.lens)) errors.push(`${where}: lens must be one of ${LENSES.join(", ")}`);
+    if (!filled(o.claim)) errors.push(`${where}: claim is empty`);
+    if (o.rule !== undefined && !RULE_ID.test(o.rule ?? ""))
+      errors.push(`${where}: rule must be a constitution id (CON-<SEC|A11Y|ARCH|LIC>-NN)`);
+    checkAnchors(o.evidence, where, errors, { required: true });
+  });
+
+  const answers = new Map();
+  doc.replies.forEach((r, i) => {
+    const where = typeof r?.objection === "string" ? `reply to ${r.objection}` : `replies[${i}]`;
+    if (!isObject(r)) {
+      errors.push(`${where}: not an object`);
+      return;
+    }
+    extraKeys(r, REPLY_KEYS, where, errors);
+    if (!OBJ_ID.test(r.objection ?? "")) errors.push(`${where}: objection must match OBJ-NN`);
+    if (!ACTIONS.includes(r.action))
+      errors.push(`${where}: action must be one of ${ACTIONS.join(", ")}`);
+    if (r.action === "accept" && !filled(r.revision))
+      errors.push(`${where}: an accept says how the plan was revised (revision)`);
+    if (r.revision !== undefined && !filled(r.revision)) errors.push(`${where}: revision is empty`);
+    if (r.note !== undefined && typeof r.note !== "string")
+      errors.push(`${where}: note must be a string`);
+    checkAnchors(r.evidence, where, errors, { required: r.action === "rebut" });
+    if (typeof r.objection === "string")
+      answers.set(r.objection, (answers.get(r.objection) || 0) + 1);
+  });
+
+  for (const id of ids) {
+    const count = answers.get(id) || 0;
+    if (count === 0)
+      protocol.push({ kind: "missing-reply", objection: id, detail: `${id} has no reply` });
+    if (count > 1) {
+      protocol.push({
+        kind: "duplicate-reply",
+        objection: id,
+        detail: `${id} has ${count} replies; each objection is answered once`,
+      });
+    }
+  }
+  for (const id of answers.keys()) {
+    if (!ids.has(id)) {
+      protocol.push({
+        kind: "orphan-reply",
+        objection: id,
+        detail: `${id} answers an objection the critique does not raise`,
+      });
+    }
+  }
+  return { errors, protocol };
+}
+
+function taskIds(plan) {
+  const list = Array.isArray(plan?.tasks)
+    ? plan.tasks
+    : Array.isArray(plan?.todos)
+      ? plan.todos
+      : [];
+  return new Set(list.map((t) => (t && t.id !== undefined ? String(t.id) : null)).filter(Boolean));
+}
+
+const norm = (s) => String(s).replace(/\s+/g, " ").trim().toLowerCase();
+
+/**
+ * A file anchor strong enough to clear a binding objection: `path:line` and a
+ * quote of that line. A bare path or a line number says a file exists, not
+ * what the reply read in it.
+ */
+const pinned = (a) => a.kind === "file" && /:\d+$/.test(String(a.ref).trim()) && filled(a.quote);
+
+/**
+ * Does one anchor hold? `inference` never does: it is the label, not a check.
+ * An `analysis` anchor is a requirement id, or the whole text of a heading
+ * (with or without its section number); a fragment of a heading is not.
+ *
+ * @returns {{ok: boolean, reason: string|null}}
+ */
+export function checkAnchor(anchor, { repo, tasks, docs, cache }) {
+  const ref = String(anchor.ref).trim();
+  if (anchor.kind === "inference") return { ok: false, reason: null };
+  if (anchor.kind === "plan") {
+    return tasks.has(ref)
+      ? { ok: true, reason: null }
+      : { ok: false, reason: `the plan has no task ${ref}` };
+  }
+  if (anchor.kind === "analysis") {
+    if (!docs.length) return { ok: false, reason: "no analysis document to check against" };
+    const want = ref.toUpperCase();
+    const heading = norm(ref);
+    const found = docs.some(
+      (d) =>
+        requirementIds(d.text).has(want) ||
+        (heading !== "" &&
+          d.text.split("\n").some((l) => {
+            if (!/^#{1,6}\s/.test(l)) return false;
+            const text = norm(l.replace(/^#+\s*/, ""));
+            return text === heading || text.replace(/^[0-9]+(?:\.[0-9]+)*\.?\s+/, "") === heading;
+          })),
+    );
+    return found
+      ? { ok: true, reason: null }
+      : { ok: false, reason: `no analysis document defines ${ref}` };
+  }
+  if (!repo) return { ok: false, reason: "no repository to resolve against" };
+  const m = ref.match(/^(.*?)(?::(\d+))?$/);
+  const c = { file: m[1], line: m[2] ? Number(m[2]) : null };
+  if (anchor.quote !== undefined) c.quote = anchor.quote;
+  const reason = resolveOne(repo, null, c, cache);
+  return { ok: reason === null, reason };
+}
+
+/**
+ * The judgement over one critique.
+ *
+ * @param {{critique: object, plan: object, docs: {path: string, text: string}[],
+ *          constitution: {path: string|null, doc: object|null}|null, repo: string|null,
+ *          prior?: object|null}} input
+ */
+export function evaluate({
+  critique,
+  plan,
+  docs = [],
+  constitution = null,
+  repo = null,
+  prior = null,
+}) {
+  const base = { gate: GATE, constitution: { path: constitution?.path ?? null } };
+  const { errors, protocol } = checkCritique(critique);
+  if (errors.length) {
+    return {
+      ...base,
+      verdict: "unreadable",
+      reason: `the critique does not match plan-critique.schema.json: ${errors.join("; ")}`,
+    };
+  }
+  const whole = digest(critique);
+  const objectionsDigest = digest(critique.objections);
+  if (prior && prior.digest && prior.digest !== whole) {
+    protocol.unshift({
+      kind: "second-round",
+      objection: null,
+      detail:
+        prior.objectionsDigest && prior.objectionsDigest !== objectionsDigest
+          ? `this run's critique was already judged (${prior.verdict} at ${prior.at}); a second critique round is refused`
+          : `the replies differ from the ones judged at ${prior.at}; a second answer to the same critique is refused`,
+    });
+  }
+
+  const ctx = { repo, tasks: taskIds(plan), docs, cache: new Map() };
+  const rules = new Map((constitution?.doc?.rules || []).map((r) => [r.id, r]));
+  const unresolvedAnchors = [];
+  const check = (list, objection, side) =>
+    (list || []).map((a) => {
+      const r = checkAnchor(a, ctx);
+      if (!r.ok && a.kind !== "inference") {
+        unresolvedAnchors.push({ objection, side, kind: a.kind, ref: a.ref, reason: r.reason });
+      }
+      return { kind: a.kind, ok: r.ok, pinned: a.kind !== "file" || pinned(a) };
+    });
+
+  const blocking = [];
+  const advisory = [];
+  for (const o of critique.objections) {
+    const label = check(o.evidence, o.id, "objection").some((a) => a.ok) ? "anchored" : "inference";
+    const replies = critique.replies.filter((r) => r.objection === o.id);
+    const reply = replies.length === 1 ? replies[0] : null;
+    const replyAnchors = reply ? check(reply.evidence, o.id, "reply") : [];
+    let ruleStatus = null;
+    if (o.rule) {
+      if (!constitution?.doc) ruleStatus = "unchecked";
+      else ruleStatus = rules.get(o.rule)?.status ?? "unknown";
+    }
+    if (ruleStatus === "binding" && reply) {
+      const resolved =
+        reply.action === "accept"
+          ? filled(reply.revision) && replyAnchors.some((a) => a.kind === "plan" && a.ok)
+          : replyAnchors.some((a) => a.ok && a.pinned);
+      if (!resolved) {
+        const loose = replyAnchors.some((a) => a.ok && !a.pinned);
+        blocking.push({
+          objection: o.id,
+          rule: o.rule,
+          lens: o.lens,
+          claim: o.claim,
+          reason:
+            reply.action === "accept"
+              ? "accepted, but no reply anchor resolves to a task of the plan"
+              : loose
+                ? "rebutted with a file anchor that lacks path:line and a quote; a binding rule needs both"
+                : "rebutted, but no reply anchor resolves",
+        });
+        continue;
+      }
+    }
+    advisory.push({
+      objection: o.id,
+      lens: o.lens,
+      claim: o.claim,
+      label,
+      ...(o.rule ? { rule: o.rule, ruleStatus } : {}),
+      reply: reply ? reply.action : null,
+    });
+  }
+
+  return {
+    ...base,
+    verdict: protocol.length || blocking.length ? "fail" : "pass",
+    round: critique.round,
+    objections: critique.objections.length,
+    protocol,
+    blocking,
+    advisory,
+    unresolvedAnchors,
+    digest: whole,
+    objectionsDigest,
+  };
+}
+
+/** One line per finding, for a preview or stderr. */
+export function describe(result) {
+  const out = [];
+  for (const p of result.protocol || []) out.push(`  ${p.kind}: ${p.detail}`);
+  for (const b of result.blocking || []) {
+    out.push(`  ${b.objection} (${b.rule}, binding): ${b.reason}`);
+  }
+  for (const a of result.advisory || []) {
+    const rule = a.rule ? `, ${a.rule} ${a.ruleStatus}` : "";
+    out.push(`  ${a.objection} [${a.lens}, ${a.label}${rule}] advisory: ${a.claim}`);
+  }
+  for (const u of result.unresolvedAnchors || []) {
+    out.push(`  ${u.objection} ${u.side} anchor ${u.kind}:${u.ref} does not resolve: ${u.reason}`);
+  }
+  return out;
+}
+
+/** The one-line verdict. */
+export function summary(result) {
+  if (result.verdict === "not-applicable" || result.verdict === "unreadable") {
+    return `${result.verdict} (${result.reason})`;
+  }
+  return `${result.verdict} (${result.objections} objection(s), ${result.blocking.length} blocking, ${result.protocol.length} protocol, ${result.advisory.length} advisory)`;
+}
+
+/** Gate verdict -> ledger verdict. */
+export const LEDGER = {
+  pass: "pass",
+  fail: "fail",
+  "not-applicable": "not-applicable",
+  unreadable: "fail",
+};
+const EXIT = { pass: 0, fail: 1, "not-applicable": 2, unreadable: 3 };
+
+function readJson(path, what) {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch (err) {
+    throw new Error(`cannot read the ${what} ${path}: ${err.message}`, { cause: err });
+  }
+}
+
+function frontMatterFeature(text) {
+  const m = String(text).match(/^---\n[\s\S]*?^feature:\s*(.+)$[\s\S]*?^---$/m);
+  return m ? m[1].trim().replace(/^["']|["']$/g, "") : null;
+}
+
+function judge(opts, state) {
+  const worktree =
+    typeof state?.worktreePath === "string" && existsSync(state.worktreePath)
+      ? state.worktreePath
+      : null;
+  const repo = opts.repo || worktree || process.cwd();
+  const planPath =
+    opts.plan ||
+    (worktree && existsSync(join(worktree, ".pipeline", "plan.json"))
+      ? join(worktree, ".pipeline", "plan.json")
+      : null);
+  let plan = null;
+  if (planPath) plan = readJson(planPath, "plan");
+  else if (Array.isArray(state?.plan?.todos)) plan = { todos: state.plan.todos };
+  if (!plan) {
+    return {
+      repo,
+      result: {
+        gate: GATE,
+        verdict: "not-applicable",
+        reason: "no plan (--plan, <worktree>/.pipeline/plan.json or state.plan.todos) to critique",
+      },
+    };
+  }
+
+  const critiquePath =
+    opts.critique || (worktree ? join(worktree, ".pipeline", "plan-critique.json") : null);
+  if (!critiquePath) throw new Error("no critique: pass --critique or a state with a worktreePath");
+  const critique = readJson(critiquePath, "critique");
+
+  const docPath = state?.analysis?.docPath;
+  const docPaths = opts.analysis.length
+    ? opts.analysis
+    : Array.isArray(docPath)
+      ? docPath
+      : typeof docPath === "string"
+        ? [docPath]
+        : [];
+  const docs = docPaths.map((path) => {
+    try {
+      return { path, text: readFileSync(path, "utf8") };
+    } catch (err) {
+      throw new Error(`cannot read the analysis document ${path}: ${err.message}`, { cause: err });
+    }
+  });
+  const feature = docs.map((d) => frontMatterFeature(d.text)).find(Boolean) || null;
+  const constitution = resolveConstitution({ explicit: opts.constitution || null, repo, feature });
+  if (constitution.error) throw new Error(constitution.error);
+
+  const prior =
+    state?.planCritique && typeof state.planCritique === "object" ? state.planCritique : null;
+  const result = evaluate({ critique, plan, docs, constitution, repo, prior });
+  return { repo, result, critiquePath, prior };
+}
+
+function print(json, result, attended) {
+  const shown = attended ? { ...result, attended: true } : result;
+  if (json) {
+    writeFileSync(1, `${JSON.stringify(shown, null, 2)}\n`);
+    return;
+  }
+  const head = `plan-critique${attended ? " (attended: advisory, the quality gates are off)" : ""}: ${summary(result)}`;
+  const failing = result.verdict === "fail" || result.verdict === "unreadable";
+  writeFileSync(attended || !failing ? 1 : 2, `${[head, ...describe(result)].join("\n")}\n`);
+}
+
+function parse(argv) {
+  const opts = { analysis: [], json: false };
+  const single = new Set(["--state", "--critique", "--plan", "--constitution", "--repo"]);
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--json") opts.json = true;
+    else if (a === "--analysis" || single.has(a)) {
+      const v = argv[++i];
+      if (v === undefined) throw new Error(`${a} needs a value`);
+      if (a === "--analysis") opts.analysis.push(v);
+      else opts[a.slice(2)] = v;
+    } else throw new Error(`unexpected argument ${a}`);
+  }
+  return opts;
+}
+
+async function main(argv) {
+  let opts;
+  try {
+    opts = parse(argv);
+  } catch (err) {
+    process.stderr.write(`plan-critique-gate: ${err.message}\n`);
+    return 3;
+  }
+  const ctx = gateContext({ statePath: opts.state || null, repo: opts.repo || process.cwd() });
+  const attended = !ctx.active;
+
+  let judged;
+  try {
+    judged = judge(opts, ctx.state);
+  } catch (err) {
+    judged = {
+      repo: opts.repo || process.cwd(),
+      result: { gate: GATE, verdict: "unreadable", reason: err.message },
+    };
+  }
+  const { result, repo } = judged;
+
+  if (!attended) {
+    const detail =
+      result.verdict === "pass" || result.verdict === "fail"
+        ? {
+            verdict: result.verdict,
+            objections: result.objections,
+            blocking: result.blocking.length,
+            protocol: result.protocol.length,
+            advisory: result.advisory.length,
+          }
+        : { verdict: result.verdict, reason: result.reason };
+    await recordQuietly(GATE, LEDGER[result.verdict], detail, {
+      repo: existsSync(repo) ? repo : null,
+      statePath: ctx.statePath,
+    });
+    if (ctx.statePath) {
+      try {
+        await persist(ctx.statePath, judged);
+      } catch (err) {
+        process.stderr.write(`plan-critique-gate: state not updated: ${err.message}\n`);
+      }
+    }
+  }
+  print(opts.json, result, attended);
+  return attended ? 0 : EXIT[result.verdict];
+}
+
+async function persist(statePath, { result, critiquePath, prior }) {
+  const judgedHere = result.verdict === "pass" || result.verdict === "fail";
+  if (judgedHere && (!prior || prior.digest === result.digest)) {
+    await updateState(statePath, (s) => ({
+      ...s,
+      planCritique: {
+        round: 1,
+        path: critiquePath,
+        digest: result.digest,
+        objectionsDigest: result.objectionsDigest,
+        verdict: result.verdict,
+        objections: result.objections,
+        blocking: result.blocking.map(({ objection, rule, reason }) => ({
+          objection,
+          rule,
+          reason,
+        })),
+        advisory: result.advisory,
+        at: new Date().toISOString(),
+      },
+    }));
+  }
+  if (result.verdict === "fail" || result.verdict === "unreadable") {
+    await park(statePath, {
+      outcome: "verification-failed",
+      gate: GATE,
+      reason: summary(result),
+    });
+  }
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => process.exit(code));
+}