@mmerterden/multi-agent-pipeline 20.2.0 → 20.3.0

package/pipeline/scripts/spec-consistency-gate.mjs

@@ -0,0 +1,469 @@
+#!/usr/bin/env node
+/**
+ * spec-consistency-gate.mjs  -  the spec, the plan and the constitution agree
+ * before anything is built on them.
+ *
+ * Three checks, all by id, no model call and no reading of prose:
+ *
+ *   1. Every requirement maps to at least one task and at least one test.
+ *   2. Every task traces back to a requirement the documents define.
+ *   3. Nothing waives a binding constitution rule. A requirement row or a task
+ *      that writes `waives CON-SEC-01` (or overrides / violates / bypasses /
+ *      ignores / exempt from / exception to / deviates from) against a rule
+ *      whose status is `binding` is a contradiction. The same against a
+ *      `proposed` rule is reported and passes; a constitution id the
+ *      constitution does not define is a gap.
+ *
+ * Requirements are the atoms `analysis-story-tree.mjs` plans from:
+ * `BR-<slug>-NN` in the global profile, `FG-NN` in the corporate one. A test is
+ * a row of the document's Test Plan section (global Section 15, corporate
+ * Section 19) that cites the id, directly or under a sub-table label that
+ * does, or a plan task of type `test` or with a file the stack adapter counts
+ * as a test file (`_stack-adapter.mjs`, testFilePatterns). A document with no
+ * development layer (a repo-less run, Locked 34) carries no test plan, so with
+ * no plan beside it the test check is reported as not run.
+ *
+ * Tasks come from the Phase 1 plan: `--plan` (planning-output), else
+ * `<worktree>/.pipeline/plan.json`, else `state.plan.todos`. A task cites a
+ * requirement through `requirements[]`, its title, its acceptance criteria or
+ * its todo text. `analysis-story-tree.mjs` calls `evaluate()` with its own
+ * story nodes as the tasks, so /multi-agent:analysis-jira runs the same check.
+ *
+ * The constitution is `--constitution`, else the project's knowledge store
+ * (constitution.mjs). None is a normal state: references are counted as
+ * unchecked, never failed.
+ *
+ * Quality gates active (lib/unattended.mjs, gatesActive): the verdict is
+ * recorded as `spec-consistency` through gate-ledger.mjs and the exit code is
+ * the verdict. Otherwise the same report is printed on stdout marked advisory,
+ * the exit is 0 and nothing is written: the attended Phase 1 consistency check
+ * (Step 12) never blocked, and this does not make it block.
+ *
+ * Usage:
+ *   spec-consistency-gate.mjs [--state <agent-state.json>] [--analysis <doc.md>]...
+ *                             [--plan <plan.json>] [--constitution <constitution.json>]
+ *                             [--repo <dir>] [--json]
+ *
+ * Exit (gates active): 0 pass · 1 gap · 2 not-applicable (no requirement) · 3 usage or unreadable input.
+ * Attended: 0, and 3 on a usage error.
+ *
+ * @module pipeline/scripts/spec-consistency-gate
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { resolveConstitution } from "./constitution.mjs";
+import { gateContext, recordQuietly } from "./gate-ledger.mjs";
+import { resolveStack, testFileMatcher } from "./_stack-adapter.mjs";
+
+export const GATE = "spec-consistency";
+const ATOM_GLOBAL = /\bBR-[a-z0-9]+(?:-[a-z0-9]+)*-\d+\b/gi;
+const ATOM_CORP = /\bFG-\d{2,3}\b/gi;
+const RULE_REF = /\bCON-[A-Z0-9]+-\d{2,3}\b/gi;
+const ID = "(CON-[A-Z0-9]+-\\d{2,3})";
+const WAIVE_VERB =
+  "waives?|waived(?:\\s+for)?|waiving|waiver\\s+(?:of|for)|overrides?|overridden|overrode|overriding|" +
+  "violates?|violated|violating|bypass(?:es|ed|ing)?|ignores?|ignored|ignoring|" +
+  "suspends?|suspended|suspending|disregards?|disregarded|disregarding|" +
+  "exempt(?:ed|ing)?\\s+from|exemption\\s+from|exception\\s+to|deviat(?:es?|ed|ing)\\s+from";
+const PASSIVE =
+  "(?:is|are|was|were|be|been|being|gets?|got|stays?|remains?)\\s+(?:temporarily\\s+|hereby\\s+)?" +
+  "(?:waived|overridden|bypassed|ignored|suspended|disregarded|violated|relaxed|lifted|exempted)";
+const WAIVER_BEFORE = new RegExp(
+  `\\b(?:${WAIVE_VERB})\\s+(?:(?:the\\s+)?(?:constitution\\s+)?rule\\s+)?${ID}\\b`,
+  "gi",
+);
+const WAIVER_AFTER = new RegExp(
+  `\\b${ID}\\s+(?:(?:will|has|have|had)\\s+)?(?:${PASSIVE}|does\\s+not\\s+apply|doesn't\\s+apply|is\\s+not\\s+applicable)\\b`,
+  "gi",
+);
+const TEST_PLAN = /^##\s+\d+\.?\s+.*\bTest Plan/i;
+const DEV_LAYER =
+  /^#{1,2}\s+.*(Files to Add|Eklenecek Dosyalar|Development Analysis|Geliştirme Analizi|Gelistirme Analizi|Test Plan)/i;
+
+function frontMatter(text) {
+  const lines = String(text).split("\n");
+  if (lines[0]?.trim() !== "---") return {};
+  const end = lines.indexOf("---", 1);
+  const fm = {};
+  for (let i = 1; i < end; i++) {
+    const m = lines[i].match(/^([a-z_]+):\s*(.*)$/);
+    if (m) fm[m[1]] = m[2].trim().replace(/^["']|["']$/g, "");
+  }
+  return fm;
+}
+
+const isCorporate = (text) => (frontMatter(text).profile || "").toLowerCase() === "corporate";
+const idsIn = (s, re) => (String(s).match(re) || []).map((x) => x.toUpperCase());
+
+/** The requirement atoms one document defines. */
+export function requirementIds(text) {
+  return new Set(idsIn(text, isCorporate(text) ? ATOM_CORP : ATOM_GLOBAL));
+}
+
+/**
+ * Requirement ids a document's Test Plan rows cite, and whether the document
+ * has a development layer at all.
+ *
+ * @returns {{ids: Set<string>, devLayer: boolean, section: boolean}}
+ */
+export function testedIds(text) {
+  const atom = isCorporate(text) ? ATOM_CORP : ATOM_GLOBAL;
+  const lines = String(text).split("\n");
+  const devLayer = lines.some((l) => DEV_LAYER.test(l));
+  const start = lines.findIndex((l) => TEST_PLAN.test(l));
+  const ids = new Set();
+  if (start === -1) return { ids, devLayer, section: false };
+  let label = [];
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (/^#{1,2}\s/.test(line)) break;
+    if (!line) continue;
+    if (line.startsWith("|")) {
+      for (const id of [...idsIn(line, atom), ...label]) ids.add(id);
+    } else if (/^#{3,6}\s/.test(line) || /^\*\*.+\*\*$/.test(line)) {
+      label = idsIn(line, atom);
+    } else {
+      label = [];
+    }
+  }
+  return { ids, devLayer, section: true };
+}
+
+/** Constitution ids a line cites, each marked when the line waives it. */
+export function constitutionRefs(text) {
+  const s = String(text);
+  const waived = new Set(
+    [...s.matchAll(WAIVER_BEFORE), ...s.matchAll(WAIVER_AFTER)].map((m) => m[1].toUpperCase()),
+  );
+  return [...new Set(idsIn(s, RULE_REF))].map((rule) => ({ rule, waives: waived.has(rule) }));
+}
+
+/**
+ * Plan tasks in the shape evaluate() reads.
+ *
+ * @param {object} plan  planning-output, or {todos: [...]}
+ * @param {(path: string) => boolean} isTestFile
+ */
+export function planTasks(plan, isTestFile = () => false) {
+  if (Array.isArray(plan?.tasks)) {
+    return plan.tasks.map((t) => {
+      const text = [t.title, ...(t.acceptanceCriteria || [])].filter(Boolean).join("\n");
+      return {
+        id: t.id ?? "(no id)",
+        ids: [
+          ...(t.requirements || []).map((x) => String(x).toUpperCase()),
+          ...idsIn(text, ATOM_GLOBAL),
+          ...idsIn(text, ATOM_CORP),
+        ],
+        text,
+        isTest: t.type === "test" || (t.files || []).some((f) => isTestFile(f)),
+      };
+    });
+  }
+  const todos = Array.isArray(plan?.todos) ? plan.todos : [];
+  return todos.map((t) => ({
+    id: t.id ?? "(no id)",
+    ids: [...idsIn(t.task, ATOM_GLOBAL), ...idsIn(t.task, ATOM_CORP)],
+    text: String(t.task ?? ""),
+    isTest: false,
+  }));
+}
+
+/** analysis-story-tree nodes in the shape evaluate() reads. */
+export function storyTasks(nodes) {
+  return (nodes || []).map((n) => ({
+    id: n.label || n.title,
+    ids: (n.sourceIds || []).map((x) => String(x).toUpperCase()),
+    text: "",
+    isTest: false,
+  }));
+}
+
+/**
+ * @param {{docs: {path: string, text: string}[], tasks: object[], taskSource: string,
+ *          constitution: {path: string|null, doc: object|null}|null}} input
+ */
+export function evaluate({ docs, tasks, taskSource, constitution }) {
+  const requirements = new Set();
+  const tested = new Set();
+  let devLayer = false;
+  for (const d of docs) {
+    for (const id of requirementIds(d.text)) requirements.add(id);
+    const t = testedIds(d.text);
+    for (const id of t.ids) tested.add(id);
+    devLayer = devLayer || t.devLayer;
+  }
+  const base = {
+    gate: GATE,
+    documents: docs.map((d) => d.path),
+    taskSource,
+    constitution: { path: constitution?.path ?? null },
+  };
+  if (requirements.size === 0) {
+    return {
+      ...base,
+      verdict: "not-applicable",
+      reason: "no requirement ids (BR-<slug>-NN or FG-NN) in the analysis document(s)",
+    };
+  }
+
+  const mapped = new Set();
+  const orphanTasks = [];
+  const undefinedRequirementRefs = [];
+  for (const t of tasks) {
+    const cited = [...new Set(t.ids)];
+    const known = cited.filter((id) => requirements.has(id));
+    for (const id of cited.filter((x) => !requirements.has(x))) {
+      undefinedRequirementRefs.push({ id, by: `task ${t.id}` });
+    }
+    if (known.length === 0) orphanTasks.push({ id: t.id, text: t.text.split("\n")[0] });
+    for (const id of known) {
+      mapped.add(id);
+      if (t.isTest) tested.add(id);
+    }
+  }
+  const sorted = [...requirements].sort();
+  const unmappedToTask = sorted.filter((id) => !mapped.has(id));
+  const testsChecked = devLayer || taskSource !== "story-tree";
+  const unmappedToTest = testsChecked ? sorted.filter((id) => !tested.has(id)) : [];
+
+  const rules = new Map((constitution?.doc?.rules || []).map((r) => [r.id, r]));
+  const contradictions = [];
+  const proposedWaivers = [];
+  const unknownRuleRefs = [];
+  let uncheckedRuleRefs = 0;
+  const judge = (refs, by) => {
+    for (const ref of refs) {
+      if (!constitution?.doc) {
+        uncheckedRuleRefs++;
+        continue;
+      }
+      const rule = rules.get(ref.rule);
+      if (!rule) unknownRuleRefs.push({ rule: ref.rule, by });
+      else if (ref.waives && rule.status === "binding") contradictions.push({ rule: ref.rule, by });
+      else if (ref.waives) proposedWaivers.push({ rule: ref.rule, by });
+    }
+  };
+  for (const d of docs) {
+    const atom = isCorporate(d.text) ? ATOM_CORP : ATOM_GLOBAL;
+    d.text.split("\n").forEach((line, i) => {
+      const refs = constitutionRefs(line);
+      if (!refs.length) return;
+      const ids = [...new Set(idsIn(line, atom))];
+      judge(refs, ids.length ? `requirement ${ids.join(", ")}` : `${d.path}:${i + 1}`);
+    });
+  }
+  for (const t of tasks) judge(constitutionRefs(t.text), `task ${t.id}`);
+
+  const gap =
+    unmappedToTask.length +
+      unmappedToTest.length +
+      orphanTasks.length +
+      undefinedRequirementRefs.length +
+      contradictions.length +
+      unknownRuleRefs.length >
+    0;
+  return {
+    ...base,
+    verdict: gap ? "gap" : "pass",
+    requirements: sorted,
+    tasks: tasks.length,
+    tests: testsChecked
+      ? { checked: true }
+      : {
+          checked: false,
+          reason: "no development layer in the document(s) and no plan beside them (Locked 34)",
+        },
+    unmappedToTask,
+    unmappedToTest,
+    orphanTasks,
+    undefinedRequirementRefs,
+    contradictions,
+    unknownRuleRefs,
+    proposedWaivers,
+    uncheckedRuleRefs,
+  };
+}
+
+/** One line per finding, for a preview or stderr. */
+export function describe(result) {
+  const out = [];
+  for (const id of result.unmappedToTask || []) out.push(`  ${id}: no task implements it`);
+  for (const id of result.unmappedToTest || []) out.push(`  ${id}: no test covers it`);
+  for (const t of result.orphanTasks || []) out.push(`  task ${t.id}: cites no requirement`);
+  for (const r of result.undefinedRequirementRefs || []) {
+    out.push(`  ${r.by}: cites ${r.id}, which no analysis document defines`);
+  }
+  for (const c of result.contradictions || []) {
+    out.push(`  ${c.by}: waives ${c.rule}, a binding constitution rule`);
+  }
+  for (const c of result.unknownRuleRefs || []) {
+    out.push(`  ${c.by}: cites ${c.rule}, which the constitution does not define`);
+  }
+  for (const c of result.proposedWaivers || []) {
+    out.push(`  ${c.by}: waives ${c.rule} (proposed, not binding)`);
+  }
+  return out;
+}
+
+/** What the verdict rests on, in one clause. */
+export function detail(result) {
+  if (result.verdict === "not-applicable" || result.verdict === "unreadable") return result.reason;
+  const tests = result.tests.checked ? "" : "; tests not checked";
+  return `${result.requirements.length} requirement(s), ${result.tasks} task(s)${tests}`;
+}
+
+/** The one-line verdict. */
+export function summary(result) {
+  return `${result.verdict} (${detail(result)})`;
+}
+
+/** Gate verdict -> ledger verdict. */
+export const LEDGER = {
+  pass: "pass",
+  gap: "fail",
+  "not-applicable": "not-applicable",
+  unreadable: "fail",
+};
+const EXIT = { pass: 0, gap: 1, "not-applicable": 2, unreadable: 3 };
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+function loadInputs(opts, state) {
+  const docPaths = opts.analysis.length
+    ? opts.analysis
+    : Array.isArray(state?.analysis?.docPath)
+      ? state.analysis.docPath
+      : [];
+  const docs = docPaths.map((path) => {
+    try {
+      return { path, text: readFileSync(path, "utf8") };
+    } catch (err) {
+      throw new Error(`cannot read the analysis document ${path}: ${err.message}`, {
+        cause: err,
+      });
+    }
+  });
+
+  const worktree =
+    typeof state?.worktreePath === "string" && existsSync(state.worktreePath)
+      ? state.worktreePath
+      : null;
+  const repo = opts.repo || worktree || process.cwd();
+  const planPath =
+    opts.plan ||
+    (worktree && existsSync(join(worktree, ".pipeline", "plan.json"))
+      ? join(worktree, ".pipeline", "plan.json")
+      : null);
+  let plan = null;
+  if (planPath) {
+    try {
+      plan = readJson(planPath);
+    } catch (err) {
+      throw new Error(`cannot read the plan ${planPath}: ${err.message}`, { cause: err });
+    }
+  } else if (Array.isArray(state?.plan?.todos)) {
+    plan = { todos: state.plan.todos };
+  }
+
+  const feature = docs.map((d) => frontMatter(d.text).feature).find(Boolean) || null;
+  const constitution = resolveConstitution({
+    explicit: opts.constitution || null,
+    repo,
+    feature,
+  });
+  if (constitution.error) throw new Error(constitution.error);
+  return { docs, plan, planPath, repo, constitution };
+}
+
+function print(json, result, advisory) {
+  const shown = advisory ? { ...result, advisory: true } : result;
+  if (json) {
+    writeFileSync(1, `${JSON.stringify(shown, null, 2)}\n`);
+    return;
+  }
+  const head = `spec-consistency${advisory ? " (advisory, the quality gates are off)" : ""}: ${summary(result)}`;
+  const lines = [
+    head,
+    ...(result.verdict === "gap" || result.proposedWaivers?.length ? describe(result) : []),
+  ];
+  writeFileSync(advisory || result.verdict !== "gap" ? 1 : 2, `${lines.join("\n")}\n`);
+}
+
+function parse(argv) {
+  const opts = { analysis: [], json: false };
+  const single = new Set(["--state", "--plan", "--constitution", "--repo"]);
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--json") opts.json = true;
+    else if (a === "--analysis" || single.has(a)) {
+      const v = argv[++i];
+      if (v === undefined) throw new Error(`${a} needs a value`);
+      if (a === "--analysis") opts.analysis.push(v);
+      else opts[a.slice(2)] = v;
+    } else throw new Error(`unexpected argument ${a}`);
+  }
+  return opts;
+}
+
+async function main(argv) {
+  let opts;
+  try {
+    opts = parse(argv);
+  } catch (err) {
+    process.stderr.write(`spec-consistency-gate: ${err.message}\n`);
+    return 3;
+  }
+  const ctx = gateContext({ statePath: opts.state || null, repo: opts.repo || process.cwd() });
+  const advisory = !ctx.active;
+  const state = ctx.state;
+
+  let result;
+  let repo = opts.repo || process.cwd();
+  try {
+    const input = loadInputs(opts, state);
+    repo = input.repo;
+    let tasks = [];
+    const taskSource = "plan";
+    if (input.plan) {
+      const ids = resolveStack(repo).adapters.map((a) => a.id);
+      tasks = planTasks(input.plan, testFileMatcher(ids));
+    } else if (input.docs.some((d) => requirementIds(d.text).size)) {
+      throw new Error(
+        "no plan found (--plan, <worktree>/.pipeline/plan.json or state.plan.todos), so no task can be checked",
+      );
+    }
+    result = evaluate({ docs: input.docs, tasks, taskSource, constitution: input.constitution });
+  } catch (err) {
+    result = { gate: GATE, verdict: "unreadable", reason: err.message };
+  }
+
+  if (!advisory) {
+    const detail =
+      result.verdict === "pass" || result.verdict === "gap"
+        ? {
+            verdict: result.verdict,
+            requirements: result.requirements.length,
+            unmappedToTask: result.unmappedToTask.length,
+            unmappedToTest: result.unmappedToTest.length,
+            orphanTasks: result.orphanTasks.length,
+            contradictions: result.contradictions.length + result.unknownRuleRefs.length,
+          }
+        : { verdict: result.verdict, reason: result.reason };
+    await recordQuietly(GATE, LEDGER[result.verdict], detail, {
+      repo: existsSync(repo) ? repo : null,
+      statePath: ctx.statePath,
+    });
+  }
+  print(opts.json, result, advisory);
+  return advisory ? 0 : EXIT[result.verdict];
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => process.exit(code));
+}