@mmerterden/multi-agent-pipeline 17.3.0 → 17.5.0

package/pipeline/scripts/phase0-exit-gate.mjs

@@ -20,6 +20,20 @@
  * `agent-state.json` is the only durable evidence they happened - so the gate asserts
  * their output too, not just `taskType`.
  *
+ * A third run got past that, because `baseBranch` and `baseFetchStatus` can be filled
+ * in by a branch that was never chosen. The remote had one PR-targetable branch, the
+ * one-option `AskUserQuestion` was refused by the host (its schema needs two), and the
+ * run announced "only candidate, continuing with it" and carried on. The branch was
+ * right; nothing was asked. `baseBranchSource` is the field that separates those two,
+ * and the dev-context picker never ran at all - no `siblings`, so Phase 4's parity
+ * cross-check had nothing to read and could not tell an empty answer from no answer.
+ *
+ * A fourth showed the same shape one layer down: `git fetch origin` failed on a
+ * restricted network, `git branch -r` printed the remote-tracking cache anyway, and a
+ * weeks-old local list was presented as the remote's answer. Degrading to local refs is
+ * correct; reporting them as remote is not, so `baseBranchEvidence.refProvenance` has to
+ * agree with `baseFetchStatus`.
+ *
  * A phase that reports success without its output is worse than one that fails:
  * every later phase then reasons from a field that is not there. So this is a
  * gate, not a lint  -  the spec already said what to write, and prose alone did
@@ -157,6 +171,120 @@ export function evaluate(state, extraInput = "") {
     );
   }
 
+  // Which rule decided the base branch. `asked` and `input` are the only two an
+  // interactive run can honestly record: `remembered`, `default` and `derived` are the
+  // autopilot resolutions, and an interactive run that reaches for them has skipped its
+  // picker. This is the assertion `baseBranch` alone cannot make - a branch announced in
+  // prose and a branch chosen by the user leave the same value behind. A branch the run
+  // derived from the issue and the user then confirmed is still `asked`; the derivation
+  // lives in `baseBranchEvidence`, which is a record, not a permission.
+  const BRANCH_SOURCES = ["asked", "input", "remembered", "default", "derived"];
+  const branchSource =
+    typeof state.baseBranchSource === "string" ? state.baseBranchSource.trim() : "";
+  if (baseBranch && !BRANCH_SOURCES.includes(branchSource)) {
+    failures.push(
+      `agent-state.json has baseBranchSource="${branchSource || "<unset>"}"; Step 3 must ` +
+        `record one of ${BRANCH_SOURCES.join(" | ")}. Unset means nothing distinguishes a ` +
+        `branch the user chose from one the run picked and announced.`,
+    );
+  }
+  const isAutopilot = state.autopilot === true;
+  const AUTOPILOT_ONLY_SOURCES = ["remembered", "default", "derived"];
+  if (!isAutopilot && AUTOPILOT_ONLY_SOURCES.includes(branchSource)) {
+    failures.push(
+      `baseBranchSource="${branchSource}" on an interactive run. Those three are autopilot ` +
+        `resolutions; an interactive run asks (Step 3 is not skippable) or takes the base ` +
+        `from the task reference. A one-candidate filter is still asked, with a second ` +
+        `option - picker-contract.md, "Two options or it is not a question".`,
+    );
+  }
+
+  // What the base branch was chosen from, and what that list was worth.
+  //
+  // Two separate silent failures live here. The first: `git fetch origin` can fail on a
+  // restricted network while `git branch -r` still prints a full, confident list - the
+  // remote-tracking cache - so a weeks-old local guess gets presented as the remote's
+  // answer. Falling back to local refs is correct; not saying so is not. The second:
+  // `derived` claims the issue named the branch, and a claim with no evidence behind it
+  // is `default` wearing a hat.
+  const evidence =
+    state.baseBranchEvidence && typeof state.baseBranchEvidence === "object"
+      ? state.baseBranchEvidence
+      : null;
+  const DEGRADED_FETCH = ["cached-stale", "local-branch"];
+  if (DEGRADED_FETCH.includes(fetchStatus)) {
+    if (!evidence) {
+      failures.push(
+        `baseFetchStatus="${fetchStatus}" but state.baseBranchEvidence is absent. A run whose ` +
+          `fetch failed listed its branches from local refs; the record of that is what stops a ` +
+          `local-only guess being read afterwards as the remote's answer.`,
+      );
+    } else if (evidence.refProvenance !== "local") {
+      failures.push(
+        `baseFetchStatus="${fetchStatus}" but baseBranchEvidence.refProvenance=` +
+          `"${evidence.refProvenance || "<unset>"}". The fetch failed, so the candidate list came ` +
+          `from local refs and must say so - "remote" here is the silent degradation this field exists to catch.`,
+      );
+    }
+  }
+  if (branchSource === "derived") {
+    const ISSUE_DERIVED = new Set(["issue-version", "linked-release"]);
+    const cands = evidence && Array.isArray(evidence.candidates) ? evidence.candidates : [];
+    const chosen = cands.find((c) => c && c.branch === baseBranch);
+    const hasIssueEvidence =
+      chosen &&
+      Array.isArray(chosen.evidence) &&
+      chosen.evidence.some((e) => e && ISSUE_DERIVED.has(e.kind));
+    if (!hasIssueEvidence) {
+      failures.push(
+        `baseBranchSource="derived" but baseBranchEvidence carries no issue-version or ` +
+          `linked-release evidence for "${baseBranch}". "Derived" names a specific claim - the ` +
+          `issue's version field or a linked release issue pointed at this branch - and without ` +
+          `that record it is the sort-order default under a better name.`,
+      );
+    }
+    if (evidence && evidence.ambiguous === true) {
+      failures.push(
+        `baseBranchSource="derived" with baseBranchEvidence.ambiguous=true. Two or more ` +
+          `candidates tied at the top score; autopilot does not break a tie by picking one.`,
+      );
+    }
+  }
+
+  // Step 5b decides where the branch lives, and `localMode` alone cannot say
+  // whether anyone decided: `false` is both "the user chose a worktree" and
+  // "nothing asked and the default stood". Same shape as baseBranchSource, and
+  // the same reason - an autopilot run resolves it rather than asking, so
+  // `autopilot` is a legal source there and nowhere else.
+  const WORKSPACE_SOURCES = ["asked", "command", "autopilot"];
+  const workspaceSource =
+    typeof state.workspaceSource === "string" ? state.workspaceSource.trim() : "";
+  if (!WORKSPACE_SOURCES.includes(workspaceSource)) {
+    failures.push(
+      `agent-state.json has workspaceSource="${workspaceSource || "<unset>"}"; Step 5b must ` +
+        `record one of ${WORKSPACE_SOURCES.join(" | ")}. Unset means nothing distinguishes a ` +
+        `worktree the user chose from one no question was asked about.`,
+    );
+  }
+  if (!isAutopilot && workspaceSource === "autopilot") {
+    failures.push(
+      `workspaceSource="autopilot" on an interactive run. Autopilot resolves the workspace ` +
+        `to a worktree because an unattended commit in the user's own checkout is what ` +
+        `worktrees prevent; an interactive run asks (Step 5b) or is told by :local / --local.`,
+    );
+  }
+
+  // Step 2b's dev-context picker writes siblings[], empty included. The empty array is
+  // the record that it ran; absent, a multi-repo task silently became a single-repo one
+  // and Phase 4's parity cross-check lost its fourth counterpart source.
+  if (!Array.isArray(state.siblings)) {
+    failures.push(
+      "agent-state.json has no siblings array. Step 2b runs the dev-context picker on " +
+        "every input type and persists the result, `[]` included; an absent field means " +
+        "the picker never ran, so extra repos and read-only counterparts were never offered.",
+    );
+  }
+
   // Worktree isolation is a standing rule: never develop in the primary checkout.
   const worktree = typeof state.worktreePath === "string" ? state.worktreePath.trim() : "";
   const projectRoot = typeof state.projectRoot === "string" ? state.projectRoot.trim() : "";

package/pipeline/scripts/review-file-filter.mjs

@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+
+/**
+ * @file review-file-filter.mjs  -  decide what the reviewers are asked to read.
+ *
+ * Phase 4 has a size cap and no exclusion list. When the diff exceeds the
+ * budget the cap truncates the LARGEST files first (`phase-4-review.md`, Step
+ * 1.9), so a regenerated lockfile or a snapshot dump does not merely waste
+ * tokens - it is the thing that survives while real code is cut. The cheapest
+ * fix is to decide what is worth reading before the cap decides what fits.
+ *
+ * Every exclusion carries the reason it was excluded, and the two lists
+ * partition the input exactly. A file that quietly disappears between the diff
+ * and the reviewer is indistinguishable from a file nobody found anything in,
+ * and that is the failure this script exists to prevent.
+ *
+ * Patterns live in `schemas/review-file-exclusions.json` so the list is data,
+ * versioned beside the code that reads it, and generic: no stack, project or
+ * company name appears in it.
+ *
+ * Glob matching comes from `glob-match.mjs`, the same module
+ * `skill-conformance.mjs` uses. Two matchers would drift, and the day they
+ * disagree a file is excluded from the review and still counted in the
+ * conformance denominator, with nothing saying so.
+ *
+ * Inputs:
+ *   (stdin)             One path per line. `git diff --name-only` output.
+ *   --files <path>      Read the path list from a file instead of stdin
+ *   --patterns <path>   Override the pattern file
+ *   --json              Emit the full report (default)
+ *   --reviewed          Print only the reviewed paths, one per line
+ *
+ * Exit codes:
+ *   0   -  filtered
+ *   2   -  the pattern list could not be read. The report still comes out with
+ *          EVERY file reviewed, because the safe direction is to read too much:
+ *          a filter that fails closed would silently review nothing.
+ *   64  -  usage error
+ *
+ * @module pipeline/scripts/review-file-filter
+ */
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { globToRegExp } from "./glob-match.mjs";
+import { unquotePathIfNeeded } from "./git-path.mjs";
+
+const DEFAULT_PATTERNS = join(import.meta.dirname, "..", "schemas", "review-file-exclusions.json");
+
+/**
+ * Read and validate the pattern list.
+ *
+ * A pattern with no reason is rejected rather than defaulted, because the
+ * default would be the thing the caller is supposed to print.
+ *
+ * @param {string} path
+ * @returns {{patterns: {glob: string, reason: string, re: RegExp}[], error: string|null}}
+ */
+export function loadPatterns(path) {
+  let doc;
+  try {
+    doc = JSON.parse(readFileSync(path, "utf-8"));
+  } catch (e) {
+    return { patterns: [], error: `cannot read ${path}: ${e.message}` };
+  }
+  const rows = Array.isArray(doc?.patterns) ? doc.patterns : null;
+  if (!rows) return { patterns: [], error: `${path} has no patterns[] array` };
+
+  const patterns = [];
+  for (const [i, r] of rows.entries()) {
+    const glob = typeof r?.glob === "string" ? r.glob.trim() : "";
+    const reason = typeof r?.reason === "string" ? r.reason.trim() : "";
+    if (!glob) return { patterns: [], error: `patterns[${i}] has no glob` };
+    if (!reason) return { patterns: [], error: `patterns[${i}] (${glob}) has no reason` };
+    patterns.push({ glob, reason, re: globToRegExp(glob) });
+  }
+  return { patterns, error: null };
+}
+
+/**
+ * Split the changed files into what the reviewers read and what they do not.
+ *
+ * @param {string[]} files
+ * @param {{glob: string, reason: string, re: RegExp}[]} patterns
+ * @returns {{reviewed: string[], excluded: {path: string, reason: string, pattern: string}[]}}
+ */
+export function partition(files, patterns) {
+  const reviewed = [];
+  const excluded = [];
+  for (const f of files) {
+    const hit = patterns.find((p) => p.re.test(f));
+    if (hit) excluded.push({ path: f, reason: hit.reason, pattern: hit.glob });
+    else reviewed.push(f);
+  }
+  return { reviewed, excluded };
+}
+
+/**
+ * @param {string} raw
+ * @returns {string[]}
+ */
+export function readPaths(raw) {
+  const seen = new Set();
+  const out = [];
+  for (const line of raw.split("\n")) {
+    // Only the line ending is stripped: a leading or trailing space is a real
+    // character in a path. The C-style quoting git applies to a non-ASCII name
+    // IS undone, because a quoted `"G\303\266..."` matches no glob and would
+    // drop the file out of the denominator without anything reporting it -
+    // exactly the bug diff-risk-score.mjs already carries a fix for.
+    const p = unquotePathIfNeeded(line.replace(/\r$/, ""));
+    if (!p || seen.has(p)) continue;
+    seen.add(p);
+    out.push(p);
+  }
+  return out;
+}
+
+function main() {
+  const argv = process.argv.slice(2);
+  // A value flag with no value is a usage error, not a silent default. Falling
+  // back would hand a caller who typo'd `--patterns` the SHIPPED list while
+  // they believed they were running their own.
+  const valueOf = (flag) => {
+    const i = argv.indexOf(flag);
+    if (i === -1) return null;
+    const v = argv[i + 1];
+    if (v === undefined || v.startsWith("--")) {
+      console.error(`review-file-filter: ${flag} needs a path`);
+      process.exit(64);
+    }
+    return v;
+  };
+  for (const a of argv) {
+    if (a.startsWith("--") && !["--files", "--patterns", "--json", "--reviewed"].includes(a)) {
+      console.error(`review-file-filter: unknown flag ${a}`);
+      process.exit(64);
+    }
+  }
+
+  const filesArg = valueOf("--files");
+  let raw;
+  try {
+    raw = filesArg ? readFileSync(filesArg, "utf-8") : readFileSync(0, "utf-8");
+  } catch (e) {
+    console.error(`review-file-filter: cannot read the file list: ${e.message}`);
+    process.exit(64);
+  }
+  const files = readPaths(raw);
+
+  const patternsPath = valueOf("--patterns") || DEFAULT_PATTERNS;
+  const { patterns, error } = loadPatterns(patternsPath);
+  const { reviewed, excluded } = error
+    ? { reviewed: files.slice(), excluded: [] }
+    : partition(files, patterns);
+
+  const report = {
+    reviewed,
+    excluded,
+    total: files.length,
+    patternsSource: patternsPath,
+    patternCount: patterns.length,
+    ...(error ? { patternsError: error } : {}),
+  };
+
+  if (argv.includes("--reviewed")) {
+    for (const f of reviewed) console.log(f);
+  } else {
+    console.log(JSON.stringify(report, null, 2));
+  }
+
+  if (error) {
+    console.error(`review-file-filter: ${error} - reviewing all ${files.length} file(s)`);
+    process.exit(2);
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}

package/pipeline/scripts/skill-conformance.mjs

@@ -44,6 +44,7 @@ import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from "
 import { dirname, join, resolve, relative, basename, extname, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { homedir } from "node:os";
+import { matchesAnyGlob } from "./glob-match.mjs";
 
 const here = dirname(fileURLToPath(import.meta.url));
 
@@ -364,37 +365,6 @@ function classifyLanguage(file) {
   return LANG_BY_EXT[extname(base).toLowerCase()] ?? "other";
 }
 
-// ---------------------------------------------------------------------------
-// Minimal glob matcher for the `**/x`, `*.ext`, `dir/**` shapes registries use.
-// ---------------------------------------------------------------------------
-function globToRegExp(glob) {
-  let re = "";
-  for (let i = 0; i < glob.length; i++) {
-    const c = glob[i];
-    if (c === "*") {
-      if (glob[i + 1] === "*") {
-        // `**/` matches zero or more path segments.
-        if (glob[i + 2] === "/") {
-          re += "(?:.*/)?";
-          i += 2;
-        } else {
-          re += ".*";
-          i += 1;
-        }
-      } else {
-        re += "[^/]*";
-      }
-    } else if (c === "?") re += "[^/]";
-    else re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
-  }
-  return new RegExp(`^${re}$`);
-}
-
-function matchesAnyGlob(file, globs) {
-  if (!globs || globs.length === 0) return true;
-  return globs.some((g) => globToRegExp(g).test(file));
-}
-
 // ---------------------------------------------------------------------------
 // Diff parsing: changed files and added lines (for the exception audit).
 // ---------------------------------------------------------------------------

package/pipeline/scripts/validate-analysis-doc.mjs

@@ -635,6 +635,59 @@ function main() {
     );
   }
 
+  // 2d-2. The diagram has to describe the document it sits in.
+  //
+  // Until now the only question asked about Section 3 was "is a mermaid block
+  // present". A sequence diagram could name a participant the document never
+  // mentions, or omit a service the document devotes a section to, and nothing
+  // said so - the diagram is the part a reader trusts most and the part nothing
+  // checked. Both directions are reported, because they are different mistakes:
+  // an invented participant is a claim the document does not support, and a
+  // missing one is a service the reader will not see in the picture.
+  //
+  // Warn, not error: a participant can legitimately be an actor ("User") or an
+  // external system named nowhere else, and failing a correct document is how a
+  // check gets switched off. The warning names the label so it is one grep to
+  // settle.
+  const participants = [];
+  let inMermaid = false;
+  for (const raw of text.split("\n")) {
+    const t = raw.trim();
+    if (/^```/.test(t)) {
+      inMermaid = /^```mermaid\b/.test(t);
+      continue;
+    }
+    if (!inMermaid) continue;
+    const m = t.match(/^(?:participant|actor)\s+(\S+)(?:\s+as\s+(.+))?$/);
+    if (m) participants.push((m[2] || m[1]).trim());
+  }
+
+  if (participants.length > 0) {
+    // The prose is everything outside the fences: a participant that only ever
+    // appears inside the diagram has no support in the document.
+    const prose = text
+      .split(/^```[\s\S]*?^```/gm)
+      .join("\n")
+      .toLowerCase();
+    // Only identifier-shaped labels are evidence. A human actor is an ordinary
+    // word in whatever language the document is written in - "User",
+    // "Kullanici", "Operator" - and its absence from the prose says nothing. A
+    // service or component is written like code: an internal capital
+    // (PaymentService) or a separator (payment-service). Testing the SHAPE
+    // rather than keeping a word list keeps this working in both languages and
+    // stops the list from rotting.
+    const looksLikeIdentifier = (x) => /[a-z][A-Z]/.test(x) || (/[._-]/.test(x) && x.length >= 6);
+    const orphans = participants.filter((p) => {
+      const raw = p.replace(/^["'`]|["'`]$/g, "");
+      return looksLikeIdentifier(raw) && !prose.includes(raw.toLowerCase());
+    });
+    if (orphans.length > 0) {
+      warns.push(
+        `Section 3 diagram names ${orphans.length} participant(s) the document never mentions: ${orphans.join(", ")}`,
+      );
+    }
+  }
+
   mark("flow-chart");
 
   // 2e. Locked 16: every Files-to-Add row is tagged. The decision says untagged

package/pipeline/scripts/validate-reviewer.mjs

@@ -11,6 +11,7 @@
 //   node validate-reviewer.mjs path/to/reviewer.json
 //   cat reviewer.json | node validate-reviewer.mjs -
 //     [--criteria path/to/criteria-manifest.json]   # enforce the rule-ID checklist
+//     [--coverage path/to/review-files.json]        # enforce the file denominator
 //
 // With --criteria, the validator also enforces the conformance contract added in
 // schema v1.1.0. Without that enforcement the field is decoration: this validator
@@ -29,13 +30,16 @@ import { readFileSync } from "node:fs";
 
 const ALLOWED_SEVERITIES = new Set(["blocking", "important", "suggestion"]);
 const ALLOWED_VERDICTS = new Set(["conformant", "violated", "not-applicable"]);
+const ALLOWED_COVERAGE = new Set(["reviewed", "skipped"]);
 const FINGERPRINT_RE = /^F:[0-9a-f]{8}$/;
 
 /** @returns {Promise<string>} */
 function readInput() {
   const arg = process.argv[2];
   if (!arg) {
-    console.error("usage: validate-reviewer.mjs <path|-> [--criteria <manifest>]");
+    console.error(
+      "usage: validate-reviewer.mjs <path|-> [--criteria <manifest>] [--coverage <report>]",
+    );
     process.exit(64);
   }
   if (arg === "-") {
@@ -81,6 +85,72 @@ function validateFinding(f, label, errors) {
   }
 }
 
+/** Files the reviewer was given. Empty set = no denominator supplied. */
+function loadReviewedFiles(path) {
+  const report = JSON.parse(readFileSync(path, "utf-8"));
+  return new Set((report?.reviewed ?? []).filter((f) => typeof f === "string" && f.length > 0));
+}
+
+/**
+ * Enforce the file checklist against the fixed denominator.
+ *
+ * The same three failures the conformance checklist has, on the other axis:
+ * a file with no row (silently unread), a row for a file outside the set
+ * (a reviewer answering about something it was not given), and `skipped`
+ * with no reason. Without this, a reviewer that opened one file of ten and a
+ * reviewer that found nothing in all ten return the identical empty findings[].
+ */
+function validateFileCoverage(parsed, reviewedFiles, errors) {
+  const rows = parsed.fileCoverage;
+  if (!Array.isArray(rows)) {
+    errors.push(
+      `fileCoverage must be an array with one row per file in the review set (${reviewedFiles.size} expected)`,
+    );
+    return;
+  }
+
+  const seen = new Map();
+  rows.forEach((row, i) => {
+    const label = `fileCoverage[${i}]`;
+    if (typeof row !== "object" || row === null) {
+      errors.push(`${label}: not an object`);
+      return;
+    }
+    const path = row.path;
+    if (typeof path !== "string" || path.length === 0) {
+      errors.push(`${label}: missing path`);
+      return;
+    }
+    if (!ALLOWED_COVERAGE.has(row.verdict)) {
+      errors.push(`${label} (${path}): bad verdict ${JSON.stringify(row.verdict)}`);
+    }
+    if (!reviewedFiles.has(path)) {
+      errors.push(
+        `${label}: "${path}" is not in the review set  -  answer only for the files you were given`,
+      );
+    }
+    seen.set(path, (seen.get(path) ?? 0) + 1);
+
+    if (row.verdict === "skipped" && (typeof row.reason !== "string" || row.reason.length < 4)) {
+      errors.push(
+        `${label} (${path}): verdict "skipped" needs a reason  -  a file dropped without one is indistinguishable from one that was read`,
+      );
+    }
+  });
+
+  for (const [path, count] of seen) {
+    if (count > 1)
+      errors.push(`fileCoverage: "${path}" appears ${count} times (expected exactly once)`);
+  }
+  const missing = [...reviewedFiles].filter((f) => !seen.has(f));
+  if (missing.length > 0) {
+    const shown = missing.slice(0, 8).join(", ");
+    errors.push(
+      `fileCoverage: ${missing.length} file(s) in the review set have no row (${shown}${missing.length > 8 ? ", ..." : ""})  -  neither read nor declared skipped`,
+    );
+  }
+}
+
 /** Rule IDs the reviewer was told to answer for. Empty set = no criteria supplied. */
 function loadSelectedRuleIds(path) {
   const manifest = JSON.parse(readFileSync(path, "utf-8"));
@@ -214,6 +284,25 @@ async function main() {
     }
   }
 
+  // File checklist, only when the orchestrator supplied a denominator.
+  const covIdx = process.argv.indexOf("--coverage");
+  if (covIdx !== -1) {
+    const covPath = process.argv[covIdx + 1];
+    if (!covPath) {
+      errors.push("--coverage given with no report path");
+    } else {
+      try {
+        const reviewedFiles = loadReviewedFiles(covPath);
+        // Everything excluded means there is nothing to answer for. Demanding an
+        // empty array would fail honest output, and the filter's excluded[] is
+        // what carries that information onward.
+        if (reviewedFiles.size > 0) validateFileCoverage(parsed, reviewedFiles, errors);
+      } catch (err) {
+        errors.push(`cannot read review-set report: ${err.message}`);
+      }
+    }
+  }
+
   // Internal consistency: approved=true AND blocking findings = contradiction
   if (
     errors.length === 0 &&