forgedock 1.42.0 → 1.43.0

package/bin/lean/conformance.mjs

@@ -7,6 +7,7 @@
  * The reviewer supplies per-row judgements; the engine computes the verdict. A review without rows (a generic
  * "PASS") is incomplete, and incomplete never merges.
  */
+import { spawnSync } from "node:child_process";
 import { OUTCOME_CODES, ENV_VALIDATE_TIMEOUT } from "../engine/outcome-codes.mjs";
 import { noComments } from "./context.mjs";
 import { activationFailures } from "./activation.mjs";
@@ -303,6 +304,8 @@ export function validateContract(c, issueText, { repro = false } = {}) {
     if (ref) problems.push(`${v.id}: \`git diff\` against the moving ${ref} tip fails as soon as the base moves on, even when the change is fine; compare against the merge base instead (\`${ref}...HEAD\` or \`$(git merge-base HEAD ${ref})\`)`);
     for (const p of tmpdirInputs(v.command)) problems.push(tmpInputProblem(v.id, p));
     if (vitestQuietFlag(v.command)) problems.push(`${v.id}: vitest 3 rejects -q/--quiet (usage error before any test runs); use --reporter=dot`);
+    const syn = shellSyntaxError(v.command);
+    if (syn) problems.push(`${v.id}: the command is not valid shell (bash -n: ${syn}); keep notes out of the command`);
   }
   if (repro) {
     const t = c.repro_test;
@@ -311,6 +314,8 @@ export function validateContract(c, issueText, { repro = false } = {}) {
     else {
       for (const p of tmpdirInputs(t.command)) problems.push(tmpInputProblem("repro_test", p));
       if (vitestQuietFlag(t.command)) problems.push("repro_test: vitest 3 rejects -q/--quiet; use --reporter=dot");
+      const syn = shellSyntaxError(t.command);
+      if (syn) problems.push(`repro_test: the command is not valid shell (bash -n: ${syn})`);
     }
   }
   if (JSON.stringify(c).length > MAX_CONTRACT_CHARS) problems.push(`the contract is over ${MAX_CONTRACT_CHARS} characters; keep quotes to the requirement sentence and descriptions short`);
@@ -320,7 +325,8 @@ export function validateContract(c, issueText, { repro = false } = {}) {
 /** forge#4519: a shell pattern as in scripts/hotspots.sh `case` (`*` crosses `/`, `?` one character), anchored. */
 const shellGlobRe = (g) => new RegExp(`^${String(g).replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".")}$`);
 
-const AMEND_KINDS = ["add-deliverable", "add-requirement", ...TRACKED, "amend-requirement", "supersedes", "decision"];
+const AMEND_KINDS = ["add-deliverable", "add-requirement", ...TRACKED, "amend-requirement", "supersedes", "decision", "substitute-validate"];
+const UNRELATED_CI_KIND = "unrelated-ci-failure";
 const GUARD_ID_RE = /^#?\d+$/;
 const AMEND_SLIPS = { "amend-requirement": "defer", limitation: "defer" };
 // #4916: near-synonyms builders emit for the per-guard `supersedes` amendment
@@ -364,16 +370,17 @@ function amendmentProblem(a, tag) {
 
 /**
  * Apply result.json amendments to the frozen contract. Kinds: add-deliverable {path}, add-requirement {requirement},
- * defer|post-deploy {requirement: id}. Every amendment needs a why. Returns {contract, applied[], toTrack[], problems[]}.
+ * defer|post-deploy {requirement: id}. Every amendment needs a why (or the engine's defaultWhy); `addedPaths` are lane-added files that count as decision anchors when a deliverable lists them. Returns {contract, applied[], toTrack[], problems[]}.
  * `generatedPaths` (forge#4519, forge.yaml maintenance.generated_paths via scripts/hotspots.sh): an add-deliverable on a
  * matching path is refused; the repo's generator or bot owns that file.
  * `decision` {decision:{choice, anchors, symbols?, supersedes?}} (#4695): every anchor must match a file of `baseFiles` (the
  * base tree; null skips the check); `supersedes` names the decision whose engine-authored preserve row it replaces and
  * drops that row; returns the records in `decisions`.
  */
-export function applyAmendments(frozen, amendments, issueText = null, { generatedPaths = [], baseFiles = null } = {}) {
+export function applyAmendments(frozen, amendments, issueText = null, { generatedPaths = [], baseFiles = null, addedPaths = [], defaultWhy = null } = {}) {
   const c = structuredClone(frozen);
   c.deliverables = c.deliverables ?? [];
+  for (const v of c.validate ?? []) if (v && typeof v === "object") delete v.substituted; // SEC-21: only a substitute-validate amendment sets it, never the frozen contract
   const applied = [], toTrack = [], problems = [], decisions = [];
   const genRes = (Array.isArray(generatedPaths) ? generatedPaths : []).filter((g) => typeof g === "string" && g).map(shellGlobRe);
   const rows = [];
@@ -392,8 +399,11 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
     }
     rows.push({ a: n, raw });
   }
-  for (const { a, raw } of rows) {
+  for (let { a, raw } of rows) {
+    // an unrelated out-of-diff CI failure is not a contract change: informational, never a problem
+    if (a?.kind === UNRELATED_CI_KIND) { applied.push({ kind: a.kind, target: String(a.target ?? "unrelated check"), why: String(a.why ?? "").trim() || "unrelated to the diff" }); continue; }
     const tag = JSON.stringify(raw ?? null).slice(0, 400);
+    if (a && blank(a.why) && !blank(defaultWhy)) a = { ...a, why: String(defaultWhy) }; // the engine demanded the change: its reason is the why
     if (blank(a?.why)) { problems.push(`amendment without why: ${tag}`); continue; }
     if (a.kind === "add-deliverable" && !blank(a.path)) {
       if (genRes.some((re) => re.test(cleanPath(a.path)))) {
@@ -405,6 +415,16 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       // package.json declares the edge (forge#4159); before, it was reported as applied but changed nothing.
       if (have) { have.change = have.change ? `${have.change}; ${a.why}` : a.why; have.amended = true; }
       else c.deliverables.push({ id: a.id ?? `A${c.deliverables.length + 1}`, path: a.path, change: a.why, amended: true });
+    } else if (a.kind === "substitute-validate" && !blank(a.validate)) {
+      // #4857: a runnable equivalent of an unrunnable frozen validate command; same id, same checks as a planned command
+      const v = (c.validate ?? []).find((x) => x.id === a.validate);
+      if (!v) { problems.push(`substitute-validate names unknown validate ${a.validate}`); continue; }
+      const bad = blank(a.command) ? "no command" : shellSyntaxError(a.command) ? "not valid shell (bash -n)" : noopCommand(a.command) ? "runs no check (true, :, exit 0, echo)" : movingBaseDiff(a.command) ? "diffs against a moving base tip" : tmpdirInputs(a.command).length ? "reads a $TMPDIR file it did not create" : null;
+      if (bad) { problems.push(`substitute-validate ${a.validate}: ${bad}`); continue; }
+      // SEC-11: a second substitute of the same id keeps the frozen command as `from`, so the engine always reruns the original
+      v.substituted = { from: v.substituted?.from ?? v.command, why: String(a.why).trim() }; v.command = String(a.command).trim();
+      applied.push({ kind: a.kind, target: a.validate, why: String(a.why).trim() });
+      continue;
     } else if (a.kind === "add-requirement" && a.requirement?.id && !blank(a.requirement?.quote)) {
       if (c.requirements.some((r) => r.id === a.requirement.id)) { problems.push(`add-requirement reuses id ${a.requirement.id}`); continue; }
       if (issueText != null && !inIssue(a.requirement.quote, issueText)) { problems.push(`add-requirement ${a.requirement.id}: quote is not verbatim from the issue`); continue; }
@@ -424,6 +444,16 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       // #4594: the requirement stays deliver and keeps its verbatim quote; the reviewer judges it against amendment.text
       const r = c.requirements.find((x) => x.id === a.requirement);
       if (!r) { problems.push(`amendment names unknown requirement ${a.requirement}`); continue; }
+      if (TRACKED.includes(r.disposition)) {
+        // SEC-31/SPEC-33: a flip that would orphan a check id or a filed tracking issue must name it and say why it is now delivered
+        const named = [r.check, r.tracked ? String(r.tracked).replace(/\D/g, "") : null].filter((k) => !blank(k));
+        const said = `${a.why} ${a.text}`;
+        const missing = named.filter((k) => !new RegExp(`(^|[^\\w-])#?${String(k).replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w-])`).test(said));
+        if (missing.length) { problems.push(`amendment flips ${r.disposition} ${r.id} to deliver but does not name its ${missing.join(", ")} (check id or tracking issue) and why it is now delivered in the diff`); continue; }
+        // #4984: deliver text on a deferred requirement puts it in scope: it is delivered, not tracked in a follow-up; amended_from keeps the original for the reviewer
+        r.amended_from ??= Object.fromEntries(["quote", "disposition", "why", "bucket", "check", "tracked", "class"].filter((k) => r[k] != null).map((k) => [k, r[k]]));
+        r.disposition = "deliver"; for (const k of ["bucket", "why", "check", "tracked", "class"]) delete r[k];
+      }
       r.amendment = { text: String(a.text).trim(), why: String(a.why).trim() }; r.amended = true;
     } else if (TRACKED.includes(a.kind) && a.requirement) {
       const r = c.requirements.find((x) => x.id === a.requirement);
@@ -437,7 +467,10 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
     } else if (a.kind === "decision") {
       const d = normaliseDecision({ ...a.decision, why: a.decision?.why ?? a.why });
       if (!d) { problems.push(amendmentProblem(a, tag)); continue; }
-      const dead = Array.isArray(baseFiles) ? deadAnchors(d.anchors, baseFiles) : [];
+      // a file the lane adds and the contract lists as a deliverable is a new file the issue declared: a valid anchor
+      const listed = new Set(c.deliverables.map((x) => x.path));
+      const added = (Array.isArray(addedPaths) ? addedPaths : []).filter((x) => listed.has(x));
+      const dead = Array.isArray(baseFiles) ? deadAnchors(d.anchors, [...baseFiles, ...added]) : [];
       if (dead.length) { problems.push(`decision anchor(s) ${dead.join(", ")} match no file on the base; anchor a path that exists there`); continue; }
       if (!blank(a.decision?.supersedes)) c.preserve = (c.preserve ?? []).filter((p) => p.decision !== a.decision.supersedes && p.id !== a.decision.supersedes);
       decisions.push(a.decision?.supersedes ? { ...d, supersedes: String(a.decision.supersedes) } : d);
@@ -452,6 +485,74 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
 const PY_MISSING_RE = /ModuleNotFoundError: No module named '([^']+)'|ImportError: No module named '?([\w.]+)'?/;
 const NODE_MISSING_RE = /Cannot find (?:module|package) '([^']+)'/;
 const CMD_NOT_FOUND_RE = /([^\s:'"`]+): command not found/, SH_NOT_FOUND_RE = /([^\s:'"`]+): not found\b/;
+/** #4857: why a command is not valid bash (`bash -n`), or null when it parses (or bash is unavailable). Catches prose left on a command. */
+export function shellSyntaxError(command) {
+  const r = spawnSync("bash", ["-n", "-c", String(command ?? "")], { encoding: "utf8", timeout: 5000 });
+  if (r.error || r.status === 0 || r.status == null) return null;
+  return String(r.stderr || "syntax error").trim().split("\n")[0].replace(/^bash: (?:line \d+: )?/, "").slice(0, 160);
+}
+const NOOP_SEGMENT_RE = /^(?:true|false|:|exit(?:\s+\d+)?|echo(?:\s.*)?|printf(?:\s.*)?|sleep(?:\s+[\d.]+)?|test\s+-?\w+|\[\s+-?\w+\s+\]|fi|done|node\s+-e\s+['"]?[\d;\s]*['"]?|python3?\s+-c\s+['"]?(?:pass|0|None)?['"]?)$/;
+const KEYWORD_RE = /^(if|then|else|elif|do|while|until)(?:\s+|$)/;
+const CONST_FALSE_RE = /^(?:false|\[\s+0\s+=\s+1\s+\]|\[\s+1\s+=\s+0\s+\]|test\s+0\s+=\s+1|test\s+-n\s+(?:""|'')|\[\s+-n\s+(?:""|'')\s+\])$/;
+/**
+ * #4979 SPEC-2: a command every segment of which is a no-op (true, :, exit N, echo, printf, sleep, a constant test,
+ * shell keywords, `node -e 0`, `python -c pass`) checks nothing. A best-effort denylist: the engine's rerun of the
+ * frozen command (`substituteVerdict`) is the real gate.
+ */
+export function noopCommand(command) {
+  // SEC-22: a leading shell keyword is stripped and the rest judged, so `if [ -f x ]; then pytest; fi` is a real check
+  const segs = String(command ?? "").split(/&&|\|\||;|\||\n/).map((s) => s.trim()).filter(Boolean);
+  const live = [];
+  let dead = false; // #4995: inside the body of an if/elif/while whose condition is a constant false
+  for (const raw of segs) {
+    let s = raw, cond = false, m;
+    while ((m = KEYWORD_RE.exec(s))) {
+      s = s.slice(m[0].length);
+      if (m[1] === "else" || m[1] === "elif") dead = false;
+      cond = m[1] === "if" || m[1] === "elif" || m[1] === "while";
+    }
+    if (s === "fi" || s === "done") dead = false;
+    if (cond) { if (CONST_FALSE_RE.test(s)) dead = true; else if (!dead && s) live.push(s); continue; }
+    if (!dead && s) live.push(s);
+  }
+  return segs.length > 0 && live.every((s) => NOOP_SEGMENT_RE.test(s));
+}
+/**
+ * #4979 SPEC-2: a substitute-validate stands only when the frozen command it replaced cannot run (died before any test,
+ * a runner usage error, a missing host tool, exit 127) or passes anyway. A frozen command that runs and fails is a real
+ * failure the substitute may not hide. `run` is the engine's rerun of the frozen command; null when none was made.
+ */
+export function substituteVerdict(run, { repoHas = () => false, command = "" } = {}) {
+  if (!run) return "unverified";
+  if (run.rc === 0 || run.rc === 127 || unrunnableValidate(run) || cliUsageError(run) || missingTool(run, { repoHas, command })) return "ok";
+  return run.rc === 124 ? "timeout" : "runs";
+}
+/**
+ * #4990 SEC-13: the paths of `removed` (deleted, or the old side of a rename, in the diff) that a frozen command names:
+ * a whitespace token equal to the path, or a directory token above it. A builder that removes the target of its frozen
+ * validate makes the engine's rerun read "unrunnable", so the substitute is judged on the diff instead.
+ */
+export function removedNamedPaths(command, removed = []) {
+  const toks = String(command ?? "").split(/\s+/).map((t) => t.replace(/^['"`(]+|['"`);]+$/g, "").replace(/^\.\//, "").replace(/\/+$/, "")).filter((t) => t && t !== ".");
+  return removed.filter((p) => toks.some((t) => p === t || p.startsWith(`${t}/`)));
+}
+/** #4990: paths the lane's diff deletes or renames away from (the old side), per `git diff --name-status -M`. */
+export async function removedPathsOf(git, base) {
+  const rows = String(await git(["diff", "--name-status", "-M", `${base}...HEAD`]).catch(() => "")).split("\n").map((l) => l.split("\t"));
+  return rows.filter((r) => /^[DR]/.test(r[0] ?? "") && r[1]).map((r) => r[1]);
+}
+const UNRUNNABLE_RE = /no tests ran|collected 0 items|no tests found|could not find (?:the )?pyproject|poetry could not find|syntax error near|unexpected EOF while looking/i;
+/** #4857: a failed rerun that died before any test ran (no tests selected, no project at the cwd, invalid shell): a defect of the frozen command. */
+export function unrunnableValidate({ rc, output } = {}) {
+  return rc !== 0 && rc !== 124 && UNRUNNABLE_RE.test(String(output ?? ""));
+}
+const BLOCK_VALIDATE_RE = /\b(?:V\d+|validate|repro)\b/i;
+const BLOCK_UNRUNNABLE_RE = /unrunnable|cannot (?:be )?run|could not (?:be )?run|can't run|not runnable|command not found|No module named|no tests (?:ran|were collected)|collected 0 items|syntax error|pyproject|(?:tool|module)s? (?:is|are) missing|fails? before any test/i;
+/** #4857: a blocked result's text that says a validate/repro command could not run: an environment defect, never a product decision. */
+export function describesUnrunnableCommand(text) {
+  const t = String(text ?? "");
+  return BLOCK_VALIDATE_RE.test(t) && BLOCK_UNRUNNABLE_RE.test(t);
+}
 // #4761: a runner rejecting its own flags is a defect of the frozen command, not of the change
 const CLI_USAGE_RE = /\bUnknown option\b|Unrecognized CLI Parameter|error: unrecognized arguments|unknown (?:shorthand )?flag\b|bad option:/i;
 const VITEST_QUIET_RE = /\bvitest\b[^&|;]*\s(?:-q|--quiet)(?=\s|$)/;
@@ -513,7 +614,7 @@ export function repoTracks(paths) {
  * `ciCovered` (#4527): the validate ids accepted only because the head is CI-verified (no local result, or a missing host
  * tool), for the trajectory; always an array.
  */
-export function precheck({ contract, result, changedFiles, ignore = [], ciVerified = false, hotspots = null, repro = null, validateRuns = null, repoHas = () => false, frozen = null, checkIds = null, guardFailures = null, newOffKeys = null }) {
+export function precheck({ contract, result, changedFiles, ignore = [], ciVerified = false, hotspots = null, repro = null, validateRuns = null, repoHas = () => false, frozen = null, checkIds = null, guardFailures = null, newOffKeys = null, substitutedRuns = null, removedPaths = [] }) {
   const failures = [], superseded = [], ciCovered = [];
   // #4626: a post-deploy check id must be on the default branch or added by this PR (`checkIds`: ids of both checks files; null = not resolved)
   if (checkIds) for (const r of contract.requirements ?? []) {
@@ -527,6 +628,16 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
   if (creep.length) failures.push({ code: C.CONFORMANCE_SCOPE_CREEP, kind: "scope_creep", detail: `changed outside the contract's deliverables: ${creep.join(", ")}` });
   const runs = Array.isArray(result?.validation) ? result.validation : [];
   for (const v of contract.validate ?? []) {
+    // #4979 SPEC-2: a substitute counts only when the frozen command it replaced cannot run (or passes); checked first, never CI-covered
+    if (v.substituted) {
+      const gone = removedNamedPaths(v.substituted.from, removedPaths);
+      if (gone.length) { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_removed_path", detail: `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\`, but the diff removes ${gone.map((x) => `\`${x}\``).join(", ")} that the frozen command names: a substitute may not replace a check by deleting its target` }); continue; }
+      const verdict = substituteVerdict(substitutedRuns?.[v.id], { repoHas, command: v.substituted.from });
+      const from = `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\``;
+      if (verdict === "unverified") { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_unverified", detail: `${from}, but the engine has no rerun of the frozen command to show it cannot run` }); continue; }
+      if (verdict === "runs") { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "validate_failed", detail: `${from}, but the frozen command runs and fails (exit ${substitutedRuns[v.id].rc}): a substitute-validate replaces only a command that cannot run, never a failing check` }); continue; }
+      if (verdict === "timeout" && !ciVerified) { failures.push({ code: ENV_VALIDATE_TIMEOUT, kind: "validate_timeout", detail: `${from}, and the frozen command timed out (exit 124) on the engine's rerun, so its substitution cannot be judged: infrastructure (host load)` }); continue; }
+    }
     const r = runs.find((x) => x?.id === v.id);
     // #4167: a head the engine verified on CI covers validate ids the builder left to CI (no result, or ci:true without an exit)
     const leftToCi = !r || (r.ci === true && r.exit === undefined);
@@ -536,6 +647,7 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
       const rerun = validateRuns?.[v.id];
       if (rerun?.rc === 0) continue;
       if (rerun && cliUsageError(rerun)) { failures.push({ code: C.CONTRACT_INVALID_COMMAND, kind: "validate_command_invalid", detail: `validate ${v.id} (\`${v.command}\`) is a CLI usage error of the runner, a defect of the frozen command rather than of the change` }); continue; }
+      if (rerun && unrunnableValidate(rerun)) { failures.push({ code: C.CONTRACT_INVALID_COMMAND, kind: "validate_command_invalid", detail: `validate ${v.id} (\`${v.command}\`) died before any test ran (no tests selected, no project at its cwd or invalid shell), a defect of the frozen command rather than of the change; substitute a runnable equivalent with a substitute-validate amendment` }); continue; }
       // #4906: the rerun timed out (even after the longer retry): host load, not a verdict on the change. A CI-verified head covers it; else an environment error.
       if (rerun?.rc === 124) {
         if (ciVerified) { ciCovered.push(v.id); continue; }
@@ -962,8 +1074,9 @@ export function followupItems(reqs, { now = Date.now() } = {}) {
 /** #4388: the follow-up script's `key\tissue` lines as {[item.key]: issue}, matched on trimmed keys (the script trims what it echoes); unmatched items omitted. */
 export function followupMap(out, items) {
   const echoed = new Map();
-  for (const l of String(out ?? "").split("\n")) { const [k, n] = l.split("\t"); if (k && /^\d+$/.test(String(n ?? "").trim())) echoed.set(k.trim(), Number(n)); }
-  return Object.fromEntries(items.filter((i) => echoed.has(String(i.key).trim())).map((i) => [i.key, echoed.get(String(i.key).trim())]));
+  const norm = (k) => String(k).replace(/`/g, "").trim(); // #4984: the script strips backticks from the label it echoes
+  for (const l of String(out ?? "").split("\n")) { const [k, n] = l.split("\t"); if (k && /^\d+$/.test(String(n ?? "").trim())) echoed.set(norm(k), Number(n)); }
+  return Object.fromEntries(items.filter((i) => echoed.has(norm(i.key))).map((i) => [i.key, echoed.get(norm(i.key))]));
 }
 
 /**

package/bin/lean/conformance.test.mjs

@@ -1,8 +1,9 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
+import { readFileSync } from "node:fs";
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
+import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
 
 const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
 const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -81,6 +82,15 @@ describe("#4655 multi-requirement amendment references", () => {
     assert.match(applyAmendments(C, [{ kind: "defer", requirement: "R1/R9", why: "w" }]).problems[0], /unknown requirement R9/);
     assert.match(applyAmendments(C, [{ kind: "scope-note", requirement: "R1", why: "w" }]).problems[0], /unknown kind 'scope-note'; valid kinds are/);
   });
+  it("an unrelated-ci-failure amendment (kind or type) is an informational no-op; other unknown kinds still fail", () => {
+    for (const a of [{ kind: "unrelated-ci-failure", target: "core rate-limit test", why: "diff touches no core file" }, { type: "unrelated-ci-failure", target: "t", why: "w" }]) {
+      const r = applyAmendments(C, [a]);
+      assert.deepEqual(r.problems, []);
+      assert.deepEqual(r.contract, C);
+      assert.equal(r.applied[0].kind, "unrelated-ci-failure");
+    }
+    assert.match(applyAmendments(C, [{ kind: "flaky-note", why: "w" }]).problems[0], /unknown kind 'flaky-note'; valid kinds are/);
+  });
   it("supersede-guard guards[] expands to per-guard supersedes", () => {
     const a = applyAmendments(C, [{ kind: "supersede-guard", guards: ["#4879", "#4838"], why: "x" }]);
     assert.deepEqual(a.problems, []);
@@ -137,6 +147,83 @@ describe("precheck (deterministic, no reviewer)", () => {
     assert.deepEqual(codes(precheck({ contract: C, result: { ...res, validation: [{ id: "V1", exit: 1 }] }, changedFiles: files })), ["conformance.unimplemented"]);
     assert.deepEqual(codes(precheck({ contract: C, result: { ...res, validation: [] }, changedFiles: files })), ["conformance.unimplemented"]);
   });
+  it("#4857 validateContract rejects a validate command that fails bash -n", () => {
+    const prose = { ...C, validate: [{ id: "V3", command: "node checks.mjs --base origin/staging (then run the scoped tsc/eslint commands it prints)" }] };
+    assert.ok(validateContract(prose, ISSUE).problems.some((p) => /V3: the command is not valid shell/.test(p)));
+    assert.ok(!validateContract({ ...C, validate: [{ id: "V1", command: "node checks.mjs --base origin/staging" }] }, ISSUE).problems.some((p) => /valid shell/.test(p)));
+    assert.equal(shellSyntaxError("echo ok && true"), null);
+    assert.ok(shellSyntaxError("echo (oops"));
+  });
+  it("#4857 a command that died before any test ran is contract.invalid_command; substitute-validate swaps a runnable one", () => {
+    assert.ok(unrunnableValidate({ rc: 5, output: "collected 0 items / 1 deselected\nno tests ran" }));
+    assert.ok(unrunnableValidate({ rc: 1, output: "Poetry could not find a pyproject.toml file in /repo" }));
+    assert.ok(!unrunnableValidate({ rc: 1, output: "AssertionError: 1 !== 2" }));
+    assert.ok(!unrunnableValidate({ rc: 0, output: "no tests ran" }));
+    const r = applyAmendments({ requirements: [], validate: [{ id: "V2", command: "poetry run pytest -k 404_probe" }] }, [{ kind: "substitute-validate", validate: "V2", command: "poetry -C services/worker run pytest tests/test_probe.py", why: "no pyproject at root" }]);
+    assert.deepEqual(r.problems, []);
+    assert.equal(r.contract.validate[0].command, "poetry -C services/worker run pytest tests/test_probe.py");
+    assert.equal(r.contract.validate[0].substituted.from, "poetry run pytest -k 404_probe");
+    const bad = applyAmendments({ requirements: [], validate: [{ id: "V2", command: "x" }] }, [{ kind: "substitute-validate", validate: "V2", command: "echo (oops", why: "w" }]);
+    assert.ok(bad.problems.some((p) => /not valid shell/.test(p)));
+    assert.ok(applyAmendments({ requirements: [], validate: [] }, [{ kind: "substitute-validate", validate: "V9", command: "true", why: "w" }]).problems.length);
+  });
+  it("#4995 noopCommand bare keywords and constant-false guards", () => {
+    for (const cmd of ["if true\nthen\n  :\nfi", "while true\ndo\n  :\ndone", "if false; then pytest; fi", "if [ 0 = 1 ]; then pytest; fi", "if test -n \"\"; then node --test a.test.mjs; fi", "while false; do pytest; done", "if false\nthen\n  pytest\nfi"]) assert.ok(noopCommand(cmd), JSON.stringify(cmd));
+    for (const cmd of ["if [ -f x ]; then pytest; fi", "for f in a b; do node --test $f; done", "if false; then :; else pytest; fi", "if true\nthen\n  pytest\nfi", "if false; then :; elif [ -f x ]; then pytest; fi"]) assert.ok(!noopCommand(cmd), JSON.stringify(cmd));
+  });
+  it("#4979 SPEC-2 a substitute that runs no check is rejected at amendment time", () => {
+    for (const cmd of ["true", ":", "exit 0", "echo ok", "true && echo done", "printf x | true"]) {
+      assert.ok(noopCommand(cmd), cmd);
+      const r = applyAmendments({ requirements: [], validate: [{ id: "V2", command: "poetry run pytest -k probe" }] }, [{ kind: "substitute-validate", validate: "V2", command: cmd, why: "w" }]);
+      assert.ok(r.problems.some((p) => /runs no check/.test(p)), cmd);
+    }
+    for (const cmd of ["node --test a.test.mjs", "echo start && pytest tests/x.py", "test -f out.json", "[ -f a ] && npm test", "bash scripts/check.sh", "if [ -f x ]; then pytest; fi", "if true; then node --test a.test.mjs; fi", "for f in a b; do node --test $f; done"]) assert.ok(!noopCommand(cmd), cmd); // SEC-22: keywords around a real check
+    for (const cmd of ["test 1", "[ 1 ]", "sleep 0", "node -e 0", "true\ntrue", "if true; then :; fi", "python -c pass", "exit 1"]) assert.ok(noopCommand(cmd), JSON.stringify(cmd)); // SEC-12
+  });
+  it("#4979 SPEC-2 a substitute stands only when the frozen command cannot run", () => {
+    const sub = applyAmendments({ ...C, validate: [{ id: "V1", command: "poetry run pytest -k leak" }] }, [{ kind: "substitute-validate", validate: "V1", command: "node --test test/leak.test.js", why: "no pyproject at root" }]).contract;
+    const res = { requirements: [{ id: "R1", status: "implemented" }, { id: "R2", status: "implemented" }], validation: [{ id: "V1", exit: 0 }] };
+    const run = (substitutedRuns) => precheck({ contract: sub, result: res, changedFiles: files, substitutedRuns });
+    assert.deepEqual(run({ V1: { rc: 1, output: "Poetry could not find a pyproject.toml file in /repo" } }).failures, [], "unrunnable original: substitute stands");
+    assert.deepEqual(run({ V1: { rc: 127, output: "poetry: command not found" } }).failures, [], "missing runner");
+    assert.deepEqual(run({ V1: { rc: 0, output: "1 passed" } }).failures, [], "original passes anyway");
+    const runs = run({ V1: { rc: 1, output: "FAILED tests/test_leak.py::test_release - AssertionError" } });
+    assert.deepEqual(runs.failures.map((f) => f.kind), ["validate_failed"], "a failing original may not be hidden");
+    assert.match(runs.failures[0].detail, /frozen command runs and fails \(exit 1\)/);
+    assert.deepEqual(run(null).failures.map((f) => f.kind), ["substitute_unverified"], "no rerun: fail closed");
+    assert.deepEqual(run({ V1: { rc: 124, output: "" } }).failures.map((f) => f.kind), ["validate_timeout"]);
+    assert.deepEqual(precheck({ contract: sub, result: res, changedFiles: files, ciVerified: true, substitutedRuns: { V1: { rc: 1, output: "AssertionError" } } }).failures.map((f) => f.kind), ["validate_failed"], "CI never covers a hidden failure");
+    assert.equal(substituteVerdict(undefined), "unverified");
+    // SEC-11: a second substitute of the same id never replaces the frozen command the engine reruns
+    const twice = applyAmendments({ ...C, validate: [{ id: "V1", command: "poetry run pytest -k leak" }] }, [
+      { kind: "substitute-validate", validate: "V1", command: "nonexistent-xyz", why: "a" },
+      { kind: "substitute-validate", validate: "V1", command: "node --test test/leak.test.js", why: "b" },
+    ]).contract;
+    assert.equal(twice.validate[0].substituted.from, "poetry run pytest -k leak");
+    assert.equal(twice.validate[0].command, "node --test test/leak.test.js");
+    // SEC-21: a planner-set substituted field on the frozen contract is dropped; only an amendment sets it
+    const planted = applyAmendments({ ...C, validate: [{ id: "V1", command: "poetry run pytest -k leak", substituted: { from: "true", why: "x" } }] }, []).contract;
+    assert.equal(planted.validate[0].substituted, undefined);
+    const plantedSub = applyAmendments({ ...C, validate: [{ id: "V1", command: "poetry run pytest -k leak", substituted: { from: "true", why: "x" } }] }, [{ kind: "substitute-validate", validate: "V1", command: "node --test test/leak.test.js", why: "b" }]).contract;
+    assert.equal(plantedSub.validate[0].substituted.from, "poetry run pytest -k leak");
+  });
+  it("#4990 a substitute whose frozen command names a removed path is rejected", async () => {
+    const sub = applyAmendments({ ...C, validate: [{ id: "V1", command: "node --test test/leak.test.js" }] }, [{ kind: "substitute-validate", validate: "V1", command: "node --test test/weak.test.js", why: "w" }]).contract;
+    const res = { requirements: [{ id: "R1", status: "implemented" }, { id: "R2", status: "implemented" }], validation: [{ id: "V1", exit: 0 }] };
+    const gone = precheck({ contract: sub, result: res, changedFiles: files, substitutedRuns: { V1: { rc: 1, output: "no tests found" } }, removedPaths: ["test/leak.test.js"] });
+    assert.deepEqual(gone.failures.map((f) => f.kind), ["substitute_removed_path"]);
+    assert.equal(gone.failures[0].code, "conformance.unimplemented");
+    assert.match(gone.failures[0].detail, /test\/leak\.test\.js/);
+    const env = precheck({ contract: sub, result: res, changedFiles: files, substitutedRuns: { V1: { rc: 1, output: "Unknown option `-q`" } }, removedPaths: ["docs/other.md"] });
+    assert.deepEqual(env.failures, [], "intact named files: env-unrunnable substitute stands");
+    assert.deepEqual(removedNamedPaths("node --test ./test/a.js 'b/c.js' tests/", ["test/a.js", "b/c.js", "tests/x.js", "z.js"]), ["test/a.js", "b/c.js", "tests/x.js"]);
+    assert.deepEqual(removedNamedPaths("pytest -k leak", ["leak.py"]), []);
+    assert.deepEqual(await removedPathsOf(async () => "D\ta.js\nR100\told.js\tnew.js\nM\tm.js\nA\tn.js", "b"), ["a.js", "old.js"]);
+  });
+  it("#4857 describesUnrunnableCommand reads a blocked summary", () => {
+    assert.ok(describesUnrunnableCommand("V2 is unrunnable: poetry found no pyproject at the repo root"));
+    assert.ok(!describesUnrunnableCommand("needs a pricing decision from the product owner"));
+  });
   it("#4761 vitest -q is a contract problem; a runner usage error is contract.invalid_command, not conformance", () => {
     const bad = { ...C, validate: [{ id: "V1", command: "pnpm --filter x exec vitest run a.test.ts -q" }] };
     assert.ok(validateContract(bad, ISSUE).problems.some((p) => /V1: vitest 3 rejects -q/.test(p)));
@@ -721,6 +808,36 @@ describe("#4594 amend-requirement", () => {
     const r = a.contract.requirements.find((x) => x.id === "R1");
     assert.equal(r.disposition, "deliver"); assert.equal(r.amendment.text, am.text); assert.equal(r.amendment.why, am.why); assert.equal(r.quote, C.requirements[0].quote);
   });
+  it("amend-requirement with text on a defer requirement flips it to deliver and files no follow-up (#4984)", () => {
+    const D = { ...C, requirements: [C.requirements[0], { ...C.requirements[1], disposition: "defer", why: "later", bucket: "new-issue", tracked: "#5" }] };
+    const a = applyAmendments(D, [{ ...am, requirement: D.requirements[1].id, why: "token stored hash-only; #5 is now delivered in the diff" }]);
+    assert.deepEqual(a.problems, []); assert.deepEqual(a.toTrack, []);
+    const r = a.contract.requirements[1];
+    assert.equal(r.disposition, "deliver"); assert.equal(r.amendment.text, am.text);
+    assert.ok(!("bucket" in r) && !("why" in r) && !("tracked" in r));
+  });
+  it("a trivial rewording of a deferred requirement keeps amended_from for the reviewer (#4996)", () => {
+    const D = { ...C, requirements: [C.requirements[0], { ...C.requirements[1], disposition: "defer", why: "later", bucket: "new-issue" }] };
+    const a = applyAmendments(D, [{ ...am, requirement: D.requirements[1].id, text: "done", why: "trivial" }]);
+    assert.deepEqual(a.problems, []);
+    const r = a.contract.requirements[1];
+    assert.equal(r.disposition, "deliver"); assert.equal(r.amended_from.disposition, "defer"); assert.equal(r.amended_from.quote, C.requirements[1].quote); assert.equal(r.amended_from.why, "later");
+  });
+  it("post-deploy flip with a check id is rejected without naming it, accepted when named (#4996)", () => {
+    const chk = "pd-check-1";
+    const D = { ...C, requirements: [C.requirements[0], { ...C.requirements[1], disposition: "post-deploy", why: "after deploy", bucket: "post-deploy", check: chk }] };
+    const bad = applyAmendments(D, [{ ...am, requirement: D.requirements[1].id }]);
+    assert.match(bad.problems[0], /does not name its pd-check-1/);
+    assert.equal(D.requirements[1].disposition, "post-deploy"); assert.equal(bad.contract.requirements[1].disposition, "post-deploy");
+    const ok = applyAmendments(D, [{ ...am, requirement: D.requirements[1].id, why: `${chk} now runs in the diff` }]);
+    assert.deepEqual(ok.problems, []); assert.equal(ok.contract.requirements[1].amended_from.check, chk);
+  });
+  it("the reviewer prompt tells it to compare amended_from with the amendment (#4996)", () => {
+    assert.match(readFileSync(new URL("./prompts.mjs", import.meta.url), "utf8"), /amended_from\.quote and amended_from\.why with amendment\.text/);
+  });
+  it("followupMap matches an item key whose backticks the script stripped (#4984)", () => {
+    assert.deepEqual(followupMap("R2: wire pd.py in\t20\n", [{ key: "R2: wire `pd.py` in" }]), { "R2: wire `pd.py` in": 20 });
+  });
   it("an unknown requirement is a problem; the frozen contract is untouched", () => {
     assert.match(applyAmendments(C, [{ ...am, requirement: "R9" }]).problems[0], /unknown requirement R9/);
     assert.equal(C.requirements[0].amendment, undefined);
@@ -856,6 +973,19 @@ describe("decision amendments (#4695)", () => {
     assert.match(a.problems[0], /match no file on the base/);
     assert.equal(a.decisions.length, 0);
   });
+  it("an anchor on a lane-added file the contract lists as a deliverable conforms; an unlisted added file does not", () => {
+    const withNew = { ...withRow, deliverables: [...withRow.deliverables, { id: "D3", path: "scripts/seed.py" }] };
+    const go = (added) => applyAmendments(withNew, [{ kind: "decision", why: "w", decision: { choice: "use B", anchors: ["scripts/seed.py"] } }], null, { baseFiles: ["src/scrape.js"], addedPaths: added });
+    assert.deepEqual(go(["scripts/seed.py"]).problems, []);
+    assert.match(go(["scripts/other.py"]).problems[0], /match no file on the base/);
+  });
+  it("a blank-why amendment takes the engine's defaultWhy; without one it is still rejected", () => {
+    const am = [{ kind: "add-deliverable", path: "x.js" }];
+    const ok = applyAmendments(C, am, null, { defaultWhy: "engine-demanded rename" });
+    assert.deepEqual(ok.problems, []);
+    assert.equal(ok.applied[0].why, "engine-demanded rename");
+    assert.match(applyAmendments(C, am).problems[0], /amendment without why/);
+  });
   it("supersedes drops the engine-authored preserve row; without it the row stays", () => {
     assert.ok(!amend({ supersedes: "d9-1" }).contract.preserve.some((p) => p.id === "H1"));
     assert.ok(amend({}).contract.preserve.some((p) => p.id === "H1"));
@@ -978,7 +1108,7 @@ describe("mutation check (#4913)", () => {
   });
   it("#4931: a real repo whose change deleted a non-test file is fully restored", async () => {
     const { execFileSync } = await import("node:child_process");
-    const { writeFileSync, existsSync, readFileSync } = await import("node:fs");
+    const { writeFileSync, existsSync } = await import("node:fs");
     const { join } = await import("node:path");
     const { tmpDir } = await import("../tests/_tmp.mjs");
     const d = tmpDir("mut-");

package/bin/lean/fixround.test.mjs

@@ -3,7 +3,7 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { MECHANICAL_KINDS, FIX_PROMPT_MAX, citesFileLine, freshFix, issueSummary, capText, TRIGGERS, REWORK_TRIGGERS, firstPassOf, formatFirstPassSummary, sessionCostDelta } from "./fixround.mjs";
-import { fixSessionPrompt } from "./prompts.mjs";
+import { fixSessionPrompt, ciFailurePrompt } from "./prompts.mjs";
 
 describe("#4363 fresh fix sessions: which rounds, and the prompt caps", () => {
   it("every mechanical kind is fresh; block/contract/conflict kinds are not", () => {
@@ -115,3 +115,11 @@ describe("#4407 sessionCostDelta: per-attempt cost from a resumed session's runn
     assert.equal(sessionCostDelta(last, { resume: "b", sessionId: "b", cost: 2 }).delta, 0.5);
   });
 });
+
+describe("#4987 CI fix prompt: unrelated failures", () => {
+  it("tells the builder an out-of-diff failing check is a note, not an amendment", () => {
+    const p = ciFailurePrompt([{ job: "core-tests" }], "ci");
+    assert.match(p, /not an amendment/);
+    assert.match(p, /unknown amendment kinds are rejected/);
+  });
+});

package/bin/lean/guardgate.mjs

@@ -4,10 +4,11 @@
  * Guard gate (#4693): execution-based, no pattern matching on diff text. A lane whose diff changes lines a merged fix
  * protected (git blame on the lane's merge base names the guard's merge commit, or a later commit that kept the lines), or
  * edits/deletes a guard's test file, reruns that guard's test files *as they exist on base* against the lane's tree. A
- * failure is `conformance.guard_broken` (builder resumed once). The only override is a `supersedes` amendment (conformance.mjs).
+ * failure is `conformance.guard_broken` (builder resumed once) unless the same base test also fails on the merge-base tree (#4968). The only override is a `supersedes` amendment (conformance.mjs).
  * Deterministic, 0 LLM calls; a lane touching nothing guarded does one local index read and no git work.
  */
 import * as nodeFs from "node:fs";
+import { tmpdir } from "node:os";
 import { dirname, extname, join } from "node:path";
 import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
 import { collectGuards, guardsIndexPath, readIndex, writeIndex } from "./guards.mjs";
@@ -94,9 +95,15 @@ export async function selectGuards({ guards, files, git, base }) {
   return { selected, mergeBase };
 }
 
-/** The command that runs one test file: the contract's own validate command with that path substituted, else a per-extension default; null when unknown. */
-export function commandFor(path, contract) {
+/**
+ * The command that runs one test file: the repo's `verify.guard_command` template for the extension (`{path}` substituted,
+ * or the path appended), else the contract's own validate command with that path substituted, else a per-extension
+ * default; null when unknown. The command runs from the worktree root with the repo-relative path.
+ */
+export function commandFor(path, contract, guardCommand = {}) {
   const ext = extname(path);
+  const tpl = String(guardCommand?.[ext.toLowerCase()] ?? "").trim();
+  if (tpl) return tpl.includes("{path}") ? tpl.replaceAll("{path}", path) : `${tpl} ${path}`;
   for (const v of contract?.validate ?? []) {
     const cmd = String(v?.command ?? "");
     if (/(?:^|\s)(?:-k|-t|--grep|--test-name-pattern|--testNamePattern)(?:\s|=)|::/.test(cmd)) continue;
@@ -112,22 +119,61 @@ export function commandFor(path, contract) {
 /** Sibling path for the base copy of a test file, so its relative imports resolve against the lane's tree. */
 export const basePathOf = (path) => { const e = extname(path); return `${path.slice(0, path.length - e.length)}__guardbase${e}`; };
 
+/** Names of the failing tests in a runner's output: TAP/node:test `not ok` lines (SKIP/TODO excluded), the spec reporter's `✖ name (Nms)`, pytest `FAILED path::name`. null when the output names none. */
+export function failingNames(output) {
+  const names = new Set();
+  for (const raw of String(output ?? "").split("\n")) {
+    const line = raw.replace(/\r$/, "");
+    let m = /^\s*not ok \d+ - (.+?)\s*$/.exec(line);
+    if (m && !/\s#\s*(?:SKIP|TODO)\b/i.test(m[1])) { names.add(m[1].replace(/\s+#\s.*$/, "").trim()); continue; }
+    m = /^\s*✖ (.+?) \([\d.]+m?s\)\s*$/.exec(line) ?? /^\s*FAILED (\S+?)(?: - .*)?\s*$/.exec(line);
+    if (m) names.add(m[1]);
+  }
+  return names.size ? [...names] : null;
+}
+
 const tailOf = (out) => String(out ?? "").split("\n").filter((l) => l.trim()).slice(-40).join("\n").slice(-1500);
 
 /**
  * Rerun the selected guards' base test files against the lane tree, in a small pool, under one wall-time cap.
  * Returns {failures, runs, skipped[], ms}. A test that cannot be run (no base copy, unknown runner, over the cap) is skipped, never a failure.
  */
-export async function runGuards({ selected, mergeBase, contract, git, runCommand, cwd, env, capMs = GUARD_CAP_MS, parallelism = 2, fs = nodeFs, now = Date.now }) {
+export async function runGuards({ selected, mergeBase, contract, guardCommand, git, runCommand, cwd, env, capMs = GUARD_CAP_MS, parallelism = 2, fs = nodeFs, now = Date.now }) {
   const t0 = now(), skipped = [], runs = [], jobs = [];
   for (const { guard } of selected) {
     for (const t of guard.tests ?? []) {
-      const path = norm(t.path), copy = basePathOf(path), command = commandFor(copy, contract);
+      const path = norm(t.path), copy = basePathOf(path), command = commandFor(copy, contract, guardCommand);
       if (!command) { skipped.push(`#${guard.issue} ${path}: no runner known for ${extname(path) || "this file"}`); continue; }
       jobs.push({ guard, path, copy, command });
     }
   }
   const made = [];
+  let base = null; // #4968: one detached worktree of the merge base, made on the first lane failure and shared by the pool
+  const baseTree = () => (base ??= (async () => {
+    let dir = null;
+    try {
+      dir = fs.mkdtempSync(join(tmpdir(), "guardbase-"));
+      await git(["worktree", "add", "--detach", dir, mergeBase]);
+      try { if (fs.existsSync(join(cwd, "node_modules"))) fs.symlinkSync(join(cwd, "node_modules"), join(dir, "node_modules")); } catch { /* best effort */ }
+      return { dir };
+    } catch (e) { return { dir, error: String(e?.message ?? e).split("\n")[0].slice(0, 80) }; }
+  })());
+  /** Decide a failed lane run against the same test on the base tree: the names the lane newly fails, [] when base fails it too, null when base passes (every failure is the lane's). */
+  const newFailures = async (j, laneOut) => {
+    const left = capMs - (now() - t0);
+    if (left <= 0) return { skip: `over the ${Math.round(capMs / 1000)}s guard cap` };
+    const t = await baseTree();
+    if (t.error) return { skip: `baseline unavailable (${t.error})` };
+    const bcmd = commandFor(j.path, contract, guardCommand); // the same runner as the lane run (verify.guard_command, #4792)
+    let b;
+    try { b = await runCommand({ command: bcmd, cwd: t.dir, timeoutMs: left, env }); } catch (e) { b = { rc: 127, output: String(e?.message ?? e) }; }
+    if (b?.rc === TIMEOUT_RC || b?.rc === 127) return { skip: `baseline unavailable (${b.rc === 127 ? "runner unavailable" : "over the guard cap"})` };
+    if (b?.rc === 0) return { fresh: null };
+    const lane = failingNames(laneOut), prior = failingNames(b?.output);
+    if (!lane || !prior) return { skip: "fails on base" };
+    const fresh = lane.filter((n) => !prior.includes(n));
+    return fresh.length ? { fresh } : { skip: "fails on base" };
+  };
   try {
     const queue = jobs.slice();
     const work = async () => {
@@ -142,12 +188,25 @@ export async function runGuards({ selected, mergeBase, contract, git, runCommand
         try { r = await runCommand({ command: j.command, cwd, timeoutMs: left, env }); } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
         if (r?.rc === TIMEOUT_RC) { skipped.push(`#${j.guard.issue} ${j.path}: over the ${Math.round(capMs / 1000)}s guard cap`); continue; }
         if (r?.rc === 127) { skipped.push(`#${j.guard.issue} ${j.path}: runner unavailable`); continue; }
-        runs.push({ issue: j.guard.issue, why: j.guard.why, test: j.path, command: j.command, rc: Number.isInteger(r?.rc) ? r.rc : 1, output: String(r?.output ?? "") });
+        const rc = Number.isInteger(r?.rc) ? r.rc : 1, output = String(r?.output ?? "");
+        let failed = rc === 0 ? null : failingNames(output);
+        if (rc !== 0) {
+          const v = await newFailures(j, output);
+          if (v.skip) { skipped.push(`guard #${j.guard.issue} ${j.path}: ${v.skip}`); continue; }
+          if (v.fresh) failed = v.fresh;
+        }
+        runs.push({ issue: j.guard.issue, why: j.guard.why, test: j.path, command: j.command, rc, output, failed });
       }
     };
     await Promise.all(Array.from({ length: Math.max(1, Math.min(parallelism, jobs.length)) }, work));
   } finally {
     for (const p of made) { try { fs.rmSync(p, { force: true }); } catch { /* best effort */ } }
+    const t = base ? await base : null;
+    if (t?.dir) {
+      try { await git(["worktree", "remove", "--force", t.dir]); } catch { /* best effort */ }
+      try { fs.rmSync(t.dir, { recursive: true, force: true }); } catch { /* best effort */ }
+      try { await git(["worktree", "prune"]); } catch { /* best effort */ }
+    }
   }
   return { failures: guardFailures(runs), runs, skipped, ms: now() - t0 };
 }
@@ -159,7 +218,7 @@ const supersedesHint = (guard) => `If this change intentionally changes the guar
 export function guardFailures(runs) {
   return (runs ?? []).filter((r) => r.rc !== 0).map((r) => ({
     code: OUTCOME_CODES.CONFORMANCE_GUARD_BROKEN, kind: "guard_broken", guard: `#${r.issue}`, test: r.test,
-    detail: `${supersedesHint(`#${r.issue}`)} guard #${r.issue} broken (${String(r.why ?? "").slice(0, 200)}): base test ${r.test} fails against your tree (\`${r.command}\`, exit ${r.rc}):\n${tailOf(r.output)}`,
+    detail: `${supersedesHint(`#${r.issue}`)} guard #${r.issue} broken (${String(r.why ?? "").slice(0, 200)}): base test ${r.test} fails against your tree (\`${r.command}\`, exit ${r.rc})${r.failed?.length ? `; failing tests: ${r.failed.slice(0, 20).join(" | ")}` : ""}:\n${tailOf(r.output)}`,
   }));
 }