forgedock 1.26.0 → 1.28.0

package/scripts/bootstrap-validate.mjs

@@ -0,0 +1,321 @@
+#!/usr/bin/env node
+/**
+ * scripts/bootstrap-validate.mjs — mechanical backlog plan validator for /bootstrap (#4171, part of #4155)
+ *
+ * Validates a JSON backlog plan BEFORE any issue is created: milestones in build order, issues with plan keys,
+ * `#{{KEY}}` placeholders standing in for not-yet-created issue numbers, planned dependencies and bodies.
+ * Codebase-agnostic: it reads only the plan, never the repo, so planned (not-yet-existing) paths are fine.
+ *
+ * Plan shape:
+ *   {
+ *     "labels":     ["backend", ...],                      // optional: when given, every issue label must be in it
+ *     "milestones": [{ "key": "M1", "title": "..." }, ...],  // build order: foundation first
+ *     "issues": [{ "key": "API", "title": "...", "milestone": "M1", "labels": ["backend"],
+ *                  "deps": ["DB"], "needsHuman": false, "ui": false, "hero": false, "body": "## Problem\n..." }],
+ *     "numbers":    { "DB": 12 }                             // optional: real issue numbers once created
+ *   }
+ *
+ * Rejects: invalid-plan, unknown-dep, cycle, forward-milestone-dep, depends-on-mismatch, unresolved-placeholder,
+ * missing-section, missing-type-tag, missing-parent, missing-labels, unknown-label, code-depends-on-needs-human,
+ * missing-design-spec (a `ui` issue lacks a design-spec item), hero-not-routed (a `hero` issue never names `/design`),
+ * missing-needs-human-section (a needs-human issue lacks setup steps, env var names or secret handling) (#4176).
+ * `Depends on` lines are read with the engine's own parser (bin/engine/phases.mjs parseDependsOn) after placeholder
+ * resolution, so the plan and engine admission agree on what blocks what.
+ *
+ * Usage:
+ *   node scripts/bootstrap-validate.mjs <plan.json>
+ *   node scripts/bootstrap-validate.mjs -            # read the plan from stdin
+ *
+ * Exit codes:
+ *   0 = plan valid
+ *   1 = plan invalid (one `<KEY>: [code] message` line per error on stderr)
+ *   2 = usage error, unreadable file or bad JSON
+ */
+
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { parseDependsOn } from "../bin/engine/phases.mjs";
+
+const NEEDS_HUMAN_LABEL = "needs-human";
+const TYPE_TAG_RE = /\[type:(?:api|unit|e2e|manual)\]/;
+const CHECKBOX_RE = /^[ \t]*[-*+][ \t]+\[[ xX]\]/;
+const PLACEHOLDER_RE = /#\{\{([^{}]*)\}\}/g;
+// Synthetic numbers for keys without a real number: far above any real issue number, so they never collide.
+const SYNTHETIC_BASE = 1_000_000_000;
+
+// Mandatory Pipeline Issue Template sections (commands/issue.md Phase 3D). Matched case-insensitively on `## ` headings,
+// ignoring a trailing parenthetical such as "Root Cause (if known)".
+const SECTIONS = ["Problem", "Root Cause", "Affected Files", "Acceptance Criteria", "Context", "Dependencies"];
+
+// UI issues carry a full design spec (#4176): each item must be named somewhere in the body.
+const DESIGN_SPEC = [
+  ["breakpoints", /breakpoint/i],
+  ["empty state", /\bempty\b/i],
+  ["loading state", /\bloading\b/i],
+  ["error state", /\berror\b/i],
+  ["offline state", /\boffline\b/i],
+  ["motion", /\bmotion\b/i],
+  ["accessibility", /\b(?:accessibility|a11y)\b/i],
+  ["dark mode", /\bdark[ -]mode\b/i],
+];
+// Needs-human issues (#4176): headings for setup steps, env vars and secret handling, plus at least one env var name.
+const NEEDS_HUMAN_PARTS = [
+  ["setup steps heading", /^#{2,6}[ \t]+.*\bsetup steps\b/im],
+  ["environment variables heading", /^#{2,6}[ \t]+.*\b(?:environment variables|env vars)\b/im],
+  ["secret handling heading", /^#{2,6}[ \t]+.*\bsecrets?\b/im],
+  ["env var name (e.g. `API_KEY`)", /`[A-Z][A-Z0-9]*_[A-Z0-9_]+`/],
+];
+
+const isObj = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
+const isNonEmptyString = (v) => typeof v === "string" && v.trim() !== "";
+const normHeading = (h) => h.replace(/\([^)]*\)\s*$/, "").trim().toLowerCase();
+
+/** True when the issue is a needs-human issue (credential, account or external input). */
+export function isNeedsHuman(issue) {
+  return issue?.needsHuman === true || (Array.isArray(issue?.labels) && issue.labels.includes(NEEDS_HUMAN_LABEL));
+}
+
+/**
+ * Replace every `#{{KEY}}` whose KEY is in `keyToNumber` with `#N`. Unknown keys are left in place.
+ * @param {string} body
+ * @param {Record<string, number>|Map<string, number>} keyToNumber
+ */
+export function resolvePlaceholders(body, keyToNumber) {
+  const map = keyToNumber instanceof Map ? keyToNumber : new Map(Object.entries(keyToNumber ?? {}));
+  return String(body ?? "").replace(PLACEHOLDER_RE, (whole, key) => {
+    const n = map.get(key);
+    return Number.isSafeInteger(n) && n > 0 ? `#${n}` : whole;
+  });
+}
+
+/** `## ` sections of a markdown body (fenced code blocks ignored): normalized heading -> content lines. */
+function sections(body) {
+  const out = new Map();
+  let current = null;
+  let fence = null;
+  for (const line of String(body).replace(/\r\n/g, "\n").split("\n")) {
+    const f = line.match(/^[ \t]*(`{3,}|~{3,})/);
+    if (f) {
+      if (!fence) fence = f[1][0];
+      else if (f[1][0] === fence) fence = null;
+      if (current) current.push(line);
+      continue;
+    }
+    const h = !fence && line.match(/^##(?!#)[ \t]+(.+?)[ \t]*#*[ \t]*$/);
+    if (h) {
+      current = [];
+      const name = normHeading(h[1]);
+      if (!out.has(name)) out.set(name, current);
+      continue;
+    }
+    if (current) current.push(line);
+  }
+  return out;
+}
+
+function checkStructure(plan) {
+  const errors = [];
+  const bad = (message) => errors.push({ issue: null, code: "invalid-plan", message });
+  if (!isObj(plan)) return [{ issue: null, code: "invalid-plan", message: "plan must be a JSON object" }];
+  if (!Array.isArray(plan.milestones) || plan.milestones.length === 0) bad("`milestones` must be a non-empty array");
+  if (!Array.isArray(plan.issues) || plan.issues.length === 0) bad("`issues` must be a non-empty array");
+  if (plan.labels !== undefined && !(Array.isArray(plan.labels) && plan.labels.every(isNonEmptyString))) {
+    bad("`labels` must be an array of strings when given");
+  }
+  if (plan.numbers !== undefined && !isObj(plan.numbers)) bad("`numbers` must be an object of KEY -> issue number");
+  if (errors.length) return errors;
+
+  const milestoneKeys = new Set();
+  plan.milestones.forEach((m, i) => {
+    if (!isObj(m) || !isNonEmptyString(m.key)) return bad(`milestones[${i}] needs a string \`key\``);
+    if (milestoneKeys.has(m.key)) bad(`duplicate milestone key ${m.key}`);
+    milestoneKeys.add(m.key);
+  });
+  const issueKeys = new Set();
+  plan.issues.forEach((it, i) => {
+    if (!isObj(it) || !isNonEmptyString(it.key)) return bad(`issues[${i}] needs a string \`key\``);
+    const at = (message) => errors.push({ issue: it.key, code: "invalid-plan", message });
+    if (issueKeys.has(it.key)) at(`duplicate issue key ${it.key}`);
+    issueKeys.add(it.key);
+    if (!isNonEmptyString(it.title)) at("needs a non-empty `title`");
+    if (!milestoneKeys.has(it.milestone)) at(`unknown milestone ${JSON.stringify(it.milestone)}`);
+    if (it.deps !== undefined && !(Array.isArray(it.deps) && it.deps.every(isNonEmptyString))) {
+      at("`deps` must be an array of issue keys");
+    }
+    if (it.labels !== undefined && !(Array.isArray(it.labels) && it.labels.every((l) => typeof l === "string"))) {
+      at("`labels` must be an array of strings");
+    }
+    if (typeof it.body !== "string") at("needs a string `body`");
+    for (const f of ["ui", "hero"]) if (it[f] !== undefined && typeof it[f] !== "boolean") at(`\`${f}\` must be a boolean`);
+  });
+  return errors;
+}
+
+/** Issue keys on dependency cycles, each with one cycle path through it (DFS over known deps). */
+function findCycles(issues, byKey) {
+  const state = new Map(); // key -> 1 visiting | 2 done
+  const stack = [];
+  const onCycle = new Map();
+  const visit = (key) => {
+    state.set(key, 1);
+    stack.push(key);
+    for (const d of byKey.get(key).deps ?? []) {
+      if (!byKey.has(d)) continue;
+      if (state.get(d) === 1) {
+        const path = [...stack.slice(stack.indexOf(d)), d];
+        for (const k of path) if (!onCycle.has(k)) onCycle.set(k, path);
+      } else if (!state.has(d)) visit(d);
+    }
+    stack.pop();
+    state.set(key, 2);
+  };
+  for (const it of issues) if (!state.has(it.key)) visit(it.key);
+  return onCycle;
+}
+
+/**
+ * Validate a backlog plan. Never throws on bad input; structural problems come back as `invalid-plan` errors.
+ * @returns {{ ok: boolean, errors: Array<{ issue: string|null, code: string, message: string }> }}
+ */
+export function validatePlan(plan) {
+  const structural = checkStructure(plan);
+  if (structural.length) return { ok: false, errors: structural };
+
+  const errors = [];
+  const { issues, milestones } = plan;
+  const err = (issue, code, message) => errors.push({ issue, code, message });
+  const byKey = new Map(issues.map((it) => [it.key, it]));
+  const milestoneIndex = new Map(milestones.map((m, i) => [m.key, i]));
+  const declaredLabels = Array.isArray(plan.labels) ? new Set(plan.labels) : null;
+
+  // Every plan key resolves: to its real number when known, else to a unique synthetic one.
+  const keyToNumber = new Map();
+  issues.forEach((it, i) => {
+    const real = plan.numbers?.[it.key];
+    keyToNumber.set(it.key, Number.isSafeInteger(real) && real > 0 ? real : SYNTHETIC_BASE + i);
+  });
+  const numberToKey = new Map([...keyToNumber].map(([k, n]) => [n, k]));
+
+  const cycles = findCycles(issues, byKey);
+
+  for (const it of issues) {
+    const deps = it.deps ?? [];
+    const myMilestone = milestoneIndex.get(it.milestone);
+
+    for (const d of deps) {
+      const dep = byKey.get(d);
+      if (!dep) {
+        err(it.key, "unknown-dep", `depends on ${d}, which is not an issue in the plan`);
+        continue;
+      }
+      const depMilestone = milestoneIndex.get(dep.milestone);
+      if (depMilestone > myMilestone) {
+        err(it.key, "forward-milestone-dep",
+          `in milestone ${it.milestone} but depends on ${d} in later milestone ${dep.milestone}`);
+      }
+      if (!isNeedsHuman(it) && isNeedsHuman(dep)) {
+        err(it.key, "code-depends-on-needs-human",
+          `code issue depends on needs-human issue ${d}; reference it for real credentials, but keep mocks so it never blocks`);
+      }
+    }
+    if (cycles.has(it.key)) err(it.key, "cycle", `dependency cycle: ${cycles.get(it.key).join(" -> ")}`);
+
+    const labels = it.labels ?? [];
+    if (labels.filter(isNonEmptyString).length === 0) err(it.key, "missing-labels", "issue has no labels");
+    if (declaredLabels) {
+      for (const l of labels) {
+        if (isNonEmptyString(l) && !declaredLabels.has(l)) err(it.key, "unknown-label", `label ${l} is not in the plan's labels`);
+      }
+    }
+
+    const resolved = resolvePlaceholders(it.body, keyToNumber);
+    const unresolved = [...new Set([...resolved.matchAll(PLACEHOLDER_RE)].map((m) => m[0]))];
+    for (const p of unresolved) err(it.key, "unresolved-placeholder", `${p} does not match any issue key in the plan`);
+
+    const declared = new Set();
+    const strays = [];
+    for (const n of parseDependsOn(resolved)) {
+      if (numberToKey.has(n)) declared.add(numberToKey.get(n));
+      else strays.push(`#${n}`);
+    }
+    const planned = new Set(deps);
+    const missing = [...planned].filter((k) => !declared.has(k));
+    const extra = [...declared].filter((k) => !planned.has(k)).concat(strays);
+    if (missing.length || extra.length) {
+      const parts = [];
+      if (missing.length) parts.push(`planned but not declared: ${missing.join(", ")}`);
+      if (extra.length) parts.push(`declared but not planned: ${extra.join(", ")}`);
+      err(it.key, "depends-on-mismatch", `\`Depends on\` lines differ from planned deps (${parts.join("; ")})`);
+    }
+
+    const secs = sections(it.body);
+    for (const name of SECTIONS) {
+      if (!secs.has(name.toLowerCase())) err(it.key, "missing-section", `missing \`## ${name}\` section`);
+    }
+    const ac = secs.get("acceptance criteria");
+    if (ac) {
+      const items = ac.filter((l) => CHECKBOX_RE.test(l));
+      if (items.length === 0) err(it.key, "missing-section", "`## Acceptance Criteria` has no `- [ ]` checklist items");
+      const untagged = items.filter((l) => !TYPE_TAG_RE.test(l));
+      if (untagged.length) {
+        err(it.key, "missing-type-tag",
+          `${untagged.length} acceptance criteria lack a [type:api|unit|e2e|manual] tag: ${untagged[0].trim()}`);
+      }
+    }
+    const ctx = secs.get("context");
+    if (ctx && !ctx.some((l) => l.includes("**Parent**"))) err(it.key, "missing-parent", "`## Context` has no `**Parent**` line");
+
+    if (it.ui === true) {
+      const lacking = DESIGN_SPEC.filter(([, re]) => !re.test(it.body)).map(([name]) => name);
+      if (lacking.length) err(it.key, "missing-design-spec", `UI issue design spec lacks: ${lacking.join(", ")}`);
+    }
+    if (it.hero === true && !/(^|[^\w/])\/design\b/.test(it.body)) {
+      err(it.key, "hero-not-routed", "hero surface issue does not route to `/design`");
+    }
+    if (isNeedsHuman(it)) {
+      const lacking = NEEDS_HUMAN_PARTS.filter(([, re]) => !re.test(it.body)).map(([name]) => name);
+      if (lacking.length) err(it.key, "missing-needs-human-section", `needs-human issue lacks: ${lacking.join(", ")}`);
+    }
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/** One `<KEY>: [code] message` line per error; plan-level errors use `(plan)`. */
+export function formatErrors(errors) {
+  return errors.map((e) => `${e.issue ?? "(plan)"}: [${e.code}] ${e.message}`).join("\n");
+}
+
+function main(argv) {
+  const arg = argv[0];
+  if (!arg || argv.length > 1) {
+    process.stderr.write("usage: node scripts/bootstrap-validate.mjs <plan.json|->\n");
+    return 2;
+  }
+  let raw;
+  try {
+    raw = readFileSync(arg === "-" ? 0 : arg, "utf8");
+  } catch (e) {
+    process.stderr.write(`bootstrap-validate: cannot read ${arg}: ${e.message}\n`);
+    return 2;
+  }
+  let plan;
+  try {
+    plan = JSON.parse(raw);
+  } catch (e) {
+    process.stderr.write(`bootstrap-validate: invalid JSON: ${e.message}\n`);
+    return 2;
+  }
+  const { ok, errors } = validatePlan(plan);
+  if (!ok) {
+    process.stderr.write(formatErrors(errors) + "\n");
+    process.stderr.write(`bootstrap-validate: ${errors.length} error(s)\n`);
+    return 1;
+  }
+  process.stdout.write(`bootstrap-validate: OK (${plan.issues.length} issues, ${plan.milestones.length} milestones)\n`);
+  return 0;
+}
+
+// Run as CLI only when invoked directly, not when imported by tests.
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/scripts/calibration.mjs

@@ -36,6 +36,12 @@
  *   --dry-run                      Compute table(s) but do not write any files
  *   --verbose                      Verbose logging
  *   --issue <number>               Compute outcome for a single issue only (debugging)
+ *   --shadow                       Instead: the per-repo conformance-gate shadow verdict distribution (forge#4160),
+ *                                  read from FORGE:REVIEW comments, and whether to switch engine.conformance to enforce
+ *                                  (--repo accepts a comma-separated list)
+ *   --threshold <share>            --shadow: recommend enforce while would-block-on-incomplete stays under this share (default 0.10)
+ *   --min-runs <n>                 --shadow: minimum shadow reviews before recommending enforce (default 20)
+ *   --json                         --shadow: print JSON instead of the text table
  *
  * Exit codes:
  *   0  — success
@@ -75,6 +81,11 @@ const SCHEMA_VERSION = 1;
 // larger and cells accumulate samples more slowly.
 const DEFAULT_PROVENANCE_MIN_SAMPLES = 5;
 
+// forge#4160: the conformance gate graduates from shadow to enforce once incomplete reviews are rare.
+const DEFAULT_SHADOW_THRESHOLD = 0.10;
+const DEFAULT_SHADOW_MIN_RUNS = 20;
+const SHADOW_VERDICTS = ['pass', 'nonconforming', 'review.incomplete', 'review.inconsistent', 'review.unparseable'];
+
 // Survival rate threshold above which a cell is classified as PROVEN (eligible
 // for optional-agent de-escalation).
 const PROVENANCE_PROVEN_THRESHOLD = 0.90;
@@ -105,6 +116,10 @@ export function parseArgs(argv) {
     dryRun: false,
     verbose: false,
     issue: null,
+    shadow: false,
+    threshold: DEFAULT_SHADOW_THRESHOLD,
+    minRuns: DEFAULT_SHADOW_MIN_RUNS,
+    json: false,
   };
 
   for (let i = 2; i < argv.length; i++) {
@@ -120,6 +135,10 @@ export function parseArgs(argv) {
     else if (arg === '--dry-run') { args.dryRun = true; }
     else if (arg === '--verbose') { args.verbose = true; }
     else if (arg === '--issue' && argv[i + 1]) { args.issue = parseInt(argv[++i], 10); }
+    else if (arg === '--shadow') { args.shadow = true; }
+    else if (arg === '--threshold' && argv[i + 1]) { args.threshold = parseFloat(argv[++i]); }
+    else if (arg === '--min-runs' && argv[i + 1]) { args.minRuns = parseInt(argv[++i], 10); }
+    else if (arg === '--json') { args.json = true; }
   }
 
   return args;
@@ -812,6 +831,98 @@ export function publishTable(table, repoPath, dryRun = false) {
 // Main pipeline
 // ---------------------------------------------------------------------------
 
+// ---------------------------------------------------------------------------
+// Shadow verdict distribution (forge#4160)
+// ---------------------------------------------------------------------------
+
+/**
+ * The shadow gate verdict in one FORGE:REVIEW comment body: "pass", or the would-block state
+ * (nonconforming | review.incomplete | review.inconsistent | review.unparseable). Null for anything else,
+ * including an enforced review.
+ * @param {string} body
+ * @returns {string|null}
+ */
+export function parseShadowVerdict(body) {
+  const b = String(body ?? '');
+  if (!b.includes('FORGE:REVIEW')) return null;
+  const m = /\*\*Gate verdict:\*\*\s*would (PASS|BLOCK \(([^)]+)\))/.exec(b);
+  if (!m) return null;
+  return m[1] === 'PASS' ? 'pass' : m[2].trim();
+}
+
+/**
+ * Per-repo counts and shares of shadow verdicts.
+ * @param {{repo: string, verdict: string}[]} records
+ * @returns {Record<string, {runs: number, counts: Record<string, number>, shares: Record<string, number>}>}
+ */
+export function shadowDistribution(records) {
+  const out = {};
+  for (const r of records ?? []) {
+    if (!r?.repo || !r?.verdict) continue;
+    const e = out[r.repo] ??= { runs: 0, counts: Object.fromEntries(SHADOW_VERDICTS.map((v) => [v, 0])), shares: {} };
+    e.runs++;
+    e.counts[r.verdict] = (e.counts[r.verdict] ?? 0) + 1;
+  }
+  for (const e of Object.values(out)) e.shares = Object.fromEntries(Object.entries(e.counts).map(([k, n]) => [k, e.runs ? Math.round((n / e.runs) * 1000) / 1000 : 0]));
+  return out;
+}
+
+/**
+ * enforce once there are at least minRuns shadow reviews and the would-block-on-incomplete share is under threshold.
+ * @param {{runs: number, counts: Record<string, number>}} entry
+ * @param {{threshold?: number, minRuns?: number}} [o]
+ * @returns {{mode: 'enforce'|'shadow', incompleteShare: number, reason: string}}
+ */
+export function recommendEnforce(entry, { threshold = DEFAULT_SHADOW_THRESHOLD, minRuns = DEFAULT_SHADOW_MIN_RUNS } = {}) {
+  const runs = entry?.runs ?? 0;
+  const share = runs ? (entry.counts?.['review.incomplete'] ?? 0) / runs : 0;
+  const pct = (x) => `${Math.round(x * 1000) / 10}%`;
+  if (runs < minRuns) return { mode: 'shadow', incompleteShare: share, reason: `only ${runs} shadow review(s); need ${minRuns}` };
+  if (share >= threshold) return { mode: 'shadow', incompleteShare: share, reason: `would-block-on-incomplete ${pct(share)} is not under ${pct(threshold)}` };
+  return { mode: 'enforce', incompleteShare: share, reason: `would-block-on-incomplete ${pct(share)} < ${pct(threshold)} over ${runs} runs` };
+}
+
+/**
+ * Shadow FORGE:REVIEW verdicts for one repo, from its issue comments.
+ * @param {string} repo
+ * @param {(path: string, flags: string[]) => string} [gh]
+ * @returns {{repo: string, issue: number|null, verdict: string}[]}
+ */
+export function fetchShadowRecords(repo, gh = ghApi) {
+  const out = gh(`repos/${repo}/issues/comments?per_page=100`, ['--paginate', '--jq', '.[] | select(.body | contains("<!-- FORGE:REVIEW -->")) | {issue_url, body}']);
+  const records = [];
+  for (const l of String(out).split('\n')) {
+    let c; try { c = JSON.parse(l); } catch { continue; }
+    const verdict = parseShadowVerdict(c?.body);
+    if (verdict) records.push({ repo, issue: Number(/\/issues\/(\d+)$/.exec(String(c.issue_url ?? ''))?.[1]) || null, verdict });
+  }
+  return records;
+}
+
+/** Text table: one row per repo with the verdict mix and the recommendation. */
+export function formatShadowReport(report) {
+  const lines = ['| Repo | Runs | ' + SHADOW_VERDICTS.join(' | ') + ' | Recommend |', '| --- | --- | ' + SHADOW_VERDICTS.map(() => '---').join(' | ') + ' | --- |'];
+  for (const [repo, e] of Object.entries(report.repos)) {
+    lines.push(`| ${repo} | ${e.runs} | ${SHADOW_VERDICTS.map((v) => `${e.counts[v] ?? 0} (${Math.round((e.shares[v] ?? 0) * 100)}%)`).join(' | ')} | ${e.recommendation.mode}: ${e.recommendation.reason} |`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * --shadow entry: per-repo distribution plus an enforce recommendation each.
+ * @returns {{shadow: true, repos: object}}
+ */
+export function runShadow(args, repoPath, gh = ghApi) {
+  const repos = String(args.repo || readForgeYaml(repoPath) || '').split(',').map((r) => r.trim()).filter(Boolean);
+  if (!repos.length) throw new Error('Cannot determine repo: no --repo flag and forge.yaml not found or missing project.owner/repo');
+  const records = repos.flatMap((r) => fetchShadowRecords(r, gh));
+  const dist = shadowDistribution(records);
+  const zero = () => Object.fromEntries(SHADOW_VERDICTS.map((v) => [v, 0]));
+  for (const r of repos) dist[r] ??= { runs: 0, counts: zero(), shares: zero() };
+  for (const e of Object.values(dist)) e.recommendation = recommendEnforce(e, { threshold: args.threshold, minRuns: args.minRuns });
+  return { shadow: true, repos: dist };
+}
+
 /**
  * Main entry point (exported for testing).
  *
@@ -821,6 +932,7 @@ export function publishTable(table, repoPath, dryRun = false) {
  */
 export async function run(args, repoPath) {
   _verbose = args.verbose;
+  if (args.shadow) return runShadow(args, repoPath);
 
   // Resolve repo name
   const repo = args.repo || readForgeYaml(repoPath);
@@ -965,6 +1077,10 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
 
   run(args, repoPath)
     .then(result => {
+      if (result?.shadow) {
+        process.stdout.write((args.json ? JSON.stringify(result, null, 2) : formatShadowReport(result)) + '\n');
+        process.exit(0);
+      }
       const table = result.table || result;
       const provenanceTable = result.provenanceTable || null;
       const errorCount = result.errorCount || 0;

package/scripts/check-devdocs-freshness.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# check-devdocs-freshness.sh — docs must move with runtime behaviour (forge#4185).
+# A PR that changes non-test files under bin/lean/ or scripts/ must, for every such file, also:
+#   (a) touch a dossier whose devdocs/index.yaml `modules:` glob matches the file, or
+#   (b) change lines inside the `## Runtime map` section of NORTHSTAR.md, or
+#   (c) add a new devdocs/decisions/*.md.
+# Exempt: a `docs-not-needed: <reason>` line (non-empty reason) in the PR body.
+# Deterministic, no LLM. bash 3.2 portable (no bash-4 builtins, no PCRE grep).
+#
+# Usage: bash scripts/check-devdocs-freshness.sh [<base-ref>] [--body-file <file>]
+#   No base-ref: the current branch's upstream (@{upstream}), else origin/HEAD (the remote default branch).
+#   PR body: --body-file, else env PR_BODY (empty when unset). Diff: <base-ref>...HEAD, run from the repo root.
+# Exit: 0 fresh / exempt / nothing to check | 1 docs missing | 2 bad usage or unknown base ref
+
+set -u
+BASE=""
+case "${1:-}" in
+  ""|--*) BASE=$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' 2>/dev/null) \
+            || BASE=$(git rev-parse --abbrev-ref origin/HEAD 2>/dev/null) || BASE="" ;;
+  *) BASE="$1"; shift ;;
+esac
+[ -n "$BASE" ] || { echo "usage: check-devdocs-freshness.sh [<base-ref>] [--body-file <file>] (no upstream or origin/HEAD to default to)" >&2; exit 2; }
+BODY="${PR_BODY:-}"
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --body-file)
+      [ -f "${2:-}" ] || { echo "check-devdocs-freshness: body file not found: ${2:-}" >&2; exit 2; }
+      BODY=$(cat "$2"); shift 2 ;;
+    *) echo "check-devdocs-freshness: unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+git rev-parse --verify --quiet "${BASE}^{commit}" >/dev/null || { echo "check-devdocs-freshness: unknown base ref: $BASE" >&2; exit 2; }
+
+CHANGES=$(git diff --no-renames --name-status "${BASE}...HEAD") || { echo "check-devdocs-freshness: git diff failed" >&2; exit 2; }
+CHANGED=$(printf '%s\n' "$CHANGES" | awk -F'\t' 'NF>=2 {print $2}')
+
+# Behaviour files: non-test files under bin/lean/ or scripts/.
+BEHAVIOUR=$(printf '%s\n' "$CHANGED" | grep -E '^(bin/lean|scripts)/' \
+  | grep -vE '\.test\.[^/]*$|(^|/)fixtures/|^scripts/tests/' || true)
+if [ -z "$BEHAVIOUR" ]; then
+  echo "OK: no behaviour change under bin/lean/ or scripts/ — devdocs freshness not required."
+  exit 0
+fi
+
+if printf '%s\n' "$BODY" | tr -d '\r' | grep -Eq '^[[:space:]]*docs-not-needed:[[:space:]]*[^[:space:]]'; then
+  echo "OK: exempt via 'docs-not-needed:' line in the PR body."
+  exit 0
+fi
+
+# (c) a new decision record.
+DECISION=$(printf '%s\n' "$CHANGES" | awk -F'\t' '$1=="A" && $2 ~ /^devdocs\/decisions\/[^\/]+\.md$/ {print $2}' | head -1)
+
+# (b) changed lines inside NORTHSTAR.md § Runtime map (line range in HEAD vs the -U0 hunks' new-side lines).
+RUNTIME_MAP=0
+if printf '%s\n' "$CHANGED" | grep -qx 'NORTHSTAR.md' && [ -f NORTHSTAR.md ]; then
+  RANGE=$(awk '/^## /{ if (s && !e) e = NR - 1; if ($0 ~ /^## Runtime map[[:space:]]*$/) s = NR } END { if (s) { if (!e) e = NR; print s, e } }' NORTHSTAR.md)
+  if [ -n "$RANGE" ]; then
+    RS=${RANGE% *}; RE=${RANGE#* }
+    RUNTIME_MAP=$(git diff -U0 "${BASE}...HEAD" -- NORTHSTAR.md | awk -v s="$RS" -v e="$RE" '
+      /^@@/ {
+        match($0, /\+[0-9]+(,[0-9]+)?/)
+        spec = substr($0, RSTART + 1, RLENGTH - 1)
+        n = split(spec, p, ",")
+        start = p[1] + 0; cnt = (n > 1) ? p[2] + 0 : 1
+        if (cnt == 0) { lo = start; hi = start + 1 } else { lo = start; hi = start + cnt - 1 }
+        if (lo <= e && hi >= s) hit = 1
+      }
+      END { print hit ? 1 : 0 }')
+  fi
+fi
+
+# (a) dossiers: glob/path pairs from devdocs/index.yaml `modules:`.
+MODULES=""
+if [ -f devdocs/index.yaml ]; then
+  MODULES=$(awk '
+    /^modules:/ { m = 1; next }
+    m && /^[^[:space:]#]/ { m = 0 }
+    m && /^[[:space:]]*-[[:space:]]/ { if (g != "" && p != "") print g "\t" p; g = ""; p = "" }
+    m && /^[[:space:]]*(-[[:space:]]+)?glob:/ { v = $0; sub(/^[^:]*:[[:space:]]*/, "", v); gsub(/["\047]/, "", v); g = v }
+    m && /^[[:space:]]*(-[[:space:]]+)?path:/ { v = $0; sub(/^[^:]*:[[:space:]]*/, "", v); gsub(/["\047]/, "", v); p = v }
+    END { if (g != "" && p != "") print g "\t" p }' devdocs/index.yaml)
+fi
+
+MISSING=""
+while IFS= read -r f; do
+  [ -n "$f" ] || continue
+  [ -n "$DECISION" ] && continue
+  [ "$RUNTIME_MAP" = "1" ] && continue
+  ok=0
+  if [ -n "$MODULES" ]; then
+    while IFS="$(printf '\t')" read -r glob dpath; do
+      [ -n "$glob" ] || continue
+      # shellcheck disable=SC2254
+      case "$f" in
+        $glob) printf '%s\n' "$CHANGED" | grep -qxF "devdocs/$dpath" && ok=1 ;;
+      esac
+    done <<EOF_MODULES
+$MODULES
+EOF_MODULES
+  fi
+  [ "$ok" = "1" ] || MISSING="${MISSING}${f}
+"
+done <<EOF_BEHAVIOUR
+$BEHAVIOUR
+EOF_BEHAVIOUR
+
+if [ -z "$MISSING" ]; then
+  if [ -n "$DECISION" ]; then echo "OK: new decision record $DECISION covers the behaviour change."
+  elif [ "$RUNTIME_MAP" = "1" ]; then echo "OK: NORTHSTAR.md § Runtime map updated."
+  else echo "OK: every behaviour change has its module dossier updated."; fi
+  exit 0
+fi
+
+echo "ERROR: behaviour changed under bin/lean/ or scripts/ without a docs update:"
+printf '%s' "$MISSING" | sed 's/^/  - /'
+echo ""
+echo "Fix with one of:"
+echo "  - edit the matching dossier (devdocs/index.yaml modules: glob -> devdocs/modules/*.md)"
+echo "  - update NORTHSTAR.md § Runtime map"
+echo "  - add devdocs/decisions/<issue>-<slug>.md"
+echo "  - or add a line 'docs-not-needed: <reason>' to the PR body"
+exit 1