forgedock 1.27.0 → 1.28.0

package/bin/lean/drift.mjs

@@ -0,0 +1,198 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Milestone drift checkpoint (forge#4161). No LLM anywhere: facts that merged lanes already computed are summed, and
+ * an issue is filed only when a threshold is crossed.
+ *
+ * - Summary: each merged milestone lane's merge-time STRUCTURE_DELTA (`result.structure`, lean.mjs recordStructure) is
+ *   folded into one `<!-- FORGE:DRIFT {json} -->` block in the GitHub milestone description (GET + PATCH per merge).
+ *   The fold is keyed by PR, so a re-fold is a no-op. Text outside the block is kept byte for byte.
+ * - Report: `scripts/milestone-drift.sh` recomputes pair growth, new edges and size growth from git (fork point to
+ *   the milestone head). The summary only attributes PRs.
+ * - Crossings (only with `orchestrate.drift` thresholds): one templated issue per crossing in the milestone,
+ *   deduplicated by its `<!-- FORGE:DRIFT:<slug>:<kind>:<key> -->` marker against the milestone's open issues.
+ * - block_dispatch (off by default): a queued issue on the same milestone lane whose Affected Files sit under a
+ *   flagged package dir is held.
+ */
+import { execFile } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const SUMMARY_MARKER = "FORGE:DRIFT";
+const SUMMARY_RE = /<!-- FORGE:DRIFT (\{[\s\S]*?\}) -->/;
+// The summary keeps the largest growth entries only, so the description stays small.
+export const MAX_GROWTH_FILES = 50;
+const SCRIPT = fileURLToPath(new URL("../../scripts/milestone-drift.sh", import.meta.url));
+
+export const emptySummary = (milestone) => ({ v: 1, milestone, merged: [], pairs: {}, deps_added: [], deps_removed: [], growth: {} });
+
+const arr = (x) => (Array.isArray(x) ? x : []);
+const edgeKey = (e) => `${e?.from}:${e?.to}`;
+
+/**
+ * Fold one merged run's structure record into the summary (pure; returns a new object). Keyed by PR: a PR already in
+ * `merged` leaves the summary unchanged.
+ */
+export function foldSummary(summary, { issue, pr, milestone, structure }) {
+  const s = summary && typeof summary === "object" ? summary : emptySummary(milestone ?? null);
+  const merged = arr(s.merged);
+  if (pr != null && merged.some((m) => m?.pr === pr)) return s;
+  const st = structure ?? {};
+  const pairs = { ...(s.pairs ?? {}) };
+  const touch = (e, field, n) => {
+    if (!e?.from || !e?.to || !(n > 0)) return;
+    const k = edgeKey(e), p = pairs[k] ?? { files_added: 0, files_removed: 0, prs: [] };
+    pairs[k] = { ...p, [field]: (p[field] ?? 0) + n, prs: field === "files_added" && pr != null && !p.prs.includes(pr) ? [...p.prs, pr] : p.prs };
+  };
+  for (const e of arr(st.new_edges)) touch(e, "files_added", Number(e?.files) || 0);
+  for (const e of arr(st.removed_edges)) touch(e, "files_removed", Number(e?.files) || 0);
+  const deps = (old, add) => [...arr(old), ...arr(add).filter((e) => e?.from && e?.to).map((e) => ({ from: e.from, to: e.to, ...(e.kind ? { kind: e.kind } : {}), pr: pr ?? null }))];
+  const growth = { ...(s.growth ?? {}) };
+  for (const g of arr(st.top_growth)) if (g?.file && Number(g.lines) > 0) growth[g.file] = (growth[g.file] ?? 0) + Number(g.lines);
+  const topGrowth = Object.fromEntries(Object.entries(growth).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, MAX_GROWTH_FILES));
+  return { v: 1, milestone: s.milestone ?? milestone ?? null, merged: [...merged, { issue: issue ?? null, pr: pr ?? null }], pairs,
+    deps_added: deps(s.deps_added, st.deps_added), deps_removed: deps(s.deps_removed, st.deps_removed), growth: topGrowth };
+}
+
+/** The FORGE:DRIFT summary in a milestone description, or null. */
+export function parseSummary(description) {
+  const m = SUMMARY_RE.exec(String(description ?? ""));
+  if (!m) return null;
+  try { const s = JSON.parse(m[1]); return s && typeof s === "object" && !Array.isArray(s) ? s : null; } catch { return null; }
+}
+
+/** `description` with its FORGE:DRIFT block replaced (or appended); the text around it is unchanged. */
+export function renderSummary(description, summary) {
+  const d = String(description ?? "");
+  // `>` never appears in the JSON's structure, so escaping it keeps `-->` out of the comment and the JSON valid.
+  const block = `<!-- ${SUMMARY_MARKER} ${JSON.stringify(summary).replace(/>/g, "\\u003e")} -->`;
+  if (SUMMARY_RE.test(d)) return d.replace(SUMMARY_RE, () => block);
+  return d ? `${d}\n\n${block}` : block;
+}
+
+export const crossingMarker = (slug, c) => `FORGE:DRIFT:${slug}:${c.kind}:${c.key}`;
+
+/** One issue per crossing: {title, body, marker}. The body is a template (numbers and files from the report). */
+export function crossingIssues(report, { slug, milestone }) {
+  const fork = String(report?.fork_point ?? "").slice(0, 12);
+  const files = arr(report?.files);
+  return arr(report?.crossings).map((c) => {
+    const marker = crossingMarker(slug, c);
+    let evidence = [];
+    if (c.kind === "pair_growth") {
+      const [from, to] = String(c.key).split("->");
+      const p = arr(report?.pairs).find((x) => x.from === from && x.to === to) ?? {};
+      evidence = [...arr(p.files).map((f) => `- \`${f}\``), ...(arr(p.prs).length ? [`- PRs: ${p.prs.map((n) => `#${n}`).join(", ")}`] : [])];
+    } else if (c.kind === "package_lines") {
+      evidence = files.filter((f) => f.package === c.key).map((f) => `- \`${f.path}\` +${f.added} -${f.removed}`);
+    } else {
+      const f = files.find((x) => x.path === c.key);
+      if (f) evidence = [`- \`${f.path}\` +${f.added} -${f.removed}`];
+    }
+    const what = { pair_growth: "value-import files", package_lines: "lines of growth", file_lines: "lines of growth" }[c.kind] ?? "growth";
+    const body = [
+      "## Problem", "",
+      `\`milestone/${slug}\` (${milestone}) crossed \`orchestrate.drift.${c.kind}\` since its fork point \`${fork}\`: \`${c.key}\` has ${c.value} ${what} (threshold ${c.threshold}). No single PR shows this; found by \`scripts/milestone-drift.sh\` after a lane merged.`, "",
+      "## Evidence", "", ...(evidence.length ? evidence : ["- (see `scripts/milestone-drift.sh " + slug + "`)"]), "",
+      "## Acceptance Criteria", "",
+      `- [ ] \`scripts/milestone-drift.sh ${slug}\` no longer reports \`${c.kind}\` for \`${c.key}\`, or the growth is recorded as intended in the repo's architecture doc`, "",
+      `<!-- ${marker} -->`,
+    ].join("\n");
+    return { title: `drift(milestone/${slug}): ${c.kind} ${c.key} ${c.value} > ${c.threshold}`, body, marker, kind: c.kind, key: c.key };
+  });
+}
+
+/**
+ * File the crossings that no open issue of the milestone carries yet: one `gh issue list` read, one create per new
+ * crossing. `seen` (markers filed earlier in this process) guards against a list that lags a fresh create.
+ * @returns {Promise<{filed: Array<{marker:string, number:?number}>, skipped: string[]}>}
+ */
+export async function fileCrossings({ gh, repo, milestone, issues, seen = new Set() }) {
+  if (!issues.length) return { filed: [], skipped: [] };
+  const open = JSON.parse(String(await gh(["issue", "list", "-R", repo, "--milestone", milestone, "--state", "open", "--limit", "500", "--json", "number,body"])) || "[]");
+  const bodies = arr(open).map((x) => String(x?.body ?? ""));
+  const filed = [], skipped = [];
+  for (const it of issues) {
+    if (seen.has(it.marker) || bodies.some((b) => b.includes(`<!-- ${it.marker} -->`))) { skipped.push(it.marker); continue; }
+    const out = String(await gh(["issue", "create", "-R", repo, "--title", it.title, "--body", it.body, "--milestone", milestone]));
+    seen.add(it.marker);
+    filed.push({ marker: it.marker, number: Number(/\/issues\/(\d+)/.exec(out)?.[1]) || null });
+  }
+  return { filed, skipped };
+}
+
+/** Package dirs a report flags: package_lines crossings and the importing side of pair_growth crossings. */
+export function flaggedPackages(report) {
+  return [...new Set(arr(report?.crossings).filter((c) => (c.kind === "package_lines" || c.kind === "pair_growth") && c.package_path).map((c) => c.package_path))].sort();
+}
+
+/** The first flagged package dir that one of `files` sits under (or names), else null. */
+export function heldBy(files, flagged) {
+  for (const d of arr(flagged)) if (arr(files).some((f) => f === d || String(f).startsWith(`${d}/`))) return d;
+  return null;
+}
+
+/** {number, description} of the milestone titled `title`, or null. One `gh api` read. */
+export async function readMilestone({ gh, repo, title }) {
+  const list = JSON.parse(String(await gh(["api", `repos/${repo}/milestones?state=all&per_page=100`])) || "[]");
+  const m = arr(list).find((x) => x?.title === title);
+  return m ? { number: m.number, description: m.description ?? "" } : null;
+}
+
+const defaultRunScript = (args, { cwd }) => new Promise((res, rej) => {
+  execFile("bash", [SCRIPT, ...args], { cwd, timeout: 60000, maxBuffer: 16 * 1024 * 1024 }, (e, stdout) => (e ? rej(e) : res(stdout)));
+});
+
+/**
+ * The orchestrate seam: `{ onMerged(issue, row, result), holds? }`. `config` is engine-cli `readDrift()` (null: the
+ * summary only, no script run, no issue, no hold). `holds` exists only with `config.blockDispatch`. Calls to onMerged
+ * are serialised, so parallel merges never race on the milestone description.
+ */
+export function makeDrift({ repo, cwd, io, config = null, stagingBranch = "staging", affectedFiles = () => [], runScript = defaultRunScript, onEvent = () => {} }) {
+  const flagged = new Map(); // lane -> flagged package dirs (from the last report of that milestone)
+  const seen = new Set();
+  let chain = Promise.resolve();
+  const thresholds = config && ["pairGrowth", "packageLines", "fileLines"].some((k) => config[k] != null);
+
+  async function check(issue, row, result) {
+    const lane = String(result?.lane ?? row?.lane ?? "");
+    const title = result?.milestone;
+    if (!lane.startsWith("milestone/") || !title || !result?.structure) return null;
+    const slug = lane.slice("milestone/".length);
+    const ms = await readMilestone({ gh: io.gh, repo, title });
+    if (!ms) { onEvent({ issue, detail: `drift: milestone "${title}" not found` }); return null; }
+    const before = parseSummary(ms.description);
+    const summary = foldSummary(before ?? emptySummary(title), { issue, pr: result.pr ?? row?.pr ?? null, milestone: title, structure: result.structure });
+    if (summary !== before) await io.gh(["api", "-X", "PATCH", `repos/${repo}/milestones/${ms.number}`, "-f", `description=${renderSummary(ms.description, summary)}`]);
+    if (!thresholds) return { summary };
+    for (const b of [lane, stagingBranch]) { try { await io.git(["fetch", "origin", b], { cwd }); } catch { /* offline: the local refs */ } }
+    const dir = mkdtempSync(join(tmpdir(), "milestone-drift-")), file = join(dir, "summary.json");
+    let report;
+    try {
+      writeFileSync(file, JSON.stringify(summary));
+      const t = (flag, v) => (v != null ? [flag, String(v)] : []);
+      report = JSON.parse(String(await runScript([slug, "--summary", file, ...t("--pair-growth", config.pairGrowth), ...t("--package-lines", config.packageLines), ...t("--file-lines", config.fileLines)], { cwd })));
+    } finally { rmSync(dir, { recursive: true, force: true }); }
+    flagged.set(lane, flaggedPackages(report));
+    const filed = await fileCrossings({ gh: io.gh, repo, milestone: title, issues: crossingIssues(report, { slug, milestone: title }), seen });
+    if (filed.filed.length) onEvent({ issue, detail: `drift: filed ${filed.filed.map((f) => (f.number ? `#${f.number}` : f.marker)).join(", ")}` });
+    return { summary, report, ...filed };
+  }
+
+  const seam = {
+    onMerged(issue, row, result) {
+      const p = chain.then(() => check(issue, row, result));
+      chain = p.catch(() => {});
+      return p;
+    },
+  };
+  if (config?.blockDispatch) {
+    seam.holds = (issue, { body, lane }) => {
+      const f = flagged.get(lane);
+      return f?.length ? heldBy(affectedFiles(body), f) : null;
+    };
+  }
+  return seam;
+}

package/bin/lean/drift.test.mjs

@@ -0,0 +1,223 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { writeFileSync } from "node:fs";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { join } from "node:path";
+import { foldSummary, emptySummary, parseSummary, renderSummary, crossingIssues, fileCrossings, flaggedPackages, heldBy, makeDrift } from "./drift.mjs";
+import { orchestrate } from "./orchestrate.mjs";
+import { readDrift } from "../engine-cli.mjs";
+import { affectedFiles } from "../orchestrate-preflight.mjs";
+
+const S1 = { deps_added: [{ from: "@s/web", to: "@s/core", kind: "dependencies", manifest: "apps/web/package.json" }], deps_removed: [],
+  new_edges: [{ from: "@s/web", to: "@s/db", file: "apps/web/a.ts", files: 2 }], removed_edges: [], top_growth: [{ file: "packages/db/big.ts", lines: 40 }], decisions: [] };
+const S2 = { deps_added: [], deps_removed: [], new_edges: [{ from: "@s/web", to: "@s/db", file: "apps/web/c.ts", files: 1 }],
+  removed_edges: [{ from: "@s/web", to: "@s/old", files: 1 }], top_growth: [{ file: "packages/db/big.ts", lines: 10 }], decisions: [] };
+const REPORT = {
+  milestone: "m1", fork_point: "0123456789abcdef",
+  pairs: [{ from: "@s/web", to: "@s/db", growth: 6, files_added: 6, files_removed: 0, files: ["apps/web/a.ts", "apps/web/c.ts"], prs: [21, 22] }],
+  files: [{ path: "packages/db/big.ts", added: 12000, removed: 0, growth: 12000, package: "@s/db" }],
+  packages: [{ name: "@s/db", path: "packages/db", added: 12000, removed: 0, growth: 12000 }],
+  crossings: [
+    { kind: "pair_growth", key: "@s/web->@s/db", value: 6, threshold: 5, package_path: "apps/web" },
+    { kind: "package_lines", key: "@s/db", value: 12000, threshold: 10000, package_path: "packages/db" },
+  ],
+};
+
+/** gh fake: one milestone, open issues, and a log of every call. */
+function fakeGh({ description = "Ship the MCP server.", open = [] } = {}) {
+  const st = { description, open: [...open], calls: [], created: [] };
+  st.gh = async (a) => {
+    st.calls.push(a);
+    if (a[0] === "api" && /milestones\?state=all/.test(a[1])) return JSON.stringify([{ number: 7, title: "Other", description: "" }, { number: 3, title: "M One", description: st.description }]);
+    if (a[0] === "api" && a[1] === "-X" && a[2] === "PATCH") { st.description = a[a.indexOf("-f") + 1].slice("description=".length); return "{}"; }
+    if (a[0] === "issue" && a[1] === "list") return JSON.stringify(st.open);
+    if (a[0] === "issue" && a[1] === "create") {
+      const n = 900 + st.created.length;
+      st.created.push({ n, title: a[a.indexOf("--title") + 1], body: a[a.indexOf("--body") + 1], milestone: a[a.indexOf("--milestone") + 1] });
+      st.open.push({ number: n, body: a[a.indexOf("--body") + 1] });
+      return `https://github.com/o/r/issues/${n}\n`;
+    }
+    throw new Error(`unexpected gh ${a.join(" ")}`);
+  };
+  return st;
+}
+const fakeGit = () => { const calls = []; return { calls, git: async (a) => { calls.push(a); return ""; } }; };
+const merged = (pr, structure = S1, lane = "milestone/m1") => ({ terminalReason: "merged", pr, structure, milestone: "M One", lane });
+
+describe("drift summary fold (forge#4161)", () => {
+  it("folds structure records keyed by PR; a re-fold is a no-op", () => {
+    const a = foldSummary(emptySummary("M One"), { issue: 1, pr: 21, milestone: "M One", structure: S1 });
+    const b = foldSummary(a, { issue: 2, pr: 22, milestone: "M One", structure: S2 });
+    assert.deepEqual(b.merged, [{ issue: 1, pr: 21 }, { issue: 2, pr: 22 }]);
+    assert.deepEqual(b.pairs, { "@s/web:@s/db": { files_added: 3, files_removed: 0, prs: [21, 22] }, "@s/web:@s/old": { files_added: 0, files_removed: 1, prs: [] } });
+    assert.deepEqual(b.deps_added, [{ from: "@s/web", to: "@s/core", kind: "dependencies", pr: 21 }]);
+    assert.deepEqual(b.growth, { "packages/db/big.ts": 50 });
+    assert.equal(foldSummary(b, { issue: 2, pr: 22, structure: S2 }), b, "same PR: unchanged");
+  });
+
+  it("the description block round-trips and keeps the text around it byte for byte", () => {
+    const desc = "Line one.\r\n<!-- FORGE:MILESTONE_INDEX: https://gist.github.com/x -->\ntrailing  ";
+    const s1 = foldSummary(emptySummary("M One"), { issue: 1, pr: 21, structure: { ...S1, top_growth: [{ file: "a-->b.ts", lines: 3 }] } });
+    const d1 = renderSummary(desc, s1);
+    assert.ok(d1.startsWith(`${desc}\n\n<!-- FORGE:DRIFT {`));
+    assert.equal((d1.split("-->").length - 1), 2, "a `-->` in a path never ends the comment");
+    assert.deepEqual(parseSummary(d1), s1);
+    const s2 = foldSummary(s1, { issue: 2, pr: 22, structure: S2 });
+    const d2 = renderSummary(d1, s2);
+    assert.equal(d2.slice(0, desc.length + 2), `${desc}\n\n`);
+    assert.deepEqual(parseSummary(d2), s2);
+    assert.equal((d2.match(/FORGE:DRIFT \{/g) ?? []).length, 1, "replaced, not appended");
+    assert.equal(parseSummary("no block"), null);
+    assert.equal(parseSummary("<!-- FORGE:DRIFT {bad} -->"), null);
+    assert.equal(renderSummary("", s1).startsWith("<!-- FORGE:DRIFT "), true);
+  });
+});
+
+describe("drift crossings (forge#4161)", () => {
+  it("one templated issue per crossing with numbers, files and a marker; flagged packages and holds", () => {
+    const issues = crossingIssues(REPORT, { slug: "m1", milestone: "M One" });
+    assert.deepEqual(issues.map((i) => i.marker), ["FORGE:DRIFT:m1:pair_growth:@s/web->@s/db", "FORGE:DRIFT:m1:package_lines:@s/db"]);
+    assert.equal(issues[0].title, "drift(milestone/m1): pair_growth @s/web->@s/db 6 > 5");
+    assert.match(issues[0].body, /`@s\/web->@s\/db` has 6 value-import files \(threshold 5\)/);
+    assert.match(issues[0].body, /- `apps\/web\/a\.ts`\n- `apps\/web\/c\.ts`\n- PRs: #21, #22/);
+    assert.match(issues[1].body, /- `packages\/db\/big\.ts` \+12000 -0/);
+    assert.ok(issues[1].body.endsWith("<!-- FORGE:DRIFT:m1:package_lines:@s/db -->"));
+    assert.deepEqual(flaggedPackages(REPORT), ["apps/web", "packages/db"]);
+    assert.equal(heldBy(["packages/db/src/x.ts"], ["packages/db"]), "packages/db");
+    assert.equal(heldBy(["packages/dbx/a.ts", "apps/webapp/a.ts"], ["packages/db", "apps/web"]), null, "a dir prefix only");
+    assert.deepEqual(crossingIssues({ ...REPORT, crossings: [] }, { slug: "m1", milestone: "M One" }), []);
+  });
+
+  it("files exactly one issue per new crossing; open drift issues and earlier filings are deduplicated", async () => {
+    const st = fakeGh({ open: [{ number: 5, body: "old\n<!-- FORGE:DRIFT:m1:package_lines:@s/db -->" }] });
+    const issues = crossingIssues(REPORT, { slug: "m1", milestone: "M One" });
+    const seen = new Set();
+    const r = await fileCrossings({ gh: st.gh, repo: "o/r", milestone: "M One", issues, seen });
+    assert.deepEqual(r, { filed: [{ marker: "FORGE:DRIFT:m1:pair_growth:@s/web->@s/db", number: 900 }], skipped: ["FORGE:DRIFT:m1:package_lines:@s/db"] });
+    assert.equal(st.created[0].milestone, "M One");
+    const again = await fileCrossings({ gh: st.gh, repo: "o/r", milestone: "M One", issues, seen });
+    assert.deepEqual(again.filed, []);
+    assert.equal(st.created.length, 1);
+    const none = await fileCrossings({ gh: st.gh, repo: "o/r", milestone: "M One", issues: [] });
+    assert.deepEqual(none, { filed: [], skipped: [] });
+    assert.equal(st.calls.filter((a) => a[0] === "issue" && a[1] === "list").length, 2, "no read without a crossing");
+  });
+});
+
+describe("makeDrift (forge#4161)", () => {
+  it("no config: the summary only (milestone GET + PATCH), no script, no fetch, no issue, no holds", async () => {
+    const st = fakeGh(), g = fakeGit(); let ran = 0;
+    const d = makeDrift({ repo: "o/r", cwd: "/x", io: { gh: st.gh, git: g.git }, config: null, runScript: async () => { ran++; return "{}"; } });
+    assert.equal(d.holds, undefined);
+    const r = await d.onMerged(1, { issue: 1, outcome: "merged", pr: 21 }, merged(21));
+    assert.deepEqual(parseSummary(st.description).merged, [{ issue: 1, pr: 21 }]);
+    assert.ok(st.description.startsWith("Ship the MCP server.\n\n"));
+    assert.equal(r.summary.pairs["@s/web:@s/db"].files_added, 2);
+    assert.deepEqual(st.calls.map((a) => `${a[0]} ${a[1]}`), ["api repos/o/r/milestones?state=all&per_page=100", "api -X"]);
+    assert.deepEqual([ran, g.calls.length], [0, 0]);
+    // same PR again: read only, no PATCH
+    await d.onMerged(1, { issue: 1, outcome: "merged", pr: 21 }, merged(21));
+    assert.equal(st.calls.filter((a) => a[2] === "PATCH").length, 1);
+    // staging lanes, results without structure or milestone: nothing
+    for (const res of [merged(23, S1, "staging"), { ...merged(24), structure: undefined }, { ...merged(25), milestone: undefined }]) assert.equal(await d.onMerged(3, {}, res), null);
+    assert.equal(st.calls.length, 3);
+  });
+
+  it("thresholds: runs the script with the summary and thresholds, files one issue per crossing once, flags packages", async () => {
+    const st = fakeGh(), g = fakeGit(); const runs = [];
+    const runScript = async (args) => {
+      const sum = JSON.parse((await import("node:fs")).readFileSync(args[args.indexOf("--summary") + 1], "utf8"));
+      runs.push({ args, merged: sum.merged.length });
+      return JSON.stringify(runs.length === 1 ? { ...REPORT, crossings: [] } : REPORT);
+    };
+    const d = makeDrift({ repo: "o/r", cwd: "/x", io: { gh: st.gh, git: g.git }, config: { pairGrowth: 5, packageLines: 10000, fileLines: null, blockDispatch: true }, stagingBranch: "staging", affectedFiles, runScript });
+    const below = await d.onMerged(1, {}, merged(21));
+    assert.deepEqual([below.filed, st.created.length], [[], 0], "below the thresholds nothing is filed");
+    assert.equal(st.calls.filter((a) => a[0] === "issue").length, 0, "and no issue read");
+    assert.deepEqual(runs[0].args.filter((x) => !x.includes("/")), ["m1", "--summary", "--pair-growth", "5", "--package-lines", "10000"]);
+    assert.deepEqual(g.calls, [["fetch", "origin", "milestone/m1"], ["fetch", "origin", "staging"]]);
+    const body = (files) => `## Affected Files\n${files.map((f) => `- \`${f}\``).join("\n")}\n`;
+    assert.equal(d.holds(9, { body: body(["packages/db/x.ts"]), lane: "milestone/m1" }), null, "no flag yet");
+    const r = await d.onMerged(2, {}, merged(22, S2));
+    assert.equal(runs[1].merged, 2, "the script sees the folded summary");
+    assert.deepEqual(r.filed.map((f) => f.number), [900, 901]);
+    assert.deepEqual(st.created.map((c) => c.title), ["drift(milestone/m1): pair_growth @s/web->@s/db 6 > 5", "drift(milestone/m1): package_lines @s/db 12000 > 10000"]);
+    await d.onMerged(3, {}, merged(23, S2));
+    assert.equal(st.created.length, 2, "a later merge with the same crossings files nothing");
+    assert.equal(d.holds(9, { body: body(["packages/db/x.ts"]), lane: "milestone/m1" }), "packages/db");
+    assert.equal(d.holds(9, { body: body(["packages/core/x.ts"]), lane: "milestone/m1" }), null, "another package");
+    assert.equal(d.holds(9, { body: body(["packages/db/x.ts"]), lane: "milestone/m2" }), null, "another milestone");
+  });
+
+  it("calls are serialised: parallel merges never lose a fold", async () => {
+    const st = fakeGh(); const slow = st.gh;
+    st.gh = async (a) => { await new Promise((r) => setTimeout(r, 5)); return slow(a); };
+    const d = makeDrift({ repo: "o/r", cwd: "/x", io: { gh: st.gh, git: async () => "" } });
+    await Promise.all([d.onMerged(1, {}, merged(21)), d.onMerged(2, {}, merged(22, S2)), d.onMerged(3, {}, merged(23))]);
+    assert.deepEqual(parseSummary(st.description).merged.map((m) => m.pr), [21, 22, 23]);
+  });
+});
+
+describe("orchestrate drift seam (forge#4161)", () => {
+  const lanesOf = (m) => async (n) => ({ ok: true, lane: m[n] ?? "staging" });
+
+  it("a merged milestone lane awaits onMerged before the batch ends; staging lanes and other outcomes do not", async () => {
+    const calls = [];
+    const drift = { onMerged: async (n, row, r) => { await new Promise((res) => setTimeout(res, 10)); calls.push([n, row.lane, r.pr]); } };
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 3, readBody: async () => "", resolveLane: lanesOf({ 1: "milestone/m1", 3: "milestone/m1" }), drift,
+      runOne: async (n) => (n === 3 ? { terminalReason: "needs-human" } : merged(20 + n, S1, n === 1 ? "milestone/m1" : "staging")) });
+    assert.deepEqual(calls, [[1, "milestone/m1", 21]]);
+    assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "needs-human"]);
+  });
+
+  it("a drift failure never changes the outcome or stops the batch", async () => {
+    const seen = [];
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, readBody: async () => "", resolveLane: lanesOf({ 1: "milestone/m1", 2: "milestone/m1" }), onUpdate: (r) => seen.push(r),
+      drift: { onMerged: async () => { throw new Error("gh down"); } }, runOne: async (n) => merged(20 + n) });
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"]]);
+    assert.deepEqual(seen.filter((r) => r.outcome === "drift").map((r) => r.detail), ["drift check failed: gh down", "drift check failed: gh down"]);
+  });
+
+  it("without drift: identical rows and store shape", async () => {
+    const saves = [];
+    const run = (extra) => orchestrate({ issues: [1, 2], parallel: 2, readBody: async (n) => (n === 2 ? "Depends on #1" : ""), resolveLane: lanesOf({ 1: "milestone/m1", 2: "milestone/m1" }),
+      store: { load: () => null, save: (s) => saves.push(Object.keys(s).sort().join(",")) }, now: () => 0, runOne: async (n) => merged(20 + n), ...extra });
+    const a = await run({}); const keysA = [...saves]; saves.length = 0;
+    const b = await run({ drift: { onMerged: async () => {} } });
+    assert.deepEqual(a, b);
+    assert.deepEqual([...new Set(keysA)], [...new Set(saves)]);
+  });
+
+  it("block_dispatch: holds only queued issues whose Affected Files touch the flagged package; they end held", async () => {
+    const bodies = { 1: "", 2: "Depends on #1\n\n## Affected Files\n- `packages/db/src/x.ts`\n", 3: "Depends on #1\n\n## Affected Files\n- `packages/core/src/y.ts`\n" };
+    const st = fakeGh();
+    const d = makeDrift({ repo: "o/r", cwd: "/x", io: { gh: st.gh, git: async () => "" }, config: { packageLines: 10000, blockDispatch: true }, affectedFiles,
+      runScript: async () => JSON.stringify({ ...REPORT, crossings: [REPORT.crossings[1]] }) });
+    const ran = [];
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async (n) => bodies[n], resolveLane: lanesOf({ 1: "milestone/m1", 2: "milestone/m1", 3: "milestone/m1" }), drift: d,
+      runOne: async (n) => { ran.push(n); return merged(20 + n); } });
+    assert.deepEqual(ran, [1, 3]);
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome, r.detail ?? null]), [[1, "merged", null], [2, "held", "drift: packages/db flagged"], [3, "merged", null]]);
+    assert.equal(st.created.length, 1);
+  });
+});
+
+describe("readDrift (forge#4161)", () => {
+  const withYaml = (yaml, f) => {
+    const dir = tmpDir("drift-yaml-");
+    if (yaml != null) writeFileSync(join(dir, "forge.yaml"), yaml);
+    return f(dir);
+  };
+  it("null without forge.yaml, without the block, or with nothing set", () => {
+    assert.equal(withYaml(null, readDrift), null);
+    assert.equal(withYaml("orchestration:\n  max_deferrals: 3\n", readDrift), null);
+    assert.equal(withYaml("orchestrate:\n  drift:\n    pair_growth: 0\n    block_dispatch: false\n", readDrift), null);
+  });
+  it("reads the orchestrate.drift thresholds, block_dispatch and generated globs", () => {
+    const y = "orchestrate:\n  drift:\n    pair_growth: 5\n    package_lines: 10000\n    file_lines: 600\n    block_dispatch: true\n    generated_globs: data/*, *.lock\n";
+    assert.deepEqual(withYaml(y, readDrift), { pairGrowth: 5, packageLines: 10000, fileLines: 600, blockDispatch: true, generatedGlobs: ["data/*", "*.lock"] });
+    assert.deepEqual(withYaml("orchestrate:\n  drift:\n    file_lines: 600\n", readDrift), { pairGrowth: null, packageLines: null, fileLines: 600, blockDispatch: false, generatedGlobs: [] });
+  });
+});

package/bin/lean/effects.mjs

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
 /** Core work-on: the real effects behind runCore's injected `io`. Argv only, no shell. */
-import { execFile, execFileSync } from "node:child_process";
+import { execFile, execFileSync, spawn } from "node:child_process";
 import { promisify } from "node:util";
 import { readFileSync, appendFileSync, mkdirSync, statSync, existsSync, rmSync, writeFileSync, readdirSync, lstatSync, readlinkSync, realpathSync } from "node:fs";
 import { homedir, hostname, freemem, loadavg, cpus, tmpdir } from "node:os";
@@ -11,7 +11,9 @@ import { fileURLToPath } from "node:url";
 import { runCliBackendAsync, resolveClaudeCliBinary, getClaudeCliCapabilities, buildLeanSessionFlags, buildReadOnlyToolFlags } from "../runner.mjs";
 import { makeIo } from "../engine-cli.mjs";
 import { makeProjector } from "../engine/projector.mjs";
-import { makeLedger } from "./ledger.mjs";
+import { makeLedger, countLiveLanes, ledgerRoot } from "./ledger.mjs";
+import { makeDepCache } from "./depcache.mjs";
+import { makeHeavyIo } from "./heavy.mjs";
 
 const pexec = promisify(execFile);
 const script = (n) => fileURLToPath(new URL(`../../scripts/${n}`, import.meta.url));
@@ -45,11 +47,15 @@ export function wrapLaneLimits(bin, args, limits) {
   return { bin: "systemd-run", args: ["--user", "--scope", "-q", ...(mem ? ["-p", `MemoryMax=${mem}`] : []), ...(cpu ? ["-p", `CPUQuota=${cpu}`] : []), "--", bin, ...args] };
 }
 
+/** #4211: a judgment call over data in the prompt only (the refactor:split grouping call): no tool can read a file. */
+export const noToolFlags = (caps) => (caps?.strictMcp ? ["--tools", ""]
+  : ["--disallowedTools", "Read", "Grep", "Glob", "Bash", "Edit", "Write", "MultiEdit", "NotebookEdit", "WebFetch", "WebSearch", "Task"]);
+
 export function makeLeanRunner({ runCli = runCliBackendAsync, probe = systemdRunAvailable } = {}) {
-  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, env = null, idleMs = 0, maxMs = 0, laneLimits = null, control = null, onSpawn = null }) => {
+  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, noTools = false, env = null, idleMs = 0, maxMs = 0, laneLimits = null, control = null, onSpawn = null }) => {
     const bin = resolveClaudeCliBinary(cwd) || "claude";
     const caps = getClaudeCliCapabilities(cwd);
-    const flags = [...buildLeanSessionFlags(caps), ...(readOnly ? buildReadOnlyToolFlags(caps) : [])];
+    const flags = [...buildLeanSessionFlags(caps), ...(noTools ? noToolFlags(caps) : readOnly ? buildReadOnlyToolFlags(caps) : [])];
     const r = await runCli({ spec: { name: `core-${role}` }, userMessage: prompt, systemPrompt: resume ? "" : system,
       args: [String(issue)], cwd, bin, model, env, sessionFlags: flags, extraArgs: resume ? ["--resume", resume] : [],
       logger: { log() {} },
@@ -159,6 +165,8 @@ export function makeHostProbe() {
     },
     loadAvg1: () => loadavg()[0],
     cores: () => cpus().length || 1,
+    /** Live lanes on this host from the ledger (#4182), this lane included once it registered. */
+    activeLanes: () => countLiveLanes({ root: ledgerRoot(), host: hostname() }),
     tmpPaths: () => [...new Set([join(homedir(), ".forge", "tmp"), tmpdir(), "/tmp"].filter((x) => existsSync(x) || x === join(homedir(), ".forge", "tmp")))],
   };
 }
@@ -184,22 +192,47 @@ export async function requiredChecksFor({ gh, repo, lane }) {
   return [...new Set(all.filter((x) => typeof x === "string" && x))];
 }
 
+/** Combined stdout+stderr kept per declared check command (tail). */
+const CHECK_OUTPUT_CAP = 64000;
+/** #4240: run a forge.yaml declared check command locally (no gh, no Actions); {rc, output}, rc 124 on timeout. */
+export function runCheckCommand({ command, cwd, timeoutMs = 300000 }) {
+  return new Promise((done) => {
+    let out = "", timedOut = false, child;
+    const add = (b) => { out += String(b); if (out.length > 2 * CHECK_OUTPUT_CAP) out = out.slice(-CHECK_OUTPUT_CAP); };
+    try { child = spawn("bash", ["-c", String(command)], { cwd, detached: true, stdio: ["ignore", "pipe", "pipe"] }); }
+    catch (e) { done({ rc: 127, output: String(e?.message ?? e) }); return; }
+    const kill = () => { try { process.kill(-child.pid, "SIGKILL"); } catch { try { child.kill("SIGKILL"); } catch { /* gone */ } } };
+    const timer = setTimeout(() => { timedOut = true; kill(); }, timeoutMs);
+    child.stdout.on("data", add); child.stderr.on("data", add);
+    child.on("error", (e) => { clearTimeout(timer); done({ rc: 127, output: `${out}${String(e?.message ?? e)}`.slice(-CHECK_OUTPUT_CAP) }); });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      const tail = out.slice(-CHECK_OUTPUT_CAP);
+      done(timedOut ? { rc: 124, output: `${tail}\n(timed out after ${Math.round(timeoutMs / 1000)}s)` } : { rc: Number.isInteger(code) ? code : 1, output: tail });
+    });
+  });
+}
+
 export function makeLeanIo({ repo }) {
   const base = makeIo();
   const run = (bin) => async (args, opts = {}) => (await pexec(bin, args, { cwd: opts.cwd, maxBuffer: 100 * 1024 * 1024, timeout: 120000 })).stdout;
   return {
-    gh: run("gh"), git: run("git"), docker: run("docker"), killGroup: async (pid, marker) => killLedgerPid(pid, marker), ledger: (opts) => makeLedger(opts), runTmp: makeRunTmp(), host: makeHostProbe(), projector: makeProjector({ gh: base.gh }), runner: makeLeanRunner(), waitCiGreen: base.waitCiGreen,
+    gh: run("gh"), git: run("git"), docker: run("docker"), killGroup: async (pid, marker) => killLedgerPid(pid, marker), ledger: (opts) => makeLedger(opts), runTmp: makeRunTmp(), host: makeHostProbe(), heavy: makeHeavyIo({ host: makeHostProbe() }), projector: makeProjector({ gh: base.gh }), runner: makeLeanRunner(), waitCiGreen: base.waitCiGreen,
+    /** Shared dependency cache (#4220): the repo's own package manager, argv only, bounded by the cache's timeout. */
+    deps: makeDepCache({ exec: async (cmd, args, { cwd, env, timeoutMs } = {}) => String((await pexec(cmd, args, { cwd, env: { ...process.env, ...(env ?? {}) }, timeout: timeoutMs ?? 0, maxBuffer: 100 * 1024 * 1024 })).stdout) }),
     /** Names of the checks the base branch requires: classic protection plus rulesets (#4158). [] when neither has any. */
     async requiredChecks(lane) { return requiredChecksFor({ gh: run("gh"), repo, lane }); },
-    /** [{name, state: success|failure|pending}] for the PR's checks. */
+    /** [{name, state: success|failure|pending, url, head}] for the PR's checks; `head` is the PR head the rollup belongs to (#4240). */
     async checkStates(pr) {
-      const j = JSON.parse(await run("gh")(["pr", "view", String(pr), "-R", repo, "--json", "statusCheckRollup"]));
+      const j = JSON.parse(await run("gh")(["pr", "view", String(pr), "-R", repo, "--json", "statusCheckRollup,headRefOid"]));
       return (j.statusCheckRollup ?? []).map((c) => {
         const v = String(c.conclusion || c.state || "").toUpperCase();
         const state = ["SUCCESS", "NEUTRAL", "SKIPPED"].includes(v) ? "success" : ["FAILURE", "ERROR", "TIMED_OUT", "CANCELLED", "ACTION_REQUIRED", "STARTUP_FAILURE"].includes(v) ? "failure" : "pending";
-        return { name: c.name ?? c.context, state };
+        return { name: c.name ?? c.context, state, url: c.detailsUrl || c.targetUrl || null, head: j.headRefOid ?? null };
       });
     },
+    /** #4240: one declared check command (`bash -c`) in `cwd`; never throws. A timeout kills the process group and gives rc 124. */
+    runCommand: runCheckCommand,
     /** #4167 verify-by-CI: `gh workflow run` on the lane branch; inputs become -f key=value. */
     async dispatchWorkflow({ workflow, ref, inputs = {} }) {
       await run("gh")(["workflow", "run", String(workflow), "-R", repo, "--ref", String(ref), ...Object.entries(inputs).flatMap(([k, v]) => ["-f", `${k}=${v}`])]);
@@ -234,10 +267,54 @@ export function makeLeanIo({ repo }) {
       for (const l of out.split("\n")) { const [k, n] = l.split("\t"); if (k && /^\d+$/.test(String(n ?? "").trim())) map[k] = Number(n); }
       return map;
     },
+    /**
+     * forge#4163: forge.yaml env.setup, once per fresh lane worktree: `bash -c <command>` in cwd with the lane env.
+     * Resolves {exit, stdout, stderr} (output tails); a non-zero exit, timeout or spawn error never throws.
+     */
+    runSetup({ cwd, command, env = null, timeoutMs = 15 * 60000 }) {
+      return new Promise((res) => {
+        execFile("bash", ["-c", command], { cwd, env: { ...process.env, ...(env ?? {}) }, timeout: timeoutMs, maxBuffer: 10 * 1024 * 1024 }, (err, stdout, stderr) => {
+          const tail = (x) => String(x ?? "").slice(-2000);
+          if (!err) return res({ exit: 0, stdout: tail(stdout), stderr: tail(stderr) });
+          const exit = Number.isInteger(err.code) ? err.code : 1;
+          const why = err.killed ? `timed out after ${Math.round(timeoutMs / 1000)}s` : typeof err.code === "string" ? err.code : "";
+          res({ exit: exit === 0 ? 1 : exit, stdout: tail(stdout), stderr: tail([stderr, why].filter(Boolean).join("\n")) });
+        });
+      });
+    },
     /** forge#4159: scripts/structure-delta.sh over base...HEAD in cwd; null on any failure (never fails a run). */
     async structureDelta({ cwd, base }) {
       try { return JSON.parse((await pexec("bash", [script("structure-delta.sh"), `${base}...HEAD`], { cwd, timeout: 30000, maxBuffer: 16 * 1024 * 1024 })).stdout); } catch { return null; }
     },
+    /** forge#4195: scripts/hotspots.sh facts for `paths` in cwd (growth since base when given); null on any failure. */
+    async hotspots({ cwd, base = null, paths = [] }) {
+      const args = [script("hotspots.sh"), ...(base ? ["--base", base] : []), "--", ...paths];
+      try { return JSON.parse((await pexec("bash", args, { cwd, timeout: 30000, maxBuffer: 16 * 1024 * 1024 })).stdout); } catch { return null; }
+    },
+    /** #4271: scripts/acceptance-red-green.sh for the frozen repro_test in cwd; {base, head} words, or null on any failure. */
+    async redGreen({ cwd, base, command, timeoutSec = 300 }) {
+      const args = [script("acceptance-red-green.sh"), "--repo-path", cwd, "--base-ref", base, "--test-cmd", command, "--timeout", String(timeoutSec)];
+      try {
+        const m = /BASE=(\w+) HEAD=(\w+)/.exec(String((await pexec("bash", args, { cwd, timeout: (timeoutSec * 2 + 60) * 1000, maxBuffer: 4 * 1024 * 1024 })).stdout ?? ""));
+        return m ? { base: m[1], head: m[2] } : null;
+      } catch { return null; }
+    },
+    /**
+     * The target repo's own docs-move-with-code rule, when it has one (scripts/check-devdocs-freshness.sh, forge#4185),
+     * over base...HEAD in cwd. null when the repo has no such script or it cannot run (exit 2+, timeout): never blocks.
+     * Otherwise {ok, output} with the script's stdout+stderr (exit 0 = ok, exit 1 = stale docs).
+     */
+    async docsFreshness({ cwd, base }) {
+      const s = join(cwd, "scripts", "check-devdocs-freshness.sh");
+      if (!existsSync(s)) return null;
+      try {
+        const r = await pexec("bash", [s, base], { cwd, timeout: 60000, maxBuffer: 4 * 1024 * 1024, env: { ...process.env, PR_BODY: "" } });
+        return { ok: true, output: String(r.stdout ?? "") };
+      } catch (e) {
+        if (e?.code !== 1) return null;
+        return { ok: false, output: `${e.stdout ?? ""}${e.stderr ?? ""}`.slice(0, 6000) };
+      }
+    },
     /** #4162: processes left running inside a lane worktree (Linux /proc). */
     proc: { inDir: async (dir) => procsInDir(dir), alive: pidAlive, kill: (pid, sig) => { try { process.kill(pid, sig); } catch { /* already gone */ } } },
     fs: {