forgedock 1.46.0 → 1.48.0

package/bin/lean/context.mjs

@@ -7,9 +7,14 @@
  * symbol from the issue, and repo-fact file pointers. Only GitHub and git are read; nothing here is ForgeDock-specific
  * beyond the FORGE:* records lanes already write. `gatherContextFacts` does the I/O; `compileContextPack` is pure, so the
  * same facts always give a byte-identical pack. The pack is a hint: it never enters the issue text contracts quote from.
+ * #5085: `attempts` also carries subject history: closed issues and merged PRs found by the issue's subject terms (codes,
+ * symbols, title words; `subjectTerms`) through GitHub search and a repo-wide `git log --grep`, whatever files they touched.
+ * The pack also reports whether the Forge Ledger recall index exists, so a missing index is never skipped silently.
  */
 import { createHash } from "node:crypto";
 import { join } from "node:path";
+import { homedir } from "node:os";
+import { fileURLToPath } from "node:url";
 import { parseDependsOn } from "../engine/phases.mjs";
 import { codeForTerminalReason } from "../engine/outcome-codes.mjs";
 import { runLogKey } from "../engine/runlog.mjs";
@@ -33,6 +38,12 @@ export const MAX_LINKS = 20;
 export const MAX_LEDGER_PRS = 8;
 /** #4401: FORGE:FACTS records (earlier lanes) read into one pack: this issue's own attempts first, then the file-path ledger. */
 export const MAX_FACT_LANES = 6;
+/** #5085: subject-history bounds per pack: GitHub searches run, rows kept. */
+export const MAX_SUBJECT_QUERIES = 3;
+export const MAX_SUBJECT_ITEMS = 8;
+/** #5085: where the Forge Ledger recall CLI ships (the ForgeDock install) and its default index file (bin/recall.mjs). */
+const RECALL_INSTALL = fileURLToPath(new URL("../recall.mjs", import.meta.url));
+const RECALL_INDEX = join(homedir(), ".forge", "index", "knowledge.jsonl");
 export const REPO_FACT_FILES = Object.freeze([REPO_FACTS_FILE, TEST_MAP_FILE]);
 const ATTEMPT_MARKERS = ["NEEDS_HUMAN", "BASE_SYNC", "FIX_CI"];
 const LINE_MAX = 200;
@@ -106,6 +117,29 @@ export function issueSymbols(body) {
 const reEscape = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 const namesSymbol = (text, symbols) => symbols.some((s) => new RegExp(`(?:^|[^\\w$])${reEscape(s)}(?:$|[^\\w$])`).test(String(text)));
 
+const STOPWORDS = new Set(("about after again also always another because before being between both cannot could does doing during each every from have having into issue issues just make makes more most must never only other over same should some such than that their them then there these they this those through under until very what when where which while will with within without would your " +
+  "fix fixes fixed bug bugs feat chore docs test tests file files when wrong error errors fail fails failed failing add adds remove update use uses").split(" "));
+const FILE_EXT = /\.(?:m?[jt]sx?|cjs|md|sh|json|ya?ml|py|go|rs|rb|css|html|txt|toml|lock|sql)$/i;
+/** #5085: subject terms split by kind: codes and symbols (exact, quoted searches) and distinctive title words. */
+function subjectParts({ title = "", body = "" } = {}) {
+  const text = `${stripComments(title)}\n${stripComments(body)}`;
+  const codes = [];
+  const add = (t) => { if (t && !codes.some((c) => c.toLowerCase() === t.toLowerCase())) codes.push(t); };
+  for (const m of text.matchAll(/`([a-z][a-z0-9_.-]{2,63})`/g)) if (/[._-]/.test(m[1]) && !FILE_EXT.test(m[1]) && /[a-z0-9]$/.test(m[1])) add(m[1]);
+  for (const m of text.matchAll(/(?<![\w/#:])([A-Z][A-Z0-9]*_[A-Z0-9_]*[A-Z0-9])(?![\w:])/g)) if (!/^FORGE_/.test(m[1])) add(m[1]);
+  for (const m of text.matchAll(/\bHTTP\s*([1-5]\d\d)\b|(?<![\w#])([1-5]xx)\b/gi)) add(m[1] ?? m[2].toLowerCase());
+  for (const s of issueSymbols(body)) if (!codes.some((c) => c.toLowerCase().endsWith(`.${s.toLowerCase()}`))) add(s);
+  const head = stripComments(title).replace(/^\s*\w+(?:\([^)]*\))?!?:\s*/, "");
+  const words = [];
+  for (const w of head.toLowerCase().match(/[a-z][a-z0-9-]{3,}/g) ?? []) if (!STOPWORDS.has(w) && !words.includes(w) && !codes.some((c) => c.toLowerCase() === w)) words.push(w);
+  return { codes: codes.slice(0, 3), words: words.slice(0, 3) };
+}
+/** #5085: the issue's subject terms, codes and symbols first, then title words; deterministic, deduped, at most 6. */
+export function subjectTerms({ title = "", body = "" } = {}) {
+  const { codes, words } = subjectParts({ title, body });
+  return [...codes, ...words];
+}
+
 /** Rerun (an earlier run left a record or salvage), decomposed child (`**Parent**`), or first attempt. */
 export function laneType({ resumed = false, attempts = 0, body = "" } = {}) {
   if (resumed || attempts > 0) return "rerun";
@@ -186,13 +220,74 @@ async function readComments(gh, repo, n) {
   return parseLines(await gh(["api", "--paginate", `repos/${repo}/issues/${n}/comments`, "--jq", ".[] | {id, body, author_association, user: {type: .user.type, login: .user.login}} | @json"]));
 }
 
+/**
+ * #5085: closed issues and merged PRs on the same subject, found by term rather than by issue number or file: up to
+ * MAX_SUBJECT_QUERIES GitHub searches (each code/symbol quoted, then the title words together) and one `git log --grep`
+ * over the whole history of `base`. `exclude` (self, linked issues) is dropped; rows are ranked by the number of
+ * matched terms, then newest first, and capped. A failed query lands in `errors`; the rest still count.
+ * @returns {Promise<{terms:string[], rows:{number:number, kind:string, status:string, title:string, matched:string[]}[], errors:string[]}>}
+ */
+export async function subjectHistory({ io, repo, issue = null, title = "", body = "", base = null, exclude = [], max = MAX_SUBJECT_ITEMS }) {
+  const { codes, words } = subjectParts({ title, body });
+  const terms = [...codes, ...words], errors = [], hits = new Map();
+  const skip = new Set([Number(issue), ...exclude.map(Number)]);
+  const hit = (n, info, matched) => {
+    if (!Number.isSafeInteger(n) || n <= 0 || skip.has(n)) return;
+    const row = hits.get(n) ?? { number: n, kind: info.kind, status: info.status, title: info.title, matched: [] };
+    if (info.src === "gh") Object.assign(row, { kind: info.kind, status: info.status, title: info.title || row.title });
+    for (const t of matched) if (!row.matched.includes(t)) row.matched.push(t);
+    hits.set(n, row);
+  };
+  if (!terms.length) return { terms, rows: [], errors };
+  const queries = [...codes.slice(0, words.length ? MAX_SUBJECT_QUERIES - 1 : MAX_SUBJECT_QUERIES).map((c) => [c]), ...(words.length ? [words] : [])].slice(0, MAX_SUBJECT_QUERIES);
+  for (const q of queries) {
+    const text = q.length === 1 && codes.includes(q[0]) ? `"${q[0]}"` : q.join(" ");
+    try {
+      const out = await io.gh(["api", "-X", "GET", "search/issues", "-f", `q=repo:${repo} is:closed ${text}`, "-f", "per_page=10", "--jq",
+        ".items[] | {number, title, state_reason, pr: (.pull_request != null), merged: (.pull_request.merged_at != null)} | @json"]);
+      for (const it of parseLines(out)) {
+        const status = it.pr ? (it.merged ? "merged" : "closed unmerged") : (it.state_reason ? String(it.state_reason) : "closed");
+        hit(Number(it.number), { src: "gh", kind: it.pr ? "PR" : "issue", status: codeOf(status.replace(" ", "-")) ? status : "closed", title: oneLine(it.title, 120) }, q);
+      }
+    } catch (e) { errors.push(`attempts: subject search ${oneLine(e?.message ?? e, 80)}`); }
+  }
+  if (base && io.git) {
+    try {
+      const greps = codes.length ? ["-E", `--grep=${codes.map(reEscape).join("|")}`] : ["--all-match", ...words.map((w) => `--grep=${reEscape(w)}`)];
+      const out = String(await io.git(["log", "-i", ...greps, "-n", "20", "--format=%x1e%H%x1f%s%x1f%b", base]));
+      for (const rec of out.split("\x1e").filter((x) => x.trim())) {
+        const [, subject = "", msg = ""] = rec.split("\x1f");
+        const pr = Number(/Merge pull request #(\d+)/.exec(subject)?.[1] ?? /\(#(\d+)\)\s*$/.exec(subject.trim())?.[1]) || null;
+        const hay = `${subject}\n${msg}`.toLowerCase();
+        const matched = (codes.length ? codes : words).filter((t) => hay.includes(t.toLowerCase()));
+        if (!pr || !matched.length || (!codes.length && matched.length < words.length)) continue;
+        const named = /^Merge pull request/.test(subject) ? msg.split("\n").map((l) => l.trim()).find(Boolean) ?? subject : subject;
+        hit(pr, { src: "git", kind: "PR", status: "merged", title: oneLine(named, 120) }, matched);
+      }
+    } catch (e) { errors.push(`attempts: subject git log ${oneLine(e?.message ?? e, 80)}`); }
+  }
+  const rows = [...hits.values()].sort((a, b) => b.matched.length - a.matched.length || b.number - a.number)
+    .slice(0, max);
+  return { terms, rows, errors };
+}
+
+/** #5085: is the Forge Ledger recall CLI (repo or ForgeDock install) and its index there? null when it cannot be probed. */
+async function recallStatus(io, dirs, index) {
+  if (typeof io?.exists !== "function") return null;
+  let cli = false;
+  for (const p of [...dirs.filter(Boolean).map((d) => `${d}/bin/recall.mjs`), RECALL_INSTALL]) if (await io.exists(p)) { cli = true; break; }
+  if (!cli) return { present: false, reason: "bin/recall.mjs missing" };
+  if (!(await io.exists(index))) return { present: false, reason: "index ~/.forge/index missing" };
+  return { present: true, reason: "" };
+}
+
 /**
  * Collect the facts for one issue's pack. `io`: {gh, git, exists}; git runs in `cwd` (the lane worktree), base is the
  * base ref; `checkout` (the main checkout, optional) is where repo-fact files absent from the worktree are looked up.
  * Every source is independent and best effort: a failing source is listed in `errors` and contributes nothing.
  */
-export async function gatherContextFacts({ io, repo, issue, iss, comments = [], cwd, checkout = null, base, sources = CONTEXT_SOURCES, days = DEFAULT_LEDGER_DAYS, now = Date.now(), pitfalls = [] }) {
-  const facts = { issue: Number(issue), links: [], attempts: [], pitfalls: [], ledger: [], facts: [], lanes: [], errors: [] };
+export async function gatherContextFacts({ io, repo, issue, iss, comments = [], cwd, checkout = null, base, sources = CONTEXT_SOURCES, days = DEFAULT_LEDGER_DAYS, now = Date.now(), pitfalls = [], recallIndex = RECALL_INDEX }) {
+  const facts = { issue: Number(issue), links: [], attempts: [], pitfalls: [], ledger: [], facts: [], lanes: [], errors: [], recall: null };
   const body = String(iss?.body ?? "");
   const on = new Set(sources);
   const step = async (name, fn) => { if (!on.has(name)) return; try { await fn(); } catch (e) { facts.errors.push(`${name}: ${oneLine(e?.message ?? e, 120)}`); } };
@@ -243,7 +338,13 @@ export async function gatherContextFacts({ io, repo, issue, iss, comments = [],
     }
     const refs = String(await git(["for-each-ref", "--format=%(refname) %(objectname:short)", `refs/forge/salvage/${issue}/`])).split("\n").map((l) => l.trim()).filter(Boolean).sort();
     for (const l of refs) { const [ref, sha] = l.split(" "); facts.attempts.push({ kind: "salvage", ref, line: `salvaged uncommitted work at ${sha ?? "?"} (git diff HEAD ${ref} --stat)` }); }
+    // #5085: earlier issues and PRs on the same subject, deduped against the graph (or, with graph off, the body's own links)
+    const exclude = [...facts.links.map((l) => l.number), ...bodyLinks(body, repo, issue), ...parseDependsOn(stripComments(body), issue), ...(parentOf(body) ? [parentOf(body)] : [])];
+    const sh = await subjectHistory({ io: { gh: io.gh, git }, repo, issue, title: String(iss?.title ?? ""), body, base, exclude });
+    facts.errors.push(...sh.errors);
+    for (const r of sh.rows) facts.attempts.push({ kind: "subject", ref: `#${r.number}`, line: `[${r.kind}] ${r.status} "${r.title}"; matched ${r.matched.join(", ")}` });
   });
+  try { facts.recall = await recallStatus(io, [...new Set([cwd, checkout])], recallIndex); } catch { facts.recall = null; }
 
   // #4375: already read and selected by the engine from the local ledger (pitfalls.mjs selectPitfalls); no I/O here
   await step("pitfalls", async () => {
@@ -367,9 +468,10 @@ export function compileContextPack(facts, { budgetBytes = DEFAULT_CONTEXT_BYTES,
     for (const f of facts?.facts ?? []) cand.push({ src: "facts", line: `- ${f} (repo facts file; read it if relevant)` });
   }
   const found = Object.fromEntries(CONTEXT_SOURCES.map((s) => [s, cand.filter((c) => c.src === s).length]));
-  const empty = { text: "", bytes: 0, items: 0, truncated: 0, sources: Object.fromEntries(CONTEXT_SOURCES.map((s) => [s, 0])), found, hash: null, pitfalls: [] };
+  const recall = facts?.recall && typeof facts.recall === "object" ? { present: Boolean(facts.recall.present), reason: oneLine(facts.recall.reason ?? "", 80) } : null;
+  const empty = { text: "", bytes: 0, items: 0, truncated: 0, sources: Object.fromEntries(CONTEXT_SOURCES.map((s) => [s, 0])), found, hash: null, pitfalls: [], recall };
   if (!cand.length) return empty;
-  const titles = { graph: "Linked issues and PRs", attempts: "Prior attempts at this issue", pitfalls: "Recurring pitfalls in this repo (past rework)", ledger: "Merged PRs on the same symbols (reverts, decisions, findings)", facts: "Facts from earlier lanes and repo facts" };
+  const titles = { graph: "Linked issues and PRs", attempts: "Prior attempts (this issue and the same subject)", pitfalls: "Recurring pitfalls in this repo (past rework)", ledger: "Merged PRs on the same symbols (reverts, decisions, findings)", facts: "Facts from earlier lanes and repo facts" };
   const head = `Context pack for #${facts.issue}`;
   const size = (s) => Buffer.byteLength(s, "utf8");
   const kept = [];
@@ -388,7 +490,7 @@ export function compileContextPack(facts, { budgetBytes = DEFAULT_CONTEXT_BYTES,
     if (rows.length) parts.push(`\n${titles[s]}:`, ...rows.map((r) => r.text));
   }
   const text = parts.join("\n");
-  return { text, bytes: size(text), items: kept.length, truncated, sources: Object.fromEntries(CONTEXT_SOURCES.map((s) => [s, kept.filter((k) => k.src === s).length])), found, hash: hash12(text), pitfalls: kept.filter((k) => k.pit).map((k) => k.pit) };
+  return { text, bytes: size(text), items: kept.length, truncated, sources: Object.fromEntries(CONTEXT_SOURCES.map((s) => [s, kept.filter((k) => k.src === s).length])), found, hash: hash12(text), pitfalls: kept.filter((k) => k.pit).map((k) => k.pit), recall };
 }
 
 /** The pack text without its pitfalls section (#4375: pitfalls are local only and never posted). */
@@ -406,7 +508,7 @@ export function publicPackText(text) {
 /** The collapsed FORGE:CONTEXT issue record body (without the marker); pitfalls show as a count only (#4375). */
 export function renderContextRecord(pack, { mode, laneType: lt }) {
   const src = CONTEXT_SOURCES.map((s) => `${s} ${pack.sources?.[s] ?? 0}/${pack.found?.[s] ?? 0}`).join(", ");
-  return `<details><summary>Context pack: ${pack.items} item(s), ${pack.bytes} bytes, ${pack.truncated} dropped (mode ${mode}, lane ${lt}, hash ${pack.hash})</summary>\n\nSources (included/found): ${src}${pack.sources?.pitfalls ? " (pitfalls are local only, not shown)" : ""}\n\n~~~~text\n${noComments(publicPackText(pack.text)).replace(/~~~~/g, "~~~")}\n~~~~\n</details>`;
+  return `<details><summary>Context pack: ${pack.items} item(s), ${pack.bytes} bytes, ${pack.truncated} dropped (mode ${mode}, lane ${lt}, hash ${pack.hash})</summary>\n\nSources (included/found): ${src}${pack.sources?.pitfalls ? " (pitfalls are local only, not shown)" : ""}${pack.recall && !pack.recall.present ? `\n\nRecall index: absent (${pack.recall.reason || "unknown"}); subject history comes from live search.` : ""}\n\n~~~~text\n${noComments(publicPackText(pack.text)).replace(/~~~~/g, "~~~")}\n~~~~\n</details>`;
 }
 
 /**

package/bin/lean/context.test.mjs

@@ -3,7 +3,7 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { normaliseFacts, renderFactsRecord, FACTS_MARKER } from "./facts.mjs";
-import { gatherContextFacts, compileContextPack, contextPackFor, contextArm, contextPackCacheKey, linkStateStamp, LINK_STAMP_MAX, parseContextPackConfig, laneType, priorAttempts, issueSymbols, bodyLinks, parentOf, renderContextRecord, DEFAULT_CONTEXT_BYTES, noComments } from "./context.mjs";
+import { gatherContextFacts, compileContextPack, contextPackFor, contextArm, contextPackCacheKey, linkStateStamp, LINK_STAMP_MAX, parseContextPackConfig, laneType, priorAttempts, issueSymbols, bodyLinks, parentOf, renderContextRecord, DEFAULT_CONTEXT_BYTES, noComments, subjectTerms, subjectHistory, MAX_SUBJECT_ITEMS, MAX_SUBJECT_QUERIES } from "./context.mjs";
 
 const REPO = "o/r";
 const BOT = { author_association: "NONE", user: { type: "Bot", login: "forge[bot]" } };
@@ -47,6 +47,7 @@ function fakes(o = {}) {
     calls.push(["gh", ...a]);
     const path = a.find((x) => String(x).startsWith("repos/"));
     if (o.ghFail) throw new Error("gh down");
+    if (a.includes("search/issues")) { const q = String(a.find((x) => String(x).startsWith("q=")) ?? ""); if (o.searchFail) throw new Error("HTTP 403 rate limit"); return o.search ? o.search(q) : ""; }
     let m;
     if ((m = /issues\/(\d+)\/timeline$/.exec(path))) return o.timeline ?? "https://api.github.com/repos/o/r 70\nhttps://api.github.com/repos/o/r 61\n";
     if ((m = /issues\/(\d+)\/comments$/.exec(path))) return (comments[m[1]] ?? []).map((c) => JSON.stringify(c)).join("\n");
@@ -56,6 +57,7 @@ function fakes(o = {}) {
   };
   const git = async (a) => {
     calls.push(["git", ...a]);
+    if (a[0] === "log" && a.some((x) => String(x).startsWith("--grep="))) return o.grepLog ?? "";
     if (a[0] === "for-each-ref") return "refs/forge/salvage/50/1700 abc1234\n";
     if (a[0] === "log") return log.map(([h, s, b, f]) => `\x1e${h}\x1f${s}\x1f${b}\x1f\n${f}\n`).join("");
     return "";
@@ -405,7 +407,7 @@ describe("#4375 recurring pitfalls in the context pack", () => {
     const r = renderContextRecord(p, { mode: "on", laneType: "first" });
     for (const x of ["no-unused-vars", "Recurring pitfalls", "src/*.ts", "devdocs/modules/lean.md is stale"]) assert.ok(!r.includes(x), x);
     assert.match(r, /pitfalls 2\/2/);
-    assert.ok(r.includes("Prior attempts at this issue:") && r.includes("Merged PRs on the same symbols"), "the other sections stay");
+    assert.ok(r.includes("Prior attempts (this issue and the same subject):") && r.includes("Merged PRs on the same symbols"), "the other sections stay");
   });
 
   it("the cache key changes when the selected pitfall signatures change", () => {
@@ -488,3 +490,83 @@ describe("#4467: priorAttempts reads App comments in gh issue view shape", () =>
     assert.equal(priorAttempts("", [ghShape("someone")]), 0);
   });
 });
+
+describe("#5085: subject history and recall status", () => {
+  const item = (n, title, extra = {}) => JSON.stringify({ number: n, title, state_reason: "completed", pr: false, merged: false, ...extra });
+  const SUBJ = { number: 50, title: "fix(fetch): TLS bypass misclassified as blocked", body: "Requests end as `tls.bypass_misclassified` in src/fetch/tls.ts.\n\n**Depends on**: #45" };
+
+  it("#5085: subject history surfaces a closed issue matched by an error code with no shared file", async () => {
+    const f = fakes({ search: (q) => (q.includes('"tls.bypass_misclassified"') ? [item(900, "Billing: classify TLS bypass"), item(45, "dep, already in graph"), item(50, "self")].join("\n") : "") });
+    const facts = await gatherContextFacts({ io: f.io, repo: REPO, issue: 50, iss: SUBJ, comments: [], cwd: "/wt", base: "origin/staging", sources: ["graph", "attempts"] });
+    const p = compileContextPack(facts);
+    assert.match(p.text, /- #900: \[issue\] completed "Billing: classify TLS bypass"; matched tls\.bypass_misclassified/);
+    assert.ok(!/- #45: \[issue\]/.test(p.text) && !/- #50: \[issue\]/.test(p.text), "self and graph links are deduped");
+    assert.ok(p.sources.attempts >= 1 && p.found.attempts >= 1, JSON.stringify(p.found));
+    assert.ok(f.calls.some((c) => c.includes("search/issues") && c.some((x) => String(x) === 'q=repo:o/r is:closed "tls.bypass_misclassified"')));
+  });
+
+  it("subjectTerms: codes, symbols, then distinctive title words; deterministic and capped", () => {
+    const t = subjectTerms({ title: "fix(context): recall prior attempts by subject, not only by issue number", body: "`ci.red` and `bin/lean/context.mjs` and `compileContextPack()` HTTP 403 <!-- `hidden.code` -->" });
+    assert.deepEqual(t, ["ci.red", "403", "compileContextPack", "recall", "prior", "attempts"]);
+    assert.ok(t.length <= 6);
+    assert.deepEqual(subjectTerms({ title: "", body: "" }), []);
+  });
+
+  it("matches title words and repo-wide git history; ranks by matched terms, caps the rows, keeps own attempts first", async () => {
+    const many = Array.from({ length: 12 }, (_, i) => item(800 + i, `thin content ${i}`)).join("\n");
+    const grepLog = `\x1e${"e".repeat(40)}\x1fMerge pull request #777 from o/fix-thin\x1fGuard thin content pages in billing\n\x1e${"f".repeat(40)}\x1fMerge pull request #778 from o/x\x1funrelated\n`;
+    const f = fakes({ search: () => many, grepLog });
+    const r = await subjectHistory({ io: f.io, repo: REPO, issue: 50, title: "Thin content billing guard", body: "", base: "origin/staging" });
+    assert.deepEqual(r.terms, ["thin", "content", "billing", "guard"].slice(0, 3));
+    assert.equal(r.rows.length, MAX_SUBJECT_ITEMS);
+    assert.ok(!r.rows.some((x) => x.number === 778), "a git hit must name the terms");
+    const g = f.calls.find((c) => c[0] === "git" && c[1] === "log");
+    assert.ok(g.includes("--all-match") && !g.includes("--"), "repo-wide, no pathspec");
+    assert.ok(f.calls.filter((c) => c.includes("search/issues")).length <= MAX_SUBJECT_QUERIES);
+    const both = await subjectHistory({ io: fakes({ search: () => item(777, "thin content PR", { pr: true, merged: true }), grepLog }).io, repo: REPO, issue: 50, title: "Thin content billing", body: "", base: "origin/staging" });
+    assert.deepEqual(both.rows.find((x) => x.number === 777), { number: 777, kind: "PR", status: "merged", title: "thin content PR", matched: ["thin", "content", "billing"] });
+    const facts = await gather(fakes({ search: () => many }), { iss: { ...ISS, title: "Thin content guard" }, sources: ["attempts"] });
+    const kinds = facts.attempts.map((a) => a.kind);
+    assert.ok(kinds.indexOf("subject") > kinds.lastIndexOf("salvage"), kinds.join(","));
+  });
+
+  it("ranks rows by matched-term count, then newest number", async () => {
+    const search = (q) => (q.includes('"a.code"') ? [item(901, "x"), item(950, "y")] : q.includes('"b.code"') ? [item(901, "x")] : []).join("\n");
+    const r = await subjectHistory({ io: fakes({ search }).io, repo: REPO, issue: 50, title: "", body: "`a.code` and `b.code`" });
+    assert.deepEqual(r.rows.map((x) => [x.number, x.matched]), [[901, ["a.code", "b.code"]], [950, ["a.code"]]]);
+  });
+
+  it("subject history is ranked and truncated within the byte budget like the other sources", async () => {
+    const many = Array.from({ length: 8 }, (_, i) => item(800 + i, `A long title about thin content billing regressions number ${i}`)).join("\n");
+    const facts = await gather(fakes({ search: () => many }), { iss: { ...ISS, title: "Thin content billing" }, sources: ["attempts"] });
+    const full = compileContextPack(facts);
+    assert.ok(full.found.attempts >= 8 + 3);
+    const small = compileContextPack(facts, { budgetBytes: 400 });
+    assert.ok(small.bytes <= 400 && small.truncated > 0, `${small.bytes} ${small.truncated}`);
+    assert.ok(small.text.includes("- #50 FORGE:STATE") && small.text.indexOf("- #50 FORGE:STATE") < (small.text.indexOf("- #80") >>> 0), "own attempts outrank subject rows");
+  });
+
+  it("a failed subject search is an error, never a throw, and an attempts-off pack makes no search", async () => {
+    const facts = await gather(fakes({ searchFail: true }), { iss: { ...ISS, title: "Thin content billing" }, sources: ["attempts"] });
+    assert.ok(facts.errors.some((e) => e.startsWith("attempts: subject search")), facts.errors.join("|"));
+    assert.ok(facts.attempts.some((a) => a.kind === "state"));
+    const g = fakes();
+    await gather(g, { sources: ["graph", "facts"] });
+    assert.ok(!g.calls.some((c) => c.includes("search/issues")));
+  });
+
+  it("a missing bin/recall.mjs or index is reported in the FORGE:CONTEXT record", async () => {
+    const absent = await gather(fakes(), { sources: ["graph"], recallIndex: "/idx/knowledge.jsonl" });
+    assert.deepEqual(absent.recall, { present: false, reason: "bin/recall.mjs missing" });
+    const p = compileContextPack(absent);
+    const r = renderContextRecord(p, { mode: "on", laneType: "first" });
+    assert.match(r, /Recall index: absent \(bin\/recall\.mjs missing\); subject history comes from live search\./);
+    assert.ok(!p.text.includes("Recall index"), "the status stays out of the pack text and its hash");
+    const noIndex = await gather({ io: { ...fakes().io, exists: async (x) => x === "/wt/bin/recall.mjs" } }, { sources: ["graph"], recallIndex: "/idx/knowledge.jsonl" });
+    assert.match(renderContextRecord(compileContextPack(noIndex), { mode: "on", laneType: "first" }), /Recall index: absent \(index ~\/\.forge\/index missing\)/);
+    const ok = await gather({ io: { ...fakes().io, exists: async (x) => x === "/wt/bin/recall.mjs" || x === "/idx/knowledge.jsonl" } }, { sources: ["graph"], recallIndex: "/idx/knowledge.jsonl" });
+    assert.deepEqual(ok.recall, { present: true, reason: "" });
+    assert.ok(!renderContextRecord(compileContextPack(ok), { mode: "on", laneType: "first" }).includes("Recall index"));
+    assert.equal(compileContextPack({ issue: 1, links: [], attempts: [], ledger: [], facts: [], recall: absent.recall }).recall.present, false, "the empty pack carries it too");
+  });
+});

package/bin/lean/coupled.mjs

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Coupled files (#5079): forge.yaml `verification.coupled_files` is a block map `{when glob: required glob[, glob]}`,
+ * "if A changes, B must change" (a digest pin, a manifest, a runtime closure, CI wiring of a new test). A key prefixed
+ * `added:` fires only for files the lane adds. Deterministic, 0 LLM calls; a failure is `conformance.unimplemented`.
+ */
+import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
+import { globToRegex } from "../../scripts/ci-local/lib.mjs";
+
+const ADDED = "added:";
+const norm = (p) => String(p ?? "").trim().replace(/^\.\//, "");
+
+/** Rules `{when, requires[], addedOnly}` from the config map; entries with no usable glob are dropped, never thrown on. */
+export function parseCoupled(map) {
+  if (!map || typeof map !== "object" || Array.isArray(map)) return [];
+  const rules = [];
+  for (const [key, val] of Object.entries(map)) {
+    const k = String(key).trim(), addedOnly = k.startsWith(ADDED);
+    const when = norm(addedOnly ? k.slice(ADDED.length) : k);
+    const requires = String(val ?? "").split(",").map((g) => norm(g.replace(/^(["'])(.*)\1$/, "$2"))).filter(Boolean);
+    if (when && requires.length) rules.push({ when, requires, addedOnly });
+  }
+  return rules;
+}
+
+/** Failures for each rule whose trigger files changed while none of a required glob's files did. */
+export function coupledFailures({ rules = [], changedFiles = [], addedFiles = [] } = {}) {
+  const changed = changedFiles.map(norm), added = addedFiles.map(norm), out = [];
+  for (const r of rules) {
+    const re = globToRegex(r.when);
+    const hits = (r.addedOnly ? added : changed).filter((f) => re.test(f));
+    if (!hits.length) continue;
+    const missing = r.requires.filter((g) => { const gre = globToRegex(g); return !changed.some((f) => gre.test(f)); });
+    if (missing.length) out.push({ code: OUTCOME_CODES.CONFORMANCE_UNIMPLEMENTED, kind: "coupled_file_missing", detail: `${r.addedOnly ? "added" : "changed"} ${hits.slice(0, 3).join(", ")} (forge.yaml verification.coupled_files) but no change to ${missing.join(", ")}` });
+  }
+  return out;
+}

package/bin/lean/coupled.test.mjs

@@ -0,0 +1,57 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { parseCoupled, coupledFailures } from "./coupled.mjs";
+import { parseVerify } from "./verify.mjs";
+import { planPrecheck } from "./precheck.mjs";
+import { parseTestMap } from "./testmap.mjs";
+import { precheck } from "./conformance.mjs";
+
+const rules = parseCoupled({
+  "infra/ssh-guard/bootstrap-body.sh": "infra/ssh-guard/launcher.sh, infra/ssh-guard/MANIFEST.sha256",
+  "added:scripts/tests/test_*.py": ".github/workflows/ci.yml",
+});
+
+describe("coupled files (#5079)", () => {
+  it("coupled files: a changed file without its pinned counterpart fails", () => {
+    const f = coupledFailures({ rules, changedFiles: ["infra/ssh-guard/bootstrap-body.sh", "infra/ssh-guard/launcher.sh"] });
+    assert.equal(f.length, 1);
+    assert.equal(f[0].kind, "coupled_file_missing");
+    assert.equal(f[0].code, "conformance.unimplemented");
+    assert.match(f[0].detail, /MANIFEST\.sha256/);
+  });
+  it("coupled files: passes when every counterpart changes, or the trigger did not", () => {
+    assert.deepEqual(coupledFailures({ rules, changedFiles: ["infra/ssh-guard/bootstrap-body.sh", "infra/ssh-guard/launcher.sh", "infra/ssh-guard/MANIFEST.sha256"] }), []);
+    assert.deepEqual(coupledFailures({ rules, changedFiles: ["README.md"] }), []);
+  });
+  it("coupled files: an added: rule fires on a new test file only", () => {
+    const t = "scripts/tests/test_x.py";
+    assert.equal(coupledFailures({ rules, changedFiles: [t], addedFiles: [t] }).length, 1);
+    assert.deepEqual(coupledFailures({ rules, changedFiles: [t], addedFiles: [] }), []);
+    assert.deepEqual(coupledFailures({ rules, changedFiles: [t, ".github/workflows/ci.yml"], addedFiles: [t] }), []);
+  });
+  it("coupled files: malformed config yields no rules", () => {
+    assert.deepEqual(parseCoupled(null), []);
+    assert.deepEqual(parseCoupled(["a"]), []);
+    assert.deepEqual(parseCoupled({ "": "x", a: "" }), []);
+  });
+  it("coupled files: precheck() carries the failures; absent they change nothing", () => {
+    const c = { requirements: [], deliverables: [], validate: [] };
+    const failures = coupledFailures({ rules, changedFiles: ["infra/ssh-guard/bootstrap-body.sh"] });
+    assert.equal(precheck({ contract: c, result: {}, changedFiles: [], coupledFailures: failures }).ok, false);
+    assert.equal(precheck({ contract: c, result: {}, changedFiles: [] }).ok, true);
+  });
+  it("coupled files: test-map entries select the touched suite and formatter per path (R3, R6)", () => {
+    const testMap = parseTestMap(JSON.stringify({ version: 1, entries: [
+      { command: "bash infra/ssh-guard/validate-manifest.sh", globs: ["infra/ssh-guard/*.sh"] },
+      { command: "black --check services/api && isort --check services/api", globs: ["services/api/**"] },
+    ] }));
+    const p = planPrecheck({ testMap, files: ["infra/ssh-guard/check-guard-deps.sh", "services/api/x.py"], learned: [] });
+    assert.deepEqual(p.commands, ["bash infra/ssh-guard/validate-manifest.sh", "black --check services/api && isort --check services/api"]);
+    assert.deepEqual(planPrecheck({ testMap, files: ["docs/a.md"], learned: [] }).commands, []);
+  });
+  it("coupled files: verify.precheck_block is opt-in", () => {
+    assert.equal(parseVerify({ precheck_block: "true" }).precheckBlock, true);
+    assert.equal("precheckBlock" in parseVerify({}), false);
+    assert.equal("precheckBlock" in parseVerify({ precheck_block: "yes" }), false);
+  });
+});

package/bin/lean/decision-engine.mjs

@@ -13,6 +13,7 @@
  * Not yet called in production: the issue-watch stage (#4765) is meant to call `settle` once per escalation (key: the escalation comment id).
  */
 import { parseForgeYaml } from "../forge-utils.mjs";
+import { READONLY_PROD_RE, EXTERNAL_NEED_RE } from "./conformance.mjs";
 import { foldDecisions, recordDecisions as recordToRegister } from "./decisions.mjs";
 
 export const VETO_LABEL = "forge:decision-veto";
@@ -80,7 +81,8 @@ const POSTDEPLOY_RE = /\b(all (remaining )?(items|checks|criteria) (are|is) post
 /**
  * Splits an escalation into decision and external parts from facts. `{terminalCode, blockType, text}`, `matrix` from
  * loadPolicy, `retryable` a set of codes recovery owns. Returns `{kind, decisions:[text], external:[{class, text}]}` with
- * kind `recovery | evidence | post-deploy | external | decision | mixed`.
+ * kind `recovery | evidence | post-deploy | host | external | decision | mixed`. #5080: `host` is read-only production access with no
+ * credential/spend/account/customer need: host work (a `host-task`), never a human ask.
  */
 export function splitEscalation({ terminalCode = null, blockType = null, text = "" } = {}, matrix = [], { retryable = [] } = {}) {
   const body = String(text ?? "");
@@ -88,6 +90,7 @@ export function splitEscalation({ terminalCode = null, blockType = null, text =
   if (terminalCode && new Set(retryable).has(terminalCode)) return out("recovery");
   if (terminalCode === "conformance.unimplemented" || EVIDENCE_RE.test(body)) return out("evidence");
   if (POSTDEPLOY_RE.test(body)) return out("post-deploy");
+  if (READONLY_PROD_RE.test(body) && !EXTERNAL_NEED_RE.test(body)) return out("host");
   const klass = (t) => matrix.find((m) => m.re.test(t))?.class ?? null;
   if (blockType === "credentials" && matrix.some((m) => m.class === "credentials")) return out("external", [], [{ class: "credentials", text: body.slice(0, 200) }]);
   const lines = body.split("\n").map((l) => l.replace(/^\s*(?:[-*]|\d+[.)])\s+/, "").trim()).filter(Boolean);
@@ -177,6 +180,7 @@ export async function settle(item, policy, deps = {}, opts = {}) {
     if (s.kind === "recovery") return { ...row, reason: "recovery owns this code" };
     if (s.kind === "external") return { ...row, action: "external", kept: true, class: s.external[0]?.class, reason: `skip: external (${s.external[0]?.class})` };
     if (s.kind === "evidence") return { ...row, action: "evidence-needed" };
+    if (s.kind === "host") return { ...row, action: "host-task" };
     if (s.kind === "post-deploy") return { ...row, action: "post-deploy-only" };
     const key = `Escalation: ${item.escalationId ?? "-"}`;
     const precedent = foldDecisions(deps.records ?? []).filter((d) => d.status !== "superseded");

package/bin/lean/decision-engine.test.mjs

@@ -35,6 +35,14 @@ test("#4766 loadPolicy: defaults, fallback order, matrix schema", () => {
   assert.equal(loadPolicy({ io: files({}) }).policyPath, null);
 });
 
+test("#5080 splitEscalation: a credentials block that is only read-only production access is host work; real credentials stay external", () => {
+  const matrix = loadPolicy({ io: files({ "AGENTS.md": "a" }) }).matrix;
+  const host = splitEscalation({ text: "D1 is gated by K1 on read-only production telemetry: a host-only prod_monitor capability this sandbox lacks", blockType: "credentials", terminalCode: "builder.blocked" }, matrix);
+  assert.equal(host.kind, "host");
+  assert.deepEqual(host.external, []);
+  assert.equal(splitEscalation({ text: "needs the production API key to read prod data", blockType: "credentials", terminalCode: "builder.blocked" }, matrix).kind, "external");
+});
+
 test("#4766 splitEscalation: the six AlterLab fixtures", () => {
   const { matrix } = policy();
   for (const [n, f] of Object.entries(FIX)) assert.equal(splitEscalation(f, matrix).kind, f.want, `#${n}`);
@@ -158,6 +166,19 @@ test("#4766 tally, telemetry mapping and the batch report 'decided' section", ()
   assert.deepEqual(formatOutcomeLists([{ issue: 7, outcome: "merged" }]), []);
 });
 
+test("#5078 the batch report lists follow-ups appended, grouped by issue across rows, the rollup named", () => {
+  const lines = formatOutcomeLists([
+    { issue: 7, outcome: "merged", followups: [{ issue: 900, class: "post-deploy", count: 2 }, { issue: 901, class: "engineering", count: 1 }] },
+    { issue: 8, outcome: "merged", followups: [{ issue: 900, class: "post-deploy", count: 1 }] },
+    { issue: 9, outcome: "merged", followups: [{ issue: 950, class: "external", count: 3 }] },
+  ]);
+  const fu = lines.filter((l) => l.startsWith("follow-ups appended:"));
+  assert.equal(fu.length, 1);
+  assert.equal(fu[0], "follow-ups appended: #900 (post-deploy rollup: 3 item(s) from #7,#8) #901 (engineering: 1 item(s) from #7) #950 (external aggregate: 3 item(s) from #9)");
+  assert.ok(!lines.some((l) => /issues filed/.test(l)));
+  assert.deepEqual(formatOutcomeLists([{ issue: 7, outcome: "merged", followups: [] }]), []);
+});
+
 test("#4826 docs say 'not yet wired' exactly while no production module calls settle", async () => {
   const { readFileSync, readdirSync } = await import("node:fs");
   const dir = new URL(".", import.meta.url);

package/bin/lean/drift.mjs

@@ -19,6 +19,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { listMilestones } from "./lane.mjs";
 
 export const SUMMARY_MARKER = "FORGE:DRIFT";
 const SUMMARY_RE = /<!-- FORGE:DRIFT (\{[\s\S]*?\}) -->/;
@@ -134,9 +135,9 @@ export function heldBy(files, flagged) {
   return null;
 }
 
-/** {number, description} of the milestone titled `title`, or null. One `gh api` read. */
+/** {number, description} of the milestone titled `title`, or null. One paginated read (lane.mjs `listMilestones`, every page). */
 export async function readMilestone({ gh, repo, title }) {
-  const list = JSON.parse(String(await gh(["api", `repos/${repo}/milestones?state=all&per_page=100`])) || "[]");
+  const list = await listMilestones({ gh, repo, state: "all" });
   const m = arr(list).find((x) => x?.title === title);
   return m ? { number: m.number, description: m.description ?? "" } : null;
 }

package/bin/lean/drift.test.mjs

@@ -5,7 +5,7 @@ import assert from "node:assert/strict";
 import { writeFileSync } from "node:fs";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { join } from "node:path";
-import { foldSummary, emptySummary, parseSummary, renderSummary, crossingIssues, fileCrossings, flaggedPackages, heldBy, makeDrift } from "./drift.mjs";
+import { foldSummary, emptySummary, parseSummary, renderSummary, crossingIssues, fileCrossings, flaggedPackages, heldBy, makeDrift, readMilestone } from "./drift.mjs";
 import { orchestrate } from "./orchestrate.mjs";
 import { readDrift } from "../engine-cli.mjs";
 import { affectedFiles } from "../orchestrate-preflight.mjs";
@@ -221,3 +221,25 @@ describe("readDrift (forge#4161)", () => {
     assert.deepEqual(withYaml("orchestrate:\n  drift:\n    file_lines: 600\n", readDrift), { pairGrowth: null, packageLines: null, fileLines: 600, blockDispatch: false, generatedGlobs: [] });
   });
 });
+
+/** #5084: a GitHub-like milestones endpoint: pages of 100; without --paginate only page 1; `--jq '.[] | @json'` prints one object per line. */
+const pagedMilestones = (all) => (a) => {
+  const state = /state=(\w+)/.exec(a[1])?.[1] ?? "open";
+  const items = all.filter((m) => state === "all" || m.state === state);
+  const pages = []; for (let i = 0; i < items.length; i += 100) pages.push(items.slice(i, i + 100));
+  if (!a.includes("--paginate")) return JSON.stringify(pages[0] ?? []);
+  if (a.includes("--jq")) return items.map((m) => JSON.stringify(m)).join("\n") + "\n";
+  return pages.map((p) => JSON.stringify(p)).join("");
+};
+const manyMilestones = (target, n = 120) => [...Array.from({ length: n }, (_, i) => ({ number: i + 1, title: `M ${i + 1}`, state: i % 2 ? "open" : "closed", description: "" })), { number: n + 1, ...target }];
+
+describe("readMilestone pagination (#5084)", () => {
+  it("(#5084) readMilestone finds a milestone on page 2", async () => {
+    const list = pagedMilestones(manyMilestones({ title: "Search & Research API", state: "open", description: "late" }));
+    const calls = [];
+    const gh = async (a) => { calls.push(a); return list(a); };
+    assert.deepEqual(await readMilestone({ gh, repo: "o/r", title: "Search & Research API" }), { number: 121, description: "late" });
+    assert.equal(calls[0][1], "repos/o/r/milestones?state=all&per_page=100");
+    assert.equal(await readMilestone({ gh, repo: "o/r", title: "Missing" }), null);
+  });
+});

package/bin/lean/effects.mjs

@@ -359,11 +359,12 @@ export function makeLeanIo({ repo, subagentModel = readSubagentModel() }) {
      * File (or append to) the parent's ONE batched follow-up issue (scripts/file-deferred-followups.sh: trusted lookup,
      * append, create, claim marker, duplicate re-check; one issue per class, #4664). items: [{key, why, class?}]; the key is the
      * item's identity (column 5, column 1 is `-`). milestone / defaultMilestone: the parent's and forge.yaml followups.default_milestone; lane: the release lane of the post-deploy rollup (#4724).
+     * batchId: the external aggregate's batch (FORGE_BATCH_ID); rollupLabel: forge.yaml followups.rollup_issue_label (FORGE_ROLLUP_LABEL) (#5078).
      * Returns {key: issueNumber}; throws on exit 2/3/4.
      */
-    async fileFollowups({ parent, items, milestone = null, defaultMilestone = null, lane = null }) {
+    async fileFollowups({ parent, items, milestone = null, defaultMilestone = null, lane = null, batchId = null, rollupLabel = null }) {
       const tsv = items.map((i) => `-\t${i.why}\t\t${i.class ?? ""}\t${i.key}`).join("\n") + "\n";
-      const env = { ...process.env, ...(milestone ? { FORGE_FOLLOWUP_MILESTONE: String(milestone) } : {}), ...(defaultMilestone ? { FORGE_FOLLOWUPS_DEFAULT_MILESTONE: String(defaultMilestone) } : {}), ...(lane ? { FORGE_RELEASE_LANE: String(lane) } : {}) }; // #4724: the post-deploy rollup is one per release lane
+      const env = { ...process.env, ...(milestone ? { FORGE_FOLLOWUP_MILESTONE: String(milestone) } : {}), ...(defaultMilestone ? { FORGE_FOLLOWUPS_DEFAULT_MILESTONE: String(defaultMilestone) } : {}), ...(lane ? { FORGE_RELEASE_LANE: String(lane) } : {}), ...(batchId ? { FORGE_BATCH_ID: String(batchId) } : {}), ...(rollupLabel ? { FORGE_ROLLUP_LABEL: String(rollupLabel) } : {}) }; // #4724: the post-deploy rollup is one per release lane
       const out = await runWithInput("bash", [script("file-deferred-followups.sh"), "--parent", String(parent), "--repo", repo], tsv, 180000, env);
       return followupMap(out, items);
     },