forgedock 1.30.0 → 1.31.0

package/bin/lean/e2e.mjs

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * e2e merge gate (#4403): a lane that adds or changes e2e specs, or whose contract has `[type:e2e]` criteria, triggers
+ * the repo's e2e run (a PR label or a workflow dispatch) and waits on it before merge. Tiered CI skips e2e on lane PRs,
+ * so the required checks alone never run those specs. All GitHub effects go through the injected io.
+ */
+import { ciPollDelayMs } from "../engine.mjs";
+import { matchesGlob, verifiableFiles, skipsCi, runCiVerify, collectFailedLogs, VERIFY_CHECK_NAME, DEFAULT_SKIP_PATHS, DEFAULT_GRACE_MS } from "./verify.mjs";
+
+export const E2E_MODES = Object.freeze(["auto", "always", "never"]);
+/** Common e2e layouts across ecosystems (Playwright, Cypress, Nest/Jest e2e, plain e2e/ trees). */
+export const E2E_DEFAULT_PATHS = Object.freeze(["e2e/**", "**/e2e/**", "*.e2e.*", "*.e2e-spec.*", "cypress/**", "playwright/**"]);
+export const NO_TRIGGER = "no e2e_label or e2e_workflow configured";
+
+const str = (v) => (v === undefined || v === null ? "" : String(v).trim());
+const list = (v) => (Array.isArray(v) ? v : str(v) ? str(v).split(",") : null)?.map((x) => String(x).trim()).filter(Boolean) ?? null;
+
+/**
+ * forge.yaml `verify:` e2e keys → {mode, label, workflow, pathsInput, checks, paths, blocking}. mode defaults to auto
+ * (inert without a label or workflow), blocking to changed (only the lane's own specs block the merge).
+ */
+export function parseE2e(sec = {}) {
+  const s = sec && typeof sec === "object" ? sec : {};
+  const mode = E2E_MODES.includes(str(s.e2e)) ? str(s.e2e) : "auto";
+  const label = /^[A-Za-z0-9._:/ -]+$/.test(str(s.e2e_label)) ? str(s.e2e_label) : null;
+  const workflow = /^[A-Za-z0-9._/-]+$/.test(str(s.e2e_workflow)) ? str(s.e2e_workflow) : null;
+  const pathsInput = /^[A-Za-z_][A-Za-z0-9_-]*$/.test(str(s.e2e_paths_input)) ? str(s.e2e_paths_input) : null;
+  const checks = list(s.e2e_checks) ?? [];
+  const paths = list(s.e2e_paths) ?? [...E2E_DEFAULT_PATHS];
+  const blocking = str(s.e2e_blocking) === "all" ? "all" : "changed";
+  return { mode, label, workflow, pathsInput, checks, paths, blocking };
+}
+
+/** Changed files (minus the engine's hand-off files) that are e2e specs per `cfg.paths`. */
+export const e2eSpecs = (files, cfg = parseE2e({})) => verifiableFiles(files).filter((f) => (cfg.paths ?? []).some((g) => matchesGlob(f, g)));
+/** True when a contract requirement quote carries a `[type:e2e]` criterion. */
+export const contractWantsE2e = (contract) => (Array.isArray(contract?.requirements) ? contract.requirements : []).some((r) => /\[type:e2e\]/i.test(String(r?.quote ?? "")));
+
+/** Whether this lane runs the e2e gate: {run, reason, specs}. */
+export function e2eWanted({ cfg, files, contract } = {}) {
+  const c = cfg ?? parseE2e({});
+  const specs = e2eSpecs(files, c);
+  if (c.mode === "never") return { run: false, reason: "verify.e2e: never", specs };
+  if (!c.label && !c.workflow) return { run: false, reason: NO_TRIGGER, specs };
+  if (c.mode === "always") {
+    return skipsCi(files, DEFAULT_SKIP_PATHS) ? { run: false, reason: "docs-only change", specs } : { run: true, reason: "verify.e2e: always", specs };
+  }
+  if (specs.length) return { run: true, reason: `${specs.length} e2e spec(s) changed`, specs };
+  if (contractWantsE2e(contract)) return { run: true, reason: "contract has [type:e2e] criteria", specs };
+  return { run: false, reason: "no e2e spec changed and no [type:e2e] criteria", specs };
+}
+
+/**
+ * The failures attributable to the lane: every failure when it changed no spec; otherwise those whose log names one of
+ * its specs (path or basename), plus any whose log is empty (cannot be attributed: fail closed).
+ */
+export function ownFailures(failures, specs) {
+  const fs = failures ?? [];
+  if (!specs?.length) return [...fs];
+  const needles = [...new Set(specs.flatMap((s) => [String(s), String(s).split("/").at(-1)]))];
+  return fs.filter((f) => { const l = String(f?.log ?? ""); return !l.trim() || needles.some((n) => l.includes(n)); });
+}
+
+/**
+ * One e2e run of `head`: {result: pass|fail|timeout|unavailable, failures, runUrl, runId, detail}. Workflow path:
+ * dispatch `cfg.workflow` (runCiVerify). Label path: label the PR, then wait on the head's e2e check runs, where a
+ * skipped run (the unlabelled tier) counts as not run.
+ */
+export async function runE2eGate({ io, pr, branch, head, cfg, specs = [], changedFiles = [], ciMaxMinutes = 360, now = () => Date.now(), sleep, random = Math.random,
+  onPoll = async () => {}, exclude = [], graceMs = DEFAULT_GRACE_MS }) {
+  const c = cfg ?? parseE2e({});
+  if (c.workflow) {
+    const v = await runCiVerify({ io, branch, head, changedFiles: specs.length ? specs : changedFiles, verify: { workflow: c.workflow, pathsInput: c.pathsInput, skipPaths: [] },
+      ciMaxMinutes, now, sleep, random, onPoll, exclude, graceMs, push: async () => {} });
+    if (v.result === "skipped") return { ...v, result: "unavailable", detail: "no changed file to run e2e for" };
+    return v;
+  }
+  const unavailable = (detail) => ({ result: "unavailable", failures: [], runUrl: null, detail });
+  if (!c.label) return unavailable(NO_TRIGGER);
+  if (!io.addPrLabel || !io.commitChecks) return unavailable("no label or commit-checks seam");
+  try { await io.addPrLabel(pr, c.label); }
+  catch (e) { return unavailable(`label ${c.label} could not be added to PR #${pr}: ${String(e?.stderr ?? e?.message ?? e).split("\n")[0].slice(0, 200)}`); }
+  const startedAt = now();
+  const isE2e = (x) => (c.checks.length ? c.checks.includes(x?.name) : x?.name !== VERIFY_CHECK_NAME);
+  for (let i = 0; ; i++) {
+    let checks = [];
+    try { checks = ((await io.commitChecks(head)) ?? []).filter(isE2e); } catch { checks = []; }
+    const ran = checks.filter((x) => String(x?.conclusion ?? "").toLowerCase() !== "skipped");
+    const names = c.checks.length ? c.checks : [...new Set(checks.map((x) => x.name))];
+    const byName = names.map((n) => ran.filter((x) => x.name === n));
+    if (names.length && byName.every((rs) => rs.length && rs.every((x) => x.state !== "pending"))) {
+      const failed = byName.flatMap((rs) => rs.filter((x) => x.state === "failure"));
+      if (!failed.length) return { result: "pass", failures: [], runUrl: null, detail: `${names.length} e2e check(s) green` };
+      return { result: "fail", failures: await collectFailedLogs(io, failed), runUrl: failed[0].url ?? null, detail: `${failed.length} e2e check(s) failed: ${failed.map((x) => x.name).join(", ")}` };
+    }
+    if (!ran.length && now() - startedAt >= graceMs) return unavailable(`no e2e check ran for ${String(head).slice(0, 12)} within ${Math.round(graceMs / 60000)} min of labelling PR #${pr} ${c.label}`);
+    if (now() - startedAt >= ciMaxMinutes * 60000) return { result: "timeout", failures: [], runUrl: null, detail: `e2e still pending after ${ciMaxMinutes} min` };
+    await onPoll();
+    await sleep(ciPollDelayMs(i, random));
+  }
+}

package/bin/lean/e2e.test.mjs

@@ -0,0 +1,86 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { parseE2e, e2eSpecs, contractWantsE2e, e2eWanted, ownFailures, runE2eGate, E2E_DEFAULT_PATHS, NO_TRIGGER } from "./e2e.mjs";
+import { VERIFY_CHECK_NAME } from "./verify.mjs";
+
+const HEAD = "abc123abc123abc123";
+const RUN = "https://github.com/o/r/actions/runs/42/job/1";
+/** Fake io + clock: `checks` is a list of successive commitChecks answers (the last repeats). */
+function fake(o = {}) {
+  const f = { clock: 0, labels: [], dispatched: [], q: [...(o.checks ?? [])] };
+  f.io = {
+    addPrLabel: async (pr, l) => { if (o.labelError) throw new Error(o.labelError); f.labels.push([pr, l]); },
+    commitChecks: async () => (f.q.length > 1 ? f.q.shift() : f.q[0] ?? []),
+    failedLogs: async () => o.log ?? "FAIL e2e/a.spec.ts",
+    dispatchWorkflow: async (a) => { f.dispatched.push(a); },
+    workflowRuns: async () => [{ databaseId: 9, headSha: HEAD, status: "completed", conclusion: o.conclusion ?? "success", url: "https://github.com/o/r/actions/runs/9", name: "e2e" }],
+  };
+  f.args = (extra = {}) => ({ io: f.io, pr: 77, branch: "feat/issue-7-lean", head: HEAD, cfg: parseE2e({ e2e_label: "full-ci", e2e_checks: "e2e" }), specs: ["e2e/a.spec.ts"],
+    ciMaxMinutes: 30, now: () => f.clock, sleep: async (ms) => { f.clock += ms; }, random: () => 0.5, ...extra });
+  return f;
+}
+const chk = (state, conclusion = state === "pending" ? null : state, name = "e2e") => ({ name, state, conclusion, url: RUN });
+
+describe("e2e config and trigger (#4403)", () => {
+  it("parseE2e defaults to auto/changed with the default paths and rejects invalid values", () => {
+    assert.deepEqual(parseE2e({}), { mode: "auto", label: null, workflow: null, pathsInput: null, checks: [], paths: [...E2E_DEFAULT_PATHS], blocking: "changed" });
+    const c = parseE2e({ e2e: "always", e2e_label: "full-ci", e2e_workflow: "e2e.yml", e2e_paths_input: "specs", e2e_checks: "e2e (1), e2e (2)", e2e_paths: "tests/e2e/**", e2e_blocking: "all" });
+    assert.deepEqual(c, { mode: "always", label: "full-ci", workflow: "e2e.yml", pathsInput: "specs", checks: ["e2e (1)", "e2e (2)"], paths: ["tests/e2e/**"], blocking: "all" });
+    assert.equal(parseE2e({ e2e: "sometimes" }).mode, "auto");
+    assert.equal(parseE2e({ e2e_workflow: "x; rm -rf /" }).workflow, null);
+    assert.equal(parseE2e({ e2e_label: "a$(b)" }).label, null);
+  });
+  it("e2eSpecs matches the default layouts and ignores .forge-lean files", () => {
+    const files = ["e2e/a.spec.ts", "apps/web/e2e/b.spec.ts", "src/x.e2e.test.ts", "test/app.e2e-spec.ts", "cypress/e2e/c.cy.ts", "playwright/d.ts", "src/a.ts", "src/a.spec.ts", ".forge-lean/e2e/x"];
+    assert.deepEqual(e2eSpecs(files), files.slice(0, 6));
+  });
+  it("e2eWanted: never never runs; no trigger never runs; auto runs on specs or [type:e2e]; always runs unless docs-only", () => {
+    const cfg = parseE2e({ e2e_label: "full-ci" });
+    const typed = { requirements: [{ quote: "Rosters render. [type:E2E]" }] };
+    assert.equal(e2eWanted({ cfg: { ...cfg, mode: "never" }, files: ["e2e/a.spec.ts"] }).run, false);
+    assert.deepEqual(e2eWanted({ cfg: parseE2e({}), files: ["e2e/a.spec.ts"] }), { run: false, reason: NO_TRIGGER, specs: ["e2e/a.spec.ts"] });
+    assert.equal(e2eWanted({ cfg, files: ["e2e/a.spec.ts"] }).run, true);
+    assert.equal(e2eWanted({ cfg, files: ["src/a.ts"] }).run, false);
+    assert.equal(contractWantsE2e(typed), true);
+    assert.equal(e2eWanted({ cfg, files: ["src/a.ts"], contract: typed }).run, true);
+    assert.equal(e2eWanted({ cfg: { ...cfg, mode: "always" }, files: ["src/a.ts"] }).run, true);
+    assert.equal(e2eWanted({ cfg: { ...cfg, mode: "always" }, files: ["README.md"] }).run, false);
+  });
+  it("ownFailures: by spec path or basename in the log; an empty log or no changed spec counts as the lane's", () => {
+    const fs = [{ job: "a", log: "FAIL e2e/a.spec.ts" }, { job: "b", log: "x/a.spec.ts:3 failed" }, { job: "c", log: "FAIL e2e/rosters.spec.ts" }, { job: "d", log: "" }];
+    assert.deepEqual(ownFailures(fs, ["e2e/a.spec.ts"]).map((f) => f.job), ["a", "b", "d"]);
+    assert.deepEqual(ownFailures(fs, []).map((f) => f.job), ["a", "b", "c", "d"]);
+  });
+});
+
+describe("runE2eGate (#4403)", () => {
+  it("label path: labels the PR; a skipped run is not a pass; waits for the real run", async () => {
+    const f = fake({ checks: [[chk("success", "skipped")], [chk("success", "skipped"), chk("pending")], [chk("success", "skipped"), chk("success")]] });
+    const r = await runE2eGate(f.args());
+    assert.equal(r.result, "pass"); assert.deepEqual(f.labels, [[77, "full-ci"]]);
+    assert.ok(f.clock > 0, "it waited");
+  });
+  it("label path: a red e2e run returns the failed log; other checks are ignored when e2e_checks is set", async () => {
+    const f = fake({ checks: [[chk("failure"), chk("failure", "failure", "lint")]], log: "FAIL e2e/a.spec.ts" });
+    const r = await runE2eGate(f.args());
+    assert.equal(r.result, "fail"); assert.deepEqual(r.failures.map((x) => x.job), ["e2e"]); assert.match(r.failures[0].log, /a\.spec\.ts/);
+  });
+  it("label path without e2e_checks waits on every check but the lane's own verify check", async () => {
+    const f = fake({ checks: [[chk("success", "success", "build"), chk("success", "skipped"), chk("success", "success", VERIFY_CHECK_NAME)], [chk("success", "success", "build"), chk("success")]] });
+    const r = await runE2eGate(f.args({ cfg: parseE2e({ e2e_label: "full-ci" }) }));
+    assert.equal(r.result, "pass"); assert.ok(f.clock > 0, "the skipped e2e kept it waiting");
+  });
+  it("label refused, or no e2e run within the grace period, is unavailable; slow e2e is a timeout", async () => {
+    assert.equal((await runE2eGate(fake({ labelError: "HTTP 422" }).args())).result, "unavailable");
+    assert.equal((await runE2eGate(fake({ checks: [[chk("success", "skipped")]] }).args())).result, "unavailable");
+    assert.equal((await runE2eGate(fake({ checks: [[chk("pending")]] }).args())).result, "timeout");
+  });
+  it("workflow path: dispatches the e2e workflow with the spec paths, wins over the label, judges the run of this head", async () => {
+    const f = fake({ conclusion: "failure" });
+    const r = await runE2eGate(f.args({ cfg: parseE2e({ e2e_label: "full-ci", e2e_workflow: "e2e.yml", e2e_paths_input: "specs" }) }));
+    assert.equal(r.result, "fail"); assert.equal(r.runId, 9); assert.deepEqual(f.labels, []);
+    assert.deepEqual(f.dispatched, [{ workflow: "e2e.yml", ref: "feat/issue-7-lean", inputs: { specs: "e2e/a.spec.ts" } }]);
+  });
+});

package/bin/lean/effects.mjs

@@ -15,8 +15,14 @@ import { makeLedger, countLiveLanes, ledgerRoot } from "./ledger.mjs";
 import { makeDepCache } from "./depcache.mjs";
 import { makeHeavyIo } from "./heavy.mjs";
 import { gatherContextFacts } from "./context.mjs";
+import { makePitfallStore } from "./pitfalls.mjs";
 import { enumeratingTests, missingEntries } from "./enumerating.mjs";
-import { sequenceCollisions, snapshotLane, verifyRenumber } from "./sequence.mjs";
+import { sequenceCollisions, snapshotLane, verifyRenumber, renumberLane, resolveSequenceConflicts } from "./sequence.mjs";
+import { duplicateExports } from "./postmerge.mjs";
+import { scopedChecks } from "./checks.mjs";
+import { makeSequenceLeaseIo } from "./seqlease.mjs";
+import { followupMap } from "./conformance.mjs";
+import { toolStats } from "./toolstats.mjs";
 
 const pexec = promisify(execFile);
 const script = (n) => fileURLToPath(new URL(`../../scripts/${n}`, import.meta.url));
@@ -55,15 +61,18 @@ export const noToolFlags = (caps) => (caps?.strictMcp ? ["--tools", ""]
   : ["--disallowedTools", "Read", "Grep", "Glob", "Bash", "Edit", "Write", "MultiEdit", "NotebookEdit", "WebFetch", "WebSearch", "Task"]);
 
 export function makeLeanRunner({ runCli = runCliBackendAsync, probe = systemdRunAvailable } = {}) {
-  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, noTools = false, env = null, idleMs = 0, maxMs = 0, laneLimits = null, control = null, onSpawn = null }) => {
+  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, noTools = false, maxTurns = null, agents = null, env = null, idleMs = 0, maxMs = 0, laneLimits = null, control = null, onSpawn = null, contextMax = 0, turnMax = 0, busy = null, busyMs = 0, onIdleExtend = null }) => {
     const bin = resolveClaudeCliBinary(cwd) || "claude";
     const caps = getClaudeCliCapabilities(cwd);
-    const flags = [...buildLeanSessionFlags(caps), ...(noTools ? noToolFlags(caps) : readOnly ? buildReadOnlyToolFlags(caps) : [])];
+    // #4391: custom subagents (the explorer) for the builder session; null leaves argv unchanged
+    const flags = [...buildLeanSessionFlags(caps), ...(noTools ? noToolFlags(caps) : readOnly ? buildReadOnlyToolFlags(caps) : []), ...(agents ? ["--agents", JSON.stringify(agents)] : [])];
     const r = await runCli({ spec: { name: `core-${role}` }, userMessage: prompt, systemPrompt: resume ? "" : system,
-      args: [String(issue)], cwd, bin, model, env, sessionFlags: flags, extraArgs: resume ? ["--resume", resume] : [],
+      args: [String(issue)], cwd, bin, model, maxTurns: caps?.maxTurns ? maxTurns : null, env, sessionFlags: flags, extraArgs: resume ? ["--resume", resume] : [],
       logger: { log() {} },
-      idleMs, maxMs, stream: idleMs > 0, control, onSpawn, wrap: laneLimits && probe() ? (b, a) => wrapLaneLimits(b, a, laneLimits) : null });
-    return { text: r.text, sessionId: r.sessionId, usage: r.usage, costUsd: r.costUsd, durationMs: r.durationMs, turns: r.turns, model: r.cliModel ?? "default" };
+      idleMs, maxMs, stream: idleMs > 0, control, onSpawn, busy, busyMs, onIdleExtend, ...(contextMax > 0 ? { contextMax } : {}), ...(turnMax > 0 ? { turnMax } : {}), wrap: laneLimits && probe() ? (b, a) => wrapLaneLimits(b, a, laneLimits) : null });
+    // #4395: a resumed session's first turn carries the prior history, so only a fresh session reports a baseline.
+    return { text: r.text, sessionId: r.sessionId, usage: r.usage, costUsd: r.costUsd, durationMs: r.durationMs, turns: r.turns, model: r.cliModel ?? "default",
+      baselineTokens: resume ? null : (r.baselineTokens ?? null), tools: toolStats(r.toolUses) }; // #4410: tool calls per turn, re-reads
   };
 }
 
@@ -240,22 +249,28 @@ export function makeLeanIo({ repo }) {
     runCommand: runCheckCommand,
     /** #4328: a read-only freeze-time probe of a validate command's tool or module (separate from runCommand). */
     probeCommand: runCheckCommand,
+    /** #4393: freeze-time execution of a validate command on the base worktree (not read-only; the engine reverts what it changes). */
+    preflightRun: runCheckCommand,
     /** #4167 verify-by-CI: `gh workflow run` on the lane branch; inputs become -f key=value. */
     async dispatchWorkflow({ workflow, ref, inputs = {} }) {
       await run("gh")(["workflow", "run", String(workflow), "-R", repo, "--ref", String(ref), ...Object.entries(inputs).flatMap(([k, v]) => ["-f", `${k}=${v}`])]);
     },
-    /** Recent workflow_dispatch runs of `workflow` on `branch`, newest first: [{databaseId, headSha, status, conclusion, url, name}]. */
+    /** Recent workflow_dispatch runs of `workflow` on `branch`, newest first: [{databaseId, headSha, status, conclusion, url, name, attempt}]. */
     async workflowRuns({ workflow, branch }) {
       return JSON.parse(await run("gh")(["run", "list", "-R", repo, "--workflow", String(workflow), "--branch", String(branch), "--event", "workflow_dispatch",
-        "--limit", "10", "--json", "databaseId,headSha,status,conclusion,url,name"]) || "[]");
+        "--limit", "10", "--json", "databaseId,headSha,status,conclusion,url,name,attempt"]) || "[]");
     },
-    /** Check runs on a commit (push-triggered CI): [{name, state: success|failure|pending, url}], same mapping as checkStates. */
+    /** #4387: rerun the failed jobs of an Actions run once (`gh run rerun <id> --failed`); throws when it cannot start. */
+    async rerunFailedJobs(runId) {
+      await run("gh")(["run", "rerun", String(runId), "--failed", "-R", repo]);
+    },
+    /** Check runs on a commit (push-triggered CI): [{name, state: success|failure|pending, url, id, conclusion}], same mapping as checkStates; `id` orders attempts (#4387). */
     async commitChecks(sha) {
-      const out = String(await run("gh")(["api", "--paginate", `repos/${repo}/commits/${sha}/check-runs`, "--jq", ".check_runs[] | {name, status, conclusion, url: .html_url} | @json"]));
+      const out = String(await run("gh")(["api", "--paginate", `repos/${repo}/commits/${sha}/check-runs`, "--jq", ".check_runs[] | {id, name, status, conclusion, url: .html_url} | @json"]));
       return out.split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l)).map((c) => {
         const v = String(c.status === "completed" ? c.conclusion ?? "" : "").toUpperCase();
         const state = ["SUCCESS", "NEUTRAL", "SKIPPED"].includes(v) ? "success" : ["FAILURE", "ERROR", "TIMED_OUT", "CANCELLED", "ACTION_REQUIRED", "STARTUP_FAILURE", "STALE"].includes(v) ? "failure" : "pending";
-        return { name: c.name, state, url: c.url ?? null };
+        return { name: c.name, state, url: c.url ?? null, id: c.id ?? null, conclusion: c.status === "completed" ? String(c.conclusion ?? "").toLowerCase() || null : null }; // #4387 id orders attempts; #4403 raw conclusion (skipped = not run)
       });
     },
     /**
@@ -280,9 +295,7 @@ export function makeLeanIo({ repo }) {
     async fileFollowups({ parent, items }) {
       const tsv = items.map((i) => `${i.key}\t${i.why}\t`).join("\n") + "\n";
       const out = await runWithInput("bash", [script("file-deferred-followups.sh"), "--parent", String(parent), "--repo", repo], tsv, 180000);
-      const map = {};
-      for (const l of out.split("\n")) { const [k, n] = l.split("\t"); if (k && /^\d+$/.test(String(n ?? "").trim())) map[k] = Number(n); }
-      return map;
+      return followupMap(out, items);
     },
     /**
      * forge#4163: forge.yaml env.setup, once per fresh lane worktree: `bash -c <command>` in cwd with the lane env.
@@ -303,6 +316,8 @@ export function makeLeanIo({ repo }) {
     async structureDelta({ cwd, base }) {
       try { return JSON.parse((await pexec("bash", [script("structure-delta.sh"), `${base}...HEAD`], { cwd, timeout: 30000, maxBuffer: 16 * 1024 * 1024 })).stdout); } catch { return null; }
     },
+    /** #4375: the repo's learned-pitfall ledger, local only (~/.forge/pitfalls, FORGE_PITFALLS_DIR). */
+    pitfalls: makePitfallStore(),
     /** forge#4275: the context pack's facts for one issue (bin/lean/context.mjs), from gh and git only; sources fail independently. */
     contextFacts: (args) => gatherContextFacts({ ...args, io: { gh: run("gh"), git: run("git"), exists: async (p) => existsSync(p) } }),
     /** forge#4195: scripts/hotspots.sh facts for `paths` in cwd (growth since base when given); null on any failure. */
@@ -316,7 +331,14 @@ export function makeLeanIo({ repo }) {
     /** #4322: sequence-numbered files the lane shares with the base (bin/lean/sequence.mjs); snapshot before, verify after a renumber. */
     sequenceCheck: ({ cwd, base }) => sequenceCollisions({ git: (a) => run("git")(a, { cwd }), base }),
     sequenceSnapshot: ({ cwd, base }) => snapshotLane({ git: (a) => run("git")(a, { cwd }), base }),
-    sequenceVerify: ({ cwd, base, before }) => verifyRenumber({ git: (a) => run("git")(a, { cwd }), base, before }),
+    sequenceVerify: ({ cwd, base, before, moves }) => verifyRenumber({ git: (a) => run("git")(a, { cwd }), base, before, moves }),
+    /** #4386: the deterministic renumber, sequence-only base-sync resolution, and the host sequence lease at the merge gate. */
+    sequenceRenumber: ({ cwd, base, laneRef, regenerate }) => renumberLane({ git: (a) => run("git")(a, { cwd }), base, cwd, laneRef, regenerate }),
+    sequenceResolve: ({ cwd, files }) => resolveSequenceConflicts({ git: (a) => run("git")(a, { cwd }), files }),
+    sequenceLease: makeSequenceLeaseIo(),
+    /** #4406: the lane's scoped checks after a base merge (bin/lean/checks.mjs; [] on failure), and same-app duplicate exports (bin/lean/postmerge.mjs). */
+    postMergeChecks: async ({ cwd, base }) => { try { return scopedChecks({ repo: cwd, base }); } catch { return []; } },
+    duplicateExports: ({ cwd, base }) => duplicateExports({ git: (a) => run("git")(a, { cwd }), cwd, base }),
     /**
      * #4271: scripts/acceptance-red-green.sh for the frozen repro_test in cwd; {base, head} words, or null on any failure.
      * `env` (#4325): the lane env (run TMPDIR, toolchain), merged over process.env, so the command sees what the builder saw.

package/bin/lean/explore.mjs

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Explorer subagent (#4391): the builder does its discovery (greps, reads, call chains) through a cheap custom subagent
+ * (`claude --agents`) in a throwaway context that returns a bounded file/line digest, so the expensive main session keeps
+ * the conclusion instead of re-reading every discovered byte on each later turn. Opt-in via forge.yaml `lean.explore: on`
+ * (default off: builder argv and prompts are unchanged); RUN_TERMINAL carries `explore` so the on/off A/B is a query.
+ */
+
+export const EXPLORER_NAME = "explorer";
+export const DEFAULT_EXPLORE_MODEL = "haiku";
+const MODEL_RE = /^[A-Za-z0-9._:[\]-]+$/;
+
+/** {mode, model} from forge.yaml lean.explore / lean.explore_model; anything but "on" is off, an invalid model is the default. */
+export function parseExploreConfig(raw = {}) {
+  const mode = String(raw?.mode ?? "").trim().toLowerCase() === "on" ? "on" : "off";
+  const m = String(raw?.model ?? "").trim();
+  return { mode, model: MODEL_RE.test(m) ? m : DEFAULT_EXPLORE_MODEL };
+}
+
+export const EXPLORER_PROMPT = `You explore a code repository for a developer who will edit it. You are read-only: never edit or write files, never commit, push or change GitHub state. Use Bash only for read commands (git log/show/diff/grep, ls, gh ... view).
+
+Answer the question you were given with ONE digest of at most 60 lines (about 6 KB). Every claim carries path:line; never paste whole files or long excerpts. Use exactly these sections, writing "none" for an empty one:
+### Files   (path:start-end: why it is relevant)
+### Call sites   (path:line: caller -> callee)
+### Tests to extend   (test path and test name, and the exact command that runs only that test)
+### Conventions to match   (naming, error handling, helpers to reuse, each with a path:line example)
+### Open questions   (what you could not determine, and where to look)`;
+
+/** The `--agents` object for the builder session, or null when off (argv unchanged). */
+export function explorerAgents(cfg) {
+  if (cfg?.mode !== "on") return null;
+  return { [EXPLORER_NAME]: { description: "Read-only code explorer: answers one focused question about the repository with a bounded file/line digest (files with line ranges, call sites, tests to extend, conventions).",
+    prompt: EXPLORER_PROMPT, tools: ["Read", "Grep", "Glob", "Bash"], model: cfg.model } };
+}
+
+const RULE = `do discovery (greps, file reads, following call chains) by calling the Agent tool with subagent_type "${EXPLORER_NAME}" and one focused question per call, and keep its digest as your notes. Open a file yourself only to edit it or to verify a digest claim.`;
+
+/** Plan-prompt rule; "" when off (prompt unchanged). */
+export function explorePlanRule(cfg) {
+  return cfg?.mode === "on" ? `\n\nExploration: ${RULE} Verify any digest claim the contract relies on before citing it.` : "";
+}
+
+/** Implement-prompt rule; "" when off (prompt unchanged). */
+export function exploreBuildRule(cfg) {
+  return cfg?.mode === "on" ? `\n\nExploration: for anything the plan's digests do not cover, ${RULE}` : "";
+}

package/bin/lean/explore.test.mjs

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { parseExploreConfig, explorerAgents, explorePlanRule, exploreBuildRule, EXPLORER_PROMPT, DEFAULT_EXPLORE_MODEL } from "./explore.mjs";
+import { makeLeanRunner } from "./effects.mjs";
+import { readExplore } from "../engine-cli.mjs";
+
+describe("explorer subagent config (#4391)", () => {
+  it("parses on|off with a default haiku model; anything else is off, an invalid model is the default", () => {
+    assert.deepEqual(parseExploreConfig(), { mode: "off", model: DEFAULT_EXPLORE_MODEL });
+    assert.deepEqual(parseExploreConfig({ mode: " ON " }), { mode: "on", model: "haiku" });
+    assert.deepEqual(parseExploreConfig({ mode: "maybe", model: "sonnet" }), { mode: "off", model: "sonnet" });
+    assert.equal(parseExploreConfig({ mode: "on", model: "x; rm -rf /" }).model, "haiku");
+  });
+
+  it("off yields no agents and empty rules; on yields a read-only explorer with the bounded digest format", () => {
+    const off = parseExploreConfig({ mode: "off" });
+    assert.equal(explorerAgents(off), null);
+    assert.equal(explorePlanRule(off), "");
+    assert.equal(exploreBuildRule(off), "");
+    const on = parseExploreConfig({ mode: "on", model: "sonnet" });
+    const a = explorerAgents(on).explorer;
+    assert.equal(a.model, "sonnet");
+    assert.deepEqual(a.tools, ["Read", "Grep", "Glob", "Bash"]);
+    assert.ok(a.description && a.prompt === EXPLORER_PROMPT);
+    for (const h of ["### Files", "### Call sites", "### Tests to extend", "### Conventions to match"]) assert.ok(EXPLORER_PROMPT.includes(h), h);
+    assert.match(EXPLORER_PROMPT, /read-only[\s\S]*at most 60 lines/);
+    for (const r of [explorePlanRule(on), exploreBuildRule(on)]) assert.match(r, /subagent_type "explorer"[\s\S]*Open a file yourself only to edit it or to verify a digest claim\./);
+  });
+
+  it("makeLeanRunner passes --agents <json> only when agents are given (argv unchanged otherwise)", async () => {
+    const seen = [];
+    const runCli = async (o) => { seen.push(o.sessionFlags); return { text: "", usage: null, costUsd: 0, durationMs: 0, turns: 0, sessionId: "s" }; };
+    const base = { role: "builder", issue: 1, cwd: process.cwd(), prompt: "p", system: "" };
+    const agents = explorerAgents(parseExploreConfig({ mode: "on" }));
+    await makeLeanRunner({ runCli })({ ...base });
+    await makeLeanRunner({ runCli })({ ...base, agents });
+    const [plain, withAgents] = seen;
+    assert.ok(!plain.includes("--agents"));
+    const i = withAgents.indexOf("--agents");
+    assert.deepEqual(withAgents.slice(0, i), plain);
+    assert.deepEqual(JSON.parse(withAgents[i + 1]), agents);
+  });
+
+  it("readExplore reads forge.yaml lean.explore / lean.explore_model; absent or invalid is undefined", () => {
+    const d = tmpDir("explore-yaml-");
+    assert.equal(readExplore(d), undefined, "no forge.yaml");
+    writeFileSync(join(d, "forge.yaml"), "engine:\n  explore: on\n");
+    assert.equal(readExplore(d), undefined, "another section's key");
+    writeFileSync(join(d, "forge.yaml"), "lean:\n  explore: maybe\n");
+    assert.equal(readExplore(d), undefined);
+    writeFileSync(join(d, "forge.yaml"), "lean:\n  explore: on  # A/B arm\n  explore_model: claude-sonnet-5-5\n");
+    assert.deepEqual(readExplore(d), { mode: "on", model: "claude-sonnet-5-5" });
+    writeFileSync(join(d, "forge.yaml"), "lean:\n  explore: off\n");
+    assert.deepEqual(readExplore(d), { mode: "off" });
+  });
+});

package/bin/lean/facts.mjs

@@ -0,0 +1,84 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * #4401: the structured FORGE:FACTS record a lane posts next to its prose. The builder's judgment (decisions, rejected
+ * approaches with why, hot spots file:line with why, conventions matched) comes from an optional result.json `facts`
+ * key; baselines are the engine's own facts (result.json validation commands and exit codes); decisions fall back to
+ * the frozen contract's. `files` (the lane's changed files) is the path index the context pack matches later lanes on.
+ * Pure: no I/O. The same caps apply on write and on read, so a hand-edited or forged comment is bounded the same way.
+ */
+import { noComments } from "./render.mjs";
+
+export const FACTS_MARKER = "FORGE:FACTS";
+export const FACTS_TEXT_MAX = 160;
+export const FACTS_LIST_MAX = 5;
+export const FACTS_FILES_MAX = 20;
+const KINDS = ["decisions", "rejected", "hotspots", "conventions", "baselines"];
+const FACTS_BLOCK_RE = /```json\n([\s\S]*?)\n```/;
+const AT_RE = /^[\w.@~+/-]+(?::\d+(?:-\d+)?)?$/;
+
+const one = (v, max = FACTS_TEXT_MAX) => {
+  const t = typeof v === "string" ? noComments(v, " ").replace(/\s+/g, " ").trim() : "";
+  return t.length > max ? `${t.slice(0, max - 1)}…` : t;
+};
+const list = (v) => (Array.isArray(v) ? v : []);
+const uniqBy = (rows, key) => { const seen = new Set(); return rows.filter((r) => { const k = key(r); if (seen.has(k)) return false; seen.add(k); return true; }); };
+
+/** The bounded, one-lined record from a raw object; every list capped, entries without their reason dropped. */
+function clean(raw) {
+  const r = raw && typeof raw === "object" ? raw : {};
+  const strs = (a) => uniqBy(list(a).map((x) => one(x)).filter(Boolean), (x) => x).slice(0, FACTS_LIST_MAX);
+  const files = uniqBy(list(r.files).map((f) => (typeof f === "string" ? f.trim().replace(/^\.\//, "") : ""))
+    .filter((f) => f && AT_RE.test(f) && !f.startsWith(".forge-lean/") && !f.split("/").includes("..")), (x) => x).slice(0, FACTS_FILES_MAX);
+  const rejected = uniqBy(list(r.rejected).map((x) => ({ approach: one(x?.approach), why: one(x?.why) })).filter((x) => x.approach && x.why), (x) => x.approach).slice(0, FACTS_LIST_MAX);
+  const hotspots = uniqBy(list(r.hotspots).map((x) => ({ at: one(x?.at, 200), why: one(x?.why) })).filter((x) => x.at && x.why && AT_RE.test(x.at) && !x.at.split("/").includes("..")), (x) => x.at).slice(0, FACTS_LIST_MAX);
+  const baselines = uniqBy(list(r.baselines).map((x) => ({ command: one(x?.command, 200), exit: x?.exit })).filter((x) => x.command && Number.isInteger(x.exit)), (x) => x.command).slice(0, FACTS_LIST_MAX);
+  return { v: 1, files, decisions: strs(r.decisions), rejected, hotspots, conventions: strs(r.conventions), baselines };
+}
+const hasContent = (f) => KINDS.some((k) => f[k].length > 0);
+
+/**
+ * The lane's FORGE:FACTS record, or null when it would carry nothing. `result` is result.json (its `facts` and
+ * `validation`), `files` the lane's changed files, `contract` the frozen contract (decision fallback).
+ */
+export function normaliseFacts(result, { files = [], contract = null } = {}) {
+  const raw = result?.facts && typeof result.facts === "object" ? result.facts : {};
+  const decisions = list(raw.decisions).some((d) => one(d)) ? raw.decisions : list(contract?.decisions).map((d) => d?.choice);
+  const baselines = list(result?.validation).map((v) => ({ command: v?.command, exit: v?.exit }));
+  const f = clean({ ...raw, decisions, baselines, files });
+  return hasContent(f) ? f : null;
+}
+
+/** JSON for a comment: `<` and backticks escaped, so builder text can never open a comment or close the fence. */
+const factsJson = (f) => JSON.stringify(f).replace(/</g, "\\u003c").replace(/`/g, "\\u0060");
+
+/** The FORGE:FACTS comment body (without the marker line): a count line plus the record as fenced JSON. */
+export function renderFactsRecord(f) {
+  return `## Lane facts\n${f.rejected.length} rejected approach(es), ${f.hotspots.length} hot spot(s), ${f.decisions.length} decision(s), ${f.conventions.length} convention(s), ${f.baselines.length} baseline(s) on ${f.files.length} file(s); read by later lanes on the same files.\n\n\`\`\`json\n${factsJson(f)}\n\`\`\``;
+}
+
+/** The record from a FORGE:FACTS comment body, re-bounded with the same caps; null for any other or malformed body. */
+export function parseFactsComment(body) {
+  const s = String(body ?? "");
+  if (!new RegExp(`^\\s*<!--\\s*${FACTS_MARKER}\\s*-->`).test(s)) return null;
+  const m = FACTS_BLOCK_RE.exec(s);
+  if (!m) return null;
+  let raw;
+  try { raw = JSON.parse(m[1]); } catch { return null; }
+  const f = clean(raw);
+  return hasContent(f) ? f : null;
+}
+
+/** The one-line pack items of a record, by kind (`ref` names its provenance, e.g. "#33, PR #101"). */
+export function factsLines(f, ref) {
+  const tag = ref ? ` (${ref})` : "";
+  return {
+    rejected: f.rejected.map((x) => `- rejected: ${x.approach} — ${x.why}${tag}`),
+    hotspots: f.hotspots.map((x) => `- hot spot ${x.at}: ${x.why}${tag}`),
+    decisions: f.decisions.map((x) => `- decision: ${x}${tag}`),
+    conventions: f.conventions.map((x) => `- convention: ${x}${tag}`),
+    baselines: f.baselines.map((x) => `- baseline: \`${x.command.replace(/`/g, "'")}\` exit ${x.exit}${tag}`),
+  };
+}
+/** Pack rank of the kinds: rejected approaches first (the knowledge most often lost), then hot spots, decisions, conventions, baselines. */
+export const FACTS_RANK = Object.freeze(["rejected", "hotspots", "decisions", "conventions", "baselines"]);

package/bin/lean/facts.test.mjs

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { normaliseFacts, renderFactsRecord, parseFactsComment, factsLines, FACTS_MARKER, FACTS_LIST_MAX, FACTS_TEXT_MAX } from "./facts.mjs";
+
+const RESULT = {
+  status: "fixed",
+  validation: [{ id: "V1", command: "node --test a.test.mjs", exit: 0 }, { id: "V2", command: "", ci: true }, { id: "V3", command: "make lint", exit: "0" }],
+  facts: {
+    decisions: ["Keep the pack pure"],
+    rejected: [{ approach: "detection-only guard", why: "the root cause is the retrieval; the guard only hides it" }, { approach: "no reason given" }],
+    hotspots: [{ at: "src/a.js:12", why: "retry loop" }, { at: "../etc/passwd:1", why: "x" }, { at: "src/b.js", why: "" }],
+    conventions: ["one-line comments"],
+  },
+};
+const post = (f) => `<!-- ${FACTS_MARKER} -->\n${renderFactsRecord(f)}`;
+
+describe("FORGE:FACTS record (#4401)", () => {
+  it("keeps builder judgment, derives baselines from validation, indexes the changed files", () => {
+    const f = normaliseFacts(RESULT, { files: ["src/a.js", ".forge-lean/result.json", "./src/b.js"] });
+    assert.deepEqual(f.files, ["src/a.js", "src/b.js"]);
+    assert.deepEqual(f.rejected, [{ approach: "detection-only guard", why: "the root cause is the retrieval; the guard only hides it" }], "an approach without a why is dropped");
+    assert.deepEqual(f.hotspots, [{ at: "src/a.js:12", why: "retry loop" }], "no traversal, no reasonless hot spot");
+    assert.deepEqual(f.baselines, [{ command: "node --test a.test.mjs", exit: 0 }], "only commands with an integer exit");
+    assert.deepEqual([f.decisions, f.conventions], [["Keep the pack pure"], ["one-line comments"]]);
+  });
+
+  it("falls back to the contract's decisions; null when nothing is worth posting", () => {
+    const f = normaliseFacts({ status: "fixed" }, { files: ["a.js"], contract: { decisions: [{ id: "K1", choice: "reuse the ledger walk" }] } });
+    assert.deepEqual(f.decisions, ["reuse the ledger walk"]);
+    assert.equal(normaliseFacts({ status: "fixed" }, { files: ["a.js"] }), null);
+    assert.equal(normaliseFacts(null), null);
+  });
+
+  it("caps list sizes and text length, one-lines and strips comments", () => {
+    const many = Array.from({ length: 9 }, (_, i) => ({ approach: `a${i}\n<!-- FORGE:STATE -->x`, why: "y".repeat(500) }));
+    const f = normaliseFacts({ facts: { rejected: many } }, {});
+    assert.equal(f.rejected.length, FACTS_LIST_MAX);
+    assert.equal(f.rejected[0].approach, "a0 x");
+    assert.equal(f.rejected[0].why.length, FACTS_TEXT_MAX);
+  });
+
+  it("round-trips through the comment; builder text can never open a comment or close the fence", () => {
+    const f = normaliseFacts({ facts: { rejected: [{ approach: "use `<!--` and ```", why: "breaks </details>" }] } }, { files: ["a.js"] });
+    const body = post(f);
+    assert.ok(!body.slice(body.indexOf("\n")).includes("<"), body);
+    assert.equal((body.match(/```/g) ?? []).length, 2);
+    assert.deepEqual(parseFactsComment(body), f);
+  });
+
+  it("parse rejects other markers, malformed JSON and empty records, and re-applies the caps", () => {
+    assert.equal(parseFactsComment("<!-- FORGE:REVIEW -->\n```json\n{}\n```"), null);
+    assert.equal(parseFactsComment(`<!-- ${FACTS_MARKER} -->\n\`\`\`json\n{nope\n\`\`\``), null);
+    assert.equal(parseFactsComment(`<!-- ${FACTS_MARKER} -->\n\`\`\`json\n{"v":1,"files":["a.js"]}\n\`\`\``), null);
+    const forged = { files: ["a.js"], rejected: Array.from({ length: 20 }, (_, i) => ({ approach: `x${i}`, why: "w" })) };
+    assert.equal(parseFactsComment(`<!-- ${FACTS_MARKER} -->\n\`\`\`json\n${JSON.stringify(forged)}\n\`\`\``).rejected.length, FACTS_LIST_MAX);
+  });
+
+  it("pack lines carry provenance", () => {
+    const l = factsLines(normaliseFacts(RESULT, { files: ["src/a.js"] }), "#33, PR #101");
+    assert.deepEqual(l.rejected, ["- rejected: detection-only guard — the root cause is the retrieval; the guard only hides it (#33, PR #101)"]);
+    assert.deepEqual(l.hotspots, ["- hot spot src/a.js:12: retry loop (#33, PR #101)"]);
+    assert.deepEqual(l.baselines, ["- baseline: `node --test a.test.mjs` exit 0 (#33, PR #101)"]);
+  });
+});