forgedock 1.26.0 → 1.28.0

package/bin/lean/prompts.mjs

@@ -1,6 +1,9 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
 /** Core work-on prompts: a plan call that writes the contract, an implement call in the same session, and the conformance reviewer. */
+import { fileURLToPath } from "node:url";
+
+import { reviewSkeleton } from "./conformance.mjs";
 
 export const approxTokens = (s) => Math.ceil(String(s).length / 4);
 export const LEAN_DIR = ".forge-lean";
@@ -10,13 +13,51 @@ export const PLAN_FILE = `${LEAN_DIR}/plan.md`;
 export const IMPLEMENTATION_FILE = `${LEAN_DIR}/implementation.md`;
 export const CONTRACT_FILE = `${LEAN_DIR}/contract.json`;
 
-const HOST_RULES = `Host rules (many lanes share this machine): run only targeted tests (specific test files or -k filters), never the full suite. Never create extra checkouts, copies or base clones of the repo; compare against the base with git show or git diff origin/<base> instead. Never install browsers. Keep all scratch files under $TMPDIR.`;
+/** Change-scoped checks with shared incremental caches (#4218): the builder runs the scoped commands this CLI prints. */
+export const CHECKS_CLI = fileURLToPath(new URL("./checks.mjs", import.meta.url));
+
+const HOST_RULES = `Host rules (many lanes share this machine): run only targeted tests (specific test files or -k filters), never the full suite. Never create extra checkouts, copies or base clones of the repo; compare against the base with git show or git diff origin/<base> instead. Never install browsers. Keep all scratch files under $TMPDIR. When $FORGE_HOST_PARALLELISM is set, cap verification workers at it: turbo --concurrency, vitest/jest --maxWorkers, playwright --workers, node --test --test-concurrency (eslint via turbo). For any typecheck, lint or test you run, first run \`node ${CHECKS_CLI} --base origin/<base>\` after your edits and run the scoped commands it prints instead of the repo's full tsc/eslint/test scripts (it widens to the full command itself when impact is unknown: lockfile, tsconfig, CI config); once they pass, run it again with --publish to share the incremental caches. node_modules/.venv may be hardlinked read-only from a shared cache: after changing a lockfile, or when an install fails with EACCES, delete that dir first (rm -rf is safe), then install; never chmod or edit files inside it.`;
+
+/** Verify-by-CI (#4167): in ci/hybrid mode the engine runs the repo's CI on GitHub runners; the host runs only light checks. */
+const LIGHT = "run only formatters and linters on the files you changed";
+const VERIFY_RUN = { ci: `${LIGHT} (no type-checks, test suites or test containers)`, hybrid: "run the targeted validate commands" };
+const VERIFY_RULES = {
+  ci: `Verification runs on CI (forge.yaml verify.mode: ci). Do NOT run type-checks (tsc and the like), test suites (pytest, jest, node --test and the like) or docker/test containers on this host: ${LIGHT}. The engine pushes the branch, runs CI and sends failures back to you. In ${RESULT_FILE} record each validate id you did not run as {"id":"V1","command":"","ci":true}.`,
+  hybrid: `Verification also runs on CI (forge.yaml verify.mode: hybrid). Run the targeted validate commands and ${LIGHT}; never the full suite and no docker/test containers. The engine pushes the branch, runs CI and sends failures back to you. Record each validate id you did not run as {"id":"V1","command":"","ci":true}.`,
+};
+const rerun = (mode) => (mode === "ci" ? `Re-run the light checks only (${LIGHT}); the engine re-verifies on CI` : "Re-run the validate commands");
+
+/** Base-sync conflict resolution: which checks to run after resolving (#4167: ci mode runs light checks only). */
+export const syncChecks = (mode = "local") => (mode === "ci" ? `${LIGHT} (the engine re-verifies on CI)` : "run the relevant tests");
+
+/** CI failed on the pushed branch (#4167): one fix round in the builder's own session. */
+export function ciFailurePrompt(failures, mode = "local") {
+  const list = failures.slice(0, 10).map((f) => `#### ${f.job ?? "check"}${f.url ? ` (${f.url})` : ""}\n${f.log ? `\`\`\`\n${f.log}\n\`\`\`` : "(no log available: open the url)"}`).join("\n\n");
+  return `CI failed on the pushed branch:\n\n${list}\n\nFix the cause in the code (or record a typed amendment with a why in ${RESULT_FILE}). ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+/** The repo's declared check commands (forge.yaml learned.test_commands) failed before the PR (#4240): fixed in-session. */
+export function precheckFailurePrompt(failures, mode = "local") {
+  const list = failures.slice(0, 10).map((f) => `#### \`${f.command}\` (exit ${f.rc})\n${f.output ? `\`\`\`\n${f.output}\n\`\`\`` : "(no output)"}`).join("\n\n");
+  return `The repo's declared check commands failed in this worktree, before the PR is opened:\n\n${list}\n\nFix the cause in the code (or record a typed amendment with a why in ${RESULT_FILE}). Re-run each failing command, ${mode === "ci" ? `${LIGHT} otherwise, ` : ""}commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+/** CI could not run (no workflow run, no checks, dispatch refused): the local fallback (#4167). */
+export function ciUnavailablePrompt() {
+  return `CI verification is unavailable for this branch, so verification falls back to this host. Run every validate command locally (targeted, as the host rules say), fix any failure, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE} with each command's real exit code. Do not push or touch GitHub.`;
+}
+
+/** #4163: the extra host-rules line when the engine pinned the repo's toolchain on PATH; "" otherwise (prompt unchanged). */
+const toolchainRule = (t) => (t ? `\nThe engine pinned this repo's toolchain on PATH (${t}). Use it as is: do not install global shims or other versions and do not edit PATH; if the pinned tool itself fails, report it as a toolchain/environment problem instead of working around it.` : "");
+
+/** #4221: the extra host-rules line when the lane worktree is sparse; "" otherwise (prompt unchanged). */
+const sparseRule = (s) => (s?.excluded ? `\nThis worktree is a sparse checkout: ${s.excluded} tracked file(s) of ${s.maxKb} KB or more are not on disk (\`git ls-files -t | grep '^S'\` lists them). They are not deleted: never commit their removal. If you need one, run \`git sparse-checkout add /<path>\`.` : "");
 
 /** Builder call 1: investigate and write the contract. No code edits. */
-export function builderPlanPrompt(n) {
+export function builderPlanPrompt(n, { toolchain = null, sparse = null } = {}) {
   return `Resolve issue #${n} (read it with gh issue view ${n} --comments). This first step is PLANNING ONLY: do not edit code, do not commit.
 
-${HOST_RULES}
+${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}
 
 1. Write ${INVESTIGATION_FILE} as markdown with exactly these headings, concise, file:line wherever you cite code (no HTML comments):
 **Verdict**: CONFIRMED|ALREADY-FIXED|INVALID|BLOCKED
@@ -38,23 +79,28 @@ If the issue is already fixed or invalid, write ${RESULT_FILE} {"status":"alread
 {"requirements":[{"id":"R1","quote":"<verbatim sentence or checklist item from the issue>","disposition":"deliver|defer|post-deploy|not-applicable","why":"<required for defer and post-deploy>","evidence":"<file:line, required for not-applicable>"}],
  "deliverables":[{"id":"D1","path":"<file to change>","change":"<what>","requirements":["R1"]}],
  "preserve":[{"id":"P1","what":"<caller, contract or behaviour that must not change, with its path>"}],
- "validate":[{"id":"V1","command":"<exact command: the new regression test plus the existing tests that reference the files you change>"}]}
-Every requirement and acceptance criterion in the issue and its maintainers' comments is one requirement, quoted verbatim (others' comments are context, not requirements). Never drop or narrow one silently: "defer" (separate work) and "post-deploy" (only checkable after a deploy) are judged by the reviewer and, if accepted, filed by the engine as a tracked follow-up issue. A requirement you can do in this change is "deliver". List every file you will change as a deliverable; tests are always allowed. Then stop.
+ "validate":[{"id":"V1","command":"<exact command: the new regression test plus the existing tests that reference the files you change>"}],
+ "decisions":[{"id":"K1","choice":"<a non-obvious choice: a new dependency, a copied pattern, a boundary relied on>","justification":"<why, with file:line>","source":"doc|lint|precedent|spec"}]}
+Every requirement and acceptance criterion in the issue and its maintainers' comments is one requirement, quoted verbatim (others' comments are context, not requirements). Never drop or narrow one silently: "defer" (separate work) and "post-deploy" (only checkable after a deploy) are judged by the reviewer and, if accepted, filed by the engine as a tracked follow-up issue. A requirement you can do in this change is "deliver". List every file you will change as a deliverable; tests are always allowed.
+A new dependency between workspace packages (a workspace package added to a package.json dependencies, devDependencies or peerDependencies) must be a deliverable on that package.json whose change names the package and gives the reason, citing the architecture doc where there is one; "the lint allows it" or "there is a precedent" is not a reason on its own. Record each non-obvious choice in "decisions" with where its justification comes from (doc, lint, precedent or spec); omit the field when there is none. Then stop.
 
 ${BLOCK_RULES}`;
 }
 
 /** Builder call 2, same session: implement the frozen contract. */
-export function builderImplementPrompt() {
-  return `The engine froze ${CONTRACT_FILE}; it is now the authority. Implement it exactly: change only deliverable paths (tests are always allowed), keep every preserve item intact, add a regression test, run every validate command, and commit. Do not push or touch GitHub.
+export function builderImplementPrompt(mode = "local", { toolchain = null, sparse = null } = {}) {
+  const remote = mode === "ci" || mode === "hybrid";
+  return `The engine froze ${CONTRACT_FILE}; it is now the authority. Implement it exactly: change only deliverable paths (tests are always allowed), keep every preserve item intact, add a regression test, ${remote ? VERIFY_RUN[mode] : "run every validate command"}, and commit. Do not push or touch GitHub.
 
-${HOST_RULES}
+${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${remote ? `\n\n${VERIFY_RULES[mode]}` : ""}
 
 The contract changes only through typed amendments in ${RESULT_FILE}, each with a why, which the engine posts on the issue:
 - {"kind":"add-deliverable","path":"<file>","why":""}: a file you must change that the contract did not list.
 - {"kind":"add-requirement","requirement":{"id":"R9","quote":"<verbatim from the issue>","disposition":"deliver|defer|post-deploy"},"why":""}: an issue requirement the contract missed.
 - {"kind":"defer|post-deploy","requirement":"R#","why":""}: a contracted requirement that becomes separate work or a post-deploy check. The engine files it as a tracked follow-up.
 Never report a deliver requirement as "deferred" or "follow-up" without an amendment.
+A new dependency between workspace packages in a package.json that no deliverable names (that package.json, with the package in its change) is blocked by the engine: add it through add-deliverable with a why naming the package, or do not add it.
+If the repository has scripts/check-devdocs-freshness.sh, its docs must move with the code: a behaviour change under the paths it checks needs the matching module dossier (devdocs/modules/*.md, per devdocs/index.yaml) updated in the same change. Run it against the base before writing the result; the engine runs it too and resumes you once if it fails.
 
 When done and committed, write ${IMPLEMENTATION_FILE} (markdown, no HTML comments):
 ### Approach
@@ -64,16 +110,48 @@ When done and committed, write ${IMPLEMENTATION_FILE} (markdown, no HTML comment
 ### Notes   (behavior changes, env vars)
 
 Finally write ${RESULT_FILE}: {"status":"fixed","summary":"one line","requirements":[{"id":"R1","status":"implemented|partial|missing","evidence":"file:line"}],"validation":[{"id":"V1","command":"","exit":0}],"tests":[{"command":"","result":"pass|fail: ...","fails_without_fix":true|false|"unknown"}],"amendments":[]}
+Docker: start containers with docker run --rm; pass --label for each entry of $FORGE_DOCKER_LABELS on every docker run, volume create, network create and build (compose: on services, volumes, networks); tear down via trap/finally.
+If you started Docker containers or volumes that are still there, list their names in ${RESULT_FILE} as "resources":{"containers":[],"volumes":[]} so the engine removes them when the lane ends.
 If you find you are blocked, write "status":"blocked" with a typed "block" instead.`;
 }
 
+/** #4271: the bug-lane plan addition (lean.repro_test prefer on a bug/type:bug issue); "" when off, so the prompt is unchanged. */
+export function reproPlanNote(on) {
+  if (!on) return "";
+  return `\n\nThis issue is a bug. When it is testable, ${CONTRACT_FILE} must also carry "repro_test":{"path":"<test file>","id":"<test name>","command":"<exact command that runs only that test>"}: a test that fails on the base and passes after the fix. The engine proves it with scripts/acceptance-red-green.sh (head's changed test files are overlaid on the base) and fails the run if it is not red on base and green on head. When it is not testable (specs, docs, workflows, environment), write "repro_test":"none","repro_test_reason":"<one line why>" instead.`;
+}
+
+/** forge#4195: hotspot facts for the plan prompt: one sentence plus one line per over-threshold file; "" when none. */
+export function hotspotPlanNote(facts) {
+  if (!facts || facts.enabled === false || !Array.isArray(facts.files)) return "";
+  const hot = facts.files.filter((f) => f?.over && !f.excluded);
+  if (!hot.length) return "";
+  return `\n\nHotspots (scripts/hotspots.sh): an Affected File below is over ${facts.file_lines} lines, so the contract must put new logic in a new or neighbouring module and give the hotspot only wiring (an import and a call); growing it by more than ${facts.growth_lines} lines needs "hotspot-exempt: <reason>" in that deliverable's change.\n${hot.map((f) => `- ${f.path}: ${f.lines} lines, ${f.commits_90d} commits/90d`).join("\n")}`;
+}
+
+/** forge#4195: the builder's growth rule with the effective thresholds; "" when the check is off. */
+export function hotspotBuildRule(facts) {
+  if (!facts || facts.enabled === false || !(facts.file_lines > 0) || !(facts.growth_lines > 0)) return "";
+  return `\n\nDon't grow a file over ${facts.file_lines} lines by more than ${facts.growth_lines} lines. Put new code next to it.`;
+}
+
 export function contractCorrectionPrompt(problems) {
   return `Your ${CONTRACT_FILE} is not valid:\n${problems.slice(0, 20).map((p) => `- ${p}`).join("\n")}\n\nFix the contract (planning only, no code, no commit) and stop. Do not push or touch GitHub.`;
 }
 
 /** The one bounded correction round, in the builder's own session. */
-export function conformanceResumePrompt(failures) {
-  return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0, 500)}`).join("\n")}\n\nFix each item in the code, or record a typed amendment with a why in ${RESULT_FILE}. Re-run the validate commands, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+export function conformanceResumePrompt(failures, mode = "local") {
+  return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0, 500)}`).join("\n")}\n\nFix each item in the code, or record a typed amendment with a why in ${RESULT_FILE}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+/** forge#4159: the one resume for package.json edges between workspace packages the contract does not name. */
+export function structurePrompt(edges, mode = "local") {
+  return `This change adds dependencies between workspace packages that the frozen contract does not name:\n${edges.slice(0, 30).map((e) => `- ${e.from} -> ${e.to} (${e.kind}, ${e.manifest})`).join("\n")}\n\nFor each one, add it to the contract with a reason, or remove it: either record {"kind":"add-deliverable","path":"<that package.json>","why":"<names the package and why this dependency is right>"} in ${RESULT_FILE}, or drop the dependency from the package.json and the code that needs it. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+/** The repo's devdocs freshness rule failed on the committed change: update the named dossier(s) and commit. */
+export function docsPrompt(output, mode = "local") {
+  return `The repository's docs freshness check (scripts/check-devdocs-freshness.sh) fails on this change:\n${String(output ?? "").slice(0, 4000)}\n\nUpdate the module dossier it names (devdocs/modules/*.md; devdocs/index.yaml maps files to dossiers) to describe the behaviour you changed, in the dossier's existing style and in a few lines. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
 }
 
 /** Human-only block reasons (#4124). Everything else agents resolve: a dependency, a split, prod data monitoring can read. */
@@ -89,13 +167,34 @@ export function blockCorrectionPrompt(problem) {
   return `Your ${RESULT_FILE} says "blocked", but ${String(problem).slice(0, 1000)}. That is not a block that needs a human.\n\n${BLOCK_RULES}\n\nEither do the work (fix, commit, update the stage notes and ${RESULT_FILE}) or write a valid typed "block". Do not push or touch GitHub.`;
 }
 
+/** refactor:split grouping call (#4211): no tools, no files; the only input is the compact symbol index. */
+export const GROUPING_SYSTEM = "You plan how to split one oversized source module into a package of smaller modules. You see only a symbol index (names, kinds, sizes, references); you have no tools and never see code. Reply with exactly one JSON object and nothing else.";
+
+/** The grouping prompt: fixed rules plus JSON of bin/lean/refactor.mjs compactIndex(). Its size depends on the index only, never on file length. */
+export function groupingPrompt(index) {
+  return `Group the top-level symbols of one module into cohesive new modules of a package. A codemod moves them verbatim and leaves the old module re-exporting every moved name.
+
+Index: "symbols" are top-level symbols {name, kind, lines, public}; "edges" are "from->to" references between them.
+
+Rules:
+- Module names are snake_case Python identifiers, unique, not equal to any symbol name.
+- Each symbol appears at most once. Symbols you leave out stay in the old module.
+- A moved symbol must not reference a symbol that stays: when you move a symbol, move everything it references (follow the edges).
+- Modules must not reference each other in a cycle; keep heavily cross-referenced symbols in one module.
+- Prefer a few modules of similar size grouped by what the symbols do.
+
+Reply with only {"<module>": ["<symbol>", ...], ...}.
+
+${JSON.stringify(index)}`;
+}
+
 export const REVIEWER_SYSTEM = `You are an independent conformance reviewer of one change. You did not write it. Read-only: use Read, Grep and Glob in the worktree.
 
-You receive the GitHub issue with its comments, the frozen contract (requirements quoted from the issue, deliverables, preserve constraints, validate commands) with typed amendments, the builder's result with validation output, the changed files, and the diff. The engine computes the verdict from your rows; a reply without rows is rejected, so fill EVERY row:
-- issue_coverage: one row per requirement or acceptance criterion in the issue, quoted verbatim, saying whether the contract covers it, narrows it, or misses it.
-- requirements: one row per contract requirement id: implemented|partial|missing|deferred|post-deploy|not-applicable, with path:line evidence and the validate id or test that proves it (validated_by). For a defer or post-deploy requirement add deferral_ok: true only if it is genuinely separate work or only checkable after a deploy, false if it belongs in this change.
-- scope: one row per changed file: the deliverable id that authorizes it (deliverables may be directories or globs), "test", "incidental" (changelog, lockfile), or "none" with what it changes.
-- preserve: one row per preserve id: intact|violated|unverified, with path:line.
+You receive the GitHub issue with its comments, the frozen contract (requirements quoted from the issue, deliverables, preserve constraints, validate commands) with typed amendments, the builder's result with validation output, the changed files, the diff, and a row skeleton with one row per contract id and changed file. The engine computes the verdict from your rows; a reply without rows is rejected, so fill the skeleton rather than composing rows: fill EVERY row (set each empty status, evidence and validated_by, and each null deferral_ok; keep the ids and files as given):
+- issue_coverage: one row per requirement or acceptance criterion in the issue, quoted verbatim, saying whether the contract covers it, narrows it, or misses it. The skeleton lists the contract's quotes; add a row for any issue requirement the contract does not quote.
+- requirements: one row per contract requirement id: implemented|partial|missing|deferred|post-deploy|not-applicable, with repo-relative path:line evidence (e.g. src/app/route.ts:12, not route.ts:12) and the validate id or test that proves it (validated_by). For a defer or post-deploy requirement add deferral_ok: true only if it is genuinely separate work or only checkable after a deploy, false if it belongs in this change.
+- scope: one row per changed file, pre-filled with the engine's authorization: the deliverable id that authorizes it (deliverables may be directories or globs), "test", "incidental" (changelog, lockfile), or "none" with what it changes. Change it only if the diff shows the file does something its deliverable does not cover.
+- preserve: one row per preserve id: intact|violated|unverified, with path:line. The skeleton starts each at "unverified"; set intact only after checking it.
 - regressions: concrete breakages only, each with path:line.
 
 Output ONLY a JSON object in a json code fence:
@@ -132,9 +231,18 @@ export function boundDiff(diff, { stat = "", base = "", maxBytes = REVIEW_DIFF_M
     + (list.length ? `\n<omitted_files>\n${list.join("\n")}\n</omitted_files>` : "");
 }
 
-export function reviewerPrompt({ issue, comments = [], contract, result, files = [], diff, stat = "", base = "" }) {
+export function reviewerPrompt({ issue, comments = [], contract, result, files = [], diff, stat = "", base = "", structure = "", ignore = [`${LEAN_DIR}/`], skeleton = null }) {
   const notes = comments.map((c) => String(c?.body ?? "")).filter((b) => !/^<!-- FORGE:/.test(b.trim()));
-  return `Issue #${issue.number}: ${issue.title}\n\n<issue_body>\n${cap(issue.body ?? "", 20000)}\n</issue_body>\n\n<issue_comments>\n${cap(notes.join("\n---\n"), 8000)}\n</issue_comments>\n\n<contract>\n${cap(JSON.stringify(contract ?? {}, null, 1), 12000)}\n</contract>\n\n<builder_result>\n${cap(JSON.stringify(result ?? {}, null, 1), 6000)}\n</builder_result>\n\n<changed_files>\n${files.join("\n")}\n</changed_files>\n\n<diff>\n${boundDiff(diff, { stat, base })}\n</diff>`;
+  const sk = JSON.stringify(skeleton ?? reviewSkeleton(contract ?? {}, files, ignore), null, 1);
+  return `Issue #${issue.number}: ${issue.title}\n\n<issue_body>\n${cap(issue.body ?? "", 20000)}\n</issue_body>\n\n<issue_comments>\n${cap(notes.join("\n---\n"), 8000)}\n</issue_comments>\n\n<contract>\n${cap(JSON.stringify(contract ?? {}, null, 1), 12000)}\n</contract>\n\n<builder_result>\n${cap(JSON.stringify(result ?? {}, null, 1), 6000)}\n</builder_result>\n\n<changed_files>\n${files.join("\n")}\n</changed_files>\n\n<row_skeleton>\n${cap(sk, 30000)}\n</row_skeleton>\n\n<diff>\n${boundDiff(diff, { stat, base })}\n</diff>${structure ? `\n<structure>${String(structure).replace(/[\r\n]+/g, " ")}</structure>` : ""}`;
+}
+
+/**
+ * #4160: the targeted re-ask after an incomplete or inconsistent reply. Only the missing or invalid ids are asked
+ * for, with the exact problems and the reviewer's previous reply; no diff is re-sent (the worktree still holds the change).
+ */
+export function reviewerRepairPrompt({ missing = [], problems = [], rows = null }) {
+  return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is repo-relative path:line.`;
 }
 
 /** Pre-filter (#4123): one read-only, single-turn check of an issue a cheap signal flagged as possibly already fixed. */

package/bin/lean/refactor.mjs

@@ -0,0 +1,276 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * refactor:split lane step (forge#4211, #4196): replaces the builder for an issue labelled `refactor:split`.
+ * Deterministic except one grouping call: scripts/symbol-index.py (Python) or scripts/symbol-index.mjs (TypeScript)
+ * indexes the target, one tool-less LLM call groups the top-level symbols from that index only (never a file body),
+ * the language's codemod mover (scripts/symbol-move.py with libcst, scripts/symbol-move.mjs with ts-morph, #4212)
+ * moves them into one package and leaves a re-export shim/barrel, and scripts/refactor-verify.sh proves the move
+ * (same test counts, every public symbol resolves from its old path, no new cycle, moved-only diff). Other languages
+ * are refused before anything runs. Every refusal leaves the worktree at its start commit.
+ * A plan over `maxLines` moved lines is cut into whole modules: this PR moves the first slice, the rest is carried in
+ * a FORGE:SPLIT-PLAN block of a chained issue that the next run applies with `symbol-move.py --extend`, no new call.
+ * runCore (lean.mjs) owns lease, worktree, PR and merge; this module only produces the verified commit.
+ */
+import { execFile } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { extname, isAbsolute, join, normalize } from "node:path";
+import { fileURLToPath } from "node:url";
+import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
+import { groupingPrompt } from "./prompts.mjs";
+
+export const SPLIT_LABEL = "refactor:split";
+export const DEFAULT_SPLIT_MAX_LINES = 4000;
+const SCRIPTS = fileURLToPath(new URL("../../scripts/", import.meta.url));
+export const SYMBOL_INDEX = join(SCRIPTS, "symbol-index.py");
+export const SYMBOL_MOVE = join(SCRIPTS, "symbol-move.py");
+export const REFACTOR_VERIFY = join(SCRIPTS, "refactor-verify.sh");
+export const SYMBOL_INDEX_MJS = join(SCRIPTS, "symbol-index.mjs");
+export const SYMBOL_MOVE_TS = join(SCRIPTS, "symbol-move.mjs");
+const LANGS = { ".py": "python", ".ts": "ts", ".tsx": "ts", ".mts": "ts", ".cts": "ts", ".js": "js", ".jsx": "js", ".mjs": "js", ".cjs": "js", ".go": "go", ".rs": "rust", ".rb": "ruby", ".java": "java" };
+const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+/** argv runner without a shell; never throws for a non-zero exit. */
+export function defaultRun(cmd, args, { cwd, env } = {}) {
+  return new Promise((resolve) => {
+    execFile(cmd, args, { cwd, env, maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+      const code = err ? (Number.isInteger(err.code) ? err.code : 127) : 0;
+      resolve({ code, stdout: String(stdout ?? ""), stderr: String(stderr ?? (err && !Number.isInteger(err.code) ? err.message : "")) });
+    });
+  });
+}
+
+/** `<!-- FORGE:MAINT:refactor:split:<path> -->` (scripts/maintenance-scan.sh), else a `**Target**: <path>` line; null otherwise. */
+export function parseSplitTarget(body) {
+  const s = String(body ?? "");
+  const m = /<!--\s*FORGE:MAINT:refactor:split:(.+?)\s*-->/.exec(s) ?? /^\s*\*\*Target\*\*:\s*`?([^`\s]+)`?/m.exec(s);
+  return m ? m[1].trim() : null;
+}
+
+/** `**Test command**: <cmd>` (backticks optional), else forge.yaml learned.test_commands joined with ` && `; null otherwise. */
+export function parseTestCmd(body, precheckCommands = []) {
+  const m = /^\s*\*\*Test command\*\*:\s*(?:`([^`]+)`|(.+))$/m.exec(String(body ?? ""));
+  const own = (m?.[1] ?? m?.[2] ?? "").trim();
+  if (own) return own;
+  const cmds = (Array.isArray(precheckCommands) ? precheckCommands : []).map((c) => String(c ?? "").trim()).filter(Boolean);
+  return cmds.length ? cmds.join(" && ") : null;
+}
+
+const validPlan = (p) => p && typeof p === "object" && !Array.isArray(p) && Object.keys(p).length > 0
+  && Object.entries(p).every(([k, v]) => IDENT.test(k) && Array.isArray(v) && v.length > 0 && v.every((n) => typeof n === "string" && n));
+
+/** The chain plan `<!-- FORGE:SPLIT-PLAN -->` + fenced json {target?, package, plan:{module:[symbols]}}; null when absent or invalid. */
+export function parseSplitPlan(body) {
+  const m = /<!--\s*FORGE:SPLIT-PLAN\s*-->\s*```(?:json)?\s*([\s\S]*?)```/.exec(String(body ?? ""));
+  if (!m) return null;
+  let j; try { j = JSON.parse(m[1]); } catch { return null; }
+  if (!j || typeof j !== "object" || !IDENT.test(String(j.package ?? "")) || !validPlan(j.plan)) return null;
+  return { target: typeof j.target === "string" && j.target ? j.target : null, package: j.package, plan: j.plan };
+}
+
+const top = (n) => String(n).split(".")[0];
+
+/** What the grouping call sees: top-level symbols {name,kind,lines,public} and deduped `from->to` edges. No spans, bodies or imports. */
+export function compactIndex(index) {
+  const symbols = (index?.symbols ?? []).filter((s) => !String(s.name).includes(".")).map((s) => ({ name: s.name, kind: s.kind, lines: s.lines, public: Boolean(s.public) }));
+  const names = new Set(symbols.map((s) => s.name));
+  const edges = new Set();
+  for (const e of index?.edges ?? []) {
+    const f = top(e.from), t = top(e.to);
+    if (f !== t && names.has(f) && names.has(t)) edges.add(`${f}->${t}`);
+  }
+  return { symbols, edges: [...edges].sort() };
+}
+
+/** The grouping reply as {module:[symbols]}: one JSON object, identifiers, known top-level symbols, none twice; null otherwise. */
+export function parseGroupingReply(text, names) {
+  const s = String(text ?? ""), a = s.indexOf("{"), b = s.lastIndexOf("}");
+  if (a < 0 || b <= a) return null;
+  let p; try { p = JSON.parse(s.slice(a, b + 1)); } catch { return null; }
+  if (!validPlan(p)) return null;
+  const known = new Set(names), seen = new Set();
+  for (const list of Object.values(p)) for (const n of list) { if (!known.has(n) || seen.has(n)) return null; seen.add(n); }
+  if (Object.keys(p).some((k) => known.has(k))) return null;
+  return p;
+}
+
+/**
+ * One PR's slice of a plan: modules ordered so a module comes after the modules it references (plan order otherwise),
+ * then whole modules until the moved lines would exceed maxLines; the first module is always taken.
+ * Returns {now, rest, lines}; rest is null when everything fits.
+ */
+export function slicePlan(plan, index, maxLines = DEFAULT_SPLIT_MAX_LINES) {
+  const ci = compactIndex(index);
+  const size = new Map(ci.symbols.map((s) => [s.name, Number(s.lines) || 0]));
+  const modOf = new Map();
+  for (const [m, list] of Object.entries(plan)) for (const n of list) modOf.set(n, m);
+  const deps = new Map(Object.keys(plan).map((m) => [m, new Set()]));
+  for (const e of ci.edges) {
+    const [f, t] = e.split("->"), mf = modOf.get(f), mt = modOf.get(t);
+    if (mf && mt && mf !== mt) deps.get(mf).add(mt);
+  }
+  const order = [], state = new Map();
+  const visit = (m) => {
+    if (state.get(m)) return; // visiting (a cycle: symbol-move refuses it) or done
+    state.set(m, 1);
+    for (const d of Object.keys(plan)) if (deps.get(m).has(d)) visit(d);
+    state.set(m, 2); order.push(m);
+  };
+  for (const m of Object.keys(plan)) visit(m);
+  const now = {}, rest = {};
+  let lines = 0, full = false;
+  for (const m of order) {
+    const l = plan[m].reduce((n, s) => n + (size.get(s) ?? 0), 0);
+    if (!full && (!Object.keys(now).length || lines + l <= maxLines)) { now[m] = plan[m]; lines += l; } else { full = true; rest[m] = plan[m]; }
+  }
+  return { now, rest: Object.keys(rest).length ? rest : null, lines };
+}
+
+const pythonCmd = (py) => {
+  const v = Array.isArray(py) ? py : String(py ?? process.env.SYMBOL_MOVE_PYTHON ?? "").trim().split(/\s+/).filter(Boolean);
+  return v.length ? v : ["python3"];
+};
+/** A script's JSON reply: the whole stdout (refactor-verify pretty-prints), else its last line. */
+const parseJson = (t) => {
+  for (const c of [String(t ?? "").trim(), String(t ?? "").trim().split("\n").filter(Boolean).at(-1) ?? ""]) {
+    try { const j = JSON.parse(c); if (j && typeof j === "object") return j; } catch { /* next candidate */ }
+  }
+  return null;
+};
+const tail = (r) => String(r?.stderr || r?.stdout || "").trim().slice(-600);
+const failureLine = (f) => (f.kind === "logic-diff" ? `logic-diff ${f.file}:${f.line} \`${f.text}\`` : f.kind === "missing-symbol" ? `missing-symbol ${f.module}.${f.name}`
+  : f.kind === "count-changed" ? `count-changed ${JSON.stringify(f.before)} -> ${JSON.stringify(f.after)}` : f.kind === "new-cycle" ? `new-cycle ${(f.cycle ?? []).join(" -> ")}` : JSON.stringify(f));
+
+function effects(ctx) {
+  const run = ctx.run ?? defaultRun;
+  const env = { ...process.env, PYTHONDONTWRITEBYTECODE: "1", ...(ctx.env ?? {}) };
+  const sh = (cmd, args) => run(cmd, args, { cwd: ctx.wt, env });
+  const git = ctx.git ?? (async (args) => { const r = await sh("git", args); if (r.code !== 0) throw new Error(`git ${args.join(" ")} failed: ${tail(r)}`); return r.stdout; });
+  const scratch = () => { const d = join(ctx.runTmp ?? join(ctx.wt, ".forge-lean"), "split"); mkdirSync(d, { recursive: true }); return d; };
+  return { sh, git, scratch };
+}
+
+/** Puts the worktree back at `head` and drops untracked files (ignored ones, such as .forge-lean/, stay). */
+async function restore(git, head) {
+  try { await git(["reset", "-q", "--hard", head]); } catch { /* best effort */ }
+  try { await git(["clean", "-fdq"]); } catch { /* best effort */ }
+}
+
+/** baseline at `ref`'s state (the worktree checked out there), back to the current branch, then verify `ref..HEAD`. */
+export async function reverifySplit(ctx) {
+  const { sh, git, scratch } = effects(ctx);
+  const head = String(await git(["rev-parse", "HEAD"])).trim();
+  const branch = String(await git(["rev-parse", "--abbrev-ref", "HEAD"])).trim();
+  const out = join(scratch(), "split-rebaseline.json");
+  await git(["checkout", "-q", "--detach", ctx.base]);
+  const b = await sh("bash", [REFACTOR_VERIFY, "baseline", "--root", ctx.wt, "--target", ctx.target, "--test-cmd", ctx.testCmd, "--out", out]);
+  await restore(git, ctx.base);
+  await git(["checkout", "-q", branch === "HEAD" ? head : branch]);
+  if (b.code !== 0) return { ok: false, code: OUTCOME_CODES.SPLIT_PLAN_INVALID, reason: `refactor-verify baseline at ${ctx.base} failed (exit ${b.code}): ${tail(b)}` };
+  const v = await sh("bash", [REFACTOR_VERIFY, "verify", "--baseline", out, "--base", ctx.base]);
+  const j = parseJson(v.stdout);
+  if (v.code === 0 && j?.ok) return { ok: true, verify: j };
+  if (v.code === 1) return { ok: false, code: OUTCOME_CODES.SPLIT_VERIFY_FAILED, reason: `refactor-verify failed:\n${(j?.failures ?? []).slice(0, 20).map((f) => `- ${failureLine(f)}`).join("\n")}`, verify: j };
+  return { ok: false, code: OUTCOME_CODES.SPLIT_UNSUPPORTED, reason: `refactor-verify could not verify (exit ${v.code}): ${tail(v)}` };
+}
+
+/**
+ * The split step. ctx: {wt, base, issue, issueBody, precheckCommands, runTmp, maxLines, run(cmd,args,{cwd,env}),
+ * callGrouper(prompt, {cwd: an empty dir}) -> reply text, git(args) -> stdout, python (argv or string; default $SYMBOL_MOVE_PYTHON, else python3)}.
+ * Returns {ok:true, target, testCmd, package, packageName, modules, rest, verify, grouped, lines} or {ok:false, code, reason}.
+ */
+export async function runSplitStep(ctx) {
+  const { sh, git, scratch } = effects(ctx);
+  const py = pythonCmd(ctx.python);
+  const fail = (code, reason) => ({ ok: false, code, reason });
+  const chain = parseSplitPlan(ctx.issueBody);
+  // ---- 1. target, language, test command, codemod tool: nothing touched yet ----
+  const raw = parseSplitTarget(ctx.issueBody) ?? chain?.target;
+  if (!raw) return fail(OUTCOME_CODES.SPLIT_PLAN_INVALID, "no split target: the issue needs a `<!-- FORGE:MAINT:refactor:split:<path> -->` marker or a `**Target**: <path>` line");
+  const target = normalize(raw.replace(/^\.\//, ""));
+  if (isAbsolute(target) || target.startsWith("..")) return fail(OUTCOME_CODES.SPLIT_PLAN_INVALID, `split target \`${raw}\` must be a path inside the repository`);
+  const ext = extname(target).toLowerCase();
+  const lang = LANGS[ext] ?? (ext.slice(1) || "unknown");
+  if (lang !== "python" && lang !== "ts") return fail(OUTCOME_CODES.SPLIT_UNSUPPORTED, `no codemod tool for ${lang}: \`${target}\` (the refactor:split movers support Python and TypeScript)`);
+  if (!existsSync(join(ctx.wt, target))) return fail(OUTCOME_CODES.SPLIT_PLAN_INVALID, `split target \`${target}\` does not exist on the base`);
+  const testCmd = parseTestCmd(ctx.issueBody, ctx.precheckCommands);
+  if (!testCmd) return fail(OUTCOME_CODES.SPLIT_PLAN_INVALID, "no test command: add a `**Test command**: <cmd>` line to the issue or forge.yaml learned.test_commands");
+  if (lang === "python") {
+    const probe = await sh(py[0], [...py.slice(1), "-c", "import libcst"]);
+    if (probe.code !== 0) return fail(OUTCOME_CODES.SPLIT_UNSUPPORTED, `no codemod tool for python: libcst is not importable by \`${py.join(" ")}\` (install it or set SYMBOL_MOVE_PYTHON)`);
+  } else {
+    // #4212: ts-morph is resolved from the target repo (or NODE_PATH); the probe reads and writes nothing
+    const probe = await sh(process.execPath, [SYMBOL_MOVE_TS, "--probe", "--src", target]);
+    if (probe.code !== 0) {
+      const why = String(parseJson(probe.stdout)?.reason ?? "");
+      return fail(OUTCOME_CODES.SPLIT_UNSUPPORTED, /^no codemod tool for ts:/.test(why) ? why : `no codemod tool for ts: the ts-morph probe failed (exit ${probe.code}): ${why || tail(probe)}`);
+    }
+  }
+  const startHead = String(await git(["rev-parse", "HEAD"])).trim();
+  const refuse = async (code, reason) => { await restore(git, startHead); return fail(code, reason); };
+  // ---- 2. symbol index ----
+  const ir = lang === "python" ? await sh("python3", [SYMBOL_INDEX, target]) : await sh(process.execPath, [SYMBOL_INDEX_MJS, target]);
+  const index = parseJson(ir.stdout);
+  if (index?.unsupported) return await refuse(OUTCOME_CODES.SPLIT_UNSUPPORTED, `no codemod tool for ${index.unsupported}: ${index.reason ?? "unsupported"}`);
+  if (ir.code !== 0 || !Array.isArray(index?.symbols)) return await refuse(OUTCOME_CODES.SPLIT_PLAN_INVALID, `symbol index failed (exit ${ir.code}): ${tail(ir)}`);
+  // ---- 3. baseline ----
+  const dir = scratch(), baseline = join(dir, "split-baseline.json");
+  const br = await sh("bash", [REFACTOR_VERIFY, "baseline", "--root", ctx.wt, "--target", target, "--test-cmd", testCmd, "--out", baseline]);
+  if (br.code !== 0) return await refuse(OUTCOME_CODES.SPLIT_PLAN_INVALID, `refactor-verify baseline failed (exit ${br.code}): ${tail(br)}`);
+  // ---- 4. plan: the chained remainder, else exactly one grouping call over the compact index ----
+  let plan, grouped = false;
+  if (chain) plan = chain.plan;
+  else {
+    const ci = compactIndex(index);
+    const reply = await ctx.callGrouper(groupingPrompt(ci), { cwd: mkdtempSync(join(dir, "group-")) }); // empty cwd: nothing to read even with a tool
+    grouped = true;
+    plan = parseGroupingReply(reply, ci.symbols.map((s) => s.name));
+    if (!plan) return await refuse(OUTCOME_CODES.SPLIT_PLAN_INVALID, `the grouping reply is not a valid {module:[symbols]} plan over the index: ${String(reply ?? "").slice(0, 300)}`);
+  }
+  // ---- 5. one PR's slice ----
+  const { now, rest, lines } = slicePlan(plan, index, ctx.maxLines ?? DEFAULT_SPLIT_MAX_LINES);
+  // ---- 6. move ----
+  const planFile = join(mkdtempSync(join(dir, "plan-")), "plan.json");
+  writeFileSync(planFile, `${JSON.stringify(now)}\n`);
+  const moveArgs = ["--src", target, "--plan", planFile, ...(chain ? ["--package", chain.package, "--extend"] : [])];
+  const mr = lang === "python" ? await sh(py[0], [...py.slice(1), SYMBOL_MOVE, ...moveArgs]) : await sh(process.execPath, [SYMBOL_MOVE_TS, ...moveArgs]);
+  const moved = parseJson(mr.stdout);
+  if (mr.code === 3 || moved?.status === "refused") {
+    const why = String(moved?.reason ?? tail(mr));
+    return await refuse(/libcst|no codemod tool/i.test(why) ? OUTCOME_CODES.SPLIT_UNSUPPORTED : OUTCOME_CODES.SPLIT_PLAN_INVALID, `symbol-move refused: ${why}`);
+  }
+  if (mr.code !== 0 || moved?.status !== "ok") return await refuse(OUTCOME_CODES.SPLIT_PLAN_INVALID, `symbol-move failed (exit ${mr.code}): ${tail(mr)}`);
+  // ---- 7. commit ----
+  const pkg = String(moved.package);
+  const packageName = lang === "python" ? pkg.split(".").at(-1) : pkg.split("/").at(-1); // dotted package | package dir
+  const tool = lang === "python" ? "libcst" : "ts-morph";
+  try {
+    await git(["add", "-A"]);
+    await git(["commit", "--no-verify", "-q", "-m", `refactor(split): move ${Object.keys(now).length} module(s) of ${target} into ${pkg}${ctx.issue ? ` (#${ctx.issue})` : ""}`]);
+  } catch (e) { return await refuse(OUTCOME_CODES.SPLIT_PLAN_INVALID, `committing the move failed: ${String(e?.message ?? e).slice(0, 300)}`); }
+  // ---- 8. verify ----
+  const vr = await sh("bash", [REFACTOR_VERIFY, "verify", "--baseline", baseline, "--base", ctx.base]);
+  const verify = parseJson(vr.stdout);
+  if (vr.code === 1) return await refuse(OUTCOME_CODES.SPLIT_VERIFY_FAILED, `refactor-verify failed; the move was reset:\n${(verify?.failures ?? []).slice(0, 20).map((f) => `- ${failureLine(f)}`).join("\n")}`);
+  if (vr.code !== 0 || !verify?.ok) return await refuse(OUTCOME_CODES.SPLIT_UNSUPPORTED, `refactor-verify could not verify (exit ${vr.code}): ${tail(vr)}`);
+  return { ok: true, target, testCmd, package: pkg, packageName, modules: moved.modules ?? now, rest, verify, grouped, lines, lang, tool };
+}
+
+const table = (modules) => ["| Module | Symbols |", "| --- | --- |", ...Object.entries(modules ?? {}).map(([m, s]) => `| \`${m}\` | ${s.length} |`)].join("\n");
+
+/** The split PR body: what moved where, the deterministic proof, the chained remainder. */
+export function renderSplitPrBody({ issue, target, package: pkg, modules, verify, rest, lines, grouped, tool }) {
+  tool ??= /\.[cm]?tsx?$/i.test(String(target ?? "")) ? "ts-morph" : "libcst"; // a rerun result carries no tool
+  const left = rest ? `\n\n## Remaining work\n${Object.keys(rest).length} module(s) (${Object.keys(rest).map((m) => `\`${m}\``).join(", ")}) move in a chained \`refactor:split\` PR filed after this one merges; each PR is mergeable on its own.` : "";
+  return `## Split\n\`${target}\` → package \`${pkg}\`, ${Object.keys(modules ?? {}).length} module(s), ${lines ?? "?"} moved line(s). Moved verbatim by ${tool === "ts-morph" ? "\`scripts/symbol-move.mjs\` (ts-morph)" : "\`scripts/symbol-move.py\` (libcst)"}; \`${target}\` keeps every unmoved statement and re-exports each moved name. Plan: ${grouped ? "one grouping call over the symbol index (no file bodies)" : "carried from the chained issue (no LLM call)"}.\n\n${table(modules)}\n\n## Verification\n\`scripts/refactor-verify.sh\`: ${verify?.ok ? "ok" : "not run"} (identical pass/skip counts, every baseline public symbol resolves from its old path, no new import cycle, diff is moved blocks and import lines only). Location-only change: no conformance review.${left}\n\nCloses #${issue}`;
+}
+
+/** The chained issue for the rest of a large split: same maintenance marker and labels, and the plan, so the next run makes no grouping call. */
+export function renderChainIssue({ parent, target, packageName, rest, testCmd = null }) {
+  const plan = JSON.stringify({ target, package: packageName, plan: rest }, null, 2);
+  return {
+    title: `split ${target} (continued)`,
+    labels: ["maintenance", SPLIT_LABEL],
+    body: `Continues the split of \`${target}\` from #${parent}: ${Object.keys(rest).length} module(s) left, moved into the existing package \`${packageName}\`.\n\n**Target**: \`${target}\`\n${testCmd ? `**Test command**: \`${testCmd}\`\n` : ""}\n<!-- FORGE:MAINT:refactor:split:${target} -->\n<!-- FORGE:SPLIT-PLAN -->\n\`\`\`json\n${plan}\n\`\`\`\n`,
+  };
+}