forgedock 1.29.1 → 1.30.0

package/bin/lean/conformance.test.mjs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, explainFailures } from "./conformance.mjs";
+import { validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, missingTool, repoTracks, tmpdirInputs } from "./conformance.mjs";
 
 const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
 const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -72,6 +72,31 @@ describe("precheck (deterministic, no reviewer)", () => {
     assert.deepEqual(codes(precheck({ contract: C, result: { ...res, validation: [{ id: "V1", exit: 1 }] }, changedFiles: files })), ["conformance.unimplemented"]);
     assert.deepEqual(codes(precheck({ contract: C, result: { ...res, validation: [] }, changedFiles: files })), ["conformance.unimplemented"]);
   });
+  it("#4323 a failed validate whose rerun could not find a host tool or module is env.tool_missing naming it, not conformance", () => {
+    const failed = { ...res, validation: [{ id: "V1", exit: 1 }] };
+    const pc = precheck({ contract: C, result: failed, changedFiles: files, validateRuns: { V1: { rc: 1, output: "Traceback...\nModuleNotFoundError: No module named 'yaml'" } } });
+    assert.deepEqual(codes(pc), ["env.tool_missing"]);
+    assert.equal(pc.failures[0].kind, "validate_env"); assert.equal(pc.failures[0].tool, "yaml");
+    assert.match(pc.failures[0].detail, /could not run: the host is missing Python module 'yaml'/);
+    assert.deepEqual(codes(precheck({ contract: C, result: failed, changedFiles: files, validateRuns: { V1: { rc: 127, output: "bash: line 1: pytest: command not found" } } })), ["env.tool_missing"]);
+    assert.deepEqual(codes(precheck({ contract: C, result: failed, changedFiles: files, ciVerified: true, validateRuns: { V1: { rc: 127, output: "bash: pytest: command not found" } } })), [], "CI-verified head: like a validate left to CI");
+    assert.deepEqual(codes(precheck({ contract: C, result: failed, changedFiles: files, validateRuns: { V1: { rc: 0, output: "" } } })), [], "a passing rerun counts as passed");
+    assert.deepEqual(codes(precheck({ contract: C, result: failed, changedFiles: files, validateRuns: { V1: { rc: 1, output: "AssertionError: 1 !== 2" } } })), ["conformance.unimplemented"]);
+    const local = precheck({ contract: C, result: failed, changedFiles: files, repoHas: repoTracks(["src/leak/__init__.py"]), validateRuns: { V1: { rc: 1, output: "ModuleNotFoundError: No module named 'leak.core'" } } });
+    assert.deepEqual(codes(local), ["conformance.unimplemented"], "a repo module the change did not create is a code defect");
+  });
+  it("#4323 missingTool: commands, python and node modules; paths, relative and repo-tracked names are not the environment", () => {
+    assert.deepEqual(missingTool({ rc: 127, output: "sh: 1: jq: not found" }), { tool: "jq", kind: "command" });
+    assert.deepEqual(missingTool({ rc: 127, output: "" }, { command: "uv run pytest -q" }), { tool: "uv", kind: "command" });
+    assert.deepEqual(missingTool({ rc: 1, output: "ImportError: No module named yaml.constructor" }), { tool: "yaml", kind: "python-module" });
+    assert.deepEqual(missingTool({ rc: 1, output: "Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'zod' imported from /x/a.mjs" }), { tool: "zod", kind: "node-module" });
+    assert.deepEqual(missingTool({ rc: 1, output: "Error: Cannot find module '@scope/pkg/sub'" }), { tool: "@scope/pkg", kind: "node-module" });
+    assert.equal(missingTool({ rc: 1, output: "Error: Cannot find module './lib/new.js'" }), null);
+    assert.equal(missingTool({ rc: 1, output: "Error: Cannot find module '/repo/src/x.mjs'" }), null);
+    assert.equal(missingTool({ rc: 127, output: "bash: ./scripts/check.sh: No such file or directory" }, { command: "./scripts/check.sh" }), null);
+    assert.equal(missingTool({ rc: 1, output: "ModuleNotFoundError: No module named 'app'" }, { repoHas: repoTracks(["app.py"]) }), null);
+    assert.equal(missingTool({ rc: 1, output: "1 test failed: expected not found" }), null, "rc != 127 needs command not found");
+  });
   it("a deliver requirement reported deferred without an amendment -> untracked_deferral; partial -> unimplemented", () => {
     const r = { ...res, requirements: [{ id: "R1", status: "deferred" }, { id: "R2", status: "partial" }] };
     assert.deepEqual(codes(precheck({ contract: C, result: r, changedFiles: files })), ["conformance.untracked_deferral", "conformance.unimplemented"]);
@@ -117,6 +142,30 @@ describe("conformance review: the engine computes the verdict from complete rows
     assert.equal(normalizeEvidencePath("b/route.ts", { changedFiles: ["app/a/route.ts", "app/b/route.ts"] }), "app/b/route.ts");
     assert.equal(decide(row("R1", { evidence: "scrape.js:7" }), ctx).review.requirements[0].evidence, "src/scrape.js:7");
   });
+  it("#4327 evidence paths with route-group, dynamic and catch-all segments are kept verbatim", () => {
+    const repo = ["apps/admin/app/(console)/layout.tsx", "apps/web/app/blog/[slug]/page.tsx", "apps/web/app/docs/[...slug]/page.tsx"];
+    const c = { ...ctx, repoFiles: repo, pathExists: (p) => repo.includes(p) };
+    for (const [ev, want] of [
+      ["`(console)/layout.tsx:12` + nav (sidebar)", "apps/admin/app/(console)/layout.tsx:12"],
+      ["`apps/admin/app/(console)/layout.tsx`:12", "apps/admin/app/(console)/layout.tsx:12"],
+      ["see apps/web/app/blog/[slug]/page.tsx:4", "apps/web/app/blog/[slug]/page.tsx:4"],
+      ["(apps/web/app/docs/[...slug]/page.tsx:9)", "apps/web/app/docs/[...slug]/page.tsx:9"],
+      ["[page](apps/web/app/docs/[...slug]/page.tsx:9)", "apps/web/app/docs/[...slug]/page.tsx:9"],
+    ]) {
+      const d = decide(row("R1", { evidence: ev }), c);
+      assert.equal(d.state, "conforms", ev); assert.equal(d.review.requirements[0].evidence, want, ev);
+    }
+    assert.deepEqual(evidencePathLine("(src/scrape.js:7), fixed"), { path: "src/scrape.js", line: "7" });
+  });
+  it("#4327 a missing route-group path is still missing; only path problems qualify for the extra re-ask", () => {
+    const c = { ...ctx, repoFiles: ["apps/admin/app/(console)/layout.tsx"], pathExists: (p) => p === "apps/admin/app/(console)/layout.tsx" };
+    const d = decide(row("R1", { evidence: "`apps/admin/app/(admin)/layout.tsx:12`" }), c);
+    assert.equal(d.state, "review.incomplete"); assert.deepEqual(d.missing, ["R1"]);
+    assert.match(d.failures[0].detail, /evidence path apps\/admin\/app\/\(admin\)\/layout\.tsx does not exist/);
+    assert.equal(evidencePathOnly(d.failures.map((f) => f.detail)), true);
+    assert.equal(evidencePathOnly(decide({ verdict: "PASS" }, ctx).failures.map((f) => f.detail)), false);
+    assert.equal(evidencePathOnly([]), false);
+  });
   it("#4160 a targeted re-ask merges by id: the repair rows complete the first reply", () => {
     const first = { ...good, requirements: [good.requirements[1]], preserve: [] };
     const d1 = decide(first, ctx);
@@ -176,6 +225,21 @@ describe("conformance review: the engine computes the verdict from complete rows
     assert.equal(movingBaseDiff("git diff HEAD~1"), null);
     assert.equal(movingBaseDiff(""), null);
   });
+  it("#4325 a validate or repro_test command reading a $TMPDIR file it does not create is rejected at freeze", () => {
+    const bad = validateContract({ ...C, validate: [{ id: "V4", command: "uv run --with-requirements $TMPDIR/api-req.txt pytest tests/x.py" }] }, ISSUE);
+    assert.equal(bad.ok, false);
+    assert.ok(bad.problems.some((x) => x.startsWith("V4: reads $TMPDIR/api-req.txt")), bad.problems.join("; "));
+    const rt = validateContract({ ...C, repro_test: { path: "t/x.py", command: "pytest -r ${TMPDIR}/req.txt t/x.py" } }, ISSUE, { repro: true });
+    assert.ok(rt.problems.some((x) => x.startsWith("repro_test: reads $TMPDIR/req.txt")), rt.problems.join("; "));
+    for (const ok of ["uv export > $TMPDIR/r.txt && uv run --with-requirements $TMPDIR/r.txt pytest", "pytest --basetemp=$TMPDIR/pt --junitxml $TMPDIR/j.xml", "mkdir -p \"$TMPDIR/d\" && cp a $TMPDIR/d/a && cat $TMPDIR/d/a", "X=$TMPDIR/cache npm test", "echo hi | tee $TMPDIR/l; grep hi $TMPDIR/l", "node --test test/leak.test.js"]) {
+      assert.equal(validateContract({ ...C, validate: [{ id: "V4", command: ok }] }, ISSUE).ok, true, ok);
+    }
+  });
+  it("#4325 tmpdirInputs lists read-before-create $TMPDIR paths once, in first-use order", () => {
+    assert.deepEqual(tmpdirInputs("cat ${TMPDIR:-/tmp}/a.json | jq . && cp $TMPDIR/b.txt out && cat $TMPDIR/a.json"), ["$TMPDIR/a.json", "$TMPDIR/b.txt"]);
+    assert.deepEqual(tmpdirInputs("cp src.txt $TMPDIR/b.txt && cat $TMPDIR/b.txt"), []);
+    assert.deepEqual(tmpdirInputs(""), []);
+  });
   it("#4160: reviewSkeleton (object form) lists every required row, scope rows pre-filled with the engine's authorization", () => {
     const k = { ...C, requirements: [...C.requirements, { id: "R9", quote: "Billing pricing is unchanged.", disposition: "defer", why: "w" }] };
     const sk = reviewSkeleton({ contract: k, changedFiles: [...files, "src/billing.js", ".forge-lean/x"], ignore: [".forge-lean/"] });
@@ -301,12 +365,22 @@ describe("repro_test on bug lanes (#4271)", () => {
   it("red on base and green on head passes; anything else fails the precheck as conformance.unimplemented", () => {
     const k = { ...C, repro_test: RT };
     assert.deepEqual(precheck({ contract: k, result: res, changedFiles: files, repro: { base: "FAILED", head: "RAN" } }), { ok: true, failures: [] });
-    for (const repro of [{ base: "RAN", head: "RAN" }, { base: "SKIPPED", head: "FAILED" }, { base: "UNRUNNABLE", head: "RAN" }, { base: "UNRUNNABLE", head: "UNRUNNABLE" }]) {
+    for (const repro of [{ base: "RAN", head: "RAN" }, { base: "SKIPPED", head: "FAILED" }, { base: "UNRUNNABLE", head: "RAN" }, { base: "SKIPPED", head: "RAN_NOTHING" }]) {
       const pc = precheck({ contract: k, result: res, changedFiles: files, repro });
       assert.deepEqual(pc.failures.map((f) => [f.code, f.kind]), [["conformance.unimplemented", "repro_red_green"]], JSON.stringify(repro));
       assert.match(pc.failures[0].detail, new RegExp(`BASE=${repro.base} HEAD=${repro.head}`));
     }
-    assert.match(reproFailures(k, {})[0].detail, /test\/leak\.test\.js releases on 429.*BASE=UNRUNNABLE HEAD=UNRUNNABLE/);
+    assert.match(reproFailures(k, { base: "SKIPPED", head: "FAILED" })[0].detail, /HEAD=FAILED \(base not run: head did not pass\)$/);
+    assert.doesNotMatch(reproFailures(k, { base: "RAN", head: "RAN" })[0].detail, /base not run/);
+  });
+  it("a harness UNRUNNABLE on head is env.repro_unrunnable, not conformance.unimplemented (#4334)", () => {
+    const k = { ...C, repro_test: RT };
+    for (const repro of [{ base: "SKIPPED", head: "UNRUNNABLE" }, { base: "UNRUNNABLE", head: "UNRUNNABLE" }, {}]) {
+      const pc = precheck({ contract: k, result: res, changedFiles: files, repro });
+      assert.deepEqual(pc.failures.map((f) => [f.code, f.kind]), [["env.repro_unrunnable", "repro_env"]], JSON.stringify(repro));
+      assert.match(pc.failures[0].detail, /test\/leak\.test\.js releases on 429 \(`node --test test\/leak\.test\.js`\): the harness could not run the test on head \(environment, not the change\)/);
+      assert.match(pc.failures[0].detail, new RegExp(`BASE=${repro.base ?? "UNRUNNABLE"} HEAD=UNRUNNABLE$`));
+    }
   });
   it("off (no repro option) is today's contract check and precheck exactly", () => {
     assert.deepEqual(validateContract(C, ISSUE), validateContract(C, ISSUE, { repro: false }));

package/bin/lean/context.mjs

@@ -111,25 +111,30 @@ export const contextPackCacheKey = (issue, headSha, cfg = {}, stamp = {}) =>
     ...(stamp?.scope != null || stamp?.attempts != null ? [String(stamp.scope ?? ""), Number(stamp.attempts) || 0] : []),
     ...(stamp?.links ? [`links:${stamp.links}`] : [])]))}.json`;
 
-/** Max referenced issues whose state is read for the cache stamp (one REST call each). */
-export const LINK_STAMP_MAX = 20;
+/** Max referenced issues whose state is read for the cache stamp (#4318: one GraphQL call for all of them). */
+export const LINK_STAMP_MAX = 100;
 /**
  * #4316: the linked-issue part of the cache stamp: a hash of `n:STATE` for each issue the body references (parent, Depends
- * on, `#N` links), so a pack never shows a linked issue as open after it closed on the same base. Timeline cross-refs are
- * left to the gather. An unreadable issue counts as `n:?`; never throws; '' when the body references nothing.
+ * on, `#N` links), so a pack never shows a linked issue as open after it closed on the same base. #4318: one GraphQL call
+ * reads every state plus the issue's own cross-reference/connected timeline count (a new relation changes the stamp too);
+ * above LINK_STAMP_MAX the highest (most recent) numbers are kept. A failed call counts every number as `n:?`; never
+ * throws; '' (and no call) when the body references nothing.
  */
 export async function linkStateStamp({ gh, repo, issue, body }) {
   const parent = parentOf(body);
   const nums = [...new Set([...(parent ? [parent] : []), ...parseDependsOn(stripComments(body), issue), ...bodyLinks(body, repo, issue)])]
-    .filter((n) => Number.isSafeInteger(n) && n > 0 && n !== Number(issue)).sort(byNum).slice(0, LINK_STAMP_MAX);
+    .filter((n) => Number.isSafeInteger(n) && n > 0 && n !== Number(issue)).sort(byNum).slice(-LINK_STAMP_MAX);
   if (!nums.length) return "";
-  const states = [];
-  for (const n of nums) {
-    let st = "?";
-    try { st = String(await gh(["api", `repos/${repo}/issues/${n}`, "--jq", ".state"])).trim().toUpperCase() || "?"; } catch { /* unreadable: n:? */ }
-    states.push(`${n}:${st}`);
-  }
-  return hash12(states.join(","));
+  const [owner, name] = String(repo ?? "").split("/");
+  const self = Number(issue);
+  const q = `query($o:String!,$n:String!){repository(owner:$o,name:$n){${Number.isSafeInteger(self) && self > 0
+    ? `self:issueOrPullRequest(number:${self}){... on Issue{timelineItems(itemTypes:[CROSS_REFERENCED_EVENT,CONNECTED_EVENT,DISCONNECTED_EVENT]){totalCount}}}` : ""}${
+    nums.map((n) => `i${n}:issueOrPullRequest(number:${n}){... on Issue{state} ... on PullRequest{state}}`).join(" ")}}}`;
+  let r = null;
+  try { r = JSON.parse(String(await gh(["api", "graphql", "-f", `query=${q}`, "-f", `o=${owner ?? ""}`, "-f", `n=${name ?? ""}`])))?.data?.repository ?? null; } catch { r = null; }
+  const states = nums.map((n) => `${n}:${String(r?.[`i${n}`]?.state ?? "").toUpperCase() || "?"}`);
+  const rel = r?.self?.timelineItems?.totalCount;
+  return hash12([...states, `rel:${Number.isInteger(rel) ? rel : "?"}`].join(","));
 }
 
 // ---- gather (I/O) ----

package/bin/lean/context.test.mjs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { gatherContextFacts, compileContextPack, contextPackFor, contextPackCacheKey, linkStateStamp, parseContextPackConfig, laneType, priorAttempts, issueSymbols, bodyLinks, parentOf, renderContextRecord, DEFAULT_CONTEXT_BYTES, noComments } from "./context.mjs";
+import { gatherContextFacts, compileContextPack, contextPackFor, contextPackCacheKey, linkStateStamp, LINK_STAMP_MAX, parseContextPackConfig, laneType, priorAttempts, issueSymbols, bodyLinks, parentOf, renderContextRecord, DEFAULT_CONTEXT_BYTES, noComments } from "./context.mjs";
 
 const REPO = "o/r";
 const BOT = { author_association: "NONE", user: { type: "Bot", login: "forge[bot]" } };
@@ -221,20 +221,51 @@ describe("#4298 context-pack cache and shared names", () => {
     assert.notEqual(k({ scope: "aaa111", attempts: 1 }), k({ scope: "aaa111", attempts: 2 }));
   });
 
+  it("#4318: above LINK_STAMP_MAX the highest numbers are kept, still in one call", async () => {
+    const calls = [];
+    const gh = async (args) => { calls.push(args); return JSON.stringify({ data: { repository: {} } }); };
+    const body = Array.from({ length: LINK_STAMP_MAX + 5 }, (_, i) => `#${i + 1}`).join(" ");
+    await linkStateStamp({ gh, repo: "o/r", issue: 9999, body });
+    assert.equal(calls.length, 1);
+    const nums = [...(calls[0].find((x) => x.startsWith("query=")) ?? "").matchAll(/i(\d+):/g)].map((m) => Number(m[1]));
+    assert.equal(nums.length, LINK_STAMP_MAX);
+    assert.equal(nums[0], 6, "the lowest numbers are dropped");
+    assert.equal(nums.at(-1), LINK_STAMP_MAX + 5);
+  });
+
   it("#4316: linked-issue states are in the key; a state change recompiles; no links keeps the old key", async () => {
     const states = { 7: "open", 8: "open", 9: "closed" };
+    let rel = 2;
     const calls = [];
-    const gh = async (args) => { calls.push(args); const n = Number(/issues\/(\d+)$/.exec(args[1])?.[1]); if (n === 9 && states.fail9) throw new Error("HTTP 502"); return `${states[n]}\n`; };
+    // #4318: one GraphQL call; an alias per referenced number, plus the issue's own timeline relation count
+    const gh = async (args) => {
+      calls.push(args);
+      if (states.fail) throw new Error("HTTP 502");
+      const q = args.find((a) => a.startsWith("query=")) ?? "";
+      const repository = { self: { timelineItems: { totalCount: rel } } };
+      for (const m of q.matchAll(/i(\d+):issueOrPullRequest/g)) repository[`i${m[1]}`] = states[m[1]] ? { state: states[m[1]].toUpperCase() } : null;
+      return JSON.stringify({ data: { repository } });
+    };
+    const aliases = (a) => [...(a.find((x) => x.startsWith("query=")) ?? "").matchAll(/i(\d+):/g)].map((m) => Number(m[1]));
     const body = "**Parent**: #7\nDepends on #8\nSee #9 and #50 (self).";
     const s1 = await linkStateStamp({ gh, repo: "o/r", issue: 50, body });
     assert.match(s1, /^[0-9a-f]{12}$/);
-    assert.deepEqual(calls.map((a) => a[1]), ["repos/o/r/issues/7", "repos/o/r/issues/8", "repos/o/r/issues/9"], "parent, depends-on and links once each, never self");
+    assert.equal(calls.length, 1, "one call for every reference");
+    assert.deepEqual(calls[0].slice(0, 2), ["api", "graphql"]);
+    assert.deepEqual(aliases(calls[0]), [7, 8, 9], "parent, depends-on and links once each, never self");
+    assert.ok(calls[0].includes("o=o") && calls[0].includes("n=r"));
     assert.equal(await linkStateStamp({ gh, repo: "o/r", issue: 50, body }), s1, "stable");
     states[8] = "closed";
     const s2 = await linkStateStamp({ gh, repo: "o/r", issue: 50, body });
     assert.notEqual(s2, s1, "a linked issue closing changes the stamp");
-    states.fail9 = true;
-    assert.match(await linkStateStamp({ gh, repo: "o/r", issue: 50, body }), /^[0-9a-f]{12}$/, "an unreadable link never throws");
+    rel = 3;
+    assert.notEqual(await linkStateStamp({ gh, repo: "o/r", issue: 50, body }), s2, "a new timeline relation changes the stamp");
+    rel = 2;
+    states.fail = true;
+    const sf = await linkStateStamp({ gh, repo: "o/r", issue: 50, body });
+    assert.match(sf, /^[0-9a-f]{12}$/, "a failed call never throws");
+    assert.notEqual(sf, s2, "a failed call is n:? for every number");
+    delete states.fail;
     assert.equal(await linkStateStamp({ gh: async () => { throw new Error("x"); }, repo: "o/r", issue: 50, body: "no links here" }), "");
     const k = (s) => contextPackCacheKey(50, "abc", cfg, s);
     assert.equal(k({ scope: "s", attempts: 0, links: "" }), k({ scope: "s", attempts: 0 }), "no links: the pre-#4316 key");

package/bin/lean/effects.mjs

@@ -15,6 +15,8 @@ import { makeLedger, countLiveLanes, ledgerRoot } from "./ledger.mjs";
 import { makeDepCache } from "./depcache.mjs";
 import { makeHeavyIo } from "./heavy.mjs";
 import { gatherContextFacts } from "./context.mjs";
+import { enumeratingTests, missingEntries } from "./enumerating.mjs";
+import { sequenceCollisions, snapshotLane, verifyRenumber } from "./sequence.mjs";
 
 const pexec = promisify(execFile);
 const script = (n) => fileURLToPath(new URL(`../../scripts/${n}`, import.meta.url));
@@ -197,12 +199,12 @@ export async function requiredChecksFor({ gh, repo, lane }) {
 
 /** Combined stdout+stderr kept per declared check command (tail). */
 const CHECK_OUTPUT_CAP = 64000;
-/** #4240: run a forge.yaml declared check command locally (no gh, no Actions); {rc, output}, rc 124 on timeout. */
-export function runCheckCommand({ command, cwd, timeoutMs = 300000 }) {
+/** #4240: run a forge.yaml declared check command locally (no gh, no Actions); {rc, output}, rc 124 on timeout. `env` (#4323) is merged over process.env. */
+export function runCheckCommand({ command, cwd, timeoutMs = 300000, env = null }) {
   return new Promise((done) => {
     let out = "", timedOut = false, child;
     const add = (b) => { out += String(b); if (out.length > 2 * CHECK_OUTPUT_CAP) out = out.slice(-CHECK_OUTPUT_CAP); };
-    try { child = spawn("bash", ["-c", String(command)], { cwd, detached: true, stdio: ["ignore", "pipe", "pipe"] }); }
+    try { child = spawn("bash", ["-c", String(command)], { cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], ...(env ? { env: { ...process.env, ...env } } : {}) }); }
     catch (e) { done({ rc: 127, output: String(e?.message ?? e) }); return; }
     const kill = () => { try { process.kill(-child.pid, "SIGKILL"); } catch { try { child.kill("SIGKILL"); } catch { /* gone */ } } };
     const timer = setTimeout(() => { timedOut = true; kill(); }, timeoutMs);
@@ -236,6 +238,8 @@ export function makeLeanIo({ repo }) {
     },
     /** #4240: one declared check command (`bash -c`) in `cwd`; never throws. A timeout kills the process group and gives rc 124. */
     runCommand: runCheckCommand,
+    /** #4328: a read-only freeze-time probe of a validate command's tool or module (separate from runCommand). */
+    probeCommand: runCheckCommand,
     /** #4167 verify-by-CI: `gh workflow run` on the lane branch; inputs become -f key=value. */
     async dispatchWorkflow({ workflow, ref, inputs = {} }) {
       await run("gh")(["workflow", "run", String(workflow), "-R", repo, "--ref", String(ref), ...Object.entries(inputs).flatMap(([k, v]) => ["-f", `${k}=${v}`])]);
@@ -306,11 +310,21 @@ export function makeLeanIo({ repo }) {
       const args = [script("hotspots.sh"), ...(base ? ["--base", base] : []), "--", ...paths];
       try { return JSON.parse((await pexec("bash", args, { cwd, timeout: 30000, maxBuffer: 16 * 1024 * 1024 })).stdout); } catch { return null; }
     },
-    /** #4271: scripts/acceptance-red-green.sh for the frozen repro_test in cwd; {base, head} words, or null on any failure. */
-    async redGreen({ cwd, base, command, timeoutSec = 300 }) {
+    /** #4322: tests enumerating the members of the given paths, and new members those tests miss (bin/lean/enumerating.mjs); [] on failure. */
+    enumeratingTests: ({ cwd, paths = [] }) => enumeratingTests({ git: (a) => run("git")(a, { cwd }), cwd, paths }),
+    enumeratingMissing: ({ cwd, base }) => missingEntries({ git: (a) => run("git")(a, { cwd }), cwd, base }),
+    /** #4322: sequence-numbered files the lane shares with the base (bin/lean/sequence.mjs); snapshot before, verify after a renumber. */
+    sequenceCheck: ({ cwd, base }) => sequenceCollisions({ git: (a) => run("git")(a, { cwd }), base }),
+    sequenceSnapshot: ({ cwd, base }) => snapshotLane({ git: (a) => run("git")(a, { cwd }), base }),
+    sequenceVerify: ({ cwd, base, before }) => verifyRenumber({ git: (a) => run("git")(a, { cwd }), base, before }),
+    /**
+     * #4271: scripts/acceptance-red-green.sh for the frozen repro_test in cwd; {base, head} words, or null on any failure.
+     * `env` (#4325): the lane env (run TMPDIR, toolchain), merged over process.env, so the command sees what the builder saw.
+     */
+    async redGreen({ cwd, base, command, timeoutSec = 300, env = null }) {
       const args = [script("acceptance-red-green.sh"), "--repo-path", cwd, "--base-ref", base, "--test-cmd", command, "--timeout", String(timeoutSec)];
       try {
-        const m = /BASE=(\w+) HEAD=(\w+)/.exec(String((await pexec("bash", args, { cwd, timeout: (timeoutSec * 2 + 60) * 1000, maxBuffer: 4 * 1024 * 1024 })).stdout ?? ""));
+        const m = /BASE=(\w+) HEAD=(\w+)/.exec(String((await pexec("bash", args, { cwd, timeout: (timeoutSec * 2 + 60) * 1000, maxBuffer: 4 * 1024 * 1024, ...(env ? { env: { ...process.env, ...env } } : {}) })).stdout ?? ""));
         return m ? { base: m[1], head: m[2] } : null;
       } catch { return null; }
     },

package/bin/lean/enumerating.mjs

@@ -0,0 +1,189 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Enumerating tests (#4322): tests whose expected list names every member of a group (a module's exports, or the files
+ * of one directory), so a new member must join the list or CI goes red. Deterministic and codebase-agnostic: git + fs,
+ * no LLM, runs none of the repo's commands. A test enumerates a group when it names at least min(3, n) members as string
+ * literals or object keys, or calls Object.keys/Object.entries on an import of the module. Never throws: failures are [].
+ */
+import fs from "node:fs";
+import { basename, dirname, join, posix } from "node:path";
+
+const SOURCE_RE = /\.(?:[cm]?[jt]sx?|py)$/;
+const TEST_RE = /(?:^|\/)(?:__tests__|tests?)\/|\.(?:test|spec)\.[^/]+$|(?:^|\/)test_[^/]+\.py$|_test\.[^/]+$/;
+const SKIP_RE = /(?:^|\/)(?:node_modules|vendor|dist|build|coverage)\//;
+const MAX_TEST_BYTES = 512 * 1024;
+const MAX_TESTS = 5000;
+const MAX_MODULES = 20, MAX_TESTS_PER_MODULE = 10;
+const MIN_NAME = 3;
+
+export const isTestPath = (p) => TEST_RE.test(p);
+const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const lines = (s) => String(s ?? "").split("\n").map((x) => x.trim()).filter(Boolean);
+const stem = (p) => basename(p).replace(/\.[^.]+$/, "").replace(/\.d$/, "");
+
+/** Exported names of a JS/TS or Python source text. */
+export function exportNames(text, path = "") {
+  const t = String(text ?? ""), out = new Set();
+  if (/\.py$/.test(path)) {
+    const all = /^__all__\s*=\s*[[(]([\s\S]*?)[\])]/m.exec(t);
+    if (all) for (const m of all[1].matchAll(/["']([A-Za-z_]\w*)["']/g)) out.add(m[1]);
+    else for (const m of t.matchAll(/^(?:async\s+)?(?:def|class)\s+([A-Za-z]\w*)/gm)) out.add(m[1]);
+    return [...out];
+  }
+  for (const m of t.matchAll(/^\s*export\s+(?:default\s+)?(?:declare\s+)?(?:abstract\s+)?(?:async\s+)?(?:function\*?|const|let|var|class|enum)\s+([A-Za-z_$][\w$]*)/gm)) out.add(m[1]);
+  for (const m of t.matchAll(/^\s*export\s+(?!type\b)\{([^}]*)\}/gm)) {
+    for (const part of m[1].split(",")) {
+      const p = part.trim().replace(/^type\s+/, "");
+      if (!p || part.trim().startsWith("type ")) continue;
+      const name = (/\bas\s+([A-Za-z_$][\w$]*)$/.exec(p)?.[1]) ?? /^[A-Za-z_$][\w$]*/.exec(p)?.[0];
+      if (name && name !== "default") out.add(name);
+    }
+  }
+  return [...out];
+}
+
+/** Line (1-based) of the first literal/key mention of any of `names` in `text`, and the distinct names mentioned. */
+function mentions(text, names) {
+  const hit = new Set();
+  let line = 0;
+  for (const n of names) {
+    if (n.length < MIN_NAME) continue;
+    const re = new RegExp(`(["'\`])${esc(n)}\\1|(?:^|[{,\\s])${esc(n)}\\s*:(?!:)`, "m");
+    const m = re.exec(text);
+    if (!m) continue;
+    hit.add(n);
+    const l = text.slice(0, m.index).split("\n").length;
+    if (!line || l < line) line = l;
+  }
+  return { hit, line };
+}
+
+/** Local names bound to an import of `module` in a test at `testPath` (namespace, default or require). */
+function importBindings(text, testPath, module, pkgs) {
+  const out = [];
+  const target = module.replace(SOURCE_RE, "");
+  const targets = new Set([target, ...(stem(module) === "index" ? [dirname(module)] : [])]);
+  const pkgNames = pkgs.filter((p) => stem(module) === "index" && [p.dir, posix.join(p.dir, "src")].includes(dirname(module))).map((p) => p.name);
+  const matches = (spec) => {
+    if (pkgNames.includes(spec)) return true;
+    if (!spec.startsWith(".")) return false;
+    const r = posix.normalize(posix.join(dirname(testPath), spec)).replace(SOURCE_RE, "");
+    return targets.has(r) || targets.has(r.replace(/\/index$/, ""));
+  };
+  for (const m of text.matchAll(/import\s+(?:\*\s+as\s+([A-Za-z_$][\w$]*)|([A-Za-z_$][\w$]*))\s+from\s+["']([^"']+)["']/g)) if (matches(m[3])) out.push(m[1] ?? m[2]);
+  for (const m of text.matchAll(/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:await\s+import|require)\(\s*["']([^"']+)["']\s*\)/g)) if (matches(m[2])) out.push(m[1]);
+  return out;
+}
+
+/** {path, text} of the repo's tracked test files (read from the working tree; absent/sparse/oversize files skipped). */
+async function loadTests(git, cwd) {
+  let files;
+  try { files = lines(await git(["-c", "core.quotepath=off", "ls-files"])); } catch { return { tests: [], files: [], pkgs: [] }; }
+  const tests = [];
+  for (const f of files) {
+    if (tests.length >= MAX_TESTS) break;
+    if (!isTestPath(f) || SKIP_RE.test(f) || !SOURCE_RE.test(f)) continue;
+    try {
+      const p = join(cwd, f);
+      if (fs.statSync(p).size > MAX_TEST_BYTES) continue;
+      tests.push({ path: f, text: fs.readFileSync(p, "utf8") });
+    } catch { /* sparse or unreadable: skip */ }
+  }
+  const pkgs = [];
+  for (const f of files.filter((x) => /(?:^|\/)package\.json$/.test(x) && !SKIP_RE.test(x))) {
+    try { const n = JSON.parse(fs.readFileSync(join(cwd, f), "utf8")).name; if (typeof n === "string" && n) pkgs.push({ name: n, dir: dirname(f) }); } catch { /* skip */ }
+  }
+  return { tests, files, pkgs };
+}
+
+/** Tests enumerating one group: [{test, line, why}]. `exports` groups also match Object.keys/entries of an import of `module`. */
+function enumerators(tests, { module, names, kind }, pkgs) {
+  const out = [];
+  const usable = names.filter((n) => n.length >= MIN_NAME);
+  if (usable.length < 2) return out;
+  const need = Math.min(3, usable.length);
+  for (const t of tests) {
+    if (t.path === module) continue;
+    const { hit, line } = mentions(t.text, usable);
+    if (hit.size >= need) { out.push({ test: t.path, line, why: `literal list of ${hit.size} sibling ${kind === "files" ? "file names" : "export names"}` }); continue; }
+    if (kind !== "exports") continue;
+    for (const b of importBindings(t.text, t.path, module, pkgs)) {
+      const m = new RegExp(`Object\\.(?:keys|entries)\\(\\s*${esc(b)}\\s*\\)`).exec(t.text);
+      if (m) { out.push({ test: t.path, line: t.text.slice(0, m.index).split("\n").length, why: `Object.keys(${b})` }); break; }
+    }
+  }
+  return out.slice(0, MAX_TESTS_PER_MODULE);
+}
+
+const fileGroup = (files, dir, exclude = new Set()) => [...new Set(files.filter((f) => dirname(f) === dir && !isTestPath(f) && !exclude.has(f) && stem(f) !== "index").map(stem))];
+
+/**
+ * Plan time: tests enumerating the exports or directory siblings of each path the issue names.
+ * @returns {Promise<Array<{module:string, test:string, line:number, why:string}>>}
+ */
+export async function enumeratingTests({ git, cwd, paths = [] }) {
+  try {
+    const { tests, files, pkgs } = await loadTests(git, cwd);
+    if (!tests.length) return [];
+    const tracked = new Set(files), out = [], seen = new Set();
+    for (const raw of [...new Set(paths.map((p) => String(p).replace(/^\.\//, "").replace(/\/$/, "")))].slice(0, MAX_MODULES)) {
+      const groups = [];
+      if (tracked.has(raw) && SOURCE_RE.test(raw) && !isTestPath(raw)) {
+        let text = ""; try { text = fs.readFileSync(join(cwd, raw), "utf8"); } catch { /* sparse */ }
+        groups.push({ module: raw, names: exportNames(text, raw), kind: "exports" });
+        groups.push({ module: dirname(raw), names: fileGroup(files, dirname(raw)), kind: "files" });
+      } else if (files.some((f) => dirname(f) === raw)) groups.push({ module: raw, names: fileGroup(files, raw), kind: "files" });
+      for (const g of groups) for (const e of enumerators(tests, g, pkgs)) {
+        const k = `${g.module}\0${e.test}`;
+        if (!seen.has(k)) { seen.add(k); out.push({ module: g.module, ...e }); }
+      }
+    }
+    return out;
+  } catch { return []; }
+}
+
+/**
+ * After implement: each new export (vs `base`) of a changed source file and each new file, paired with every test that
+ * enumerates its existing siblings but never names it. @returns {Promise<Array<{symbol, module, test, line}>>}
+ */
+export async function missingEntries({ git, cwd, base }) {
+  try {
+    const changed = lines(await git(["-c", "core.quotepath=off", "diff", "--name-only", "--diff-filter=AM", `${base}...HEAD`]));
+    const added = new Set(lines(await git(["-c", "core.quotepath=off", "diff", "--name-only", "--diff-filter=A", `${base}...HEAD`])));
+    const groups = [];
+    for (const f of changed) {
+      if (isTestPath(f) || SKIP_RE.test(f)) continue;
+      if (SOURCE_RE.test(f)) {
+        let head = "", old = "";
+        try { head = fs.readFileSync(join(cwd, f), "utf8"); } catch { continue; }
+        if (!added.has(f)) { try { old = String(await git(["show", `${base}:${f}`])); } catch { old = ""; } }
+        const now = exportNames(head, f), was = new Set(exportNames(old, f));
+        const fresh = now.filter((n) => !was.has(n) && n.length >= MIN_NAME);
+        if (fresh.length) groups.push({ module: f, names: now.filter((n) => was.has(n)), fresh, kind: "exports" });
+      }
+    }
+    if (!groups.length && !added.size) return [];
+    const { tests, files, pkgs } = await loadTests(git, cwd);
+    if (!tests.length) return [];
+    const newFiles = [...added].filter((f) => !isTestPath(f) && !SKIP_RE.test(f));
+    for (const dir of [...new Set(newFiles.map(dirname))]) {
+      const fresh = newFiles.filter((f) => dirname(f) === dir).map(stem).filter((s) => s.length >= MIN_NAME && s !== "index");
+      if (fresh.length) groups.push({ module: dir, names: fileGroup(files, dir, added), fresh, kind: "files" });
+    }
+    const out = [];
+    for (const g of groups.slice(0, MAX_MODULES)) {
+      for (const e of enumerators(tests, g, pkgs)) {
+        const text = tests.find((t) => t.path === e.test)?.text ?? "";
+        for (const s of g.fresh) if (!new RegExp(`(?<![\\w$])${esc(s)}(?![\\w$])`).test(text)) out.push({ symbol: s, module: g.module, test: e.test, line: e.line });
+      }
+    }
+    return out;
+  } catch { return []; }
+}
+
+/** Drop entries the builder justified in result.json "not_members" [{symbol, test, why}] (a non-empty why is required). */
+export function unjustified(missing, notMembers) {
+  const ok = new Set((Array.isArray(notMembers) ? notMembers : []).filter((n) => n && String(n.why ?? "").trim()).map((n) => `${n.symbol}\0${n.test}`));
+  return (missing ?? []).filter((m) => !ok.has(`${m.symbol}\0${m.test}`));
+}

package/bin/lean/enumerating.test.mjs

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { exportNames, enumeratingTests, missingEntries, unjustified } from "./enumerating.mjs";
+import { enumeratingPlanNote, enumeratingBuildRule, missingEntriesPrompt, builderPlanPrompt } from "./prompts.mjs";
+
+const sh = (cwd, ...a) => execFileSync("git", a, { cwd, encoding: "utf8", env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" } });
+const gitIn = (cwd) => async (a) => sh(cwd, ...a);
+const put = (root, files) => { for (const [p, t] of Object.entries(files)) { mkdirSync(dirname(join(root, p)), { recursive: true }); writeFileSync(join(root, p), t); } };
+function repo(files) {
+  const d = tmpDir("enum-");
+  sh(d, "init", "-q", "-b", "main");
+  put(d, files);
+  sh(d, "add", "-A"); sh(d, "commit", "-qm", "base");
+  return d;
+}
+
+const CORE = "export async function getUser() {}\nexport const listUsers = () => [];\nexport function deleteUser() {}\nexport { helperA as renamedHelper, type Foo } from './x';\n";
+const FILES = {
+  "packages/core/package.json": JSON.stringify({ name: "@acme/core" }),
+  "packages/core/src/index.ts": CORE,
+  "packages/core/test/registry.test.ts": "import * as core from '../src/index';\nconst SERVICE_TESTS = { getUser: 1, listUsers: 1 };\nconst NOT_A_SERVICE = ['deleteUser', 'renamedHelper'];\nexpect(Object.keys(core).filter((k) => !(k in SERVICE_TESTS) && !NOT_A_SERVICE.includes(k))).toEqual([]);\n",
+  "packages/web/test/pkg.test.ts": "import * as c from '@acme/core';\nit('all', () => expect(Object.keys(c).length).toBe(4));\n",
+  "packages/core/test/unrelated.test.ts": "import { getUser } from '../src/index';\nit('x', () => getUser());\n",
+  "locales/en/common.json": "{}", "locales/en/auth.json": "{}", "locales/en/billing.json": "{}",
+  "tests/catalog.test.ts": "const NAMESPACES = ['common', 'auth', 'billing'];\n",
+};
+
+describe("enumerating tests (#4322)", () => {
+  it("reads JS/TS and Python exports", () => {
+    assert.deepEqual(exportNames(CORE, "a.ts").sort(), ["deleteUser", "getUser", "listUsers", "renamedHelper"]);
+    assert.deepEqual(exportNames("__all__ = ['a', \"bb\"]\ndef c(): pass", "m.py"), ["a", "bb"]);
+    assert.deepEqual(exportNames("def run():\n  pass\nclass Job: pass\ndef _private(): pass", "m.py"), ["run", "Job"]);
+  });
+
+  it("plan: lists every test enumerating the exports of a module the issue names (literal list, Object.keys of a relative or package import)", async () => {
+    const d = repo(FILES);
+    const list = await enumeratingTests({ git: gitIn(d), cwd: d, paths: ["packages/core/src/index.ts"] });
+    const tests = list.filter((e) => e.module === "packages/core/src/index.ts").map((e) => e.test).sort();
+    assert.deepEqual(tests, ["packages/core/test/registry.test.ts", "packages/web/test/pkg.test.ts"]);
+    assert.ok(!list.some((e) => e.test.endsWith("unrelated.test.ts")), "a test importing one export does not enumerate them");
+    assert.match(list.find((e) => e.test.endsWith("registry.test.ts")).why, /literal list of [34] sibling export names/);
+    assert.match(list.find((e) => e.test.endsWith("pkg.test.ts")).why, /^Object\.keys\(c\)$/);
+    const files = await enumeratingTests({ git: gitIn(d), cwd: d, paths: ["locales/en"] });
+    assert.deepEqual(files.map((e) => [e.module, e.test, e.line]), [["locales/en", "tests/catalog.test.ts", 1]]);
+  });
+
+  it("plan note and build rule cite the list; no enumerating tests means no new plan step (prompt unchanged)", async () => {
+    const d = repo(FILES);
+    assert.deepEqual(await enumeratingTests({ git: gitIn(d), cwd: d, paths: ["README.md", "packages/core/package.json"] }), []);
+    assert.equal(enumeratingPlanNote([]), ""); assert.equal(enumeratingPlanNote(null), "");
+    assert.equal(enumeratingBuildRule([]), "");
+    const list = [{ module: "packages/core/src/index.ts", test: "packages/core/test/registry.test.ts", line: 2, why: "literal list of 4 sibling export names" }];
+    assert.match(enumeratingPlanNote(list), /Enumerating tests[\s\S]*Affected Files[\s\S]*validate command[\s\S]*- packages\/core\/src\/index\.ts -> packages\/core\/test\/registry\.test\.ts:2/);
+    const rule = enumeratingBuildRule(list);
+    assert.match(rule, /one test-list edit per new symbol/);
+    assert.match(rule, /"not_members"/);
+    assert.match(rule, /registry\.test\.ts:2/);
+    assert.ok(!builderPlanPrompt(7).includes("Enumerating tests"));
+  });
+
+  it("after implement: a new export missing from an enumerating test is reported; a listed one or a justified not-member is not", async () => {
+    const d = repo(FILES);
+    sh(d, "checkout", "-qb", "lane");
+    put(d, { "packages/core/src/index.ts": `${CORE}export function getProfile() {}\nexport function updateProfile() {}\n` });
+    put(d, { "packages/core/test/registry.test.ts": FILES["packages/core/test/registry.test.ts"].replace("listUsers: 1", "listUsers: 1, updateProfile: 1") });
+    put(d, { "locales/en/profile.json": "{}" });
+    sh(d, "add", "-A"); sh(d, "commit", "-qm", "lane");
+    const miss = await missingEntries({ git: gitIn(d), cwd: d, base: "main" });
+    const keys = miss.map((m) => `${m.symbol} -> ${m.test}`).sort();
+    assert.deepEqual(keys, [
+      "getProfile -> packages/core/test/registry.test.ts",
+      "getProfile -> packages/web/test/pkg.test.ts",
+      "profile -> tests/catalog.test.ts",
+      "updateProfile -> packages/web/test/pkg.test.ts",
+    ]);
+    const left = unjustified(miss, [{ symbol: "getProfile", test: "packages/web/test/pkg.test.ts", why: "count test, not a list" }, { symbol: "profile", test: "tests/catalog.test.ts", why: "" }]);
+    assert.equal(left.length, 3, "a not-member needs a non-empty why");
+    assert.match(missingEntriesPrompt(miss), /getProfile \(packages\/core\/src\/index\.ts\) -> packages\/core\/test\/registry\.test\.ts:\d+[\s\S]*not_members/);
+  });
+
+  it("a lane that adds no enumerable symbol gets nothing; a broken git never throws", async () => {
+    const d = repo(FILES);
+    sh(d, "checkout", "-qb", "lane");
+    put(d, { "packages/core/src/index.ts": `${CORE}// comment only\n` });
+    sh(d, "add", "-A"); sh(d, "commit", "-qm", "lane");
+    assert.deepEqual(await missingEntries({ git: gitIn(d), cwd: d, base: "main" }), []);
+    const bad = async () => { throw new Error("boom"); };
+    assert.deepEqual(await missingEntries({ git: bad, cwd: d, base: "main" }), []);
+    assert.deepEqual(await enumeratingTests({ git: bad, cwd: d, paths: ["x"] }), []);
+  });
+});