forgedock 1.44.0 → 1.45.0

package/bin/engine-cli.mjs

@@ -525,10 +525,12 @@ export function readEnvConfig(cwd = process.cwd()) {
     const e = y?.env ?? {};
     const tr = y?.verification?.test_runners; // #4857: {path prefix: runner command prefix} for validate commands
     const runners = tr && typeof tr === "object" && !Array.isArray(tr) ? Object.fromEntries(Object.entries(tr).filter(([k, v]) => k && typeof v === "string" && v.trim())) : {};
+    const rm = readYamlMap(cwd, "validate", "recipes"); // #5020: y?.validate?.recipes, a block map {match regex: replacement} applied to frozen validate and repro commands
+    const recipes = Object.entries(rm ?? {}).map(([match, replace]) => ({ match, replace }));
     const s = (v) => (typeof v === "string" && v.trim() ? v.trim() : undefined);
     const kb = typeof e.sparse_max_kb === "string" && /^\d+$/.test(e.sparse_max_kb.trim()) ? Number(e.sparse_max_kb) : e.sparse_max_kb;
     return { ...(s(e.toolchain) ? { toolchainOverride: s(e.toolchain) } : {}), ...(s(e.setup) ? { envSetup: s(e.setup) } : {}),
-      ...(Number.isInteger(kb) && kb >= 0 ? { sparseMaxKb: kb } : {}), ...(Object.keys(runners).length ? { testRunners: runners } : {}) };
+      ...(Number.isInteger(kb) && kb >= 0 ? { sparseMaxKb: kb } : {}), ...(Object.keys(runners).length ? { testRunners: runners } : {}), ...(recipes.length ? { validateRecipes: recipes } : {}) };
   } catch { return {}; }
 }
 export const ENGINE_PATHS = ["core", "full", "lean", "auto"];
@@ -1762,12 +1764,13 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
         return (n) => execMod.placeLane({ exec, verifyMode, labels: labelsOf.get(n) ?? [], weight, headroom: headroom(), needs: needsOf.get(n) ?? [], relays });
       })();
       // #4540: provider free slots, refreshed in the background (admission is sync); unknown (null) never blocks.
-      let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity; // #4547: online = agents alive, null unknown
+      let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity, capFailed = false; // #5048: capFailed = last probe rejected // #4547: online = agents alive, null unknown
       const capacity = () => {
+        if (capFailed) throw new Error("capacity probe failed"); // #5048: until the next successful sample, so every caller sees it (release review CONC-1)
         const t = deps.now ? deps.now() : Date.now();
         if (typeof provider.capacity === "function" && t - capLast >= (deps.pollMs ?? 5000)) {
           capLast = t;
-          Promise.resolve().then(() => provider.capacity()).then((c) => { capSample = c && Number.isInteger(c.free) ? c.free : null; onlineSample = c && Number.isInteger(c.online) ? c.online : null; capAt++; }, () => { capSample = null; onlineSample = null; capAt++; });
+          Promise.resolve().then(() => provider.capacity()).then((c) => { capFailed = false; capSample = c && Number.isInteger(c.free) ? c.free : null; onlineSample = c && Number.isInteger(c.online) ? c.online : null; capAt++; }, () => { capSample = null; onlineSample = null; capFailed = true; capAt++; });
         }
         return capSample === null && onlineSample === null ? null : { free: capSample, online: onlineSample, at: capAt };
       };

package/bin/lean/conformance.mjs

@@ -431,7 +431,7 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       // #4857: a runnable equivalent of an unrunnable frozen validate command; same id, same checks as a planned command
       const v = (c.validate ?? []).find((x) => x.id === a.validate);
       if (!v) { problems.push(`substitute-validate names unknown validate ${a.validate}`); continue; }
-      const bad = blank(a.command) ? "no command" : shellSyntaxError(a.command) ? "not valid shell (bash -n)" : noopCommand(a.command) ? "runs no check (true, :, exit 0, echo)" : movingBaseDiff(a.command) ? "diffs against a moving base tip" : tmpdirInputs(a.command).length ? "reads a $TMPDIR file it did not create" : null;
+      const bad = blank(a.command) ? "no command" : substituteShellProblem(a.command) ? "not valid shell (bash -n)" : noopCommand(a.command) ? "runs no check (true, :, exit 0, echo)" : movingBaseDiff(a.command) ? "diffs against a moving base tip" : tmpdirInputs(a.command).length ? "reads a $TMPDIR file it did not create" : null;
       if (bad) { problems.push(`substitute-validate ${a.validate}: ${bad}`); continue; }
       // SEC-11: a second substitute of the same id keeps the frozen command as `from`, so the engine always reruns the original
       v.substituted = { from: v.substituted?.from ?? v.command, why: String(a.why).trim() }; v.command = String(a.command).trim();
@@ -480,7 +480,9 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       if (!r) { problems.push(`amendment names unknown requirement ${a.requirement}`); continue; }
       if (!TRACKED.includes(r.disposition)) {
         if (a.bucket != null) r.bucket = a.bucket; // #4724
-        if (a.kind === "post-deploy" && !CHECK_ID_RE.test(String(a.check ?? r.check ?? ""))) { problems.push(`amendment post-deploy ${a.requirement} needs a check id (check)`); continue; }
+        if (a.kind === "post-deploy" && !CHECK_ID_RE.test(String(a.check ?? r.check ?? ""))) { // #5020: no runnable check: a manual or credentialed action is deferred to the orchestrator, never a conformance failure
+          r.disposition = "defer"; r.why = a.why; r.amended = true; r.bucket = String(a.bucket ?? "").startsWith("owner:") ? a.bucket : ORCHESTRATOR_BUCKET; toTrack.push(r); continue;
+        }
         r.disposition = a.kind; r.why = a.why; r.amended = true; r.bucket ??= "post-deploy"; toTrack.push(r); // #4724: an amendment without a bucket defaults to post-deploy
         if (a.kind === "post-deploy") r.check = a.check ?? r.check; // #4626
       }
@@ -508,6 +510,7 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
 
 const PY_MISSING_RE = /ModuleNotFoundError: No module named '([^']+)'|ImportError: No module named '?([\w.]+)'?/;
 const NODE_MISSING_RE = /Cannot find (?:module|package) '([^']+)'/;
+const POETRY_NOT_FOUND_RE = /^\s*Command not found: (\S+)/m;
 const CMD_NOT_FOUND_RE = /([^\s:'"`]+): command not found/, SH_NOT_FOUND_RE = /([^\s:'"`]+): not found\b/;
 /** #4857: why a command is not valid bash (`bash -n`), or null when it parses (or bash is unavailable). Catches prose left on a command. */
 export function shellSyntaxError(command) {
@@ -515,6 +518,49 @@ export function shellSyntaxError(command) {
   if (r.error || r.status === 0 || r.status == null) return null;
   return String(r.stderr || "syntax error").trim().split("\n")[0].replace(/^bash: (?:line \d+: )?/, "").slice(0, 160);
 }
+/** #5020: like shellSyntaxError, but fails closed: when bash cannot run the check the command is not shown to parse. */
+export function substituteShellProblem(command) {
+  const r = spawnSync("bash", ["-n", "-c", String(command ?? "")], { encoding: "utf8", timeout: 5000 });
+  if (r.error || r.status == null) return "bash -n could not run, so the command is unverified";
+  return r.status === 0 ? null : shellSyntaxError(command) ?? "syntax error";
+}
+export const ORCHESTRATOR_BUCKET = "owner:orchestrator";
+/**
+ * #5020: a post-deploy requirement that names no check id (a manual or production-credential action) becomes a `defer` with an
+ * `owner:orchestrator` bucket, the legal disposition the engine files as a follow-up. Pure; returns {contract, converted[]}.
+ */
+export function deferChecklessPostDeploy(contract) {
+  const converted = [];
+  const requirements = (Array.isArray(contract?.requirements) ? contract.requirements : []).map((r) => {
+    if (r?.disposition !== "post-deploy" || CHECK_ID_RE.test(String(r?.check ?? ""))) return r;
+    converted.push(r.id);
+    return { ...r, disposition: "defer", bucket: String(r.bucket ?? "").startsWith("owner:") ? r.bucket : ORCHESTRATOR_BUCKET, why: r.why ?? "post-deploy action with no registered check: deferred to the orchestrator" };
+  });
+  return { contract: converted.length ? { ...contract, requirements } : contract, converted };
+}
+/**
+ * #5020: apply forge.yaml `validate.recipes` ([{match, replace}], match a regex source) to each validate command and the
+ * repro_test command at freeze, first matching recipe per command. Pure; returns {contract, applied[{id, from, to}]}.
+ */
+export function applyValidateRecipes(contract, recipes) {
+  const list = (Array.isArray(recipes) ? recipes : []).map((r) => { try { return typeof r?.replace === "string" && r?.match ? { re: new RegExp(String(r.match)), replace: r.replace } : null; } catch { return null; } }).filter(Boolean);
+  if (!list.length || !contract || typeof contract !== "object") return { contract, applied: [] };
+  const applied = [];
+  const rewrite = (id, cmd) => {
+    if (typeof cmd !== "string") return cmd;
+    for (const { re, replace } of list) if (re.test(cmd)) { const to = cmd.replace(re, replace); if (to !== cmd) applied.push({ id, from: cmd, to }); return to; }
+    return cmd;
+  };
+  const out = { ...contract };
+  if (Array.isArray(contract.validate)) out.validate = contract.validate.map((v) => (v && typeof v === "object" ? { ...v, command: rewrite(v.id, v.command) } : v));
+  const t = contract.repro_test;
+  if (t && typeof t === "object" && !Array.isArray(t)) out.repro_test = { ...t, command: rewrite("repro_test", t.command) };
+  return { contract: applied.length ? out : contract, applied };
+}
+/** #5020: the env.* verdict for a repro command that failed on head because a tool is missing (e.g. an uninstalled Poetry env); null when the failure is the test's own. */
+export function reproEnvMiss(run, opts) {
+  return run && run.rc !== 0 && run.rc !== 124 ? missingTool(run, opts) : null;
+}
 const NOOP_SEGMENT_RE = /^(?:true|false|:|exit(?:\s+\d+)?|echo(?:\s.*)?|printf(?:\s.*)?|sleep(?:\s+[\d.]+)?|test\s+-?\w+|\[\s+-?\w+\s+\]|fi|done|node\s+-e\s+['"]?[\d;\s]*['"]?|python3?\s+-c\s+['"]?(?:pass|0|None)?['"]?)$/;
 const KEYWORD_RE = /^(if|then|else|elif|do|while|until)(?:\s+|$)/;
 const CONST_FALSE_RE = /^(?:false|\[\s+0\s+=\s+1\s+\]|\[\s+1\s+=\s+0\s+\]|test\s+0\s+=\s+1|test\s+-n\s+(?:""|'')|\[\s+-n\s+(?:""|'')\s+\])$/;
@@ -645,6 +691,8 @@ export function missingTool({ rc, output } = {}, { repoHas = () => false, comman
     const tool = scoped ? spec.split("/").slice(0, 2).join("/") : spec;
     return local(tool) || (scoped && !tool.includes("/")) ? null : { tool, kind: "node-module" };
   }
+  const po = POETRY_NOT_FOUND_RE.exec(out)?.[1]; // #5020: `poetry run pytest` in an env that was never installed
+  if (po) return local(po) ? null : { tool: po, kind: "command" };
   const cmd = CMD_NOT_FOUND_RE.exec(out)?.[1] ?? (rc === 127 ? SH_NOT_FOUND_RE.exec(out)?.[1] ?? String(command).trim().split(/\s+/)[0] : null);
   if (!cmd || cmd.includes("/") || local(cmd)) return null;
   return { tool: cmd, kind: "command" };
@@ -688,6 +736,8 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
   for (const v of contract.validate ?? []) {
     // #4979 SPEC-2: a substitute counts only when the frozen command it replaced cannot run (or passes); checked first, never CI-covered
     if (v.substituted) {
+      const shellBad = substituteShellProblem(v.command); // #5020: re-checked before the substitute is used
+      if (shellBad) { failures.push({ code: C.CONTRACT_INVALID_COMMAND, kind: "validate_command_invalid", detail: `validate ${v.id} substitute \`${v.command}\` is not valid shell (bash -n: ${shellBad})` }); continue; }
       if (removedPaths === null) { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_unverified", detail: `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\`, but the engine could not read the diff's removed paths to show the frozen command's target survives` }); continue; }
       const gone = removedNamedPaths(v.substituted.from, removedPaths, baseFiles);
       if (gone.length) { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_removed_path", detail: `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\`, but the diff removes ${gone.map((x) => `\`${x}\``).join(", ")} that the frozen command names: a substitute may not replace a check by deleting its target` }); continue; }

package/bin/lean/conformance.test.mjs

@@ -3,7 +3,7 @@
 import { readFileSync } from "node:fs";
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
+import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables, applyValidateRecipes, deferChecklessPostDeploy, substituteShellProblem, reproEnvMiss, ORCHESTRATOR_BUCKET } from "./conformance.mjs";
 
 const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
 const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -910,9 +910,10 @@ describe("post-deploy check ids (#4626)", () => {
     assert.ok(validateContract(pd({ check: "Bad Id" }), ISSUE).problems.some((p) => /post-deploy needs a check id/.test(p)));
     assert.ok(!validateContract(pd({ check: "login-smoke" }), ISSUE).problems.some((p) => /check id/.test(p)));
   });
-  it("applyAmendments: a post-deploy amendment needs a check id", () => {
+  it("applyAmendments: a post-deploy amendment without a check id is deferred to the orchestrator (#5020)", () => {
     const bad = applyAmendments(C, [{ kind: "post-deploy", requirement: "R1", why: "w" }], ISSUE);
-    assert.ok(bad.problems.some((p) => /needs a check id/.test(p)));
+    assert.deepEqual(bad.problems, []);
+    assert.equal(bad.contract.requirements[0].disposition, "defer");
     const ok = applyAmendments(C, [{ kind: "post-deploy", requirement: "R1", why: "w", check: "login-smoke" }], ISSUE);
     assert.equal(ok.contract.requirements[0].check, "login-smoke");
   });
@@ -1213,3 +1214,56 @@ describe("builder amendment slips (#5036)", () => {
     assert.match(applyAmendments(C, [{ kind: "flaky-note", why: "w" }]).problems[0], /unknown kind 'flaky-note'; valid kinds are .*substitute-validate/);
   });
 });
+
+describe("#5020 freeze-time environment defects", () => {
+  const base = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], validate: [{ id: "V1", command: "cd services/api && poetry run pytest tests/x.py" }, { id: "V2", command: "node --test a.test.mjs" }], repro_test: { path: "t.py", id: "t", command: "cd services/api && poetry run pytest t.py" } };
+  const recipes = [{ match: "poetry run pytest", replace: "uv run --with-requirements r.txt pytest" }];
+
+  it("applyValidateRecipes rewrites matching validate and repro commands and leaves the rest", () => {
+    const { contract, applied } = applyValidateRecipes(base, recipes);
+    assert.equal(contract.validate[0].command, "cd services/api && uv run --with-requirements r.txt pytest tests/x.py");
+    assert.equal(contract.validate[1].command, "node --test a.test.mjs");
+    assert.match(contract.repro_test.command, /uv run --with-requirements r\.txt pytest t\.py/);
+    assert.deepEqual(applied.map((a) => a.id), ["V1", "repro_test"]);
+    assert.equal(base.validate[0].command.includes("poetry"), true, "input is not mutated");
+  });
+  it("applyValidateRecipes ignores no recipes and invalid regexes", () => {
+    assert.equal(applyValidateRecipes(base, null).contract, base);
+    assert.equal(applyValidateRecipes(base, [{ match: "(", replace: "x" }]).applied.length, 0);
+  });
+  it("Poetry's missing-env output is env.tool_missing material, and a repro head that dies on it is an env miss", () => {
+    const run = { rc: 1, output: "Creating virtualenv api-py3.12\n\nCommand not found: pytest\n" };
+    assert.deepEqual(missingTool(run), { tool: "pytest", kind: "command" });
+    assert.deepEqual(reproEnvMiss(run), { tool: "pytest", kind: "command" });
+    assert.equal(reproEnvMiss({ rc: 1, output: "1 failed" }), null);
+    // release review BUG-1: a module the repo tracks is a real failure of the change, not an env miss
+    const mod = { rc: 1, output: "ModuleNotFoundError: No module named 'mypkg'" };
+    assert.deepEqual(reproEnvMiss(mod, { command: "pytest t.py" }), { tool: "mypkg", kind: "python-module" });
+    assert.equal(reproEnvMiss(mod, { command: "pytest t.py", repoHas: repoTracks(["src/mypkg/__init__.py"]) }), null);
+    assert.equal(reproEnvMiss({ rc: 0, output: "" }), null);
+  });
+  it("a post-deploy requirement with no check id is deferred to the orchestrator; one with a check is untouched", () => {
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "post-deploy", why: "needs prod creds" }, { id: "R2", quote: "q", disposition: "post-deploy", check: "login-smoke", why: "w" }] };
+    const { contract, converted } = deferChecklessPostDeploy(c);
+    assert.deepEqual(converted, ["R1"]);
+    assert.equal(contract.requirements[0].disposition, "defer");
+    assert.equal(contract.requirements[0].bucket, ORCHESTRATOR_BUCKET);
+    assert.equal(contract.requirements[1].disposition, "post-deploy");
+    assert.deepEqual(bucketProblems(contract.requirements[0]), []);
+  });
+  it("a post-deploy amendment with no check becomes a tracked defer instead of a problem", () => {
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], deliverables: [], validate: [] };
+    const r = applyAmendments(c, [{ kind: "post-deploy", requirement: "R1", why: "manual prod action" }], "q");
+    assert.deepEqual(r.problems, []);
+    assert.equal(r.contract.requirements[0].disposition, "defer");
+    assert.equal(r.contract.requirements[0].bucket, ORCHESTRATOR_BUCKET);
+  });
+  it("a substitute-validate that is not valid shell is rejected", () => {
+    assert.match(substituteShellProblem("pytest tests/ && ("), /./);
+    assert.equal(substituteShellProblem("pytest tests/"), null);
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], deliverables: [], validate: [{ id: "V1", command: "poetry run pytest" }] };
+    const r = applyAmendments(c, [{ kind: "substitute-validate", validate: "V1", command: "uv run pytest tests/ && (", why: "poetry env missing" }], "q");
+    assert.ok(r.problems.some((p) => /not valid shell/.test(p)));
+    assert.equal(r.contract.validate[0].command, "poetry run pytest");
+  });
+});

package/bin/lean/execution.mjs

@@ -562,15 +562,18 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
     for (;;) {
       try { const v = await fn(); st.waited = 0; st.tries = 0; return v; } catch (e) {
         if (!isTransportError(e)) throw e;
-        if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true });
+        // release review BUG-6: a 401 that outlasts the window is an auth failure (capped sandbox.auth), not an uncapped unreachable pool
+        if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true, ...(e?.status === 401 ? { auth: true, hop: e.hop } : {}) });
         const d = Math.min(pollMs * 2 ** Math.min(st.tries++, 10), 30000, Math.max(1, unreachableWindowMs - st.waited));
         st.waited += d;
         await sleep(d);
       }
     }
   };
-  const unreachable = (h, e) => ({ terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
-    detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
+  const unreachable = (h, e) => (e?.auth
+    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${h.id} kept answering 401 (${e.hop === "agent" ? "the pool agent refused the router" : "check the pool token"}): ${errText(e)}; kept for adopt on resume` }
+    : { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
+      detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
   const tailOf = async (h) => { try { return typeof logTail === "function" ? String(await logTail(h) ?? "") : ""; } catch { return ""; } };
   const settleClone = (r, id) => (r?.terminalReason === "engine-error" && r.code === undefined && isCloneFailure(r.detail) ? cloneDeferred(id, errLine(r.detail)) : r);
   const wait = async (h, g = guard({ waited: 0, tries: 0 })) => {
@@ -590,7 +593,7 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
   // A start failure repeats on a relaunch: typed, never engine-error (#4254). Retryable (capacity) defers with backoff.
   // #4685: a refused or missing credential defers with its own code (a fixed token is picked up on the retry), never needs-human.
   const startFailed = (id, e) => ([401, 403].includes(e?.status)
-    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)` }
+    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; ${e?.hop === "agent" ? "the pool agent refused the router (re-enroll or fix the agent token)" : "check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)"}` }
     : e?.retryable === true
     ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_CAPACITY, detail: `sandbox ${id} could not start (capacity): ${errText(e)}`, ...(Number(e.retryInSeconds) > 0 ? { retryInSeconds: Number(e.retryInSeconds) } : {}) }
     : isCloneFailure(errText(e)) ? cloneDeferred(id, errText(e))

package/bin/lean/execution.test.mjs

@@ -116,6 +116,27 @@ describe("providerLanes: an unreachable provider is retried, then deferred witho
     assert.deepEqual(ledgerFor(7).entries("sandbox").map((e) => e.value), ["fake-7"]);
   });
 
+  it("#5053 a 401 on status/result of a launched sandbox is retried, then ends sandbox.auth (capped), never engine-error", async () => {
+    const e401 = () => Object.assign(new Error("sandbox status: HTTP 401 (agent a1 auth)"), { status: 401, hop: "agent", transient: true });
+    const mk = (failPolls) => { const prov = flaky(0); let polls = 0; prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
+      prov.status = async (h) => { if (polls++ < failPolls) throw e401(); return prov.live.has(h.issue) ? "running" : "exited"; }; return prov; };
+    const ok = mk(2); ok.results[7] = { terminalReason: "merged", pr: 1 }; ok.live.delete(7);
+    assert.equal((await lanes(ok).l.runOne(7, "staging")).terminalReason, "merged");
+    const down = mk(1e9);
+    const r = await lanes(down, { unreachableWindowMs: 5000 }).l.runOne(7, "staging");
+    // release review BUG-6: a persistent 401 is sandbox.auth (capped by max_deferrals), not the uncapped sandbox.unreachable
+    assert.deepEqual([r.terminalReason, r.code, r.retryInSeconds], ["deferred", "sandbox.auth", 300]);
+    assert.match(r.detail, /agent a1 auth/);
+    assert.match(r.detail, /pool agent refused the router/);
+  });
+
+  it("#5053 launch 401 from the agent hop says the agent refused, not the forgedock token", async () => {
+    const prov = fakeProvider(); prov.start = async () => { throw Object.assign(new Error("sandbox launch: HTTP 401 (agent a1 auth)"), { status: 401, hop: "agent" }); };
+    const r = await lanes(prov).l.runOne(7, "staging");
+    assert.equal(r.code, "sandbox.auth");
+    assert.match(r.detail, /agent a1 auth.*pool agent refused the router/);
+  });
+
   it("unreachable: adopt applies the same tolerance, and a non-transport error is still engine-error", async () => {
     const down = flaky(1e9);
     const { l } = lanes(down, { unreachableWindowMs: 3000 });

package/bin/lean/lean.mjs

@@ -15,7 +15,7 @@ import { ciPollDelayMs, ENGINE_VERSION, DEFAULT_MAX_SESSION_LIMIT_PAUSES, DEFAUL
 import { openDependency, laneMergedPrs, dependencyOnLane, isTrustedComment, decomposedSubissues, isDecomposedRecord, reopenRevokedDecomposition } from "../engine/phases.mjs";
 import { createHash, randomBytes } from "node:crypto";
 import { otherLaneOwner } from "./lanelock.mjs";
-import { resolveForgedockPaths, describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
+import { applyValidateRecipes, deferChecklessPostDeploy, reproEnvMiss, resolveForgedockPaths, describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
 import { checkAdmission, hostParallelism, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
 import { namedPaths, isTrustedIssueComment } from "./prefilter.mjs";
 import { checksAt } from "./postdeploy-check.mjs";
@@ -64,7 +64,7 @@ import { buildResumeDigest } from "./resume.mjs";
 import { parsePredicate, evalPredicate, premiseAnnotation, registerAsk, registryPkgVersion, closedEdgeNote, onlyAsk, fileRemainder } from "./needs-human-premise.mjs";
 import { lightEligibility } from "./light.mjs";
 import { untrackToolState, unstageToolState, isToolState, clearLaneScratch } from "./toolstate.mjs";
-import { ensureSignedOff } from "./signoff.mjs";
+import { ensureSignedOff, salvageBranch, dropSalvageTracking } from "./signoff.mjs";
 import { createLedger, flattenPhases } from "./phaseledger.mjs";
 import { runLightStep, runLightTests, lightBuildPrompt, upgradeNote, renderLiteReceipt } from "./lightlane.mjs";
 
@@ -225,7 +225,7 @@ export function isProtectedLaneBranch(b, { issue, lane = null, branches = {}, de
  */
 export async function runCore(o) {
   const { issue, repo, lane: requestedLane = null, laneOverride = false, branches = {}, cwd, dir, io, builderModel = null, reviewerModel = DEFAULT_REVIEWER_MODEL, fixModel = null,
-    toolchainOverride = null, testRunners = null, envSetup = null, ciMaxMinutes = 360, conformance = "shadow", conformanceKinds = null, conformanceMutation = "off", reviewIncomplete = "needs-human", reproTest = "prefer", closeParent = false, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, cliIdleMinutes = 15, cliMaxMinutes = 180, cliBusyMinutes = 45, contextCeilingTokens = 120000, turnBudgets = null, laneLimits = null, maxMergeRetries = 5, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
+    toolchainOverride = null, testRunners = null, validateRecipes = null, envSetup = null, ciMaxMinutes = 360, conformance = "shadow", conformanceKinds = null, conformanceMutation = "off", reviewIncomplete = "needs-human", reproTest = "prefer", closeParent = false, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, cliIdleMinutes = 15, cliMaxMinutes = 180, cliBusyMinutes = 45, contextCeilingTokens = 120000, turnBudgets = null, laneLimits = null, maxMergeRetries = 5, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
   const now = io.now ?? (() => Date.now());
   const vcfg = { ...parseVerify({}), ...(o.verify ?? {}) };
   const learnedCommands = (Array.isArray(o.precheck?.commands) ? o.precheck.commands : []).map((c) => String(c ?? "").trim()).filter(Boolean);
@@ -657,8 +657,8 @@ export async function runCore(o) {
   const gitPush = async (args) => {
     if (!args.includes("--delete")) { await untrackToolState({ git, base, ev, keep: await keepPaths() });
       try { await signOffUnpushed(); }
-      catch (e) { // #4905: a failed rewrite must not cost the lane's finished commits: push the branch as-is before reporting
-        if (!e?.noIdentity) { try { await git(["push", "-u", "origin", branch]); ev({ event: "DCO_RESCUE_PUSH", branch }); } catch (pe) { ev({ event: "DCO_RESCUE_PUSH_FAILED", error: String(pe?.message ?? pe).slice(0, 200) }); } }
+      catch (e) { // #4905: a failed rewrite must not cost the lane's finished commits: push them as-is before reporting; #5047: to a salvage ref, never the lane branch (the sign-off range excludes origin/<branch>)
+        if (!e?.noIdentity) { try { const sha = String(await git(["rev-parse", "HEAD"])).trim(), ref = salvageBranch(issue, sha); await git(["push", "origin", `${sha}:refs/heads/${ref}`]); laneHead = { branch, sha, pushedTo: ref }; ev({ event: "DCO_RESCUE_PUSH", branch: ref }); } catch (pe) { ev({ event: "DCO_RESCUE_PUSH_FAILED", error: String(pe?.message ?? pe).slice(0, 200) }); } }
         throw e;
       }
     } // #4590: never push .forge-lean/ or tool caches this branch added
@@ -679,14 +679,14 @@ export async function runCore(o) {
       if (ahead) { laneHead = { branch, sha: String(await git(["rev-parse", "HEAD"])).trim(), pushedTo: null }; await gitPush(["push", "-u", "origin", branch]); laneHead.pushedTo = branch; }
       ev({ event: "CHECKPOINT", reason, dirty: Boolean(dirty), pushed: Boolean(ahead) });
       return true;
-    } catch (e) { ev({ event: "CHECKPOINT_FAILED", reason, error: String(e?.message ?? e).slice(0, 300) }); return laneHead ? await rescuePush(reason, e?.pushRejected) : false; }
+    } catch (e) { ev({ event: "CHECKPOINT_FAILED", reason, error: String(e?.message ?? e).slice(0, 300) }); return laneHead ? await rescuePush(reason, e?.pushRejected || e?.engineCode === OUTCOME_CODES.ENGINE_DCO_REWRITE) : false; }
   }
   /** #4958: the normal push failed (sign-off rewrite, rejection): push HEAD as is to the lane branch, else to a salvage branch, so a torn-down remote sandbox never takes the commits with it. */
   async function rescuePush(reason, rejected) {
     // #5014: gitPush untracks tool state and rewrites unpushed commits for DCO before it pushes, so HEAD may have moved since the capture
     try { const now = String(await git(["rev-parse", "HEAD"])).trim(); if (now) laneHead.sha = now; } catch { /* keep the captured sha */ }
     // a rejected lane-branch push (protected branch, permissions) would be rejected again: go straight to the salvage branch
-    for (const ref of [...(rejected ? [] : [branch]), `forge-salvage/${issue}-${laneHead.sha.slice(0, 10)}`]) {
+    for (const ref of [...(rejected ? [] : [branch]), salvageBranch(issue, laneHead.sha)]) {
       try {
         await git(["push", "origin", `${laneHead.sha}:refs/heads/${ref}`]);
         laneHead.pushedTo = ref;
@@ -1089,14 +1089,14 @@ export async function runCore(o) {
 
     // ---- 2. worktree (resume an open PR / remote branch) ----
     try { await io.git(["fetch", "origin", lane], { cwd }); } catch { /* offline: use the local ref */ }
-    let resumed = null;
+    let resumed = null, fromSalvage = false;
     try {
       const prs = JSON.parse(String(await ghc(["pr", "list", "-R", repo, "--head", branch, "--state", "open", "--json", "number,headRefOid,baseRefName"])) || "[]");
       const remote = String(await io.git(["ls-remote", "--heads", "origin", branch], { cwd })).trim();
       if (prs.length || remote) resumed = { pr: prs[0]?.number ?? null, base: prs[0]?.baseRefName ?? null };
       if (!prs.length && !remote) { // #4958: commits the exit rescue could only push to a salvage branch resume from there
         const salv = String(await io.git(["ls-remote", "--heads", "origin", `refs/heads/forge-salvage/${issue}-*`], { cwd })).split("\n").map((l) => l.trim().split(/\s+/)[1]).filter(Boolean).sort().at(-1);
-        if (salv) { await io.git(["fetch", "origin", `${salv}:refs/remotes/origin/${branch}`], { cwd }); resumed = { pr: null, base: null }; ev({ event: "RESUME_FROM_SALVAGE", ref: salv }); }
+        if (salv) { await io.git(["fetch", "origin", `${salv}:refs/remotes/origin/${branch}`], { cwd }); resumed = { pr: null, base: null }; fromSalvage = true; ev({ event: "RESUME_FROM_SALVAGE", ref: salv }); }
       }
     } catch { /* no resume info: start fresh */ }
     // An open PR on another base (a hand-retargeted PR, or one opened before lane resolution) is never merged (#4125).
@@ -1183,6 +1183,7 @@ export async function runCore(o) {
       wtInfo = await addWorktree(["worktree", "add", "-B", branch, wt, base], base);
     }
     wtCreated = true;
+    if (fromSalvage) await dropSalvageTracking({ git, branch }); // #5047: the salvaged commits are unpublished: the sign-off range must include them
     if (wtInfo?.sparse) {
       sparse = { excluded: wtInfo.excluded, maxKb: sparseMaxKb };
       ev({ event: "WORKTREE_SPARSE", excluded: wtInfo.excluded, excluded_bytes: wtInfo.bytes, max_kb: sparseMaxKb });
@@ -1286,7 +1287,13 @@ export async function runCore(o) {
         return null;
       }
       const rg = await heavyStep("precheck", () => io.redGreen({ cwd: wt, base, command: String(t.command), env: laneEnv() })).catch(() => null);
-      const words = rg?.base && rg?.head ? rg : { base: "UNRUNNABLE", head: "UNRUNNABLE" };
+      let words = rg?.base && rg?.head ? rg : { base: "UNRUNNABLE", head: "UNRUNNABLE" };
+      if (words.head === "FAILED" && io.runCommand) { // #5020: a head that failed because a tool or env is missing is env.repro_unrunnable, not a red-green verdict
+        const hr = await io.runCommand({ command: String(t.command), cwd: wt, timeoutMs: 120000, env: laneEnv() }).then((x) => x?.r ?? x).catch(() => null);
+        // release review BUG-1: a module the repo tracks is the change's own failure, never an env miss (same repoHas as the validate rerun)
+        const repoHas = repoTracks(String(await git(["ls-files"]).catch(() => "")).split("\n").map((x) => x.trim()).filter(Boolean));
+        if (reproEnvMiss(hr, { command: String(t.command), repoHas })) words = { ...words, head: "UNRUNNABLE" };
+      }
       ev({ event: "REPRO_RED_GREEN", ...words });
       T.decisions.push(`repro_test ${t.path ?? ""}${t.id ? ` ${t.id}` : ""}: BASE=${words.base} HEAD=${words.head}`);
       return words;
@@ -1391,6 +1398,14 @@ export async function runCore(o) {
      * #4328: probe the host for the tools and modules a contract's validate commands need; #4393: then flag `$NAME`s the lane
      * env leaves empty and run each remaining command once on the base worktree (io.preflightRun). The misses; [] without either hook.
      */
+    /** #5020: forge.yaml validate.recipes rewrite frozen commands; a check-less post-deploy requirement becomes a defer to the orchestrator. */
+    function freezeRewrites(c) {
+      const r = applyValidateRecipes(c, validateRecipes);
+      for (const x of r.applied) T.decisions.push(`validate recipe rewrote ${x.id}: \`${x.from}\` -> \`${x.to}\``);
+      const d = deferChecklessPostDeploy(r.contract);
+      if (d.converted.length) T.decisions.push(`Post-deploy with no registered check deferred to the orchestrator: ${d.converted.join(", ")}`);
+      return d.contract;
+    }
     async function preflightValidates(contract) {
       if ((!io.probeCommand && !io.preflightRun) || !contract || typeof contract !== "object") return [];
       const files = String(await git(["ls-files"]).catch(() => "")).split("\n").map((x) => x.trim()).filter(Boolean);
@@ -1667,6 +1682,7 @@ export async function runCore(o) {
           return o.contract;
         };
         let contract = dropScope(await readContract());
+        contract = freezeRewrites(contract);
         await refreshIssue("contract validation");
         let vc = validateContract(contract, issueText, { repro: reproOn() });
         const missed = await preflightValidates(contract); // #4328: validate commands the host cannot run share the one correction
@@ -1675,7 +1691,7 @@ export async function runCore(o) {
           const problems = [...vc.problems, ...preflightProblems(missed), ...testRunnerProblems(contract, testRunners)];
           T.decisions.push(`Contract rejected (${problems[0]}); resumed once`);
           await builder(++attemptN, contractCorrectionPrompt(problems), sessionId, false, planPrompt, { trigger: "contract_correction" });
-          contract = dropScope(await readContract());
+          contract = freezeRewrites(dropScope(await readContract()));
           await refreshIssue("contract validation");
           vc = validateContract(contract, issueText, { repro: reproOn() });
           if (missed.length) misses = await preflightValidates(contract);
@@ -2140,7 +2156,8 @@ export async function runCore(o) {
       const sig = JSON.stringify(a.applied);
       if (a.applied.length && sig !== lastAmend) {
         lastAmend = sig;
-        await post("FORGE:AMENDMENT", renderAmendments(a.applied, {}, { contract: hash, amendments: all }));
+        try { await post("FORGE:AMENDMENT", renderAmendments(a.applied, {}, { contract: hash, amendments: all })); } // #5020: retried by ghw; a still-failing post is an anomaly, never an engine-error
+        catch (e) { T.anomalies.push(`FORGE:AMENDMENT comment not posted: ${String(e?.message ?? e).slice(0, 160)}`); }
         T.decisions.push(`Contract amended: ${a.applied.map((x) => `${x.kind} ${x.target}`).join(", ")}`);
       }
       pendingDecisions = a.decisions; // #4737: persisted only once the PR merges

package/bin/lean/lean.test.mjs

@@ -3898,6 +3898,34 @@ describe("batch robustness (#4112-#4115)", () => {
     assert.ok(traj(w).some((c) => /Lane commits pushed: branch `forge-salvage\/7-/.test(c)));
   });
 
+  it("#5047 a failed DCO rewrite rescue-pushes to a salvage ref, never the lane branch", async () => {
+    const w = world();
+    const g = w.io.git;
+    const SEP = "\x1f", END = "\x1e";
+    w.io.git = async (a, o2) => {
+      if (a[0] === "log" && a.some((x) => String(x).startsWith("--format=%H"))) return `deadbeef${SEP}x@localhost${SEP}x@localhost${SEP}${SEP}parent${END}`;
+      if (a[0] === "config") return "Lane Bot";
+      if (a[0] === "filter-branch") throw new Error("boom");
+      return g(a, o2);
+    };
+    const r = await run(w);
+    const pushes = w.calls.filter((c) => c[0] === "git" && c[1] === "push").map((c) => c.join(" "));
+    assert.ok(pushes.length > 0);
+    assert.ok(pushes.some((p) => /origin [0-9a-z]+:refs\/heads\/forge-salvage\/7-/.test(p)), "the commits are published under a salvage ref");
+    assert.ok(!pushes.some((p) => /origin feat\/issue-7-lean$|:refs\/heads\/feat\/issue-7-lean$/.test(p)), `never the lane branch: ${pushes.join(" | ")}`);
+    assert.match(String(r.laneHead?.pushedTo), /^forge-salvage\/7-/);
+  });
+
+  it("#5047 a lane resumed from a salvage branch drops the origin/<branch> tracking ref so the sign-off range covers its commits", async () => {
+    const w = world({ prList: [] });
+    const g = w.io.git;
+    w.io.git = async (a, o2) => (a[0] === "ls-remote" && a.at(-1) === "refs/heads/forge-salvage/7-*" ? "abc\trefs/heads/forge-salvage/7-1234567890\n" : g(a, o2));
+    await run(w);
+    const wtAdd = w.calls.findIndex((c) => c[1] === "worktree" && c[2] === "add");
+    const drop = w.calls.findIndex((c) => c[1] === "update-ref" && c[2] === "-d" && c[3] === "refs/remotes/origin/feat/issue-7-lean");
+    assert.ok(wtAdd >= 0 && drop > wtAdd, "dropped after the worktree is built from it");
+  });
+
   it("#4958 when every rescue push is rejected the head is reported as NOT pushed", async () => {
     const msg = "remote: error: GH006: Protected branch update failed for refs/heads/x";
     const w = world({ pushErrors: [msg, msg, msg, msg] });
@@ -4048,6 +4076,13 @@ describe("batch robustness (#4112-#4115)", () => {
     assert.equal(n, 1);
   });
 
+  it("#5020 forge.yaml validate.recipes rewrite the frozen validate command and the freeze records it", async () => {
+    const w = world({});
+    const r = await run(w, { validateRecipes: [{ match: "^node --test ", replace: "node --test --test-reporter=dot " }] });
+    assert.equal(r.terminalReason, "merged");
+    assert.ok(w.comments.some((c) => c.includes("FORGE:CONTRACT") && c.includes("node --test --test-reporter=dot test/foo.test.js")));
+  });
+
   // ---- #4112 ----
   const fakeCli = async (name, body) => {
     const { writeFileSync, chmodSync } = await import("node:fs");

package/bin/lean/orchestrate-blocked.test.mjs

@@ -97,3 +97,26 @@ describe("remote lanes with a registry lacking the #4964 methods", () => {
     assert.deepEqual(calls, [], "a remote lane never takes a host slot");
   });
 });
+
+describe("#5048: the pool is unreachable only after a failed capacity probe", () => {
+  const run = async (capacity) => {
+    const notes = [];
+    const registry = { tryAcquire: () => ({ ok: true }), offer() {}, withdraw() {}, release() {}, setLanePid() {}, adoptSlot() {}, holder: () => null, notePool: (b, st) => notes.push(st === null ? "clear" : "down") };
+    const rows = await orchestrate({ issues: [4, 5, 6], parallel: 1, pollMs: 5, trusted: () => true, readBody: async () => "", sched: { registry, batch: "B", repo: "o/r" },
+      remote: { max: 1, isRemote: () => true, ...(capacity ? { capacity } : {}) },
+      runOne: async (n, lane, ctx) => { ctx.onStart?.({ handle: { id: `s-${n}` }, remote: true }); return merged; } });
+    assert.ok(rows.every((r) => r.outcome === "merged"));
+    return notes;
+  };
+  it("unknown capacity (null) does not mark the pool unreachable", async () => {
+    assert.deepEqual(await run(() => null), []);
+  });
+  it("a provider without capacity() never marks the pool unreachable", async () => {
+    assert.deepEqual(await run(null), []);
+  });
+  it("a throwing probe marks it down and a later successful sample clears it", async () => {
+    let calls = 0;
+    const notes = await run(() => { if (++calls === 1) throw new Error("boom"); return { free: 1, online: 1, at: calls }; });
+    assert.deepEqual(notes, ["down", "clear"]);
+  });
+});

package/bin/lean/orchestrate.mjs

@@ -516,9 +516,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       const adm = admitRemote();
       if (adm.ok) {
         // #4540: the provider's free slots (`remote.capacity() -> {free, at}`, sync, null = unknown) cap admissions per sample `at`; the issue queues here instead of a 503.
-        let cap = null;
-        try { cap = remote.capacity?.() ?? null; } catch { cap = null; }
-        if (typeof remote.capacity === "function") notePool(cap === null, "capacity probe failed");
+        let cap = null, probeFailed = false;
+        try { cap = remote.capacity?.() ?? null; } catch { probeFailed = true; }
+        // #5048: only a failed probe (a throw) marks the pool down; null is unknown (first pass, no capacity()) and leaves the state; a sample clears it.
+        if (probeFailed) notePool(true, "capacity probe failed"); else if (cap !== null) notePool(false);
         // #4788: zero live agents for longer than `remote.poolOfflineMin` (default 10): one loud batch event, none while online > 0.
         if (cap && cap.online === 0) {
           poolOfflineSince ??= now();

package/bin/lean/postdeploy-check.test.mjs

@@ -96,7 +96,9 @@ describe("post-deploy named checks (#4626)", () => {
     assert.ok(validateContract(c, Q).problems.some((p) => /needs a check id/.test(p)));
     assert.ok(!validateContract({ ...c, requirements: [{ ...c.requirements[0], check: "smoke" }] }, Q).problems.some((p) => /check id/.test(p)));
     const frozen = { ...c, requirements: [{ id: "R1", quote: Q, disposition: "deliver" }] };
-    assert.ok(applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w" }], Q).problems.some((p) => /check id/.test(p)));
+    const deferred = applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w" }], Q); // #5020: no check id defers to the orchestrator
+    assert.deepEqual(deferred.problems, []);
+    assert.equal(deferred.contract.requirements[0].disposition, "defer");
     const ok = applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w", check: "smoke" }], Q);
     assert.equal(ok.contract.requirements[0].check, "smoke");
     const pc = precheck({ contract: { ...c, requirements: [{ ...c.requirements[0], check: "ghost" }] }, result: { requirements: [] }, changedFiles: [], checkIds: new Set(["smoke"]) });

package/bin/lean/preflight.mjs

@@ -101,6 +101,12 @@ function moduleNeeds(exe, args) {
  * probe (tool) a module probe depends on: skip the module probe when its interpreter is missing. A name `repoHas(name)` holds
  * or a deliverable's change text names (the change adds it) is not probed; paths and `$...` words are never probed.
  */
+/** #5020: the contract's runnable commands: each validate entry plus the repro_test, which fails the same ways. */
+export function contractCommands(contract) {
+  const out = (Array.isArray(contract?.validate) ? contract.validate : []).filter((v) => v?.id && typeof v.command === "string");
+  const t = contract?.repro_test;
+  return t && typeof t === "object" && typeof t.command === "string" && t.command.trim() ? [...out, { id: "repro_test", command: t.command }] : out;
+}
 export function preflightProbes(contract, { repoHas = () => false, cap = PREFLIGHT_CAP } = {}) {
   const changes = (Array.isArray(contract?.deliverables) ? contract.deliverables : []).map((d) => String(d?.change ?? "")).join("\n");
   const named = (name) => new RegExp(`(^|[^\\w@/.-])${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w/-])`).test(changes);
@@ -115,8 +121,7 @@ export function preflightProbes(contract, { repoHas = () => false, cap = PREFLIG
       : `node -e ${sq(NODE_PROBE)} ${sq(tool)} >/dev/null 2>&1`;
     byKey.set(key, { ids: [id], tool, kind, interp, needs: kind === "command" ? null : interp, probe });
   };
-  for (const v of Array.isArray(contract?.validate) ? contract.validate : []) {
-    if (!v?.id || typeof v.command !== "string") continue;
+  for (const v of contractCommands(contract)) {
     for (const seg of shellSegments(v.command)) {
       let i = 0;
       while (i < seg.length && (/^[A-Za-z_]\w*=/.test(seg[i]) || ["!", "time", "exec", "command", "env"].includes(seg[i]))) i++;
@@ -238,7 +243,7 @@ const porcelain = (out) => new Map(String(out ?? "").split("\n").filter((l) => l
  */
 export async function runPreflight({ contract, run, git, env = {}, repoHas = () => false, knownPath = () => false, skipIds = [], now = () => Date.now(), perCommandMs = RUN_PROBE_MS, budgetMs = RUN_BUDGET_MS }) {
   const misses = [], skipped = [], reverted = [];
-  const todo = (Array.isArray(contract?.validate) ? contract.validate : []).filter((v) => v?.id && typeof v.command === "string" && !skipIds.includes(v.id));
+  const todo = contractCommands(contract).filter((v) => !skipIds.includes(v.id));
   if (!todo.length) return { misses, skipped, reverted };
   const status = async () => porcelain(await git(["status", "--porcelain", "--untracked-files=all"]).catch(() => ""));
   const before = await status(), deadline = now() + budgetMs;

package/bin/lean/preflight.test.mjs

@@ -4,7 +4,7 @@ import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { testRunnerProblems, shellSegments, pythonImports, nodeImports, preflightProbes, preflightProblems, runProbeMiss, unsetEnvVars, runPreflight, knownToContract } from "./preflight.mjs";
+import { testRunnerProblems, shellSegments, pythonImports, nodeImports, preflightProbes, preflightProblems, runProbeMiss, unsetEnvVars, runPreflight, knownToContract, contractCommands } from "./preflight.mjs";
 import { runCheckCommand } from "./effects.mjs";
 import { repoTracks } from "./conformance.mjs";
 import { tmpDir } from "../tests/_tmp.mjs";
@@ -120,3 +120,15 @@ describe("testRunnerProblems (#4857)", () => {
     assert.deepEqual(testRunnerProblems(C("pytest services/worker/t.py"), null), []);
   });
 });
+
+describe("#5020 repro_test is probed like a validate command", () => {
+  const c = { validate: [{ id: "V1", command: "node --test a.test.mjs" }], repro_test: { path: "t.py", id: "t", command: "madeuptool --run t.py" } };
+  it("contractCommands lists validate entries then the repro_test", () => {
+    assert.deepEqual(contractCommands(c).map((x) => x.id), ["V1", "repro_test"]);
+    assert.deepEqual(contractCommands({ repro_test: "none" }), []);
+  });
+  it("preflightProbes probes the repro command's tool", () => {
+    const { probes } = preflightProbes(c);
+    assert.ok(probes.some((p) => p.tool === "madeuptool" && p.ids.includes("repro_test")));
+  });
+});

package/bin/lean/prompts.mjs

@@ -246,7 +246,7 @@ export function contextPackNote(pack) {
 /** #4857: validate-command rules for the plan prompt (the implement session resumes it), kept out of the shared templates (their token caps are guarded). */
 export function unrunnableValidateNote(testRunners = null) {
   const map = testRunners && typeof testRunners === "object" ? Object.entries(testRunners).map(([k, v]) => `${k}/ -> ${v}`).join("; ") : "";
-  return `\n\nValidate and repro commands must pass bash -n (no prose notes), select tests that exist, and run from the right cwd${map ? ` (forge.yaml verification.test_runners: ${map})` : ""}. An unrunnable validate (missing tool, wrong cwd, no tests selected) is never a block or "human" product-decision: run an equivalent through the repo's toolchain and record {"kind":"substitute-validate","validate":"V#","command":"<runnable equivalent>","why":""} in ${RESULT_FILE} amendments.`;
+  return `\n\nValidate and repro commands must pass bash -n (no prose notes), select tests that exist, and run from the right cwd${map ? ` (forge.yaml verification.test_runners: ${map})` : ""}. An unrunnable validate (missing tool, wrong cwd, no tests selected) is never a block or "human" product-decision: run an equivalent through the repo's toolchain and record {"kind":"substitute-validate","validate":"V#","command":"<runnable equivalent>","why":""} in ${RESULT_FILE} amendments. A post-deploy requirement needs a registered check id; one that is a manual or production-credential action with no runnable check is disposition "defer" with bucket "owner:<action>" (the engine also converts a check-less post-deploy to a defer to the orchestrator). forge.yaml validate.recipes rewrite matching commands at freeze.`;
 }
 
 export function reproPlanNote(on) {

package/bin/lean/sandbox-provider.contract.test.mjs

@@ -189,6 +189,16 @@ describe("remote adapter (#4254)", () => {
     await assert.rejects(() => q.status(h), /ECONNRESET/); // #4672: a transport error propagates; providerLanes retries it for the unreachable window
   });
 
+  it("#5053 httpTransport: a 401 on status is transient and names its hop; a launch 401 is not transient", async () => {
+    const reply = (status, body) => ({ status, ok: status < 300, text: async () => JSON.stringify(body), headers: { get: () => null } });
+    const t = httpTransport({ endpoint: "https://pool.example", token: TOKEN, fetchFn: async () => reply(401, { error: "unauthorized", hop: "agent", agent: "a1", reason: "agent auth: router token rejected by agent" }) });
+    await assert.rejects(t.status("x"), (e) => e.transient === true && e.hop === "agent" && e.reason === "router token rejected by agent" && /agent a1 auth: router token rejected/.test(e.message) && !e.message.includes(TOKEN));
+    await assert.rejects(t.result("x"), (e) => e.transient === true && e.status === 401);
+    await assert.rejects(t.launch({ id: "x" }), (e) => e.transient !== true && e.status === 401);
+    const r = httpTransport({ endpoint: "https://pool.example", fetchFn: async () => reply(401, { hop: "router" }) });
+    await assert.rejects(r.status("x"), /router auth/);
+  });
+
   it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
     const calls = [];
     const reply = (status, body, headers = {}) => ({ status, ok: status >= 200 && status < 300, text: async () => (body === undefined ? "" : JSON.stringify(body)), headers: { get: (k) => headers[k] ?? null } });
@@ -320,10 +330,11 @@ describe("pool provider (#4546)", () => {
 
 describe("remote capacity (GET /status, #4540)", () => {
   const mk = (res) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => { if (res instanceof Error) throw res; if (url.endsWith("/agents")) return { status: 404, ok: false, text: async () => "", headers: { get: () => null } }; assert.equal(url, "https://pool.example/v1/status"); return { status: res.status, ok: res.status < 400, text: async () => res.body ?? "", headers: { get: () => null } }; } });
-  it("free integer is reported; 404, malformed, negative and transport errors are unknown (null)", async () => {
+  it("free integer is reported; 404, malformed and negative are unknown (null); an unreachable router throws (#5048)", async () => {
     assert.deepEqual(await mk({ status: 200, body: '{"free":3}' }).capacity(), { free: 3 });
     assert.deepEqual(await mk({ status: 200, body: '{"free":0}' }).capacity(), { free: 0 });
-    for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }, new Error("down")]) assert.equal(await mk(r).capacity(), null);
+    for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }]) assert.equal(await mk(r).capacity(), null);
+    await assert.rejects(() => mk(new Error("down")).capacity(), (e) => e.unreachable === true && /pool router unreachable/.test(e.message));
   });
   it("#4788: no online on /status: derived from /agents (ok && !paused); an unreadable /agents stays unknown", async () => {
     const mkA = (status, agents) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => {
@@ -337,12 +348,12 @@ describe("remote capacity (GET /status, #4540)", () => {
     assert.deepEqual(await mkA(200, down).capacity(), { online: 0, total: 1 });
     assert.equal(await mkA(404, "nope").capacity(), null);
   });
-  it("remoteProvider.capacity delegates, and is null without a transport capability or on a throw", async () => {
+  it("remoteProvider.capacity delegates, is null without a transport capability, and passes a transport error on (#5048)", async () => {
     const batch = openBatch(join(tmpDir("cap-"), "batches"), "bt");
     const p = (transport) => remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null });
     assert.deepEqual(await p({ capacity: async () => ({ free: 1 }) }).capacity(), { free: 1 });
     assert.equal(await p({}).capacity(), null);
-    assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
+    await assert.rejects(() => p({ capacity: async () => { throw new Error("x"); } }).capacity(), /x/);
   });
 });
 

package/bin/lean/sandbox-remote.mjs

@@ -60,8 +60,14 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     return { status: res.status, ok: res.ok, data, retryAfter: res.headers?.get?.("retry-after") ?? null };
   };
   const fail = (what, r) => {
-    const e = new Error(`sandbox ${what}: HTTP ${r.status}`);
+    // #5053: a 401 names the hop that answered it (router or agent; the pool tags the body), never the token.
+    const hop = r.status === 401 && ["router", "agent"].includes(r.data?.hop) ? r.data.hop : null;
+    const who = hop === "agent" && typeof r.data?.agent === "string" ? `agent ${r.data.agent.slice(0, 64)}` : hop;
+    const reason = hop && typeof r.data?.reason === "string" ? r.data.reason.replace(/^(router|agent) auth: /, "").replace(/[^\x20-\x7e]/g, "").slice(0, 80) : "";
+    const e = new Error(`sandbox ${what}: HTTP ${r.status}${who ? ` (${who} auth${reason ? `: ${reason}` : ""})` : ""}`);
     e.status = r.status;
+    if (hop) e.hop = hop;
+    if (reason) e.reason = reason;
     if ([502, 503, 504].includes(r.status)) e.transient = true; // #4672: a router restart or gateway blip, retried by the lane poll
     if (r.status === 429 || r.status === 503) { e.retryable = true; const n = Number(r.retryAfter); if (Number.isFinite(n) && n > 0) e.retryInSeconds = n; }
     return e;
@@ -74,12 +80,12 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async status(id) {
       const r = await call("GET", p(id));
       if (r.status === 404) return "gone";
-      if (!r.ok) throw fail("status", r);
+      if (!r.ok) throw Object.assign(fail("status", r), r.status === 401 ? { transient: true } : {}); // #5053: the sandbox launched, so a 401 is retried like an unreachable pool
       const s = String(r.data?.state ?? "");
       return s === "running" || s === "pending" || s === "starting" ? "running" : s === "gone" ? "gone" : "exited";
     },
     async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
-    async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
+    async result(id) { const r = await call("GET", `${p(id)}/result`); if (r.status === 401) throw Object.assign(fail("result", r), { transient: true }); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
     async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
     async stop(id) { const r = await call("POST", `${p(id)}/stop`); if (r.ok) return true; if (r.status === 404 || r.status === 405) return false; throw fail("stop", r); },
     async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
@@ -88,19 +94,23 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async capacity() {
       // `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
       const ok = (v) => Number.isInteger(v) && v >= 0;
-      let out = {};
+      let out = {}, reached = false, lastErr = null;
       try {
         const r = await call("GET", "/status");
+        reached = true;
         if (r.ok) { const { free, online, total } = r.data ?? {}; out = { ...(ok(free) ? { free } : {}), ...(ok(online) ? { online } : {}), ...(ok(total) ? { total } : {}) }; }
-      } catch { out = {}; }
+      } catch (e) { out = {}; lastErr = e; }
       if (out.online === undefined) {
         // #4788: a router without /status liveness: online = agents answering and not paused, as bin/pool/router.mjs status() does.
         try {
           const r = await call("GET", "/agents");
+          reached = true;
           const list = r.ok && Array.isArray(r.data?.agents) ? r.data.agents : null;
           if (list) out = { ...out, online: list.filter((a) => a?.ok === true && !a.paused).length, total: list.length };
-        } catch { /* liveness stays unknown */ }
+        } catch (e) { lastErr = e; /* liveness stays unknown */ }
       }
+      // #5048 (release review REG-2): a router that answered neither call is unreachable: throw, so the pool is marked down
+      if (!reached) throw Object.assign(new Error(`pool router unreachable: ${String(lastErr?.message ?? lastErr).slice(0, 160)}`), { unreachable: true });
       return Object.keys(out).length ? out : null;
     },
   };
@@ -218,7 +228,9 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
       return r ?? null;
     },
     async capacity() {
-      try { return (await transport.capacity?.()) ?? null; } catch { return null; }
+      // #5048: no capacity() is unknown (null); a transport error propagates, so the caller can report the pool down
+      if (typeof transport.capacity !== "function") return null;
+      return (await transport.capacity()) ?? null;
     },
     /** #4612: a graceful stop at the lane's checkpoint; false when unsupported or failed. Never destroys the sandbox. */
     async stop(h) {

package/bin/lean/signoff.mjs

@@ -11,6 +11,17 @@ const SEP = "\x1f", END = "\x1e";
 const STASH_REF = "refs/forgedock/dco-stash";
 const PLACEHOLDER_RE = /@localhost$/i;
 
+/**
+ * #5047: where a lane's commits go when the sign-off rewrite failed. Never the lane branch: ensureSignedOff excludes everything on
+ * origin/<branch>, so unsigned commits published there could never be rewritten. A salvage branch is outside that range.
+ */
+export const salvageBranch = (issue, sha) => `forge-salvage/${issue}-${String(sha).slice(0, 10)}`;
+
+/** #5047: a lane resumed from a salvage branch has no published lane branch: its tracking ref must not exclude the commits from the sign-off range. */
+export async function dropSalvageTracking({ git, branch }) {
+  try { await git(["update-ref", "-d", `refs/remotes/origin/${branch}`]); return true; } catch { return false; }
+}
+
 /** True when the commit must be rewritten: placeholder author or committer, or no sign-off naming its author. */
 export function needsSignoff({ authorEmail, committerEmail, signedOff }) {
   if (PLACEHOLDER_RE.test(authorEmail ?? "") || PLACEHOLDER_RE.test(committerEmail ?? "")) return true;

package/bin/lean/signoff.test.mjs

@@ -7,7 +7,7 @@ import { execFileSync } from "node:child_process";
 import { readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { ensureSignedOff, needsSignoff, offendersOf } from "./signoff.mjs";
+import { ensureSignedOff, needsSignoff, offendersOf, salvageBranch, dropSalvageTracking } from "./signoff.mjs";
 
 const CLEAN = Object.fromEntries(Object.entries(process.env).filter(([k]) => !/^GIT_(AUTHOR|COMMITTER)_/.test(k)));
 const sh = (cwd, ...a) => execFileSync("git", a, { cwd, encoding: "utf8", env: { ...CLEAN, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_SYSTEM: "/dev/null" } }).trim();
@@ -143,4 +143,34 @@ describe("ensureSignedOff", () => {
     await assert.rejects(ensureSignedOff({ git, lane: "staging", branch: "feat/x", env: {} }), (e) => e.engineCode === "engine.dco_rewrite" && e.noIdentity === false && /boom/.test(e.message));
     assert.equal(readFileSync(join(f.wt, "base.txt"), "utf8"), "churn\n");
   });
+  it("#5047 unsigned commits on a salvage ref are rewritten once the lane is resumed from it, and the signed rewrite pushes as the lane branch", async () => {
+    const f = fixture();
+    const bad = f.commit("a.txt", "unsigned placeholder", "placeholder");
+    const ref = salvageBranch(7, bad);
+    assert.equal(ref, `forge-salvage/7-${bad.slice(0, 10)}`);
+    f.sh("push", "-q", "origin", `${bad}:refs/heads/${ref}`);
+    // the recovery lane fetches the salvage branch as origin/<branch>, builds the worktree from it, then drops that tracking ref
+    f.sh("fetch", "-q", "origin", `${ref}:refs/remotes/origin/feat/x`);
+    assert.equal((await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} })).rewritten, 0, "with the tracking ref the commits are excluded (the defect)");
+    assert.equal(await dropSalvageTracking({ git: f.git, branch: "feat/x" }), true);
+    const r = await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} });
+    assert.equal(r.rewritten, 1);
+    assert.match(f.sh("log", "-1", "--format=%B"), /Signed-off-by: Lane Bot <lane@example.com>/);
+    f.sh("push", "-q", "-u", "origin", "feat/x");
+    assert.notEqual(f.sh("rev-parse", "origin/feat/x"), bad);
+  });
+  it("#5047 an unsigned commit pushed to a salvage ref is not on the lane branch's exclusion range", async () => {
+    const f = fixture();
+    const bad = f.commit("a.txt", "unsigned placeholder", "placeholder");
+    f.sh("push", "-q", "origin", `${bad}:refs/heads/${salvageBranch(7, bad)}`);
+    assert.equal((await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} })).rewritten, 1);
+  });
+  it("#5047 signed history already pushed on origin/<branch> is never rewritten", async () => {
+    const f = fixture();
+    f.commit("s.txt", "signed", "--signoff");
+    f.sh("push", "-q", "-u", "origin", "feat/x");
+    const head = f.sh("rev-parse", "HEAD");
+    assert.equal((await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} })).rewritten, 0);
+    assert.equal(f.sh("rev-parse", "HEAD"), head);
+  });
 });

package/bin/pool/agent.mjs

@@ -240,7 +240,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
   async function sandboxCred(id, kind, req) {
     const r = own(id);
     const m = /^Bearer\s+(\S.*)$/i.exec(req.headers.authorization ?? "");
-    if (!r || r.state === "exited" || !m || !sameSecret(m[1].trim(), r.credToken)) return { status: 401, body: { error: "unauthorized" } };
+    if (!r || r.state === "exited" || !m || !sameSecret(m[1].trim(), r.credToken)) return { status: 401, body: { error: "unauthorized", hop: "agent", reason: "agent auth: bad sandbox credential" } };
     if (!CRED_TTL[kind]) return { status: 404, body: { error: "unknown credential" } };
     const c = credCache[kind];
     if (!c || now() - c.at > CRED_TTL[kind]) {
@@ -256,7 +256,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
   async function sandboxRelay(id, rid, req) {
     const r = own(id);
     const m = /^Bearer\s+(\S.*)$/i.exec(req.headers.authorization ?? "");
-    if (!r || r.state === "exited" || !m || !sameSecret(m[1].trim(), r.credToken)) return { status: 401, body: { error: "unauthorized" } };
+    if (!r || r.state === "exited" || !m || !sameSecret(m[1].trim(), r.credToken)) return { status: 401, body: { error: "unauthorized", hop: "agent", reason: "agent auth: bad sandbox credential" } };
     const lane = r.alias ?? r.id; // #4935: a claimed warm sandbox posts under its warm id, but the router queue and the dispatcher use the lane id
     const path = `/agent/relay/${encodeURIComponent(lane)}${rid ? `/${encodeURIComponent(rid)}` : ""}`;
     const x = rid ? await call(cfg.router_url, "GET", path, { token: cfg.token, timeoutMs: 20000 })
@@ -400,7 +400,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
       if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body); }
       const relay = (req.method === "POST" && /^\/sandboxes\/([^/]+)\/relay$/.exec(url.pathname)) || (req.method === "GET" && /^\/sandboxes\/([^/]+)\/relay\/([^/]+)$/.exec(url.pathname));
       if (relay) { const r = await sandboxRelay(decodeURIComponent(relay[1]), relay[2] ? decodeURIComponent(relay[2]) : null, req); return send(res, r.status, r.body ?? {}); }
-      if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized" });
+      if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized", hop: "agent", reason: "agent auth: router token rejected by agent" });
       for (const [match, fn] of routes) {
         const p = match(req, url);
         if (!p) continue;

package/bin/pool/router.mjs

@@ -147,13 +147,15 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
     catch (e) { log.warn?.(`forge.yaml for ${spec.repo.slug} unreadable: ${e.message}`); return spec; }
   }
 
+  /** #5053: an agent's 401 reaches the dispatcher through this router; tag it so it is not mistaken for a router auth failure. */
+  const agentBody = (r, name) => (r.status === 401 ? { ...(r.data && typeof r.data === "object" ? r.data : {}), error: "unauthorized", hop: "agent", agent: name, reason: typeof r.data?.reason === "string" ? r.data.reason : "agent auth: router token rejected by agent" } : r.data ?? {});
   async function launch(spec) {
     if (typeof spec?.id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(spec.id)) return { status: 400, body: { error: "spec.id required ([A-Za-z0-9._-], max 128)" } };
     const known = own(sbx(), spec.id);
     if (known && own(agents(), known.agent)) {
       known.touched = now(); store.save();
       const r = await agentCall(own(agents(), known.agent), "POST", "/sandboxes", { spec: withForgeYaml(spec), ...sizeFor(spec) });
-      return { status: r.status, body: r.data, retryAfter: r.retryAfter };
+      return { status: r.status, body: agentBody(r, known.agent), retryAfter: r.retryAfter };
     }
     const size = sizeFor(spec);
     const fits = (await capacities()).filter((c) => c.ok && !c.paused && c.cpus_free >= size.cpus && c.mem_free_gb >= size.mem_gb)
@@ -162,7 +164,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
       const r = await agentCall(own(agents(), c.name), "POST", "/sandboxes", { spec: withForgeYaml(spec), ...size }, 120000);
       if (r.status === 503) continue; // raced with another launch; try the next host
       if (r.ok) { sbx()[spec.id] = { agent: c.name, issue: spec.issue ?? null, batch: spec.batch ?? null, createdAt: now(), touched: now() }; store.save(); }
-      return { status: r.status, body: { ...(r.data ?? {}), agent: c.name } };
+      return { status: r.status, body: { ...agentBody(r, c.name), agent: c.name } };
     }
     return { status: 503, body: { error: "no host has a free reservation", lane: size }, retryAfter: cfg.retry_after_s ?? 60 };
   }
@@ -190,7 +192,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
     try {
       const r = await agentCall(a, method, path);
       if (method === "DELETE" && (r.ok || r.status === 404)) { delete sbx()[id]; relayQ.delete(id); store.save(); }
-      return { status: r.status, body: r.data };
+      return { status: r.status, body: agentBody(r, s.agent) };
     } catch (e) { return { status: 502, body: { error: `agent ${s.agent} unreachable: ${e.message}` } }; }
   }
 
@@ -214,7 +216,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
   }
 
   function enroll(body) {
-    if (!enrollOk(body?.token)) return { status: 401, body: { error: "bad enrollment token" } };
+    if (!enrollOk(body?.token)) return { status: 401, body: { error: "bad enrollment token", hop: "router", reason: "router auth: bad enrollment token" } };
     const name = String(body.name ?? "").toLowerCase().replace(/[^a-z0-9-]/g, "");
     let url;
     try { url = new URL(String(body.url)); } catch { return { status: 400, body: { error: "url required" } }; }
@@ -321,14 +323,23 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
   async function handle(req, res) {
     const url = new URL(req.url, "http://router");
     if (!url.pathname.startsWith("/health")) log.info?.(`${req.socket.remoteAddress} ${req.method} ${url.pathname}`); // query (tokens) never logged
+    // #5053: every rejected request is logged with its status; the sandbox id comes from the path, the repo slug from a launch body (never tokens or the query).
+    const meta = { id: /^\/sandboxes\/([^/]+)/.exec(url.pathname)?.[1] };
+    const writeHead = res.writeHead.bind(res);
+    res.writeHead = (status, ...a) => {
+      if (status >= 400 && url.pathname !== "/health") log.warn?.(`rejected ${req.method} ${url.pathname} -> ${status}${meta.id && meta.id !== "claim" ? ` sandbox=${meta.id.slice(0, 128)}` : ""}${meta.repo ? ` repo=${meta.repo}` : ""}`);
+      return writeHead(status, ...a);
+    };
     try {
       for (const [match, r] of routes) {
         const p = match(req, url);
         if (!p) continue;
-        if (r.auth === "forgedock" && !bearerOk(req, cfg.forgedock_token)) return send(res, 401, { error: "unauthorized" });
-        if (r.auth === "agent" && !agentFor(req)) return send(res, 401, { error: "unauthorized" });
+        if (r.auth === "forgedock" && !bearerOk(req, cfg.forgedock_token)) return send(res, 401, { error: "unauthorized", hop: "router", reason: "router auth: bad or missing token" });
+        if (r.auth === "agent" && !agentFor(req)) return send(res, 401, { error: "unauthorized", hop: "router", reason: "router auth: bad or missing token" });
         if (r.auth === "enroll" && !enrollOk(url.searchParams.get("token"))) return send(res, 401, "unauthorized\n");
         const body = req.method === "POST" ? await readJson(req) : null;
+        if (typeof body?.repo?.slug === "string") meta.repo = body.repo.slug.slice(0, 200);
+        if (typeof body?.id === "string") meta.id = body.id;
         const out = await r.fn(p, body, url, req);
         if (out.bundle) return sendBundle(res);
         if (out.image) return sendImage(res, out.image);

package/docs/CONFIG.md

@@ -1358,6 +1358,7 @@ verification:
 | `consumer_sweep.mode` | string | No | `advisory` (default): the validate consumer sweep (`scripts/consumer-sweep.sh`, V4.6) records unaddressed referencing sites and never fails. `blocking`: unaddressed sites fail validate with `consumer_unaddressed`, build runs one repair round, and sites still unaddressed are recorded as `CONSUMERS: unresolved` and passed to review as known risk (never `needs-human`). Flip to `blocking` only after `bash scripts/consumer-sweep.sh --eval-history 20` shows a mean of at most 2 flagged sites per merge and none above 10 on your repo |
 | `consumer_sweep.budget_seconds` | integer | No | Wall-clock budget for the sweep (default `60`). When it runs out the sweep stops with `BUDGET_EXCEEDED`, reports the partial result, and never fails, in either mode |
 | `test_runners` | map (path prefix → runner) | No | Default unset (no runner enforced). For repos whose validate commands need a managed runner (uv, poetry, pnpm workspace). The lean engine checks it at contract freeze (`testRunnerProblems`, `bin/lean/preflight.mjs`): a validate command that touches a configured path prefix without containing that prefix's runner string is reported as a contract problem, instead of failing later as `contract.invalid_command` / `env.tool_missing`. Paths in the command are relative to the runner's cwd. Write it as a block-style map: `parseForgeYaml` does not read inline `{ ... }` maps. |
+| `validate.recipes` | map (regex → replacement) | No | Default unset. A top-level `validate:` block with a block-style `recipes:` map, e.g. `"poetry run pytest": "uv run --with-requirements r.txt pytest"`. Each key is a regex source; the first recipe that matches a frozen validate or `repro_test` command rewrites it at contract freeze (`applyValidateRecipes`), recorded as a decision. Use it for commands whose environment is not installed in lane sandboxes. |
 
 **Commands that use this section**: `quality-gate`, `validate` (V2 runs `verification.commands`; V4.8 runs the local CI gate; health checks and `consumer_sweep`), `work-on`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "forgedock",
-  "version": "1.44.0",
+  "version": "1.45.0",
   "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
   "bin": {
     "forgedock": "bin/forgedock.mjs"