forgedock 1.44.0 → 1.46.0

package/bin/engine-cli.mjs

@@ -525,10 +525,12 @@ export function readEnvConfig(cwd = process.cwd()) {
     const e = y?.env ?? {};
     const tr = y?.verification?.test_runners; // #4857: {path prefix: runner command prefix} for validate commands
     const runners = tr && typeof tr === "object" && !Array.isArray(tr) ? Object.fromEntries(Object.entries(tr).filter(([k, v]) => k && typeof v === "string" && v.trim())) : {};
+    const rm = readYamlMap(cwd, "validate", "recipes"); // #5020: y?.validate?.recipes, a block map {match regex: replacement} applied to frozen validate and repro commands
+    const recipes = Object.entries(rm ?? {}).map(([match, replace]) => ({ match, replace }));
     const s = (v) => (typeof v === "string" && v.trim() ? v.trim() : undefined);
     const kb = typeof e.sparse_max_kb === "string" && /^\d+$/.test(e.sparse_max_kb.trim()) ? Number(e.sparse_max_kb) : e.sparse_max_kb;
     return { ...(s(e.toolchain) ? { toolchainOverride: s(e.toolchain) } : {}), ...(s(e.setup) ? { envSetup: s(e.setup) } : {}),
-      ...(Number.isInteger(kb) && kb >= 0 ? { sparseMaxKb: kb } : {}), ...(Object.keys(runners).length ? { testRunners: runners } : {}) };
+      ...(Number.isInteger(kb) && kb >= 0 ? { sparseMaxKb: kb } : {}), ...(Object.keys(runners).length ? { testRunners: runners } : {}), ...(recipes.length ? { validateRecipes: recipes } : {}) };
   } catch { return {}; }
 }
 export const ENGINE_PATHS = ["core", "full", "lean", "auto"];
@@ -1762,12 +1764,13 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
         return (n) => execMod.placeLane({ exec, verifyMode, labels: labelsOf.get(n) ?? [], weight, headroom: headroom(), needs: needsOf.get(n) ?? [], relays });
       })();
       // #4540: provider free slots, refreshed in the background (admission is sync); unknown (null) never blocks.
-      let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity; // #4547: online = agents alive, null unknown
+      let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity, capFailed = false; // #5048: capFailed = last probe rejected // #4547: online = agents alive, null unknown
       const capacity = () => {
+        if (capFailed) throw new Error("capacity probe failed"); // #5048: until the next successful sample, so every caller sees it (release review CONC-1)
         const t = deps.now ? deps.now() : Date.now();
         if (typeof provider.capacity === "function" && t - capLast >= (deps.pollMs ?? 5000)) {
           capLast = t;
-          Promise.resolve().then(() => provider.capacity()).then((c) => { capSample = c && Number.isInteger(c.free) ? c.free : null; onlineSample = c && Number.isInteger(c.online) ? c.online : null; capAt++; }, () => { capSample = null; onlineSample = null; capAt++; });
+          Promise.resolve().then(() => provider.capacity()).then((c) => { capFailed = false; capSample = c && Number.isInteger(c.free) ? c.free : null; onlineSample = c && Number.isInteger(c.online) ? c.online : null; capAt++; }, () => { capSample = null; onlineSample = null; capFailed = true; capAt++; });
         }
         return capSample === null && onlineSample === null ? null : { free: capSample, online: onlineSample, at: capAt };
       };

package/bin/lean/conformance.mjs

@@ -431,7 +431,7 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       // #4857: a runnable equivalent of an unrunnable frozen validate command; same id, same checks as a planned command
       const v = (c.validate ?? []).find((x) => x.id === a.validate);
       if (!v) { problems.push(`substitute-validate names unknown validate ${a.validate}`); continue; }
-      const bad = blank(a.command) ? "no command" : shellSyntaxError(a.command) ? "not valid shell (bash -n)" : noopCommand(a.command) ? "runs no check (true, :, exit 0, echo)" : movingBaseDiff(a.command) ? "diffs against a moving base tip" : tmpdirInputs(a.command).length ? "reads a $TMPDIR file it did not create" : null;
+      const bad = blank(a.command) ? "no command" : substituteShellProblem(a.command) ? "not valid shell (bash -n)" : noopCommand(a.command) ? "runs no check (true, :, exit 0, echo)" : movingBaseDiff(a.command) ? "diffs against a moving base tip" : tmpdirInputs(a.command).length ? "reads a $TMPDIR file it did not create" : null;
       if (bad) { problems.push(`substitute-validate ${a.validate}: ${bad}`); continue; }
       // SEC-11: a second substitute of the same id keeps the frozen command as `from`, so the engine always reruns the original
       v.substituted = { from: v.substituted?.from ?? v.command, why: String(a.why).trim() }; v.command = String(a.command).trim();
@@ -480,7 +480,9 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
       if (!r) { problems.push(`amendment names unknown requirement ${a.requirement}`); continue; }
       if (!TRACKED.includes(r.disposition)) {
         if (a.bucket != null) r.bucket = a.bucket; // #4724
-        if (a.kind === "post-deploy" && !CHECK_ID_RE.test(String(a.check ?? r.check ?? ""))) { problems.push(`amendment post-deploy ${a.requirement} needs a check id (check)`); continue; }
+        if (a.kind === "post-deploy" && !CHECK_ID_RE.test(String(a.check ?? r.check ?? ""))) { // #5020: no runnable check: a manual or credentialed action is deferred to the orchestrator, never a conformance failure
+          r.disposition = "defer"; r.why = a.why; r.amended = true; r.bucket = String(a.bucket ?? "").startsWith("owner:") ? a.bucket : ORCHESTRATOR_BUCKET; toTrack.push(r); continue;
+        }
         r.disposition = a.kind; r.why = a.why; r.amended = true; r.bucket ??= "post-deploy"; toTrack.push(r); // #4724: an amendment without a bucket defaults to post-deploy
         if (a.kind === "post-deploy") r.check = a.check ?? r.check; // #4626
       }
@@ -508,6 +510,7 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
 
 const PY_MISSING_RE = /ModuleNotFoundError: No module named '([^']+)'|ImportError: No module named '?([\w.]+)'?/;
 const NODE_MISSING_RE = /Cannot find (?:module|package) '([^']+)'/;
+const POETRY_NOT_FOUND_RE = /^\s*Command not found: (\S+)/m;
 const CMD_NOT_FOUND_RE = /([^\s:'"`]+): command not found/, SH_NOT_FOUND_RE = /([^\s:'"`]+): not found\b/;
 /** #4857: why a command is not valid bash (`bash -n`), or null when it parses (or bash is unavailable). Catches prose left on a command. */
 export function shellSyntaxError(command) {
@@ -515,6 +518,49 @@ export function shellSyntaxError(command) {
   if (r.error || r.status === 0 || r.status == null) return null;
   return String(r.stderr || "syntax error").trim().split("\n")[0].replace(/^bash: (?:line \d+: )?/, "").slice(0, 160);
 }
+/** #5020: like shellSyntaxError, but fails closed: when bash cannot run the check the command is not shown to parse. */
+export function substituteShellProblem(command) {
+  const r = spawnSync("bash", ["-n", "-c", String(command ?? "")], { encoding: "utf8", timeout: 5000 });
+  if (r.error || r.status == null) return "bash -n could not run, so the command is unverified";
+  return r.status === 0 ? null : shellSyntaxError(command) ?? "syntax error";
+}
+export const ORCHESTRATOR_BUCKET = "owner:orchestrator";
+/**
+ * #5020: a post-deploy requirement that names no check id (a manual or production-credential action) becomes a `defer` with an
+ * `owner:orchestrator` bucket, the legal disposition the engine files as a follow-up. Pure; returns {contract, converted[]}.
+ */
+export function deferChecklessPostDeploy(contract) {
+  const converted = [];
+  const requirements = (Array.isArray(contract?.requirements) ? contract.requirements : []).map((r) => {
+    if (r?.disposition !== "post-deploy" || CHECK_ID_RE.test(String(r?.check ?? ""))) return r;
+    converted.push(r.id);
+    return { ...r, disposition: "defer", bucket: String(r.bucket ?? "").startsWith("owner:") ? r.bucket : ORCHESTRATOR_BUCKET, why: r.why ?? "post-deploy action with no registered check: deferred to the orchestrator" };
+  });
+  return { contract: converted.length ? { ...contract, requirements } : contract, converted };
+}
+/**
+ * #5020: apply forge.yaml `validate.recipes` ([{match, replace}], match a regex source) to each validate command and the
+ * repro_test command at freeze, first matching recipe per command. Pure; returns {contract, applied[{id, from, to}]}.
+ */
+export function applyValidateRecipes(contract, recipes) {
+  const list = (Array.isArray(recipes) ? recipes : []).map((r) => { try { return typeof r?.replace === "string" && r?.match ? { re: new RegExp(String(r.match)), replace: r.replace } : null; } catch { return null; } }).filter(Boolean);
+  if (!list.length || !contract || typeof contract !== "object") return { contract, applied: [] };
+  const applied = [];
+  const rewrite = (id, cmd) => {
+    if (typeof cmd !== "string") return cmd;
+    for (const { re, replace } of list) if (re.test(cmd)) { const to = cmd.replace(re, replace); if (to !== cmd) applied.push({ id, from: cmd, to }); return to; }
+    return cmd;
+  };
+  const out = { ...contract };
+  if (Array.isArray(contract.validate)) out.validate = contract.validate.map((v) => (v && typeof v === "object" ? { ...v, command: rewrite(v.id, v.command) } : v));
+  const t = contract.repro_test;
+  if (t && typeof t === "object" && !Array.isArray(t)) out.repro_test = { ...t, command: rewrite("repro_test", t.command) };
+  return { contract: applied.length ? out : contract, applied };
+}
+/** #5020: the env.* verdict for a repro command that failed on head because a tool is missing (e.g. an uninstalled Poetry env); null when the failure is the test's own. */
+export function reproEnvMiss(run, opts) {
+  return run && run.rc !== 0 && run.rc !== 124 ? missingTool(run, opts) : null;
+}
 const NOOP_SEGMENT_RE = /^(?:true|false|:|exit(?:\s+\d+)?|echo(?:\s.*)?|printf(?:\s.*)?|sleep(?:\s+[\d.]+)?|test\s+-?\w+|\[\s+-?\w+\s+\]|fi|done|node\s+-e\s+['"]?[\d;\s]*['"]?|python3?\s+-c\s+['"]?(?:pass|0|None)?['"]?)$/;
 const KEYWORD_RE = /^(if|then|else|elif|do|while|until)(?:\s+|$)/;
 const CONST_FALSE_RE = /^(?:false|\[\s+0\s+=\s+1\s+\]|\[\s+1\s+=\s+0\s+\]|test\s+0\s+=\s+1|test\s+-n\s+(?:""|'')|\[\s+-n\s+(?:""|'')\s+\])$/;
@@ -645,6 +691,8 @@ export function missingTool({ rc, output } = {}, { repoHas = () => false, comman
     const tool = scoped ? spec.split("/").slice(0, 2).join("/") : spec;
     return local(tool) || (scoped && !tool.includes("/")) ? null : { tool, kind: "node-module" };
   }
+  const po = POETRY_NOT_FOUND_RE.exec(out)?.[1]; // #5020: `poetry run pytest` in an env that was never installed
+  if (po) return local(po) ? null : { tool: po, kind: "command" };
   const cmd = CMD_NOT_FOUND_RE.exec(out)?.[1] ?? (rc === 127 ? SH_NOT_FOUND_RE.exec(out)?.[1] ?? String(command).trim().split(/\s+/)[0] : null);
   if (!cmd || cmd.includes("/") || local(cmd)) return null;
   return { tool: cmd, kind: "command" };
@@ -688,6 +736,8 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
   for (const v of contract.validate ?? []) {
     // #4979 SPEC-2: a substitute counts only when the frozen command it replaced cannot run (or passes); checked first, never CI-covered
     if (v.substituted) {
+      const shellBad = substituteShellProblem(v.command); // #5020: re-checked before the substitute is used
+      if (shellBad) { failures.push({ code: C.CONTRACT_INVALID_COMMAND, kind: "validate_command_invalid", detail: `validate ${v.id} substitute \`${v.command}\` is not valid shell (bash -n: ${shellBad})` }); continue; }
       if (removedPaths === null) { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_unverified", detail: `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\`, but the engine could not read the diff's removed paths to show the frozen command's target survives` }); continue; }
       const gone = removedNamedPaths(v.substituted.from, removedPaths, baseFiles);
       if (gone.length) { failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "substitute_removed_path", detail: `validate ${v.id} substituted \`${v.command}\` for the frozen \`${v.substituted.from}\`, but the diff removes ${gone.map((x) => `\`${x}\``).join(", ")} that the frozen command names: a substitute may not replace a check by deleting its target` }); continue; }

package/bin/lean/conformance.test.mjs

@@ -3,7 +3,7 @@
 import { readFileSync } from "node:fs";
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
+import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables, applyValidateRecipes, deferChecklessPostDeploy, substituteShellProblem, reproEnvMiss, ORCHESTRATOR_BUCKET } from "./conformance.mjs";
 
 const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
 const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -910,9 +910,10 @@ describe("post-deploy check ids (#4626)", () => {
     assert.ok(validateContract(pd({ check: "Bad Id" }), ISSUE).problems.some((p) => /post-deploy needs a check id/.test(p)));
     assert.ok(!validateContract(pd({ check: "login-smoke" }), ISSUE).problems.some((p) => /check id/.test(p)));
   });
-  it("applyAmendments: a post-deploy amendment needs a check id", () => {
+  it("applyAmendments: a post-deploy amendment without a check id is deferred to the orchestrator (#5020)", () => {
     const bad = applyAmendments(C, [{ kind: "post-deploy", requirement: "R1", why: "w" }], ISSUE);
-    assert.ok(bad.problems.some((p) => /needs a check id/.test(p)));
+    assert.deepEqual(bad.problems, []);
+    assert.equal(bad.contract.requirements[0].disposition, "defer");
     const ok = applyAmendments(C, [{ kind: "post-deploy", requirement: "R1", why: "w", check: "login-smoke" }], ISSUE);
     assert.equal(ok.contract.requirements[0].check, "login-smoke");
   });
@@ -1213,3 +1214,56 @@ describe("builder amendment slips (#5036)", () => {
     assert.match(applyAmendments(C, [{ kind: "flaky-note", why: "w" }]).problems[0], /unknown kind 'flaky-note'; valid kinds are .*substitute-validate/);
   });
 });
+
+describe("#5020 freeze-time environment defects", () => {
+  const base = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], validate: [{ id: "V1", command: "cd services/api && poetry run pytest tests/x.py" }, { id: "V2", command: "node --test a.test.mjs" }], repro_test: { path: "t.py", id: "t", command: "cd services/api && poetry run pytest t.py" } };
+  const recipes = [{ match: "poetry run pytest", replace: "uv run --with-requirements r.txt pytest" }];
+
+  it("applyValidateRecipes rewrites matching validate and repro commands and leaves the rest", () => {
+    const { contract, applied } = applyValidateRecipes(base, recipes);
+    assert.equal(contract.validate[0].command, "cd services/api && uv run --with-requirements r.txt pytest tests/x.py");
+    assert.equal(contract.validate[1].command, "node --test a.test.mjs");
+    assert.match(contract.repro_test.command, /uv run --with-requirements r\.txt pytest t\.py/);
+    assert.deepEqual(applied.map((a) => a.id), ["V1", "repro_test"]);
+    assert.equal(base.validate[0].command.includes("poetry"), true, "input is not mutated");
+  });
+  it("applyValidateRecipes ignores no recipes and invalid regexes", () => {
+    assert.equal(applyValidateRecipes(base, null).contract, base);
+    assert.equal(applyValidateRecipes(base, [{ match: "(", replace: "x" }]).applied.length, 0);
+  });
+  it("Poetry's missing-env output is env.tool_missing material, and a repro head that dies on it is an env miss", () => {
+    const run = { rc: 1, output: "Creating virtualenv api-py3.12\n\nCommand not found: pytest\n" };
+    assert.deepEqual(missingTool(run), { tool: "pytest", kind: "command" });
+    assert.deepEqual(reproEnvMiss(run), { tool: "pytest", kind: "command" });
+    assert.equal(reproEnvMiss({ rc: 1, output: "1 failed" }), null);
+    // release review BUG-1: a module the repo tracks is a real failure of the change, not an env miss
+    const mod = { rc: 1, output: "ModuleNotFoundError: No module named 'mypkg'" };
+    assert.deepEqual(reproEnvMiss(mod, { command: "pytest t.py" }), { tool: "mypkg", kind: "python-module" });
+    assert.equal(reproEnvMiss(mod, { command: "pytest t.py", repoHas: repoTracks(["src/mypkg/__init__.py"]) }), null);
+    assert.equal(reproEnvMiss({ rc: 0, output: "" }), null);
+  });
+  it("a post-deploy requirement with no check id is deferred to the orchestrator; one with a check is untouched", () => {
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "post-deploy", why: "needs prod creds" }, { id: "R2", quote: "q", disposition: "post-deploy", check: "login-smoke", why: "w" }] };
+    const { contract, converted } = deferChecklessPostDeploy(c);
+    assert.deepEqual(converted, ["R1"]);
+    assert.equal(contract.requirements[0].disposition, "defer");
+    assert.equal(contract.requirements[0].bucket, ORCHESTRATOR_BUCKET);
+    assert.equal(contract.requirements[1].disposition, "post-deploy");
+    assert.deepEqual(bucketProblems(contract.requirements[0]), []);
+  });
+  it("a post-deploy amendment with no check becomes a tracked defer instead of a problem", () => {
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], deliverables: [], validate: [] };
+    const r = applyAmendments(c, [{ kind: "post-deploy", requirement: "R1", why: "manual prod action" }], "q");
+    assert.deepEqual(r.problems, []);
+    assert.equal(r.contract.requirements[0].disposition, "defer");
+    assert.equal(r.contract.requirements[0].bucket, ORCHESTRATOR_BUCKET);
+  });
+  it("a substitute-validate that is not valid shell is rejected", () => {
+    assert.match(substituteShellProblem("pytest tests/ && ("), /./);
+    assert.equal(substituteShellProblem("pytest tests/"), null);
+    const c = { requirements: [{ id: "R1", quote: "q", disposition: "deliver" }], deliverables: [], validate: [{ id: "V1", command: "poetry run pytest" }] };
+    const r = applyAmendments(c, [{ kind: "substitute-validate", validate: "V1", command: "uv run pytest tests/ && (", why: "poetry env missing" }], "q");
+    assert.ok(r.problems.some((p) => /not valid shell/.test(p)));
+    assert.equal(r.contract.validate[0].command, "poetry run pytest");
+  });
+});

package/bin/lean/decision-engine.mjs

@@ -160,7 +160,18 @@ export function classify(item, policy, opts = {}) {
 export async function settle(item, policy, deps = {}, opts = {}) {
   const row = { issue: item?.issue ?? null, escalationId: item?.escalationId ?? null, kind: null, action: "skipped" };
   try {
-    if (item.escalationId && (item.comments ?? []).some((c) => /FORGE:DECISION/.test(String(c?.body ?? "")) && String(c.body).includes(`Escalation: ${item.escalationId}`))) return { ...row, reason: "already handled" };
+    const prior = item.escalationId ? (item.comments ?? []).filter((c) => /FORGE:DECISION/.test(String(c?.body ?? "")) && String(c.body).includes(`Escalation: ${item.escalationId}`)) : [];
+    if (prior.length) {
+      // release #5068 BUG-1: a full live decision whose label clear or requeue failed is finished, not skipped (the caller
+      // only lists issues still labelled needs-human); a vetoed, shadow, escalated or mixed one stays handled
+      const body = (c) => String(c?.body ?? "");
+      const torn = prior.some((c) => /^<!-- FORGE:DECISION -->\s*\n## Decision\n/.test(body(c)) && !/External part stays needs-human/.test(body(c)))
+        && !prior.some((c) => /## Decision vetoed/.test(body(c)));
+      if (!torn) return { ...row, reason: "already handled" };
+      await deps.setLabels({ issue: item.issue, remove: [NEEDS_HUMAN_LABEL] });
+      await deps.requeue(item.issue);
+      return { ...row, action: "decided", resumed: true, reason: "finished an interrupted live decision" };
+    }
     const s = splitEscalation(item, policy.matrix, opts);
     row.kind = s.kind;
     if (s.kind === "recovery") return { ...row, reason: "recovery owns this code" };

package/bin/lean/execution.mjs

@@ -562,15 +562,18 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
     for (;;) {
       try { const v = await fn(); st.waited = 0; st.tries = 0; return v; } catch (e) {
         if (!isTransportError(e)) throw e;
-        if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true });
+        // release review BUG-6: a 401 that outlasts the window is an auth failure (capped sandbox.auth), not an uncapped unreachable pool
+        if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true, ...(e?.status === 401 ? { auth: true, hop: e.hop } : {}) });
         const d = Math.min(pollMs * 2 ** Math.min(st.tries++, 10), 30000, Math.max(1, unreachableWindowMs - st.waited));
         st.waited += d;
         await sleep(d);
       }
     }
   };
-  const unreachable = (h, e) => ({ terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
-    detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
+  const unreachable = (h, e) => (e?.auth
+    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${h.id} kept answering 401 (${e.hop === "agent" ? "the pool agent refused the router" : "check the pool token"}): ${errText(e)}; kept for adopt on resume` }
+    : { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
+      detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
   const tailOf = async (h) => { try { return typeof logTail === "function" ? String(await logTail(h) ?? "") : ""; } catch { return ""; } };
   const settleClone = (r, id) => (r?.terminalReason === "engine-error" && r.code === undefined && isCloneFailure(r.detail) ? cloneDeferred(id, errLine(r.detail)) : r);
   const wait = async (h, g = guard({ waited: 0, tries: 0 })) => {
@@ -590,7 +593,7 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
   // A start failure repeats on a relaunch: typed, never engine-error (#4254). Retryable (capacity) defers with backoff.
   // #4685: a refused or missing credential defers with its own code (a fixed token is picked up on the retry), never needs-human.
   const startFailed = (id, e) => ([401, 403].includes(e?.status)
-    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)` }
+    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; ${e?.hop === "agent" ? "the pool agent refused the router (re-enroll or fix the agent token)" : "check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)"}` }
     : e?.retryable === true
     ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_CAPACITY, detail: `sandbox ${id} could not start (capacity): ${errText(e)}`, ...(Number(e.retryInSeconds) > 0 ? { retryInSeconds: Number(e.retryInSeconds) } : {}) }
     : isCloneFailure(errText(e)) ? cloneDeferred(id, errText(e))

package/bin/lean/execution.test.mjs

@@ -116,6 +116,27 @@ describe("providerLanes: an unreachable provider is retried, then deferred witho
     assert.deepEqual(ledgerFor(7).entries("sandbox").map((e) => e.value), ["fake-7"]);
   });
 
+  it("#5053 a 401 on status/result of a launched sandbox is retried, then ends sandbox.auth (capped), never engine-error", async () => {
+    const e401 = () => Object.assign(new Error("sandbox status: HTTP 401 (agent a1 auth)"), { status: 401, hop: "agent", transient: true });
+    const mk = (failPolls) => { const prov = flaky(0); let polls = 0; prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
+      prov.status = async (h) => { if (polls++ < failPolls) throw e401(); return prov.live.has(h.issue) ? "running" : "exited"; }; return prov; };
+    const ok = mk(2); ok.results[7] = { terminalReason: "merged", pr: 1 }; ok.live.delete(7);
+    assert.equal((await lanes(ok).l.runOne(7, "staging")).terminalReason, "merged");
+    const down = mk(1e9);
+    const r = await lanes(down, { unreachableWindowMs: 5000 }).l.runOne(7, "staging");
+    // release review BUG-6: a persistent 401 is sandbox.auth (capped by max_deferrals), not the uncapped sandbox.unreachable
+    assert.deepEqual([r.terminalReason, r.code, r.retryInSeconds], ["deferred", "sandbox.auth", 300]);
+    assert.match(r.detail, /agent a1 auth/);
+    assert.match(r.detail, /pool agent refused the router/);
+  });
+
+  it("#5053 launch 401 from the agent hop says the agent refused, not the forgedock token", async () => {
+    const prov = fakeProvider(); prov.start = async () => { throw Object.assign(new Error("sandbox launch: HTTP 401 (agent a1 auth)"), { status: 401, hop: "agent" }); };
+    const r = await lanes(prov).l.runOne(7, "staging");
+    assert.equal(r.code, "sandbox.auth");
+    assert.match(r.detail, /agent a1 auth.*pool agent refused the router/);
+  });
+
   it("unreachable: adopt applies the same tolerance, and a non-transport error is still engine-error", async () => {
     const down = flaky(1e9);
     const { l } = lanes(down, { unreachableWindowMs: 3000 });

package/bin/lean/issuewatch-decisions.mjs

@@ -0,0 +1,175 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * #4793: the north-star decision engine (#4766) as an issue-watch stage. Each round the watcher lists the open `needs-human`
+ * issues that match its selectors, reads the newest trusted `FORGE:NEEDS_HUMAN` comment of each (its comment id is the escalation
+ * id the engine is idempotent on) and the typed `terminalCode` from a trusted `FORGE:STATE`, and hands it to `settle`. Active when
+ * `decisions.mode: live`, or when the watch sets `sweep_needs_human: true`; escalations that predate the watch are a backlog and
+ * only a `sweep_needs_human` watch touches them. `--dry-run` classifies and writes nothing. Handled issues count against the
+ * watch's `max_issues`, issues sent back to a lane against `max_launches` (`claimsUsed`, shared with the watcher's own caps).
+ * Only live mode acts: a post-deploy-only escalation is folded into `postdeploy_rollup` and closed, an evidence escalation on a repo
+ * that declares `prod_watch` sources goes back to a lane with the prod monitor capability; in shadow mode each only posts.
+ * Gh, the clock, the judge, the lane launch and the rollup `fold` are injected. The judge is the only LLM call.
+ */
+import { settle, classify, tally, isVetoed, revertVeto, effectiveMode, NEEDS_HUMAN_LABEL } from "./decision-engine.mjs";
+import { codeOf } from "./context.mjs";
+import { noComments } from "./render.mjs";
+import { listAssociations } from "./gh-assoc.mjs";
+import { parseProdWatch } from "./prodwatch.mjs";
+import { resolveOwnLogins } from "./prefilter.mjs";
+
+const ROWS_MAX = 200;
+const LIST_LIMIT = "200";
+const MAX_ATTEMPTS = 3; // a judge/gh failure is retried this many rounds, then the escalation is left to a human
+const JUDGE_ROUND_MAX = 20; // judge calls per round, whether or not the watch sets max_issues
+const MS_PER_HOUR = 3600_000;
+const commentId = (c) => /#issuecomment-(\d+)/.exec(String(c?.url ?? ""))?.[1] ?? c?.id ?? null;
+const marked = (body, m) => new RegExp(`^\\s*<!--\\s*FORGE:${m}\\b`).test(String(body ?? ""));
+
+/** True when this watch runs the stage: `decisions.mode: live` (policy from loadPolicy) or `sweep_needs_human: true`. */
+export const decisionsActive = (watch, policy) => policy?.mode === "live" || watch.sweep_needs_human === true;
+
+/** What the watch has used up: `issues` (claimed or handled by the stage, for max_issues) and `launches` (claimed or sent back to a lane, for max_launches). */
+export function claimsUsed(state) {
+  const d = state.decisions;
+  return { issues: new Set([...(state.claimed ?? []), ...(d?.issues ?? [])]).size, launches: new Set([...(state.claimed ?? []), ...(d?.launched ?? [])]).size };
+}
+
+/** `{policy, hasSource}`: the repo's decisions policy and whether forge.yaml declares read-only data access (`prod_watch` sources). */
+export function decisionContext({ loadPolicy, readFile, cwd = "." }) {
+  const policy = loadPolicy({ io: { readFile }, cwd });
+  let hasSource = false;
+  try { hasSource = Object.keys(parseProdWatch(readFile(`${cwd}/forge.yaml`)).sources ?? {}).length > 0; } catch { /* no forge.yaml or no prod_watch block */ }
+  return { policy, hasSource };
+}
+
+/** The typed terminal code of the newest FORGE:STATE record (trusted comments, then the issue body, whose author the stage already vetted). */
+export function terminalCodeOf(body, comments, trusted) {
+  for (const t of [...(comments ?? []).filter(trusted).map((c) => c.body).reverse(), body]) {
+    const m = /FORGE:STATE\s*(\{[^\n]*\})/.exec(String(t ?? ""));
+    if (!m) continue;
+    try { const st = JSON.parse(m[1]); return codeOf(st.terminalCode) ?? codeOf(st.terminalReason); } catch { /* next record */ }
+  }
+  return null;
+}
+
+/** An issue's open escalation: `{id, text, at}` from its newest trusted FORGE:NEEDS_HUMAN comment, or null. */
+export function escalationOf(comments, trusted) {
+  const c = [...(comments ?? [])].reverse().find((x) => marked(x.body, "NEEDS_HUMAN") && trusted(x));
+  return c && commentId(c) ? { id: String(commentId(c)), text: noComments(c.body).replace(/^\s*## Needs human\s*/i, "").trim(), at: Date.parse(c.createdAt) || 0 } : null;
+}
+
+const dec = (state) => (state.decisions ??= { rows: [], handled: {}, attempts: {}, issues: [], launched: [], live: [], stats: { decided: 0, reversed: 0 } });
+
+/** Counts for the watcher status: decided, evidence-gathered, folded, external-kept (decision-engine `tally`). */
+export const decisionCounts = (state) => tally(state.decisions?.rows ?? []);
+
+const ghJson = async (io, args) => JSON.parse(String(await io.gh(args)));
+const poster = (io, repo) => async (_marker, body, issue) => { await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", `<!-- FORGE:DECISION -->\n${body}`]); };
+const labeller = (io, repo) => async ({ issue, add = [], remove = [] }) => {
+  if (!add.length && !remove.length) return;
+  await io.gh(["issue", "edit", String(issue), "-R", repo, ...add.flatMap((l) => ["--add-label", l]), ...remove.flatMap((l) => ["--remove-label", l])]);
+};
+
+/**
+ * One round. `ctx`: `{policy, hasSource}`; `shared`: issuewatch's `{matches(issue, selectors), fold({issue, watch, repo, io, state}), labelNames, isBot, TRUSTED}`;
+ * `launch(issue)→bool` sends the issue back to a lane (the watcher's per-issue launch; false leaves it to normal selection).
+ * `io.judge(q)→answer` is the judgment call (none: every decision part escalates). Returns `{rows}`; with `dryRun` the rows are
+ * classifications and nothing is written to GitHub or `state`. Never throws on one issue: a failure is retried, then left to a human.
+ */
+export async function decisionStep({ watch, repo, io, state, ctx, shared, launch = () => false, dryRun = false }) {
+  const d = dec(state);
+  const startedAt = state.started ?? io.now();
+  if (!dryRun) state.started ??= startedAt;
+  let own = [];
+  try { own = (await resolveOwnLogins(io.gh)).logins; } catch { /* only the associations count */ }
+  const bare = (l) => String(l ?? "").trim().toLowerCase().replace(/^app\//, "").replace(/\[bot\]$/, "");
+  const trusted = (c) => shared.TRUSTED.has(c?.authorAssociation) || (bare(c?.author?.login) !== "" && own.includes(bare(c.author.login))); // never another CONTRIBUTOR
+  const round = { judged: 0 };
+  const rio = { ...io, judge: io.judge && ((q) => { if (round.judged >= JUDGE_ROUND_MAX) throw new Error("judge cap for this round reached"); round.judged++; return io.judge(q); }) };
+  const rows = [];
+  const issues = await ghJson(io, ["issue", "list", "-R", repo, "--state", "open", "--label", NEEDS_HUMAN_LABEL, "--limit", LIST_LIMIT, "--json", "number,title,body,labels,milestone,author,createdAt"]);
+  const assoc = await listAssociations(io.gh, repo);
+  const sel = { ...watch, labels: (watch.labels ?? []).filter((l) => l !== NEEDS_HUMAN_LABEL), exclude_labels: (watch.exclude_labels ?? []).filter((l) => l !== NEEDS_HUMAN_LABEL) };
+  for (const raw of issues) {
+    const i = { ...raw, authorAssociation: assoc.get(raw.number) };
+    if (!shared.TRUSTED.has(i.authorAssociation) || !shared.matches(i, sel)) continue;
+    let view; // release #5068 MAT-2: one unreadable issue is retried next round, never ends the round for the rest
+    try { view = await ghJson(io, ["issue", "view", String(i.number), "-R", repo, "--json", "comments,body,labels"]); } catch (e) { rows.push({ issue: i.number, action: "error", reason: String(e?.message ?? e).slice(0, 200) }); continue; }
+    const esc = escalationOf(view.comments, trusted);
+    if (!esc || d.handled[esc.id]) continue;
+    const backlog = esc.at < startedAt;
+    if (backlog && !watch.sweep_needs_human && !dryRun) continue;
+    const item = { issue: i.number, repo, escalationId: esc.id, text: esc.text, title: i.title, terminalCode: terminalCodeOf(view.body, view.comments, trusted), comments: (view.comments ?? []).filter(trusted) }; // release #5068 SEC-1: a forged FORGE:DECISION can neither finish nor suppress a decision
+    if (dryRun) { rows.push({ ...classify(item, ctx.policy), backlog }); continue; }
+    if (round.judged >= JUDGE_ROUND_MAX) break;
+    const used = claimsUsed(state);
+    if (watch.max_issues && used.issues >= watch.max_issues) break;
+    if (watch.max_launches && used.launches >= watch.max_launches) break;
+    const row = await handle({ item, issue: i, watch, repo, io: rio, state, ctx, shared, launch, d });
+    if (row.action === "error") { // not handled: retried next round, bounded
+      d.attempts[esc.id] = (d.attempts[esc.id] ?? 0) + 1;
+      if (d.attempts[esc.id] >= MAX_ATTEMPTS) d.handled[esc.id] = "error";
+      rows.push(row);
+      continue;
+    }
+    d.handled[esc.id] = row.action;
+    if (row.action === "skipped") continue;
+    d.rows = [...d.rows, { issue: row.issue, escalationId: esc.id, action: row.action, kept: row.kept ?? false }].slice(-ROWS_MAX);
+    if (!d.issues.includes(i.number)) d.issues.push(i.number);
+    rows.push(row);
+  }
+  if (!dryRun) await vetoes({ repo, io, ctx, d, shared });
+  return { rows };
+}
+
+async function handle({ item, issue, watch, repo, io, state, ctx, shared, launch, d }) {
+  const n = item.issue;
+  try {
+    const post = poster(io, repo), setLabels = labeller(io, repo);
+    const back = async () => { if ((await launch(issue)) && !d.launched.includes(n)) d.launched.push(n); }; // label cleared: a lane takes it again; release #5068 MAT-4: counted only when it does
+    const deps = { judge: (q) => (io.judge ? io.judge(q) : null), post, setLabels, requeue: back, records: (await io.records?.()) ?? [], stats: d.stats, emit: io.emit };
+    const row = await settle(item, ctx.policy, deps);
+    if (row.action === "error") return row;
+    if (row.action === "decided" && !row.resumed) { d.stats.decided++; if (!row.kept) d.live.push({ issue: n, escalationId: item.escalationId, at: io.now() }); }
+    const key = `Escalation: ${item.escalationId}`;
+    if (row.action !== "post-deploy-only" && row.action !== "evidence-needed") return row;
+    if (effectiveMode(ctx.policy, d.stats).mode !== "live") { // shadow: say what would happen, change nothing
+      await post(null, `## Decision (shadow)\n${key}\nWould ${row.action === "post-deploy-only" ? "fold into the release checklist and close" : "send back to a prod-monitor lane"}; nothing changed.`, n);
+      return { ...row, action: "shadow" };
+    }
+    if (row.action === "post-deploy-only") {
+      if (!watch.postdeploy_rollup) return { ...row, action: "post-deploy-kept", reason: "no postdeploy_rollup to fold into" };
+      await shared.fold({ issue: { number: n, title: item.title ?? `#${n}` }, watch, repo, io, state });
+      await post(null, `## Post-deploy only\n${key}\nEvery remaining item is post-deploy: folded into #${watch.postdeploy_rollup} (the release checklist) and closed.`, n);
+      await setLabels({ issue: n, remove: [NEEDS_HUMAN_LABEL] });
+      await io.gh(["issue", "close", String(n), "-R", repo, "--reason", "completed"]);
+      return { ...row, folded: true };
+    }
+    if (!ctx.hasSource) return { ...row, action: "evidence-kept", reason: "no read-only data access declared in forge.yaml (prod_watch.sources)" };
+    await post(null, `## Evidence needed\n${key}\nThe block is missing read-only production data and forge.yaml declares a read-only source (prod_watch.sources): back to a lane that has the prod monitor capability (placement prod_monitor), needs-human cleared.`, n);
+    await setLabels({ issue: n, remove: [NEEDS_HUMAN_LABEL] });
+    await back();
+    return row;
+  } catch (e) {
+    return { issue: n, escalationId: item.escalationId, action: "error", reason: String(e?.message ?? e).slice(0, 200) };
+  }
+}
+
+/** A live decision vetoed inside its window (label or reply) is reverted once; a window that ended retires the record. */
+async function vetoes({ repo, io, ctx, d, shared }) {
+  const keep = [];
+  for (const r of d.live) {
+    try {
+      const v = await ghJson(io, ["issue", "view", String(r.issue), "-R", repo, "--json", "comments,labels"]);
+      if (isVetoed({ labels: shared.labelNames(v), comments: v.comments, decidedAt: r.at, vetoHours: ctx.policy.vetoHours, now: io.now() })) {
+        await revertVeto({ issue: r.issue, escalationId: r.escalationId }, { setLabels: labeller(io, repo), post: poster(io, repo), emit: io.emit });
+        d.stats.reversed++;
+      } else if (io.now() < r.at + ctx.policy.vetoHours * MS_PER_HOUR) keep.push(r);
+    } catch { keep.push(r); }
+  }
+  d.live = keep;
+}
+
+/** Dry-run text: one line per escalation with its classification. */
+export const formatDecisionDryRun = (rows) => rows.length ? rows.map((r) => `#${r.issue} ${r.kind}${r.backlog ? " (backlog)" : ""}: ${r.decisions} decision part(s)${r.external.length ? `, external ${r.external.join(",")}` : ""}`).join("\n") : "no needs-human escalations to classify";