forgedock 1.36.0 → 1.37.0

package/bin/lean/decisions.test.mjs

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { normaliseDecision, renderDecision, parseDecisionComment, foldDecisions, anchorMatches, deadAnchors, staleDecisions, historyRows, withDecisionRows, supersede, readDecisionIndex, recordDecisions, HISTORY_ROWS_MAX } from "./decisions.mjs";
+
+const D = (o) => normaliseDecision({ choice: "use A not B", why: "B leaks", anchors: ["src/a/**"], ...o }, { issue: 10, n: 1 });
+
+describe("decisions (#4695)", () => {
+  it("round-trips through the marker line and ignores prose FORGE:DECISION comments", () => {
+    const d = D({ symbols: ["foo"], anchors: ["src/a/x.js", "lib/-->evil"] });
+    const m = renderDecision(d);
+    const body = `<!-- ${m.marker} -->\n${m.body}`;
+    assert.deepEqual(parseDecisionComment(body), d);
+    assert.equal(parseDecisionComment("<!-- FORGE:DECISION -->\n## Decision (batch 1)\n**What**: x"), null);
+    assert.equal(parseDecisionComment("<!-- FORGE:DECISION {nope} -->"), null);
+    assert.equal(normaliseDecision({ choice: "x", anchors: [] }), null);
+  });
+  it("matches files, directory prefixes and globs", () => {
+    assert.ok(anchorMatches("src/a.js", "src/a.js"));
+    assert.ok(anchorMatches("src/a/", "src/a/b/c.js"));
+    assert.ok(anchorMatches("src/**", "src/a/b.js"));
+    assert.ok(anchorMatches("src/*.js", "src/a.js"));
+    assert.ok(!anchorMatches("src/*.js", "src/a/b.js"));
+    assert.deepEqual(deadAnchors(["src/a.js", "gone/**"], ["src/a.js"]), ["gone/**"]);
+  });
+  it("history rows: only anchored deliverables, capped at 5, deterministic", () => {
+    const recs = Array.from({ length: 8 }, (_, i) => normaliseDecision({ id: `d${i}`, choice: `c${i}`, anchors: ["src/**"], issue: i + 1 }));
+    const a = historyRows(recs, ["src/x.js"]), b = historyRows([...recs].reverse(), ["src/x.js"]);
+    assert.equal(a.length, HISTORY_ROWS_MAX);
+    assert.deepEqual(a, b);
+    assert.deepEqual(a.map((r) => r.id), ["H1", "H2", "H3", "H4", "H5"]);
+    assert.equal(a[0].decision, "d7");
+    assert.ok(a.every((r) => r.source === "decision" && r.what.includes("supersedes")));
+    assert.deepEqual(historyRows(recs, ["docs/x.md"]), []);
+    assert.deepEqual(historyRows([], ["src/x.js"]), []);
+  });
+  it("withDecisionRows leaves a contract with no hit untouched (same object)", () => {
+    const c = { preserve: [{ id: "P1", what: "w" }], deliverables: [{ id: "D1", path: "docs/x.md" }] };
+    assert.equal(withDecisionRows(c, [D()]), c);
+    const c2 = withDecisionRows({ ...c, deliverables: [{ id: "D1", path: "src/a/x.js" }] }, [D()]);
+    assert.deepEqual(c2.preserve.map((p) => p.id), ["P1", "H1"]);
+  });
+  it("supersession links both records and the old one stops binding", () => {
+    const old = D({ id: "d10-1" }), next = D({ id: "d11-1", choice: "use B" });
+    const r = supersede(old, next);
+    assert.equal(r.old.status, "superseded");
+    assert.equal(r.old.superseded_by, "d11-1");
+    assert.equal(r.next.supersedes, "d10-1");
+    assert.deepEqual(historyRows([old, r.old], ["src/a/x.js"]), []);
+    assert.equal(foldDecisions([old, r.old]).length, 1);
+  });
+  it("lists stale decisions with dead anchors, never superseded ones", () => {
+    const live = D({ id: "a" }), dead = D({ id: "b", anchors: ["src/a/**", "old/x.js"] }), gone = D({ id: "c", anchors: ["old/y.js"], status: "superseded", superseded_by: "z" });
+    const s = staleDecisions([live, dead, gone], ["src/a/q.js"]);
+    assert.deepEqual(s.map((x) => [x.decision.id, x.dead]), [["b", ["old/x.js"]]]);
+  });
+  it("recordDecisions posts the new and the superseded old record and updates the index", async () => {
+    process.env.FORGEDOCK_DECISIONS_DIR = tmpDir("dec-");
+    const posts = [], post = async (marker, body, on) => { posts.push({ marker, on }); };
+    const first = await recordDecisions([{ choice: "use A", anchors: ["src/**"], why: "w" }], { issue: 10, repo: "o/r", post });
+    assert.equal(first[0].id, "d10-1");
+    const second = await recordDecisions([{ choice: "use B", anchors: ["src/**"], why: "w2", supersedes: "d10-1" }], { issue: 11, repo: "o/r", post });
+    assert.equal(second[0].supersedes, "d10-1");
+    assert.deepEqual(posts.map((p) => p.on), [10, 11, 10]);
+    const idx = foldDecisions(readDecisionIndex("o/r"));
+    assert.equal(idx.find((d) => d.id === "d10-1").status, "superseded");
+    assert.equal(idx.find((d) => d.id === "d11-1").supersedes, "d10-1");
+  });
+});

package/bin/lean/effects.mjs

@@ -18,7 +18,7 @@ import { gatherContextFacts } from "./context.mjs";
 import { makePitfallStore } from "./pitfalls.mjs";
 import { enumeratingTests, missingEntries } from "./enumerating.mjs";
 import { resolveBaseConflicts } from "./conflictres.mjs";
-import { sequenceCollisions, snapshotLane, verifyRenumber, renumberLane, resolveSequenceConflicts, generatorProblems } from "./sequence.mjs";
+import { sequenceCollisions, snapshotLane, verifyRenumber, renumberLane, resolveSequenceConflicts, generatorProblems, journalOrderProblems } from "./sequence.mjs";
 import { duplicateExports } from "./postmerge.mjs";
 import { englishCopies } from "./i18n.mjs";
 import { scopedChecks } from "./checks.mjs";
@@ -30,9 +30,9 @@ import { detectApiShape } from "./apishape.mjs";
 const pexec = promisify(execFile);
 const script = (n) => fileURLToPath(new URL(`../../scripts/${n}`, import.meta.url));
 /** Argv-only child with stdin input; resolves stdout, rejects on a non-zero exit with stderr attached. */
-export function runWithInput(bin, args, input, timeout = 120000) {
+export function runWithInput(bin, args, input, timeout = 120000, env = undefined) {
   return new Promise((res, rej) => {
-    const child = execFile(bin, args, { timeout, maxBuffer: 10 * 1024 * 1024 }, (err, stdout, stderr) => {
+    const child = execFile(bin, args, { timeout, maxBuffer: 10 * 1024 * 1024, ...(env ? { env } : {}) }, (err, stdout, stderr) => {
       if (err) { err.stdout = stdout; err.stderr = stderr; rej(err); } else res(String(stdout));
     });
     // A child that exits before reading stdin closes the pipe: EPIPE must reject via the exit callback, never crash the lane.
@@ -50,14 +50,25 @@ export function systemdRunAvailable() {
   }
   return systemdProbe;
 }
-const MEM_RE = /^\d+(\.\d+)?[KMGT]?$/i, CPU_RE = /^\d+%$/;
-/** Cap the whole lane process tree: spawn under `systemd-run --user --scope -p MemoryMax=.. -p CPUQuota=..`. Unset or invalid limits leave the spawn unchanged. */
-export function wrapLaneLimits(bin, args, limits) {
+const MEM_RE = /^\d+(\.\d+)?[KMGT]?$/i, CPU_RE = /^\d+%$/, TASKS_RE = /^\d+$/;
+/** #4690: default `TasksMax` of a lane scope; `tasks_max: 0|off` (or any non-count) turns the cap off. */
+export const DEFAULT_LANE_TASKS_MAX = 1024;
+let scopeSeq = 0;
+/**
+ * Cap the whole lane process tree: spawn under `systemd-run --user --scope --unit=.. -p TasksMax=.. [-p MemoryMax=..] [-p CPUQuota=..]`.
+ * #4690: `TasksMax` is on by default (one runaway lane cannot fork-bomb the host); unset limits use the default, invalid ones are dropped.
+ * Returns `unit` (the scope name) when wrapped, so the runner can read the scope's limit events.
+ */
+export function wrapLaneLimits(bin, args, limits, unit = `forgedock-lane-${process.pid}-${++scopeSeq}`) {
   const mem = limits?.memory_max && MEM_RE.test(String(limits.memory_max)) ? String(limits.memory_max) : null;
   const cpu = limits?.cpu_quota && CPU_RE.test(String(limits.cpu_quota)) ? String(limits.cpu_quota) : null;
-  if (!mem && !cpu) return { bin, args };
-  return { bin: "systemd-run", args: ["--user", "--scope", "-q", ...(mem ? ["-p", `MemoryMax=${mem}`] : []), ...(cpu ? ["-p", `CPUQuota=${cpu}`] : []), "--", bin, ...args] };
+  const raw = limits?.tasks_max === undefined || limits?.tasks_max === null || limits?.tasks_max === "" ? DEFAULT_LANE_TASKS_MAX : String(limits.tasks_max).trim();
+  const tasks = TASKS_RE.test(raw) && Number(raw) > 0 ? Number(raw) : null;
+  if (!mem && !cpu && !tasks) return { bin, args };
+  return { bin: "systemd-run", unit, args: ["--user", "--scope", "-q", `--unit=${unit}`, ...(tasks ? ["-p", `TasksMax=${tasks}`] : []), ...(mem ? ["-p", `MemoryMax=${mem}`] : []), ...(cpu ? ["-p", `CPUQuota=${cpu}`] : []), "--", bin, ...args] };
 }
+/** #4690: the one line a run logs when lanes get no scope (systemd-run --user --scope unavailable); null when they are capped. */
+export const laneCapNotice = (probe = systemdRunAvailable) => (probe() ? null : "lanes are uncapped: `systemd-run --user --scope` is unavailable here, so no TasksMax/MemoryMax/CPUQuota applies to lane process trees");
 
 /** #4211: a judgment call over data in the prompt only (the refactor:split grouping call): no tool can read a file. */
 export const noToolFlags = (caps) => (caps?.strictMcp ? ["--tools", ""]
@@ -81,7 +92,7 @@ export function makeLeanRunner({ runCli = runCliBackendAsync, probe = systemdRun
     const r = await runCli({ spec: { name: `core-${role}` }, userMessage: prompt, systemPrompt: resume ? "" : system,
       args: [String(issue)], cwd, bin, model, maxTurns: caps?.maxTurns ? maxTurns : null, env, sessionFlags: flags, extraArgs: resume ? ["--resume", resume] : [],
       logger: { log() {} },
-      idleMs, maxMs, stream: idleMs > 0, control, onSpawn, busy, busyMs, onIdleExtend, ...(contextMax > 0 ? { contextMax } : {}), ...(turnMax > 0 ? { turnMax } : {}), wrap: laneLimits && probe() ? (b, a) => wrapLaneLimits(b, a, laneLimits) : null });
+      idleMs, maxMs, stream: idleMs > 0, control, onSpawn, busy, busyMs, onIdleExtend, ...(contextMax > 0 ? { contextMax } : {}), ...(turnMax > 0 ? { turnMax } : {}), wrap: probe() ? (b, a) => wrapLaneLimits(b, a, laneLimits) : null });
     // #4395: a resumed session's first turn carries the prior history, so only a fresh session reports a baseline.
     return { text: r.text, sessionId: r.sessionId, usage: r.usage, costUsd: r.costUsd, durationMs: r.durationMs, turns: r.turns, model: r.cliModel ?? "default",
       baselineTokens: resume ? null : (r.baselineTokens ?? null), tools: toolStats(r.toolUses) }; // #4410: tool calls per turn, re-reads
@@ -328,11 +339,14 @@ export function makeLeanIo({ repo }) {
     async transitionLabel(issue, state) { await pexec("bash", [script("transition-label.sh"), String(issue), "-R", repo, state], { timeout: 60000 }); },
     /**
      * File (or append to) the parent's ONE batched follow-up issue (scripts/file-deferred-followups.sh: trusted lookup,
-     * append, create, claim marker, duplicate re-check). items: [{key, why}]. Returns {key: issueNumber}; throws on exit 3/4.
+     * append, create, claim marker, duplicate re-check; one issue per class, #4664). items: [{key, why, class?}]; the key is the
+     * item's identity (column 5, column 1 is `-`). milestone / defaultMilestone: the parent's and forge.yaml followups.default_milestone; lane: the release lane of the post-deploy rollup (#4724).
+     * Returns {key: issueNumber}; throws on exit 2/3/4.
      */
-    async fileFollowups({ parent, items }) {
-      const tsv = items.map((i) => `${i.key}\t${i.why}\t`).join("\n") + "\n";
-      const out = await runWithInput("bash", [script("file-deferred-followups.sh"), "--parent", String(parent), "--repo", repo], tsv, 180000);
+    async fileFollowups({ parent, items, milestone = null, defaultMilestone = null, lane = null }) {
+      const tsv = items.map((i) => `-\t${i.why}\t\t${i.class ?? ""}\t${i.key}`).join("\n") + "\n";
+      const env = { ...process.env, ...(milestone ? { FORGE_FOLLOWUP_MILESTONE: String(milestone) } : {}), ...(defaultMilestone ? { FORGE_FOLLOWUPS_DEFAULT_MILESTONE: String(defaultMilestone) } : {}), ...(lane ? { FORGE_RELEASE_LANE: String(lane) } : {}) }; // #4724: the post-deploy rollup is one per release lane
+      const out = await runWithInput("bash", [script("file-deferred-followups.sh"), "--parent", String(parent), "--repo", repo], tsv, 180000, env);
       return followupMap(out, items);
     },
     /**
@@ -381,7 +395,8 @@ export function makeLeanIo({ repo }) {
     conflictResolve: ({ cwd, files, cfg }) => resolveBaseConflicts({ git: (a) => run("git")(a, { cwd }), cwd, files, cfg }),
     sequenceLease: makeSequenceLeaseIo(),
     /** #4591: lane-added migrations the repo's generator did not produce (no journal entry or snapshot); [] on failure. */
-    migrationGenerator: ({ cwd, base }) => generatorProblems({ git: (a) => run("git")(a, { cwd }), base }),
+    /** #4661: plus journal `when` stamps not above the base max (newest-only migrators skip them). */
+    migrationGenerator: async ({ cwd, base }) => { const git = (a) => run("git")(a, { cwd }); return [...await generatorProblems({ git, base }), ...await journalOrderProblems({ git, base })]; },
     /** #4406: the lane's scoped checks after a base merge (bin/lean/checks.mjs; [] on failure), and same-app duplicate exports (bin/lean/postmerge.mjs). */
     postMergeChecks: async ({ cwd, base }) => { try { return scopedChecks({ repo: cwd, base }); } catch { return []; } },
     duplicateExports: ({ cwd, base }) => duplicateExports({ git: (a) => run("git")(a, { cwd }), cwd, base }),

package/bin/lean/execution.mjs

@@ -62,6 +62,16 @@ const safeRegExp = (src) => { try { return new RegExp(src, "i"); } catch { retur
 const ENV_RE = /^[A-Z_][A-Z0-9_]*$/;
 const IMAGE_RE = /^[A-Za-z0-9][A-Za-z0-9._\/:@-]*$/;
 const readJson = (file) => { try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; } };
+const TRANSPORT_CODES = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EAI_AGAIN", "EPIPE", "ENOTFOUND", "EHOSTUNREACH", "ENETUNREACH", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
+/** #4672: a failure of the link to the provider (connection refused/reset, `fetch failed`, timeout, 502/503/504), not of the lane. */
+export function isTransportError(e) {
+  if (!e || e.unreachable) return false;
+  if (e.transient === true || [502, 503, 504].includes(e.status)) return true;
+  const code = e.code ?? e.cause?.code;
+  if (TRANSPORT_CODES.has(code)) return true;
+  if (e.name === "AbortError" || e.name === "TimeoutError") return true;
+  return /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|socket hang up|network error|timed? ?out/i.test(String(e.message ?? ""));
+}
 const errText = (e) => String(e?.stderr || e?.message || e).split("\n")[0].slice(0, 300);
 
 /**
@@ -98,7 +108,9 @@ export function parseExecution(sec = {}) {
   const patRaw = Array.isArray(s.host_capability_patterns) ? s.host_capability_patterns : str(s.host_capability_patterns) ? str(s.host_capability_patterns).split(",") : [];
   const hostCapabilityPatterns = patRaw.map((x) => { const t = str(x), i = t.indexOf(":"); return { name: t.slice(0, i).trim().toLowerCase(), pattern: t.slice(i + 1).trim() }; })
     .filter((x) => CAPABILITY_RE.test(x.name) && x.pattern && safeRegExp(x.pattern));
-  return { mode, maxRemote, provider, image, secrets, budget, ...(hostCapabilities.length ? { hostCapabilities } : {}), ...(hostCapabilityPatterns.length ? { hostCapabilityPatterns } : {}), ...(endpoint ? { endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(cacheStore ? { cacheStore } : {}), ...(allowActions ? { allowActions } : {}), ...(fallbackLocal ? { fallbackLocal } : {}), ...(heavy !== null ? { heavy } : {}) };
+  const uw = Number(str(s.unreachable_window_min));
+  const unreachableWindowMin = str(s.unreachable_window_min) !== "" && Number.isFinite(uw) && uw >= 0 ? uw : null; // unset: providerLanes defaults to 5
+  return { mode, maxRemote, provider, image, secrets, budget, ...(hostCapabilities.length ? { hostCapabilities } : {}), ...(hostCapabilityPatterns.length ? { hostCapabilityPatterns } : {}), ...(endpoint ? { endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(cacheStore ? { cacheStore } : {}), ...(allowActions ? { allowActions } : {}), ...(fallbackLocal ? { fallbackLocal } : {}), ...(heavy !== null ? { heavy } : {}), ...(unreachableWindowMin !== null ? { unreachableWindowMin } : {}) };
 }
 
 const round1 = (x) => Math.round(x * 10) / 10;
@@ -429,7 +441,8 @@ export function providerByName(name, opts) {
     try { forgeYaml = opts?.cwd ? readFileSync(join(opts.cwd, "forge.yaml"), "utf8") : null; } catch { forgeYaml = null; }
     // The router's token is resolved under a fixed name so remoteProvider redacts it like any `token_env` credential.
     const tokenEnv = exec.tokenEnv ?? (router?.token ? "FORGEDOCK_POOL_TOKEN" : undefined);
-    const credentials = (n) => (n === "FORGEDOCK_POOL_TOKEN" && !exec.tokenEnv ? router?.token ?? null : resolveCredential(n, { env }));
+    // #4685: an empty env/keychain lookup of a configured token_env falls back to the router's token file.
+    const credentials = (n) => (n === "FORGEDOCK_POOL_TOKEN" && !exec.tokenEnv ? router?.token ?? null : resolveCredential(n, { env }) ?? router?.token ?? null);
     const settings = { ...exec, ...(exec.endpoint ? {} : router ? { endpoint: router.endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(forgeYaml ? { forgeYaml } : {}) };
     return remoteProvider({ batch: opts.batch, repo: opts.repo, settings, name: "pool", env, credentials, transport: remoteTransportFor(settings, { transport: opts?.transport, env, credentials }), ...(opts?.now ? { now: opts.now } : {}) });
   }
@@ -443,7 +456,7 @@ export function providerByName(name, opts) {
  * returns that sandbox's ledger (ledger.mjs `makeLedger`).
  */
 export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env, secrets = [], pollMs = 5000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
-  costPerMinute = 0, now = () => Date.now() }) {
+  costPerMinute = 0, now = () => Date.now(), unreachableWindowMs = 300000 }) {
   const cpm = nonNeg(Number(costPerMinute)) ?? 0;
   // #4255: the sandbox's usage: the provider's `usage()`, else the billable minutes it put on the handle (#4254), else wall
   // clock since the handle's startedAt; cost is the provider's, else minutes x cost_per_minute.
@@ -463,20 +476,38 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
     if (ok) { try { led?.release("sandbox", h.id); led?.remove(); } catch { /* the reaper retries */ } }
     return ok;
   };
-  const wait = async (h) => {
+  // #4672: a transport failure (router restart, refused/reset connection, 502/503/504) is retried with backoff until the lane has
+  // waited `unreachableWindowMs` in a row without one successful poll; then it ends sandbox.unreachable with the sandbox kept.
+  const guard = (st) => async (fn) => {
+    for (;;) {
+      try { const v = await fn(); st.waited = 0; st.tries = 0; return v; } catch (e) {
+        if (!isTransportError(e)) throw e;
+        if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true });
+        const d = Math.min(pollMs * 2 ** Math.min(st.tries++, 10), 30000, Math.max(1, unreachableWindowMs - st.waited));
+        st.waited += d;
+        await sleep(d);
+      }
+    }
+  };
+  const unreachable = (h, e) => ({ terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
+    detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
+  const wait = async (h, g = guard({ waited: 0, tries: 0 })) => {
     for (;;) {
-      const r = await provider.fetchResult(h);
+      const r = await g(() => provider.fetchResult(h));
       if (r) return r;
-      const st = await provider.status(h);
+      const st = await g(() => provider.status(h));
       if (st !== "running") {
-        const last = await provider.fetchResult(h); // it may have written the result just before exiting
+        const last = await g(() => provider.fetchResult(h)); // it may have written the result just before exiting
         return last ?? { terminalReason: "engine-error", detail: `sandbox ${h.id} ${st} without a result` };
       }
       await sleep(pollMs);
     }
   };
   // A start failure repeats on a relaunch: typed, never engine-error (#4254). Retryable (capacity) defers with backoff.
-  const startFailed = (id, e) => (e?.retryable === true
+  // #4685: a refused or missing credential defers with its own code (a fixed token is picked up on the retry), never needs-human.
+  const startFailed = (id, e) => ([401, 403].includes(e?.status)
+    ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)` }
+    : e?.retryable === true
     ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_CAPACITY, detail: `sandbox ${id} could not start (capacity): ${errText(e)}`, ...(Number(e.retryInSeconds) > 0 ? { retryInSeconds: Number(e.retryInSeconds) } : {}) }
     : { terminalReason: "needs-human", code: OUTCOME_CODES.SANDBOX_START_FAILED, detail: `sandbox ${id} failed to start: ${errText(e)}` });
   return {
@@ -486,8 +517,12 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
       const led = ledgerFor(n);
       const base = { provider: provider.name, id: provider.idFor(n), issue: n, startedAt: now() };
       // A sandbox an earlier attempt left behind (same id) goes first, or the new one could not start.
-      if ((led?.entries("sandbox") ?? []).some((e) => e.value === base.id) && !(await tear(led, base))) {
-        return { terminalReason: "engine-error", detail: `sandbox ${base.id} from an earlier attempt could not be torn down` };
+      const left = (led?.entries("sandbox") ?? []).find((e) => e.value === base.id);
+      if (left) {
+        // #4672: a sandbox kept as unreachable is adopted when the provider answers again; it is torn down only when it is gone.
+        const kept = await adoptLeft(n, left, ctx);
+        if (kept) return kept;
+        if (!(await tear(led, base))) return { terminalReason: "engine-error", detail: `sandbox ${base.id} from an earlier attempt could not be torn down` };
       }
       led?.register("sandbox", base.id, { provider: provider.name, handle: base }); // before it exists
       let h = base, r, started = false;
@@ -498,12 +533,15 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
         ctx.onStart?.({ ...(Number.isInteger(h.pid) ? { pid: h.pid } : {}), handle: h, remote: true });
         r = await wait(h);
       } catch (e) {
+        if (started && e?.unreachable) return withUsage(unreachable(h, e), h); // #4672: no tear, the ledger keeps it
         r = started ? { terminalReason: "engine-error", detail: `sandbox ${base.id}: ${errText(e)}` } : startFailed(base.id, e);
       }
       await tear(led, h); // every path, after the result is in (tear never throws)
       return started ? withUsage(r, h) : r; // a sandbox that never started used nothing
     },
-    adopt: async (n, rec, ctx = {}) => {
+    adopt: (n, rec, ctx = {}) => adoptSaved(n, rec, ctx),
+  };
+  async function adoptSaved(n, rec, ctx = {}) {
       const saved = rec?.handle;
       if (!saved?.id || saved.provider !== provider.name) return null;
       // Usage runs from the sandbox's real start (persisted on the handle), not from this adopt.
@@ -511,15 +549,21 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
       const led = ledgerFor(n);
       led?.register("sandbox", h.id, { provider: provider.name, handle: { provider: h.provider, id: h.id, issue: h.issue, ...(Number.isFinite(h.startedAt) ? { startedAt: h.startedAt } : {}) } }); // now owned by this dispatcher
       let r;
+      const g = guard({ waited: 0, tries: 0 });
       try {
-        const ended = await provider.fetchResult(h);
-        if (!ended && (await provider.status(h)) !== "running") r = null; // relaunched
-        else { if (!ended) ctx.onLive?.(); r = await wait(h); } // #4504: a live sandbox is awaited in the background
+        const ended = await g(() => provider.fetchResult(h));
+        if (!ended && (await g(() => provider.status(h))) !== "running") r = null; // relaunched
+        else { if (!ended) ctx.onLive?.(); r = await wait(h, g); } // #4504: a live sandbox is awaited in the background
       } catch (e) {
+        if (e?.unreachable) return withUsage(unreachable(h, e), h); // #4672: same tolerance; no tear, left for the next adopt
         r = { terminalReason: "engine-error", detail: `sandbox ${h.id}: ${errText(e)}` };
       }
       await tear(led, h);
       return withUsage(r, h);
-    },
-  };
+  }
+  // A leftover ledger entry of this lane's sandbox: its result if it can still be had, else null (torn down by the caller).
+  async function adoptLeft(n, left, ctx) {
+    const r = await adoptSaved(n, { handle: left.handle }, ctx).catch(() => null);
+    return r ?? null;
+  }
 }

package/bin/lean/execution.test.mjs

@@ -6,7 +6,7 @@ import assert from "node:assert/strict";
 import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { parseExecution, SANDBOX_PROVIDERS, detectNeeds, hostCapabilityBlock, placement, placeLane, validationWeight, hostHeadroom, scopedEnv, providerLanes, containerProvider, providerByName, openExecutionBudget } from "./execution.mjs";
+import { parseExecution, SANDBOX_PROVIDERS, detectNeeds, hostCapabilityBlock, placement, placeLane, validationWeight, hostHeadroom, scopedEnv, providerLanes, isTransportError, containerProvider, providerByName, openExecutionBudget } from "./execution.mjs";
 import { makeLedger, countLiveLanes } from "./ledger.mjs";
 import { openBatch } from "./batch.mjs";
 import { orchestrate, formatTable, formatSandboxSummary } from "./orchestrate.mjs";
@@ -24,7 +24,7 @@ function fakeProvider(opts = {}) {
     async start({ issue, env, secrets }) {
       p.calls.push(["start", issue]);
       p.ledgerAtStart = opts.ledger?.entries("sandbox").map((e) => e.value) ?? null;
-      if (opts.startThrows) throw new Error("boom: no capacity");
+      if (opts.startThrows) throw Object.assign(new Error("boom: no capacity"), opts.startThrows === 401 ? { status: 401 } : {});
       p.live.add(issue);
       p.env = env; p.secrets = secrets;
       if (opts.finish) { const r = opts.finish(issue); if (r !== undefined) { p.results[issue] = r; p.live.delete(issue); } }
@@ -81,6 +81,56 @@ describe("execution config (#4253)", () => {
   });
 });
 
+describe("providerLanes: an unreachable provider is retried, then deferred without teardown (#4672)", () => {
+  const flaky = (failPolls) => {
+    const prov = fakeProvider(); let polls = 0;
+    prov.live.add(7);
+    const gate = () => { if (polls++ < failPolls) throw Object.assign(new TypeError("fetch failed"), { cause: { code: "ECONNREFUSED" } }); };
+    prov.fetchResult = async (h) => { gate(); return prov.results[h.issue] ?? null; };
+    prov.status = async (h) => { gate(); prov.calls.push(["status", h.issue]); return prov.live.has(h.issue) ? "running" : "exited"; };
+    return prov;
+  };
+  const lanes = (prov, extra = {}) => { const root = tmpDir("exec-unreach-"); const ledgerFor = ledgerIn(root); const slept = [];
+    return { slept, ledgerFor, l: providerLanes({ provider: prov, laneArgs: () => [], ledgerFor, pollMs: 1000, now: () => 1000, sleep: async (ms) => { slept.push(ms); if (slept.length === 6) { prov.results[7] = { terminalReason: "merged", pr: 9 }; prov.live.delete(7); } }, ...extra }) }; };
+
+  it("unreachable: N failed polls then recovery yields the real result", async () => {
+    const prov = flaky(3); prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
+    const { l, slept } = lanes(prov);
+    const r = await l.runOne(7, "staging");
+    assert.equal(r.terminalReason, "merged");
+    assert.ok(slept.length >= 3 && slept[1] > slept[0]); // backoff
+    assert.equal(prov.calls.filter((c) => c[0] === "teardown").length, 1); // only after the result
+  });
+
+  it("unreachable: down past the window yields sandbox.unreachable, no teardown, ledger kept", async () => {
+    const prov = flaky(1e9); prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
+    const { l, ledgerFor } = lanes(prov, { unreachableWindowMs: 5000 });
+    const r = await l.runOne(7, "staging");
+    assert.deepEqual([r.terminalReason, r.code], ["deferred", "sandbox.unreachable"]);
+    assert.equal(prov.calls.some((c) => c[0] === "teardown"), false);
+    assert.deepEqual(ledgerFor(7).entries("sandbox").map((e) => e.value), ["fake-7"]);
+  });
+
+  it("unreachable: adopt applies the same tolerance, and a non-transport error is still engine-error", async () => {
+    const down = flaky(1e9);
+    const { l } = lanes(down, { unreachableWindowMs: 3000 });
+    const r = await l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } });
+    assert.equal(r.code, "sandbox.unreachable");
+    assert.equal(down.calls.some((c) => c[0] === "teardown"), false);
+    const ok = flaky(2); ok.results[7] = { terminalReason: "merged", pr: 1 };
+    assert.equal((await lanes(ok).l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } })).terminalReason, "merged");
+    const bad = fakeProvider(); bad.fetchResult = async () => { throw new Error("boom"); };
+    assert.equal((await lanes(bad).l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } })).terminalReason, "engine-error");
+  });
+
+  it("unreachable: classifies transport errors and parses the window key", () => {
+    for (const e of [new TypeError("fetch failed"), Object.assign(new Error("x"), { code: "ECONNRESET" }), Object.assign(new Error("x"), { status: 502 }), Object.assign(new Error("x"), { name: "AbortError" })]) assert.equal(isTransportError(e), true);
+    for (const e of [new Error("boom"), Object.assign(new Error("HTTP 404"), { status: 404 })]) assert.equal(isTransportError(e), false);
+    assert.equal(parseExecution({}).unreachableWindowMin, undefined); // providerLanes defaults it to 5 min
+    assert.equal(parseExecution({ unreachable_window_min: "2" }).unreachableWindowMin, 2);
+  });
+});
+
 describe("providerLanes: sandbox ledger and teardown on every exit path (#4253)", () => {
   const run = async ({ finish, startThrows, teardownOk, pollMs = 1 } = {}) => {
     const root = tmpDir("exec-ledger-");
@@ -118,6 +168,13 @@ describe("providerLanes: sandbox ledger and teardown on every exit path (#4253)"
     assert.equal(led.read(), null);
   });
 
+  it("#4685 start refused with HTTP 401: deferred sandbox.auth with a long retry, never needs-human", async () => {
+    const { r } = await run({ startThrows: 401 });
+    assert.equal(r.terminalReason, "deferred");
+    assert.equal(r.code, "sandbox.auth");
+    assert.equal(r.retryInSeconds, 300);
+  });
+
   it("start throws: typed needs-human sandbox.start_failed row (#4254), teardown still runs", async () => {
     const { r, prov, led } = await run({ startThrows: true });
     assert.equal(r.terminalReason, "needs-human");

package/bin/lean/gatewatch.mjs

@@ -0,0 +1,31 @@
+/**
+ * forge#4656: ship-gate watchdog. A gate (a settle pass, a hold list, `/milestone ship` pre-flight) that waits on an issue the
+ * scheduler will not admit never resolves; this pure module names those waits so the batch reports `GATE_DEADLOCK` within one pass
+ * instead of polling silently. No I/O.
+ */
+
+/**
+ * `gates`: [{ name, waitsOn: [issue...], since?: ms }]; `admissible(n)`: true when the scheduler could still run or finish `n`;
+ * `now`: ms. Returns one { issue, gate, ageMs } per (gate, issue) wait that cannot resolve, in gate order.
+ */
+export function detectGateDeadlock({ gates, admissible, now = Date.now(), firstSeen = null }) {
+  const out = [];
+  for (const g of Array.isArray(gates) ? gates : []) {
+    for (const n of Array.isArray(g?.waitsOn) ? g.waitsOn : []) {
+      let ok = true;
+      try { ok = admissible(n) !== false; } catch { ok = true; }
+      if (ok) continue;
+      const key = `${g.name}#${n}`;
+      if (firstSeen && !firstSeen.has(key)) firstSeen.set(key, Number.isFinite(g.since) ? g.since : now);
+      const since = firstSeen?.get(key) ?? (Number.isFinite(g.since) ? g.since : now);
+      out.push({ issue: n, gate: String(g.name), ageMs: Math.max(0, now - since) });
+    }
+  }
+  return out;
+}
+
+/** One `pipeline-status` line: `GATE_DEADLOCK issue=#N gate=G age=12m`. */
+export function formatGateDeadlock(d) {
+  const m = Math.round((Number(d.ageMs) || 0) / 60000);
+  return `GATE_DEADLOCK issue=#${d.issue} gate=${d.gate} age=${m}m`;
+}

package/bin/lean/gatewatch.test.mjs

@@ -0,0 +1,59 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { detectGateDeadlock, formatGateDeadlock } from "./gatewatch.mjs";
+import { orchestrate } from "./orchestrate.mjs";
+import { classifyNeedsHuman, classifyLegacyNeedsHuman, parkedLabel } from "../engine/outcome-codes.mjs";
+
+describe("#4656 gate watchdog", () => {
+  it("a milestone ship gate waiting on a parked issue yields GATE_DEADLOCK with issue, gate and age", () => {
+    const firstSeen = new Map([["ship#7", 0]]);
+    const d = detectGateDeadlock({ gates: [{ name: "ship", waitsOn: [7, 8] }], admissible: (n) => n !== 7, now: 120000, firstSeen });
+    assert.deepEqual(d, [{ issue: 7, gate: "ship", ageMs: 120000 }]);
+    assert.equal(formatGateDeadlock(d[0]), "GATE_DEADLOCK issue=#7 gate=ship age=2m");
+  });
+  it("an admissible wait is not a deadlock", () => {
+    assert.deepEqual(detectGateDeadlock({ gates: [{ name: "ship", waitsOn: [8] }], admissible: () => true }), []);
+  });
+  it("the batch runner reports GATE_DEADLOCK once per (gate, issue) for a parked issue on the gate, instead of waiting", async () => {
+    const rows = [], decisions = [];
+    const runOne = async (n) => n === 1
+      ? { terminalReason: "needs-human", code: "conformance.unimplemented", detail: "unknown requirement", pr: 9 }
+      : { terminalReason: "merged", pr: 10 };
+    const _unused = async (n) => n === 1
+      ? { terminalReason: "needs-human", code: "ci.red", detail: "lint failed", pr: 9 }
+      : { terminalReason: "merged", pr: 10 };
+    const relabels = [];
+    const res = await orchestrate({ issues: [1, 2], parallel: 1, runOne, readBody: async () => "", onUpdate: (r) => rows.push(r), decide: (d) => decisions.push(d),
+      getGates: () => [{ name: "ship", waitsOn: [1] }], recovery: { maxAttempts: 1 }, relabelNeedsHuman: async (a) => { relabels.push(a); return true; }, pollMs: 1, sleep: async () => {} });
+    const dl = rows.filter((r) => r.outcome === "GATE_DEADLOCK");
+    assert.equal(dl.length, 1);
+    assert.match(dl[0].detail, /GATE_DEADLOCK issue=#1 gate=ship/);
+    assert.ok(Array.isArray(res));
+    assert.deepEqual(relabels.map((r) => r.parked?.code), ["conformance.unimplemented"]);
+  });
+});
+
+describe("#4656 parked class and migration", () => {
+  it("parked is opt-in and leaves the default classes untouched", () => {
+    const row = { outcome: "needs-human", code: "conformance.unimplemented" };
+    assert.equal(classifyNeedsHuman(row), "terminal");
+    assert.equal(classifyNeedsHuman(row, { parked: true }), "parked");
+    assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: "builder.blocked" }, { parked: true }), "human");
+    assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: "ci.red" }, { parked: true }), "recoverable");
+    assert.equal(classifyNeedsHuman({ outcome: "needs-human" }, { parked: true }), "terminal");
+  });
+  it("legacy needs-human classifies from the terminal annotation and defaults to external", () => {
+    assert.deepEqual(classifyLegacyNeedsHuman("terminal: ci.red after final round"), { kind: "parked", code: "ci.red" });
+    assert.deepEqual(classifyLegacyNeedsHuman("waiting for the AWS account"), { kind: "external" });
+    assert.deepEqual(classifyLegacyNeedsHuman(null), { kind: "external" });
+    assert.equal(parkedLabel("ci.red"), "parked:ci.red");
+  });
+});
+
+describe("#4656 report routing", () => {
+  it("a parked row past recovery is listed on the remediation queue line, not unrecovered or escalations", async () => {
+    const { formatOutcomeLists } = await import("./orchestrate.mjs");
+    const lines = formatOutcomeLists([{ issue: 1, outcome: "needs-human", code: "conformance.unimplemented", pr: 9, parkedQueued: true }]);
+    assert.deepEqual(lines, ["remediation queue (/work-on:remediate): #1 (parked:conformance.unimplemented PR #9)"]);
+  });
+});