forgedock 1.24.3 → 1.24.4

package/bin/engine/outcome-codes.mjs

@@ -36,6 +36,9 @@ export const OUTCOME_CODES = Object.freeze({
   BASE_CONFLICT: "base.conflict",
   RATE_LIMITED: "rate_limited",
   HOST_BUSY: "host.busy",
+  MERGE_TRANSIENT: "merge.transient",
+  PUSH_TRANSIENT: "push.transient",
+  CLI_IDLE: "cli.idle",
   PATH_SWITCH_CONFLICT: "path.switch_conflict",
   ENGINE_ERROR: "engine.error",
   INFRA_DISK_QUOTA: "infra.disk_quota",

package/bin/engine-cli.mjs

@@ -113,6 +113,24 @@ export function readAdmission(cwd = process.cwd()) {
       max_load_per_core: num(e.max_load_per_core ?? e.admission_max_load_per_core) };
   } catch { return {}; }
 }
+/**
+ * forge.yaml lane caps and CLI timeouts for the core path (#4115, #4112). The parser flattens one nested level, so
+ * `pipeline:`/`engine:` then `lane_limits:` with `memory_max`/`cpu_quota` arrives as `memory_max`/`cpu_quota` (or flat
+ * `lane_limits_memory_max`/`lane_limits_cpu_quota`). `engine.cli_idle_minutes` (default 15) and `engine.cli_max_minutes` (default 180).
+ */
+export function readCoreLimits(cwd = process.cwd()) {
+  try {
+    const y = parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8")) ?? {};
+    const pick = (k) => { for (const sec of [y.pipeline, y.engine]) { const v = sec?.[k] ?? sec?.[`lane_limits_${k}`]; if (v !== undefined && v !== "") return String(v).trim(); } return undefined; };
+    const mem = pick("memory_max"), cpu = pick("cpu_quota");
+    const posNum = (v) => { const n = Number(v); return Number.isFinite(n) && n > 0 ? n : undefined; };
+    return {
+      ...(mem || cpu ? { laneLimits: { ...(mem ? { memory_max: mem } : {}), ...(cpu ? { cpu_quota: cpu } : {}) } } : {}),
+      ...(posNum(y.engine?.cli_idle_minutes) ? { cliIdleMinutes: posNum(y.engine.cli_idle_minutes) } : {}),
+      ...(posNum(y.engine?.cli_max_minutes) ? { cliMaxMinutes: posNum(y.engine.cli_max_minutes) } : {}),
+    };
+  } catch { return {}; }
+}
 /** forge.yaml `engine.merge_wait` (required|all|none); undefined when absent or invalid. */
 export function readMergeWait(cwd = process.cwd()) {
   try {
@@ -480,7 +498,7 @@ export async function runFromCli(argv, deps = {}) {
     const leanIo = deps.leanIo ?? (await import("./lean/effects.mjs")).makeLeanIo({ repo: leanRepo });
     const lres = await runCore({ issue, repo: leanRepo, lane, cwd: process.cwd(), dir, io: leanIo, path: enginePath, onProgress,
       emitTelemetry: deps.emitTelemetry ?? makeEmitTelemetry({ repo: leanRepo }),
-      admission: readAdmission(), ...(readMergeWait() ? { mergeWait: readMergeWait() } : {}), ...(model ? { builderModel: model } : {}), ...(readLeanSetting("reviewer_model") ? { reviewerModel: readLeanSetting("reviewer_model") } : {}) });
+      admission: readAdmission(), ...readCoreLimits(), ...(readMergeWait() ? { mergeWait: readMergeWait() } : {}), ...(model ? { builderModel: model } : {}), ...(readLeanSetting("reviewer_model") ? { reviewerModel: readLeanSetting("reviewer_model") } : {}) });
     if (lres.code === "host.busy") { console.log(`issue #${issue} → deferred (${lres.detail})`); return lres; }
     console.log(`issue #${issue} → ${lres.terminalReason}`);
     if (!["merged", "invalid"].includes(lres.terminalReason)) console.log(lres.detail ?? "");
@@ -673,7 +691,7 @@ export async function orchestrateFromCli(argv, deps = {}) {
     issues, parallel,
     readBody: async (n) => JSON.parse(await leanIo.gh(["issue", "view", String(n), "-R", repo, "--json", "body"])).body,
     runOne: (n) => runCore({ issue: n, repo, lane, cwd: process.cwd(), dir, io: leanIo, agentId: `orch_${process.pid}_${n}`, emitTelemetry,
-      admission: readAdmission(), ...(readMergeWait() ? { mergeWait: readMergeWait() } : {}), ...(model ? { builderModel: model } : {}), ...(readLeanSetting("reviewer_model") ? { reviewerModel: readLeanSetting("reviewer_model") } : {}) }),
+      admission: readAdmission(), ...readCoreLimits(), ...(readMergeWait() ? { mergeWait: readMergeWait() } : {}), ...(model ? { builderModel: model } : {}), ...(readLeanSetting("reviewer_model") ? { reviewerModel: readLeanSetting("reviewer_model") } : {}) }),
     onUpdate: (r) => console.log(`#${r.issue} → ${r.outcome}${r.detail && r.outcome !== "merged" ? ` (${r.detail})` : ""}`),
   });
   console.log("\n" + formatTable(rows));

package/bin/lean/effects.mjs

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
 /** Core work-on: the real effects behind runCore's injected `io`. Argv only, no shell. */
-import { execFile } from "node:child_process";
+import { execFile, execFileSync } from "node:child_process";
 import { promisify } from "node:util";
 import { readFileSync, appendFileSync, mkdirSync, statSync, existsSync, rmSync, writeFileSync, readdirSync, lstatSync } from "node:fs";
 import { homedir, hostname, freemem, loadavg, cpus, tmpdir } from "node:os";
@@ -16,14 +16,32 @@ const pexec = promisify(execFile);
 const script = (n) => fileURLToPath(new URL(`../../scripts/${n}`, import.meta.url));
 
 /** Claude CLI session runner: fresh session, or `--resume <id>` for the builder. Resolves usage + session_id. */
-export function makeLeanRunner({ runCli = runCliBackendAsync } = {}) {
-  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, env = null }) => {
+let systemdProbe = null;
+/** `systemd-run --user --scope` usable here? Probed once and cached. */
+export function systemdRunAvailable() {
+  if (systemdProbe === null) {
+    try { execFileSync("systemd-run", ["--user", "--scope", "-q", "true"], { stdio: "ignore", timeout: 5000 }); systemdProbe = true; } catch { systemdProbe = false; }
+  }
+  return systemdProbe;
+}
+const MEM_RE = /^\d+(\.\d+)?[KMGT]?$/i, CPU_RE = /^\d+%$/;
+/** Cap the whole lane process tree: spawn under `systemd-run --user --scope -p MemoryMax=.. -p CPUQuota=..`. Unset or invalid limits leave the spawn unchanged. */
+export function wrapLaneLimits(bin, args, limits) {
+  const mem = limits?.memory_max && MEM_RE.test(String(limits.memory_max)) ? String(limits.memory_max) : null;
+  const cpu = limits?.cpu_quota && CPU_RE.test(String(limits.cpu_quota)) ? String(limits.cpu_quota) : null;
+  if (!mem && !cpu) return { bin, args };
+  return { bin: "systemd-run", args: ["--user", "--scope", "-q", ...(mem ? ["-p", `MemoryMax=${mem}`] : []), ...(cpu ? ["-p", `CPUQuota=${cpu}`] : []), "--", bin, ...args] };
+}
+
+export function makeLeanRunner({ runCli = runCliBackendAsync, probe = systemdRunAvailable } = {}) {
+  return async ({ role, issue, cwd, model, resume, system, prompt, readOnly = false, env = null, idleMs = 0, maxMs = 0, laneLimits = null, control = null }) => {
     const bin = resolveClaudeCliBinary(cwd) || "claude";
     const caps = getClaudeCliCapabilities(cwd);
     const flags = [...buildLeanSessionFlags(caps), ...(readOnly ? buildReadOnlyToolFlags(caps) : [])];
     const r = await runCli({ spec: { name: `core-${role}` }, userMessage: prompt, systemPrompt: resume ? "" : system,
       args: [String(issue)], cwd, bin, model, env, sessionFlags: flags, extraArgs: resume ? ["--resume", resume] : [],
-      logger: { log() {} } });
+      logger: { log() {} },
+      idleMs, maxMs, stream: idleMs > 0, control, wrap: laneLimits && probe() ? (b, a) => wrapLaneLimits(b, a, laneLimits) : null });
     return { text: r.text, sessionId: r.sessionId, usage: r.usage, costUsd: r.costUsd, durationMs: r.durationMs, turns: r.turns, model: r.cliModel ?? "default" };
   };
 }

package/bin/lean/lean.mjs

@@ -12,6 +12,7 @@ import { codeForTerminalReason, OUTCOME_CODES } from "../engine/outcome-codes.mj
 import { ciPollDelayMs, ENGINE_VERSION, DEFAULT_MAX_SESSION_LIMIT_PAUSES, DEFAULT_SESSION_LIMIT_WAIT_MS, parseLeaseHolder, sanitizeLeaseHost } from "../engine.mjs";
 import { openDependency } from "../engine/phases.mjs";
 import { checkAdmission, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
+import { retrying, PUSH_TRANSIENT_RE, PUSH_PERMANENT_RE, GH_TRANSIENT_RE, MERGE_TRANSIENT_RE, MERGE_CONFLICT_RE } from "./retry.mjs";
 import { renderInvestigation, renderPlan, renderImplementation, renderReview, renderPrBody, renderTrajectory, capComment, sanitizeMd } from "./render.mjs";
 import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, PLAN_FILE, IMPLEMENTATION_FILE, builderPrompt, resumeBuilderPrompt, reviewerPrompt, parseVerdict } from "./prompts.mjs";
 
@@ -33,7 +34,7 @@ const num = (v) => (Number.isFinite(v) ? v : null);
  */
 export async function runCore(o) {
   const { issue, repo, lane, cwd, dir, io, builderModel = null, reviewerModel = DEFAULT_REVIEWER_MODEL,
-    ciMaxMinutes = 360, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
+    ciMaxMinutes = 360, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, cliIdleMinutes = 15, cliMaxMinutes = 180, laneLimits = null, maxMergeRetries = 5, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
   const now = io.now ?? (() => Date.now());
   const sleep = io.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
   const base = `origin/${lane}`;
@@ -46,6 +47,9 @@ export async function runCore(o) {
   const ev = (e) => appendEvent(dir, issue, e);
   const emit = async (rec) => { try { await emitTelemetry?.({ ...rec, issue }); } catch { /* never fails a run */ } };
   const ghc = (args) => io.gh(args);
+  const sleepFn = io.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  /** gh writes (comment, pr create, issue edit/close): retried for 5xx and secondary rate limits (#4114). */
+  const ghw = (args) => retrying(() => io.gh(args), { transient: GH_TRANSIENT_RE, attempts: 4, sleep: sleepFn, random });
   const git = (args) => io.git(args, { cwd: wt });
 
   async function attempt(phase, n, r) {
@@ -59,10 +63,10 @@ export async function runCore(o) {
   }
   const post = async (marker, body) => {
     const marks = (Array.isArray(marker) ? marker : [marker]).map((m) => `<!-- ${m} -->`).join("\n");
-    const out = String(await ghc(["issue", "comment", String(issue), "-R", repo, "--body", capComment(`${marks}\n${body}`)])).trim();
+    const out = String(await ghw(["issue", "comment", String(issue), "-R", repo, "--body", capComment(`${marks}\n${body}`)])).trim();
     return /https?:\/\/\S+/.exec(out)?.[0] ?? null;
   };
-  const label = async (state) => { try { await io.transitionLabel(issue, state); } catch { /* labels are best effort */ } };
+  const label = async (state) => { try { await retrying(() => io.transitionLabel(issue, state), { transient: GH_TRANSIENT_RE, attempts: 4, sleep: sleepFn, random }); } catch { /* labels are best effort */ } };
   const usageLine = () => `cost $${usage.cost_usd.toFixed(4)}, tokens ${usage.input} in / ${usage.output} out / ${usage.cache} cache, wall ${Math.round((now() - t0) / 60000 * 10) / 10} min`;
 
   // ---- lease (same FORGE:STATE lease the full path uses) ----
@@ -190,11 +194,76 @@ export async function runCore(o) {
     }
   }
 
+  /** git push with transient retry (#4114): 4 attempts; permission/protected-branch rejections fail at once. */
+  const gitPush = async (args) => {
+    try { return await retrying(() => git(args), { transient: PUSH_TRANSIENT_RE, permanent: PUSH_PERMANENT_RE, attempts: 4, sleep: sleepFn, random }); }
+    catch (e) {
+      if (e?.transientExhausted) { e.deferCode = OUTCOME_CODES.PUSH_TRANSIENT; e.deferDetail = `git push kept failing transiently after 4 attempts: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`; }
+      else if (e?.permanent) e.pushRejected = true;
+      throw e;
+    }
+  };
+
+  /** After a timeout/idle kill: commit uncommitted work as a checkpoint and push the lane branch so a rerun resumes from it (#4112). */
+  async function checkpoint(reason) {
+    try {
+      const dirty = String(await git(["status", "--porcelain"])).trim();
+      if (dirty) { await git(["add", "-A"]); await git(["commit", "--no-verify", "-m", `wip(forge): checkpoint after ${reason}`]); }
+      const ahead = String(await git(["diff", "--name-only", `${base}...HEAD`])).trim();
+      if (ahead) await gitPush(["push", "-u", "origin", branch]);
+      ev({ event: "CHECKPOINT", reason, dirty: Boolean(dirty), pushed: Boolean(ahead) });
+    } catch (e) { ev({ event: "CHECKPOINT_FAILED", reason, error: String(e?.message ?? e).slice(0, 300) }); }
+  }
+  const CONTINUE_NOTE = "\n\nThe branch already holds work from an earlier session that was interrupted (see the latest `wip(forge): checkpoint` commit). Continue from it: read `git log` and `git diff origin/BASE...HEAD`, do not start over.";
+
+  /** Continuous admission (#4115): while the builder runs, pause its process group when the host is over the ceiling, resume when under. */
+  function startGovernor(control) {
+    if (!io.host || o.governor === false || o.admission?.enabled === false) return { stop() {} };
+    const pollMs = o.governorPollMs ?? 15000, maxPauseMs = o.maxPauseMs ?? 600000;
+    let stopped = false, paused = false, forced = false, pausedAt = 0, wake = null, t = null;
+    const loop = (async () => {
+      while (!stopped) {
+        await new Promise((res) => { wake = res; t = setTimeout(res, pollMs); });
+        clearTimeout(t);
+        if (stopped) break;
+        let ok = true;
+        try { ok = (await checkAdmission({ cfg: o.admission, host: io.host })).ok; } catch { ok = true; }
+        const nowMs = Date.now();
+        if (!ok && !paused && !forced && control.pause) { if (control.pause()) { paused = true; pausedAt = nowMs; ev({ event: "LANE_PAUSED", reason: "host over ceiling" }); onProgress({ event: "phase_paused", phase: "builder", reason: "host-busy", detail: "host over the admission ceiling; lane paused" }); } }
+        else if (ok && paused) { control.resume?.(); paused = false; forced = false; ev({ event: "LANE_RESUMED", reason: "host recovered" }); }
+        else if (ok) forced = false;
+        else if (paused && nowMs - pausedAt >= maxPauseMs) { control.resume?.(); paused = false; forced = true; ev({ event: "LANE_RESUMED", reason: "max pause reached" }); }
+      }
+    })();
+    return { async stop() { stopped = true; wake?.(); clearTimeout(t); await loop; if (paused) { control.resume?.(); paused = false; } } };
+  }
+
   async function builder(n, prompt, resume, live = false) {
+    let timeoutRetries = 0, p = prompt;
+    for (;;) {
+      try { return await builderOnce(n, p, resume, live); } catch (e) {
+        if (!(e?.code === "CLI_IDLE" || e?.code === "CLI_TIMEOUT")) throw e;
+        const idle = e.code === "CLI_IDLE";
+        const reason = idle ? "idle timeout" : "timeout";
+        ev({ event: idle ? "CLI_IDLE_KILL" : "CLI_TIMEOUT_KILL", reason, lastTool: e.lastTool ?? null, children: e.children ?? [] });
+        await checkpoint(reason);
+        if (timeoutRetries++ >= 1) { e.deferCode = idle ? OUTCOME_CODES.CLI_IDLE : OUTCOME_CODES.CLI_TIMEOUT; e.deferDetail = `builder ${reason}; work checkpointed to ${branch}; rerun resumes from it. ${String(e.message).slice(0, 300)}`; throw e; }
+        T.anomalies.push(`builder ${reason}: checkpointed, retried once`);
+        T.decisions.push(`Builder ${reason}: uncommitted work checkpointed to the lane branch; retrying once from the checkpoint`);
+        p = builderPrompt(issue) + CONTINUE_NOTE.replace("BASE", base); resume = null;
+      }
+    }
+  }
+
+  async function builderOnce(n, prompt, resume, live = false) {
     onProgress({ event: "phase_enter", phase: "build" });
     await renew();
-    const call = Promise.resolve(callRunner({ role: "builder", issue, cwd: wt, model: builderModel, resume, system: "", prompt }));
+    const control = {};
+    const governor = startGovernor(control);
+    const call = Promise.resolve(callRunner({ role: "builder", issue, cwd: wt, model: builderModel, resume, system: "", prompt, control,
+      idleMs: Math.round(cliIdleMinutes * 60000), maxMs: Math.round(cliMaxMinutes * 60000), laneLimits }));
     let r;
+    call.catch(() => governor.stop());
     if (live && io.fs.stat) {
       let settled = false;
       const tracked = call.then((v) => { settled = true; return v; }, (e) => { settled = true; throw e; });
@@ -206,7 +275,8 @@ export async function runCore(o) {
         if (!settled) { try { await livePoll(); } catch { /* live posting is best effort */ } }
       }
       r = await tracked;
-    } else r = await call;
+    } else { try { r = await call; } finally { /* governor stops below */ } }
+    await governor.stop();
     await renew();
     if (r?.sessionId) sessionId = r.sessionId;
     await attempt("implement", n, r);
@@ -309,9 +379,11 @@ export async function runCore(o) {
 
     // ---- 3. builder ----
     let result = null, attemptN = 0, reviewN = 0;
-    if (!(resumed && (await changedFiles()).length > 0)) {
+    const lastSubject = resumed ? String(await git(["log", "-1", "--format=%s"]).catch(() => "")).trim() : "";
+    const fromCheckpoint = lastSubject.startsWith("wip(forge): checkpoint");
+    if (!(resumed && (await changedFiles()).length > 0) || fromCheckpoint) {
       await label("building");
-      await builder(++attemptN, builderPrompt(issue), null, true);
+      await builder(++attemptN, builderPrompt(issue) + (fromCheckpoint ? CONTINUE_NOTE.replace("BASE", base) : ""), null, true);
       result = await readResult();
       const files0 = await changedFiles();
       const evidence = (Array.isArray(result?.evidence) ? result.evidence : []).filter((x) => typeof x === "string" && x.trim());
@@ -322,7 +394,7 @@ export async function runCore(o) {
       if (closing && !evidence.length) return await needsHuman(`Builder reported "${result.status}" without evidence, so the issue is not closed automatically.\n\n${result.summary ?? "(no summary)"}`, OUTCOME_CODES.BUILDER_NO_EVIDENCE);
       if (closing) {
         T.plan = "skipped"; T.implement = "skipped"; T.review = "skipped"; T.pr = "none"; T.merge = "n/a";
-        await ghc(["issue", "close", String(issue), "-R", repo, "--reason", "not planned"]);
+        await ghw(["issue", "close", String(issue), "-R", repo, "--reason", "not planned"]);
         await label("invalid");
         return await finish("invalid", result.summary ?? result.status);
       }
@@ -368,8 +440,8 @@ export async function runCore(o) {
 
     // ---- 5. push + PR ----
     const files = await changedFiles();
-    await git(["push", "-u", "origin", branch]);
-    const prOut = resumed?.pr ? `/pull/${resumed.pr}` : String(await ghc(["pr", "create", "-R", repo, "--base", lane, "--head", branch,
+    await gitPush(["push", "-u", "origin", branch]);
+    const prOut = resumed?.pr ? `/pull/${resumed.pr}` : String(await ghw(["pr", "create", "-R", repo, "--base", lane, "--head", branch,
       "--title", `${iss.title} (#${issue})`, "--body", renderPrBody({ issue, investigation: await readStage(STAGES[0]), plan: await readStage(STAGES[1]), implementation: await readStage(STAGES[2]), result, gitFiles: files })])).trim();
     const pr = Number(/\/pull\/(\d+)/.exec(prOut)?.[1] ?? /(\d+)\s*$/.exec(prOut)?.[1]);
     if (!Number.isInteger(pr)) return await needsHuman(`Could not parse the PR number from: ${prOut.slice(0, 200)}`);
@@ -388,7 +460,7 @@ export async function runCore(o) {
     T.decisions.push(`merge_wait=${mergeWait}: ${policyText}`);
 
     // ---- base-sync (#4100): merge the lane into the branch when the PR conflicts; merge only, never rebase or force-push ----
-    let syncs = 0;
+    let syncs = 0, mergeRetries = 0;
     const MAX_SYNCS = 2;
     const gitOut = async (args) => { try { return String(await git(args)).trim(); } catch { return ""; } };
     /** CONFLICTING / DIRTY, or UNKNOWN after one short re-poll. Unreadable state counts as not conflicting. */
@@ -434,14 +506,14 @@ export async function runCore(o) {
       }
       const after = await gitOut(["rev-parse", "HEAD"]);
       if (soft && clean && after === before) { syncs--; return null; } // UNKNOWN state and nothing to merge: not a conflict
-      if (after !== before) await git(["push", "origin", branch]); // plain push of a merge commit: never force
+      if (after !== before) await gitPush(["push", "origin", branch]); // plain push of a merge commit: never force
       if (!clean) {
         let rv = await doReview();
         if (rv?.fail) {
           T.decisions.push(`Base-sync re-review FAIL: builder resumed with the reason`);
           await builder(++attemptN, resumeBuilderPrompt(rv.reason), sessionId ?? null);
           if (await unresolved()) return await needsHuman("Base-sync re-review fix left conflict markers.", OUTCOME_CODES.BASE_CONFLICT);
-          if ((await gitOut(["rev-parse", "HEAD"])) !== after) await git(["push", "origin", branch]);
+          if ((await gitOut(["rev-parse", "HEAD"])) !== after) await gitPush(["push", "origin", branch]);
           rv = await doReview();
         }
         if (!rv) return await needsHuman("Reviewer returned no parseable verdict after base-sync.", OUTCOME_CODES.REVIEW_NO_VERDICT);
@@ -500,11 +572,19 @@ export async function runCore(o) {
         await ghc(["pr", "merge", String(pr), "-R", repo, "--merge", "--match-head-commit", localHead]);
         break;
       } catch (e) {
-        if (!/not mergeable|merge conflict|conflict/i.test(String(e?.stderr ?? "") + String(e?.message ?? ""))) throw e;
+        const msg = `${e?.stderr ?? ""}${e?.message ?? ""}`;
+        if (MERGE_TRANSIENT_RE.test(msg) && !/not mergeable|merge conflict/i.test(msg)) {
+          // "Base branch was modified", GraphQL and 5xx are transient: back off, then the loop top re-reads mergeable and the head.
+          if (++mergeRetries > maxMergeRetries) return await finish("deferred", `gh pr merge kept failing transiently (${mergeRetries - 1} retries): ${msg.slice(0, 300)}`, { pr, code: OUTCOME_CODES.MERGE_TRANSIENT }, OUTCOME_CODES.MERGE_TRANSIENT);
+          T.anomalies.push(`gh pr merge transient failure (retry ${mergeRetries}): ${msg.split("\n")[0].slice(0, 120)}`);
+          await sleepFn(Math.min(60000, 1000 * 2 ** (mergeRetries - 1)) + Math.floor(random() * 250));
+          continue;
+        }
+        if (!MERGE_CONFLICT_RE.test(msg)) throw e;
         const r = await baseSync("gh pr merge reported a conflict"); if (r) return r;
       }
     }
-    await ghc(["issue", "close", String(issue), "-R", repo, "--reason", "completed"]).catch(() => {});
+    await ghw(["issue", "close", String(issue), "-R", repo, "--reason", "completed"]).catch(() => {});
     await label("merged");
     return await finish("merged", `PR #${pr}`, { pr });
   } catch (e) {
@@ -512,6 +592,8 @@ export async function runCore(o) {
       const when = e.resetAt || (Number.isFinite(e.resetAtEpochMs) ? new Date(e.resetAtEpochMs).toISOString() : "unknown");
       return finish("deferred", `usage limit hit; resets at ${when}`, { code: OUTCOME_CODES.RATE_LIMITED, resetAt: when, resetAtEpochMs: Number.isFinite(e.resetAtEpochMs) ? e.resetAtEpochMs : null }, OUTCOME_CODES.RATE_LIMITED);
     }
+    if (e?.deferCode) return finish("deferred", e.deferDetail ?? String(e.message), { code: e.deferCode }, e.deferCode);
+    if (e?.pushRejected) return await needsHuman(`git push was rejected (no retry): ${String(e?.stderr ?? e?.message ?? e).slice(0, 400)}`);
     return finish("engine-error", e?.message ?? String(e));
   }
 }

package/bin/lean/lean.test.mjs

@@ -9,7 +9,8 @@ import { orchestrate, parseDeps, formatTable } from "./orchestrate.mjs";
 import { REVIEWER_SYSTEM, approxTokens, builderPrompt, reviewerPrompt, parseVerdict } from "./prompts.mjs";
 import { mapRecord } from "../engine/telemetry-seam.mjs";
 import { readLog } from "../engine/runlog.mjs";
-import { makeRunTmp } from "./effects.mjs";
+import { makeRunTmp, wrapLaneLimits, systemdRunAvailable, makeLeanRunner } from "./effects.mjs";
+import { retrying, PUSH_TRANSIENT_RE } from "./retry.mjs";
 import { checkAdmission } from "./admission.mjs";
 import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
 import { readAdmission } from "../engine-cli.mjs";
@@ -30,7 +31,7 @@ function world(o = {}) {
     comments: [], labels: [], closed: [], edits: [], prs: [], merges: [], calls: [], telemetry: [], files: {}, head: "aaaa1111aaaa1111",
     changed: o.changed ?? ["src/foo.js", "test/foo.test.js"], reviews: [...(o.reviews ?? [])], waits: [...(o.waits ?? [])], clock: 0,
     result: o.result ?? { status: "fixed", summary: "fixed foo", evidence: [], tests: ["test/foo.test.js"] },
-    prList: o.prList ?? [], remote: o.remote ?? "", resumeResult: o.resumeResult, seq: [], mtimes: {}, envs: [], prView: [...(o.prView ?? [])], mergeResults: [...(o.mergeResults ?? [])], mergeErrors: [...(o.mergeErrors ?? [])], conflictFiles: [], mergeInProgress: false, markers: false, mergeAttempts: 0,
+    prList: o.prList ?? [], remote: o.remote ?? "", resumeResult: o.resumeResult, seq: [], mtimes: {}, envs: [], pushes: 0, pushErrors: [...(o.pushErrors ?? [])], commentErrors: [...(o.commentErrors ?? [])], dirty: o.dirty ?? false, lastSubject: o.lastSubject ?? "", prView: [...(o.prView ?? [])], mergeResults: [...(o.mergeResults ?? [])], mergeErrors: [...(o.mergeErrors ?? [])], conflictFiles: [], mergeInProgress: false, markers: false, mergeAttempts: 0,
     stages: { ...STAGE_MD, ...(o.stages ?? {}) },
   };
   const put = (name, text) => { const p = `${WT}/.forge-lean/${name}`; if (w.files[p] !== text) w.mtimes[p] = Date.now(); w.files[p] = text; };
@@ -43,6 +44,7 @@ function world(o = {}) {
       if (a[0] === "api") return "";
       if (k === "pr list") return JSON.stringify(w.prList);
       if (k === "issue view") return JSON.stringify({ number: 7, title: "foo breaks", body: "Bug in foo", state: "OPEN" });
+      if (k === "issue comment" && w.commentErrors.length) throw new Error(w.commentErrors.shift());
       if (k === "issue comment") { w.comments.push(a[a.indexOf("--body") + 1]); w.seq.push("comment"); return `https://github.com/o/r/issues/7#issuecomment-${w.comments.length}\n`; }
       if (k === "issue close") { w.closed.push(a); w.seq.push("close"); return ""; }
       if (k === "issue edit") { w.edits.push(a); return ""; }
@@ -59,6 +61,9 @@ function world(o = {}) {
         w.lockFails--; const e = new Error("git failed"); e.stderr = "error: could not lock config file .git/config: File exists"; throw e;
       }
       if (a[0] === "rev-parse" && a[1] === "--git-path") return "/repo/.git/info/exclude";
+      if (a[0] === "status" && a[1] === "--porcelain") return w.dirty ? " M src/foo.js\n" : "";
+      if (a[0] === "log" && a.includes("--format=%s")) return w.lastSubject;
+      if (a[0] === "push") { w.pushes++; if (w.pushErrors.length) { const m = w.pushErrors.shift(); throw Object.assign(new Error(m), { stderr: m }); } }
       if (a[0] === "merge" && a[1] === "--no-edit") {
         const r = w.mergeResults.shift() ?? "clean";
         if (r === "clean") { w.head = `cccc${w.calls.length}`.padEnd(16, "0"); return ""; }
@@ -739,7 +744,7 @@ describe("base-sync (#4100)", () => {
   });
 
   it("an unrelated gh pr merge error is not a base-sync", async () => {
-    const w = world({ mergeErrors: ["HTTP 500 boom"] });
+    const w = world({ mergeErrors: ["GraphQL: validation failed (422)"] });
     const r = await run(w);
     assert.equal(r.terminalReason, "engine-error");
     assert.ok(!syncComment(w));
@@ -979,3 +984,215 @@ describe("host admission (#4105)", () => {
     assert.match(spec, /rate_limited[\s\S]*stop launching\s+new lanes until that reset time/);
   });
 });
+
+describe("batch robustness (#4112-#4115)", () => {
+  const mergeCount = (w) => w.merges.length;
+
+  // ---- #4115 ----
+  it("#4115a the builder prompt pins targeted tests, no extra checkouts, no browsers, scratch in $TMPDIR (within budget)", () => {
+    const p = builderPrompt(7);
+    for (const needle of ["run only targeted tests", "never the full suite", "Never create extra checkouts, copies or base clones", "git show or git diff origin/<base>", "Never install browsers", "$TMPDIR"]) assert.ok(p.includes(needle), needle);
+    assert.ok(approxTokens(p) <= 2800, `builder prompt is ${approxTokens(p)} tokens`);
+  });
+
+  it("#4115b lane limits wrap the spawn in systemd-run; unset or invalid limits do not", () => {
+    const w = wrapLaneLimits("claude", ["--print", "x"], { memory_max: "3G", cpu_quota: "150%" });
+    assert.deepEqual(w, { bin: "systemd-run", args: ["--user", "--scope", "-q", "-p", "MemoryMax=3G", "-p", "CPUQuota=150%", "--", "claude", "--print", "x"] });
+    assert.deepEqual(wrapLaneLimits("claude", ["a"], null), { bin: "claude", args: ["a"] });
+    assert.deepEqual(wrapLaneLimits("claude", ["a"], { memory_max: "lots; rm -rf /", cpu_quota: "x" }), { bin: "claude", args: ["a"] });
+    assert.equal(wrapLaneLimits("claude", ["a"], { memory_max: "1G" }).args.includes("CPUQuota=150%"), false);
+  });
+
+  it("#4115b the runner wraps only when limits are set AND systemd-run probes available (clean fallback)", async () => {
+    const seen = [];
+    const runCli = async (o) => { seen.push(o.wrap ? o.wrap("claude", ["a"]).bin : null); return { text: "", usage: null, costUsd: 0, durationMs: 0, turns: 0, sessionId: "s" }; };
+    const base = { role: "builder", issue: 1, cwd: process.cwd(), prompt: "p", system: "" };
+    await makeLeanRunner({ runCli, probe: () => true })({ ...base, laneLimits: { memory_max: "3G" } });
+    await makeLeanRunner({ runCli, probe: () => false })({ ...base, laneLimits: { memory_max: "3G" } });
+    await makeLeanRunner({ runCli, probe: () => true })({ ...base });
+    assert.deepEqual(seen, ["systemd-run", null, null]);
+  });
+
+  it("#4115b integration: a lane under systemd-run --scope really runs in a capped scope (skipped without systemd-run)", { skip: !systemdRunAvailable() }, async () => {
+    const { writeFileSync, chmodSync } = await import("node:fs");
+    const { runCliBackendAsync } = await import("../runner.mjs");
+    const d = tmpDir("core-scope-");
+    writeFileSync(`${d}/fake-claude`, '#!/bin/sh\nprintf \'{"result":"%s"}\\n\' "$(cat /proc/self/cgroup | tr -d \'\\n\' | tr / :)"\n'); chmodSync(`${d}/fake-claude`, 0o755);
+    const r = await runCliBackendAsync({ spec: { name: "t" }, userMessage: "go", args: ["1"], cwd: d, bin: `${d}/fake-claude`, logger: { log() {} }, wrap: (b, a) => wrapLaneLimits(b, a, { memory_max: "512M", cpu_quota: "100%" }) });
+    assert.match(r.text, /\.scope/);
+  });
+
+  const hostFor = (state) => ({ memAvailableMb: () => 99999, freeMb: async () => 99999, tmpPaths: () => ["/t"], loadAvg1: () => state.load(), cores: () => 2 });
+  const slowBuilder = (w, ms, hook) => { const orig = w.io.runner; w.io.runner = async (c) => { if (c.role !== "builder") return orig(c); hook?.(c); await new Promise((r) => setTimeout(r, ms)); return orig(c); }; };
+
+  it("#4115c the governor pauses the builder's process group over the ceiling and resumes under it; never pauses past the max", async () => {
+    const w = world(); const calls = [];
+    let load = 20;
+    w.io.host = hostFor({ load: () => load });
+    slowBuilder(w, 400, (c) => { c.control.pause = () => { calls.push("pause"); return true; }; c.control.resume = () => { calls.push("resume"); load = 1; return true; }; setTimeout(() => { load = 1; }, 150); });
+    // host starts under the ceiling at admission, then goes over while the builder runs
+    let first = true; const lf = w.io.host.loadAvg1; w.io.host.loadAvg1 = () => { if (first) { first = false; return 1; } return lf(); };
+    const r = await run(w, { governorPollMs: 20, admission: { max_load_per_core: 4 } });
+    assert.equal(r.terminalReason, "merged");
+    assert.deepEqual(calls.slice(0, 2), ["pause", "resume"]);
+    // max pause: stays over the whole time; resumed anyway after maxPauseMs and not re-paused
+    const w2 = world(); const calls2 = [];
+    w2.io.host = hostFor({ load: () => 20 }); let f2 = true; const l2 = w2.io.host.loadAvg1; w2.io.host.loadAvg1 = () => { if (f2) { f2 = false; return 1; } return l2(); };
+    slowBuilder(w2, 500, (c) => { c.control.pause = () => { calls2.push("pause"); return true; }; c.control.resume = () => { calls2.push("resume"); return true; }; });
+    await run(w2, { governorPollMs: 20, maxPauseMs: 100, admission: { max_load_per_core: 4 } });
+    assert.deepEqual(calls2, ["pause", "resume"], "one pause, one forced resume, no flapping");
+  });
+
+  it("#4115c 3 CPU-heavy lanes on a 2-core host: admission refuses the third, the governor pauses/resumes the running ones, load stays bounded", async () => {
+    const st = { running: 0, paused: 0, peak: 0 };
+    const load = () => (st.running - st.paused) * 1.5; // each active lane adds 1.5 load; 2 cores; ceiling 1.0 per core
+    const lane = async (i) => {
+      const w = world(); w.io.host = hostFor({ load });
+      w.io.runner = (orig => async (c) => {
+        if (c.role !== "builder") return orig(c);
+        st.running++; let paused = false;
+        c.control.pause = () => { if (paused) return false; paused = true; st.paused++; return true; };
+        c.control.resume = () => { if (!paused) return false; paused = false; st.paused--; return true; };
+        st.peak = Math.max(st.peak, load());
+        await new Promise((r) => setTimeout(r, 300));
+        if (paused) { paused = false; st.paused--; }
+        st.running--;
+        return orig(c);
+      })(w.io.runner);
+      return { w, r: await run(w, { issue: 7, governorPollMs: 80, maxPauseMs: 60, admission: { max_load_per_core: 1 } }) };
+    };
+    const a = lane(1); await new Promise((r) => setTimeout(r, 20)); const b = lane(2); await new Promise((r) => setTimeout(r, 20)); const c = lane(3);
+    const res = await Promise.all([a, b, c]);
+    assert.equal(res[2].r.code, "host.busy", "third lane refused at admission while two run");
+    assert.deepEqual(res.slice(0, 2).map((x) => x.r.terminalReason), ["merged", "merged"]);
+    assert.ok(st.peak <= 2 * 1.0 + 1.0, `observed load stays near the ceiling (${st.peak})`);
+  });
+
+  // ---- #4113 ----
+  it("#4113 a merge failing once with 'Base branch was modified' then succeeding ends merged in a single run", async () => {
+    const w = world({ mergeErrors: ["GraphQL: Base branch was modified. Review and try the merge again. (mergePullRequest)"] });
+    const r = await run(w);
+    assert.equal(r.terminalReason, "merged");
+    assert.equal(w.mergeAttempts, 2);
+    assert.match(traj(w)[0], /gh pr merge transient failure/);
+  });
+
+  it("#4113 persistent transient merge failures end deferred merge.transient, never engine-error", async () => {
+    const w = world({ mergeErrors: Array(20).fill("HTTP 502: Bad Gateway") });
+    const r = await run(w, { maxMergeRetries: 3 });
+    assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "merge.transient");
+    assert.equal(w.mergeAttempts, 4);
+    assert.equal(w.edits.some((e) => e.includes("needs-human")), false);
+  });
+
+  it("#4113 conflicts still route through base-sync", async () => {
+    const w = world({ prView: [{ mergeable: "MERGEABLE", mergeStateStatus: "CLEAN" }], mergeErrors: ["GraphQL: Pull request is not mergeable (mergePullRequest)"], mergeResults: ["clean"] });
+    assert.equal((await run(w)).terminalReason, "merged");
+    assert.ok(w.comments.some((c) => c.includes("FORGE:BASE_SYNC")));
+  });
+
+  // ---- #4114 ----
+  it("#4114 a push failing once with 'remote unpack failed' then succeeding completes the run", async () => {
+    const w = world({ pushErrors: ["error: remote unpack failed: unable to create temporary object directory"] });
+    const r = await run(w);
+    assert.equal(r.terminalReason, "merged");
+    assert.equal(w.pushes, 2);
+  });
+
+  it("#4114 exhausted push retries end deferred push.transient after 4 attempts", async () => {
+    const w = world({ pushErrors: Array(10).fill("fatal: the remote end hung up unexpectedly\nerror: RPC failed; HTTP 503") });
+    const r = await run(w);
+    assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "push.transient");
+    assert.equal(w.pushes, 4);
+    assert.equal(w.prs.length, 0);
+  });
+
+  it("#4114 a protected-branch or permission rejection fails immediately with no retries", async () => {
+    for (const msg of ["remote: error: GH006: Protected branch update failed for refs/heads/x", "fatal: unable to access: The requested URL returned error: 403", "! [remote rejected] x -> x (pre-receive hook declined)"]) {
+      const w = world({ pushErrors: [msg, msg, msg] });
+      const r = await run(w);
+      assert.equal(w.pushes, 1, msg);
+      assert.equal(r.terminalReason, "needs-human");
+    }
+  });
+
+  it("#4114 gh writes (comment) retry on 5xx and secondary rate limits; non-transient errors do not", async () => {
+    const w = world({ commentErrors: ["HTTP 502: Bad Gateway", "You have exceeded a secondary rate limit. Please wait a few minutes"] });
+    assert.equal((await run(w)).terminalReason, "merged");
+    assert.ok(w.comments.length >= 5);
+    let n = 0;
+    await assert.rejects(retrying(async () => { n++; throw new Error("HTTP 404 Not Found"); }, { transient: PUSH_TRANSIENT_RE, attempts: 4, sleep: async () => {} }), /404/);
+    assert.equal(n, 1);
+  });
+
+  // ---- #4112 ----
+  const fakeCli = async (name, body) => {
+    const { writeFileSync, chmodSync } = await import("node:fs");
+    const d = tmpDir("core-cli-"); writeFileSync(`${d}/${name}`, `#!/bin/sh\n${body}\n`); chmodSync(`${d}/${name}`, 0o755);
+    return { d, bin: `${d}/${name}` };
+  };
+  const TOOL = `echo '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","input":{"command":"pytest -x"}}]}}'`;
+  const RESULT = `echo '{"type":"result","result":"all done","session_id":"s-9","total_cost_usd":0.5,"num_turns":3,"usage":{"input_tokens":1,"output_tokens":2}}'`;
+
+  it("#4112 no stream activity past the idle threshold is killed as CLI_IDLE; the last tool call and live children are recorded", async () => {
+    const { runCliBackendAsync } = await import("../runner.mjs");
+    const { d, bin } = await fakeCli("idle", `${TOOL}\nsleep 30 &\nwait`);
+    const t0 = Date.now();
+    await assert.rejects(runCliBackendAsync({ spec: { name: "t" }, userMessage: "go", args: ["1"], cwd: d, bin, logger: { log() {} }, stream: true, idleMs: 400, maxMs: 20000 }), (e) => {
+      assert.equal(e.code, "CLI_IDLE");
+      assert.equal(e.lastTool.name, "Bash"); assert.match(e.lastTool.input, /pytest -x/);
+      assert.ok(e.children.some((c) => /sleep 30/.test(c.cmd)), JSON.stringify(e.children));
+      return true;
+    });
+    assert.ok(Date.now() - t0 < 8000, "killed promptly, not at the ceiling");
+  });
+
+  it("#4112 a session that keeps making progress is not idle-killed; the final stream result is the envelope", async () => {
+    const { runCliBackendAsync } = await import("../runner.mjs");
+    const { d, bin } = await fakeCli("busy", `for i in 1 2 3 4 5 6 7 8; do ${TOOL}; sleep 0.15; done\n${RESULT}`);
+    const r = await runCliBackendAsync({ spec: { name: "t" }, userMessage: "go", args: ["1"], cwd: d, bin, logger: { log() {} }, stream: true, idleMs: 500, maxMs: 20000 });
+    assert.equal(r.text, "all done"); assert.equal(r.sessionId, "s-9"); assert.equal(r.costUsd, 0.5);
+  });
+
+  it("#4112 the absolute ceiling still kills a session that never goes idle (CLI_TIMEOUT)", async () => {
+    const { runCliBackendAsync } = await import("../runner.mjs");
+    const { d, bin } = await fakeCli("forever", `while true; do ${TOOL}; sleep 0.1; done`);
+    await assert.rejects(runCliBackendAsync({ spec: { name: "t" }, userMessage: "go", args: ["1"], cwd: d, bin, logger: { log() {} }, stream: true, idleMs: 800, maxMs: 700 }), (e) => e.code === "CLI_TIMEOUT");
+  });
+
+  it("#4112 idle kill: uncommitted work is checkpointed and pushed, the builder retries once from the checkpoint, run log names the last tool", async () => {
+    const w = world({ dirty: true }); const dir = tmpDir("core-idle-");
+    const orig = w.io.runner; let n = 0;
+    w.io.runner = async (c) => { if (c.role === "builder" && n++ === 0) throw Object.assign(new Error("idle"), { code: "CLI_IDLE", lastTool: { name: "Bash", input: "pytest" }, children: [{ pid: 9, cmd: "pytest" }] }); return orig(c); };
+    const r = await run(w, { dir });
+    assert.equal(r.terminalReason, "merged");
+    const g = w.calls.filter((c) => c[0] === "git").map((c) => c.slice(1));
+    assert.ok(g.some((x) => x[0] === "add" && x[1] === "-A"));
+    assert.ok(g.some((x) => x[0] === "commit" && x.includes("wip(forge): checkpoint after idle timeout")));
+    assert.ok(w.pushes >= 2, "checkpoint push plus the final push");
+    assert.match(builders(w)[0][4], /Continue from it/);
+    const killEv = readLog(dir, 7).find((e) => e.event === "CLI_IDLE_KILL");
+    assert.equal(killEv.lastTool.name, "Bash"); assert.equal(killEv.children[0].cmd, "pytest");
+    assert.match(traj(w)[0], /builder idle timeout: checkpointed, retried once/);
+  });
+
+  it("#4112 a second idle or timeout kill ends deferred with cli.idle / cli.timeout (not engine-error), work checkpointed", async () => {
+    for (const [code, expected] of [["CLI_IDLE", "cli.idle"], ["CLI_TIMEOUT", "cli.timeout"]]) {
+      const w = world({ dirty: true });
+      w.io.runner = async (c) => { if (c.role === "builder") throw Object.assign(new Error(code), { code }); return { text: "" }; };
+      const r = await run(w);
+      assert.equal(r.terminalReason, "deferred", code); assert.equal(r.code, expected);
+      assert.equal(w.edits.some((e) => e.includes("needs-human")), false);
+      assert.ok(w.pushes >= 1);
+    }
+  });
+
+  it("#4112 a rerun resumes from the checkpoint branch and tells the builder to continue", async () => {
+    const w = world({ prList: [], remote: "abc\trefs/heads/feat/issue-7-lean", lastSubject: "wip(forge): checkpoint after idle timeout" });
+    const r = await run(w);
+    assert.equal(r.terminalReason, "merged");
+    assert.equal(builders(w).length, 1, "builder runs again (a checkpoint is not finished work)");
+    assert.match(builders(w)[0][4], /Continue from it/);
+    assert.ok(w.calls.some((c) => c[0] === "git" && c[1] === "worktree" && c[2] === "add" && c.includes("origin/feat/issue-7-lean")));
+  });
+});

package/bin/lean/prompts.mjs

@@ -12,6 +12,8 @@ export const IMPLEMENTATION_FILE = `${LEAN_DIR}/implementation.md`;
 export function builderPrompt(n) {
   return `Resolve issue #${n} (read it with gh issue view ${n} --comments). If it is already fixed or invalid on this code, explain why with file:line evidence and change nothing. Otherwise implement a complete fix with a regression test following repo conventions, run the relevant tests, and commit. Do not push or touch GitHub.
 
+Host rules (many lanes share this machine): run only targeted tests for the code you changed (specific test files or -k filters), never the full suite. Never create extra checkouts, copies or base clones of the repo; compare against the base with git show or git diff origin/<base> instead. Never install browsers. Keep all scratch files under $TMPDIR.
+
 The engine publishes three stage notes to the issue for you. Write each as markdown with exactly these headings, in this order, concise and concrete (file:line wherever you cite code). All files live in ${LEAN_DIR}/ (git-excluded, never commit them). Do not write HTML comments.
 
 1. BEFORE coding, write ${INVESTIGATION_FILE}:

package/bin/lean/retry.mjs

@@ -0,0 +1,31 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/** Transient-failure retry for the core work-on's network writes (git push, gh writes, gh pr merge). */
+
+export const PUSH_TRANSIENT_RE = /remote unpack failed|index-pack failed|RPC failed|unexpected disconnect|early EOF|the remote end hung up|HTTP\/?\s?5\d\d|returned error: 5\d\d|\b50[0234]\b|internal server error|bad gateway|service unavailable|gateway time-?out|connection (?:reset|timed out|refused)|timed out|could not resolve host|broken pipe|ETIMEDOUT|ECONNRESET|temporarily unavailable/i;
+export const PUSH_PERMANENT_RE = /protected branch|GH006|GH013|repository rule violations|permission (?:denied|to .* denied)|denied to|\b403\b|authentication failed|not allowed|hook declined|pre-receive hook declined|\[remote rejected\]|\[rejected\]|non-fast-forward|fetch first/i;
+export const GH_TRANSIENT_RE = /HTTP 5\d\d|\b50[0234]\b|secondary rate limit|rate limit exceeded|abuse detection|something went wrong|internal server error|bad gateway|service unavailable|ETIMEDOUT|ECONNRESET|connection reset|timed out|\bEOF\b|temporarily unavailable/i;
+export const MERGE_TRANSIENT_RE = new RegExp(`${GH_TRANSIENT_RE.source}|base branch was modified|head branch was modified|mergeability.*unknown|try again`, "i");
+export const MERGE_CONFLICT_RE = /not mergeable|merge conflict|conflict/i;
+
+export const errText = (e) => `${e?.stderr ?? ""}\n${e?.message ?? ""}\n${e?.stdout ?? ""}`;
+
+/**
+ * Run `fn` up to `attempts` times. A `permanent` match throws immediately with `e.permanent = true`; a `transient`
+ * match retries with exponential backoff; anything else throws as-is. When every attempt failed transiently the last
+ * error carries `transientExhausted = true`.
+ */
+export async function retrying(fn, { transient, permanent = null, attempts = 4, sleep, baseMs = 1000, random = Math.random }) {
+  let last;
+  for (let i = 0; i < attempts; i++) {
+    try { return await fn(i); } catch (e) {
+      const t = errText(e);
+      if (permanent && permanent.test(t)) { e.permanent = true; throw e; }
+      if (!transient.test(t)) throw e;
+      last = e;
+      if (i < attempts - 1) await sleep(baseMs * 2 ** i + Math.floor(random() * 250));
+    }
+  }
+  last.transientExhausted = true;
+  throw last;
+}

package/bin/runner.mjs

@@ -1925,15 +1925,31 @@ export function runCliBackend({
   }
 }
 
+/** Descendant processes of `pid` (pid + command line), best effort. Used to record what was alive when a session was killed. */
+export function listDescendants(pid, { psFn = null } = {}) {
+  try {
+    const out = psFn ? psFn() : execSync("ps -eo pid=,ppid=,args=", { encoding: "utf-8", timeout: 3000, maxBuffer: 8 * 1024 * 1024 });
+    const rows = String(out).split("\n").map((l) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(l)).filter(Boolean).map((m) => ({ pid: Number(m[1]), ppid: Number(m[2]), cmd: m[3].slice(0, 200) }));
+    const kids = []; const seen = new Set([Number(pid)]);
+    for (let grew = true; grew;) { grew = false; for (const r of rows) if (!seen.has(r.pid) && seen.has(r.ppid)) { seen.add(r.pid); kids.push({ pid: r.pid, cmd: r.cmd }); grew = true; } }
+    return kids.slice(0, 30);
+  } catch { return []; }
+}
+
 /**
  * Async twin of runCliBackend (core work-on live comments): the CLI runs under child_process.spawn so the caller's event
  * loop keeps turning while the session runs. It returns the SAME envelope (and throws the same errors) because the
  * argv/env building and the envelope parsing are the sync function itself: pass 1 captures the spawn arguments, the
  * child runs asynchronously, pass 2 replays the collected result through runCliBackend's `spawnFn` seam.
- * `onSpawn(child)` receives the ChildProcess (e.g. to kill it).
+ *
+ * Extras (#4112, #4115): `idleMs` kills a session with no stdout activity (CLI_IDLE; the last tool call and the live
+ * child processes ride on the error); `maxMs` is the absolute ceiling (CLI_TIMEOUT); `stream` switches the CLI to
+ * stream-json so progress is observable (the final `result` line is the envelope); `wrap(bin,args)` rewrites the spawn
+ * (cgroup caps); `control` receives {pause, resume} (SIGSTOP/SIGCONT of the child's whole process group) and paused time
+ * does not count as idle. `onSpawn(child)` receives the ChildProcess.
  */
 export async function runCliBackendAsync(opts) {
-  const { spawnFn: _ignored, onSpawn = null, ...rest } = opts;
+  const { spawnFn: _ignored, onSpawn = null, idleMs = 0, maxMs = 0, stream = false, wrap = null, control = null, ...rest } = opts;
   let captured = null;
   try {
     runCliBackend({ ...rest, spawnFn: (bin, args, o) => {
@@ -1944,33 +1960,82 @@ export async function runCliBackendAsync(opts) {
       return { status: 0, stdout: "{}", stderr: "" };
     } });
   } catch (e) { if (!captured) throw e; }
-  const { bin, args, o, sysIdx, sys } = captured;
+  let { bin, args } = captured;
+  const { o, sysIdx, sys } = captured;
   let sysDir = null;
   if (sysIdx >= 0 && sys !== null) {
     sysDir = mkdtempSync(join(os.tmpdir(), "forgedock-cli-system-prompt-"));
     args[sysIdx + 1] = join(sysDir, "system-prompt.txt");
     writeFileSync(args[sysIdx + 1], sys, "utf-8");
   }
-  let collected;
+  if (stream) {
+    const j = args.indexOf("--output-format");
+    if (j >= 0) { args[j + 1] = "stream-json"; if (!args.includes("--verbose")) args.splice(j + 2, 0, "--verbose"); }
+  }
+  if (typeof wrap === "function") ({ bin, args } = wrap(bin, args));
+  const ceiling = maxMs > 0 ? maxMs : o.timeout;
+  let collected, idleInfo = null;
   try {
     collected = await new Promise((resolveP) => {
-      let stdout = "", stderr = "", settled = false, timedOut = false;
+      let stdout = "", stderr = "", settled = false, timedOut = false, paused = false;
+      let lineBuf = "", resultLine = null, lastTool = null;
       const max = o.maxBuffer ?? DEFAULT_SPAWN_MAX_BUFFER_BYTES;
-      let timer = null;
-      const finish = (r) => { if (settled) return; settled = true; clearTimeout(timer); resolveP(r); };
+      const startedAt = Date.now();
+      let lastActivity = startedAt, pausedAt = 0, pausedTotal = 0, timer = null;
+      const finish = (r) => { if (settled) return; settled = true; clearInterval(timer); resolveP(r); };
       let child;
-      try { child = spawn(bin, args, { cwd: o.cwd, env: o.env, stdio: ["ignore", "pipe", "pipe"] }); }
+      try { child = spawn(bin, args, { cwd: o.cwd, env: o.env, stdio: ["ignore", "pipe", "pipe"], detached: true }); }
       catch (error) { return finish({ status: null, signal: null, pid: 0, error, stdout: "", stderr: "" }); }
+      const signalGroup = (sig) => { try { process.kill(-child.pid, sig); } catch { try { child.kill(sig); } catch { /* gone */ } } };
+      const ctl = {
+        pause() { if (paused || settled) return false; paused = true; pausedAt = Date.now(); signalGroup("SIGSTOP"); return true; },
+        resume() { if (!paused) return false; paused = false; const d = Date.now() - pausedAt; pausedTotal += d; lastActivity = Date.now(); signalGroup("SIGCONT"); return true; },
+        get paused() { return paused; },
+      };
+      if (control && typeof control === "object") Object.assign(control, { pause: ctl.pause, resume: ctl.resume, isPaused: () => paused });
       onSpawn?.(child);
-      timer = setTimeout(() => { timedOut = true; try { child.kill("SIGTERM"); } catch { /* gone */ } }, o.timeout);
-      const add = (cur, d) => (cur.length >= max ? cur : cur + d.toString("utf-8"));
-      child.stdout.on("data", (d) => { stdout = add(stdout, d); });
-      child.stderr.on("data", (d) => { stderr = add(stderr, d); });
+      const kill = () => { if (paused) ctl.resume(); signalGroup("SIGTERM"); setTimeout(() => signalGroup("SIGKILL"), 5000).unref?.(); };
+      const tick = Math.max(20, Math.min(1000, (idleMs || 4000) / 4));
+      timer = setInterval(() => {
+        const now = Date.now();
+        if (!paused && idleMs > 0 && now - lastActivity >= idleMs && !idleInfo) {
+          idleInfo = { lastTool, children: listDescendants(child.pid), idleMs, pid: child.pid };
+          kill();
+        } else if (!timedOut && !idleInfo && now - startedAt - pausedTotal - (paused ? now - pausedAt : 0) >= ceiling) { timedOut = true; kill(); }
+      }, tick);
+      timer.unref?.();
+      const noteLine = (line) => {
+        if (!line.startsWith("{") || line.length > 4_000_000) return;
+        let j; try { j = JSON.parse(line); } catch { return; }
+        if (j?.type === "result") resultLine = line;
+        else if (j?.type === "assistant" && Array.isArray(j.message?.content)) {
+          for (const c of j.message.content) if (c?.type === "tool_use") lastTool = { name: c.name, input: JSON.stringify(c.input ?? {}).slice(0, 200), at: Date.now() };
+        }
+      };
+      child.stdout.on("data", (d) => {
+        lastActivity = Date.now();
+        if (stream) {
+          lineBuf += d.toString("utf-8");
+          let nl;
+          while ((nl = lineBuf.indexOf("\n")) >= 0) { noteLine(lineBuf.slice(0, nl).trim()); lineBuf = lineBuf.slice(nl + 1); }
+          if (lineBuf.length > 4_000_000) lineBuf = "";
+        } else if (stdout.length < max) stdout += d.toString("utf-8");
+      });
+      child.stderr.on("data", (d) => { lastActivity = Date.now(); if (stderr.length < max) stderr += d.toString("utf-8"); });
       child.on("error", (error) => finish({ status: null, signal: null, pid: child.pid ?? 0, error, stdout, stderr }));
-      child.on("close", (status, signal) => finish({ status: timedOut ? null : status, signal, pid: child.pid ?? 0, stdout, stderr,
-        ...(timedOut ? { error: Object.assign(new Error("spawnSync ETIMEDOUT"), { code: "ETIMEDOUT" }) } : {}) }));
+      child.on("close", (status, signal) => {
+        if (stream && lineBuf.trim()) noteLine(lineBuf.trim());
+        finish({ status: timedOut ? null : status, signal, pid: child.pid ?? 0, stdout: stream ? (resultLine ?? "") : stdout, stderr,
+          ...(timedOut ? { error: Object.assign(new Error("spawnSync ETIMEDOUT"), { code: "ETIMEDOUT" }) } : {}) });
+      });
     });
   } finally { if (sysDir) rmSync(sysDir, { recursive: true, force: true }); }
+  if (idleInfo) {
+    const mins = Math.round((idleInfo.idleMs / 60000) * 10) / 10;
+    const err = new Error(`claude CLI had no stream activity for ${mins} min and was killed (idle). Last tool: ${idleInfo.lastTool ? `${idleInfo.lastTool.name} ${idleInfo.lastTool.input}` : "none"}. Live children: ${idleInfo.children.map((c) => `${c.pid}:${c.cmd}`).join("; ") || "none"}`);
+    err.code = "CLI_IDLE"; err.lastTool = idleInfo.lastTool; err.children = idleInfo.children; err.idleMs = idleInfo.idleMs;
+    throw err;
+  }
   return runCliBackend({ ...rest, spawnFn: () => collected });
 }
 

package/docs/CONFIG.md

@@ -482,6 +482,9 @@ engine:
 | `<key>_max_turns` | integer | No | Positive integer passed as `--max-turns`. Invalid values are ignored. |
 
 | `path` | string | No | `core` (default), `full`, `lean` or `auto`. `core` is the lean work-on (`bin/lean/`): one plain builder session, one read-only two-question reviewer, GitHub as the register (see RELEASE-NOTES v1.22). `lean` and `auto` are aliases of `core`; `full` runs the legacy multi-phase pipeline and is required for `--retry` and `--wait-lease`. The `--path` flag overrides. `lean_reviewer_model` sets the reviewer model (default `claude-sonnet-5-5`); the builder uses `--model` or the CLI default. `forgedock orchestrate <issues...> [--parallel K] [--lane L]` runs the core per issue (K default 3, lane default `staging`) and honours `**Depends on**` ordering among the given issues. |
+| `cli_idle_minutes` | number | No | Core builder idle timeout (#4112): the builder CLI runs with `--output-format stream-json` and is killed (code `cli.idle`) when it shows no stream activity for this many minutes. The run log records the last tool call and the live child processes. Default 15. |
+| `cli_max_minutes` | number | No | Absolute ceiling for a core builder session (code `cli.timeout`), replacing the fixed 60-minute kill. A session that keeps making progress is never idle-killed. Default 180. On either kill, uncommitted work is committed as `wip(forge): checkpoint after <reason>` and the lane branch is pushed (with the push retry), the builder retries once from the checkpoint, and a second kill ends `deferred`; a rerun resumes from the branch. |
+| `lane_limits` | map | No | Cap each core lane's whole process tree (#4115), under `pipeline:` or `engine:`: `memory_max` (for example `3G`) and `cpu_quota` (for example `150%`). When set and `systemd-run --user --scope` works (probed once), the builder CLI is spawned as `systemd-run --user --scope -q -p MemoryMax=... -p CPUQuota=... -- claude ...`; elsewhere the caps are ignored. Write it nested or flat (`lane_limits_memory_max`, `lane_limits_cpu_quota`). While a builder runs the engine polls host load and memory every 15 s against the `admission` thresholds: over them it SIGSTOPs the lane's process group, under them SIGCONT, and never pauses longer than 10 minutes. |
 | `admission` | map | No | Host-resource admission for the core work-on (#4105), checked before the lease, worktree or any GitHub call. When a check fails the run ends `deferred` (code `host.busy`, exit 75) with a last line such as `issue #N → deferred (host busy: mem 1.2G < 2G; retry in 60s)`; no lease is taken and nothing is posted. Keys: `enabled` (default `true`; `false` disables admission), `min_free_mem_mb` (default 2048; `MemAvailable` from `/proc/meminfo`, else `os.freemem()`), `min_free_tmp_mb` (default 1024; free space of the run TMPDIR filesystem, `~/.forge/tmp`, and of `/tmp`, via `statfs`), `max_load_per_core` (default 4; 1-minute load average divided by cores). Write it nested (`engine:` then `admission:` with the keys indented) or flat as `admission_enabled`, `admission_min_free_mem_mb`, `admission_min_free_tmp_mb`, `admission_max_load_per_core`; the parser reads one nested level into `engine`. `/orchestrate` requeues a host-busy lane and lowers K by 1 for 5 minutes. |
 | `merge_wait` | string | No | Core work-on merge gate. `required` (default): wait only for the status checks the base branch requires under branch protection (`gh api repos/{o}/{r}/branches/{base}/protection/required_status_checks`, contexts and checks); with no protection or no required checks (404 or empty) merge right after review PASS. A failing required check is `needs-human`; a pending one is polled with backoff and never `needs-human`. `all`: wait for every check (`scripts/wait-ci-green.sh`). `none`: merge right after review. Always `--match-head-commit` at the reviewed head. The policy used is recorded in the `FORGE:WORK-ON` comment. |
 | `ci_pending_max_minutes` | integer | No | Minutes the engine waits for CI when the review phase ends BLOCKED with "checks still pending", or the remediate phase holds at its M7 re-gate on "CI pending" (the merge gate's wait window ended while CI was still running). Default: 90. Invalid values are ignored. This key is flat like the others but is not part of the `<key>_model` / `<key>_max_turns` pattern. |

package/docs/OUTCOME-CODES.md

@@ -34,6 +34,9 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
 | `ci.timeout` | CI timed out | reserved | `REVIEW_RESULT` carries no CI field yet |
 | `base.conflict` | Base-branch conflict blocked the PR | reserved | `REVIEW_RESULT` carries no conflict field yet |
 | `path.switch_conflict` | The full path was started on an issue that has an open core-path PR (`feat/issue-<n>-lean`); refused to avoid a second implementation. Rerun on the default path (base-sync) or close that PR first | engine | `bin/engine.mjs` admission |
+| `merge.transient` | `gh pr merge` kept failing with a transient error (`Base branch was modified`, GraphQL or 5xx) after the retry budget; the run is `deferred` and a rerun resumes at the merge | lean | `bin/lean/lean.mjs` |
+| `push.transient` | `git push` kept failing with a transient error (`remote unpack failed`, `index-pack failed`, `RPC failed`, 5xx, reset, timeout) after 4 attempts; `deferred`, rerun resumes | lean | `bin/lean/lean.mjs` |
+| `cli.idle` | The builder CLI showed no stream activity for `engine.cli_idle_minutes` and was killed; uncommitted work was checkpointed to the lane branch; `deferred` after one automatic retry | lean | `bin/runner.mjs` |
 | `host.busy` | Core run deferred at admission: free memory, free tmp space or load per core outside `engine.admission` limits; no lease taken, nothing posted | lean | `bin/lean/admission.mjs` |
 | `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
 | `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "forgedock",
-  "version": "1.24.3",
+  "version": "1.24.4",
   "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
   "bin": {
     "forgedock": "bin/forgedock.mjs"