forgedock 1.35.0 → 1.37.0

package/bin/lean/retry.mjs

@@ -43,3 +43,31 @@ export async function ghCacheRetry(call, cache) {
     return await call(cache?.env?.());
   }
 }
+
+/** #4662: an expired or revoked token (HTTP 401). 403 is a missing scope or a rate limit, which a refresh does not fix. */
+export const GH_AUTH_RE = /HTTP 401|\b401\b[^\n]{0,40}(?:unauthorized|bad credentials)|bad credentials|requires authentication|authentication required|gh auth login|try authenticating/i;
+export const GH_AUTH_RETRY_SECONDS = 120;
+
+/** Runs scripts/refresh-bot-token.sh (or $FORGE_REFRESH_TOKEN_SCRIPT); resolves on exit 0, rejects otherwise. */
+export async function runRefreshScript(env = process.env) {
+  const { execFile } = await import("node:child_process");
+  const script = env.FORGE_REFRESH_TOKEN_SCRIPT || new URL("../../scripts/refresh-bot-token.sh", import.meta.url).pathname;
+  return new Promise((res, rej) => execFile("bash", [script], { env, timeout: 60000 }, (err, _out, stderr) => (err ? rej(Object.assign(err, { stderr })) : res())));
+}
+
+/**
+ * #4662: one refresh-and-retry for an auth failure. `refresh` runs once per `state` (shared across calls, so a dead token
+ * does not spawn a refresh per gh call); when the call still fails with an auth error the error carries
+ * `deferCode: "gh.auth"` so the lane ends `deferred` (requeued in `retryInSeconds`), never `engine-error`.
+ */
+export async function refreshAuthOnce(call, { refresh, state = { refreshed: false } }) {
+  try { return await call(); } catch (e) {
+    if (!GH_AUTH_RE.test(errText(e))) throw e;
+    if (!state.refreshed) {
+      state.refreshed = true;
+      try { await refresh(); state.ok = true; } catch { state.ok = false; }
+      if (state.ok) { try { const r = await call(); state.refreshed = false; return r; } catch (e2) { if (!GH_AUTH_RE.test(errText(e2))) throw e2; e = e2; } }
+    }
+    throw Object.assign(e, { deferCode: "gh.auth", deferDetail: `gh auth failed (HTTP 401) after a token refresh: ${errText(e).trim().split("\n")[0].slice(0, 200)}`, retryInSeconds: GH_AUTH_RETRY_SECONDS });
+  }
+}

package/bin/lean/sandbox-provider.contract.test.mjs

@@ -186,9 +186,7 @@ describe("remote adapter (#4254)", () => {
     const flaky = { ...fakeTransport(), status: async () => { throw new Error("ECONNRESET"); } };
     const q = remoteProvider({ batch, repo: "o/r", transport: flaky, env: {} });
     const h = await q.start({ issue: 6, args: [] });
-    const seen = [];
-    for (let i = 0; i < 5; i++) seen.push(await q.status(h));
-    assert.deepEqual(seen, ["running", "running", "running", "running", "gone"]);
+    await assert.rejects(() => q.status(h), /ECONNRESET/); // #4672: a transport error propagates; providerLanes retries it for the unreachable window
   });
 
   it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
@@ -303,6 +301,13 @@ describe("pool provider (#4546)", () => {
     assert.equal(Buffer.from(spec.env.CS_FORGE_YAML_B64, "base64").toString(), "verify:\n  mode: ci\n");
   });
 
+  it("#4685 a configured token_env that resolves empty falls back to the router token", async () => {
+    const { cfg } = setupPool();
+    const batch = openBatch(tmpDir("pool-b3-"), "bt");
+    const c = await launchOnce(() => providerByName("pool", { batch, repo: "o/r", cwd: tmpDir("pool-cwd3-"), exec: parseExecution({ mode: "remote", provider: "pool", token_env: "NO_SUCH_TOKEN_4685" }), env: {}, poolConfigPath: cfg }));
+    assert.equal(c.init.headers.authorization, "Bearer pool-token-789");
+  });
+
   it("an explicit endpoint wins; no forge.yaml adds no env; a missing router config is misconfigured", async () => {
     const { dir } = setupPool();
     const batch = openBatch(tmpDir("pool-b2-"), "bt");
@@ -328,3 +333,31 @@ describe("remote capacity (GET /status, #4540)", () => {
     assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
   });
 });
+
+describe("remote stop at the checkpoint (#4612)", () => {
+  it("httpTransport.stop: POST .../stop, 2xx true, 404/405 false, other statuses throw", async () => {
+    const calls = [];
+    const mk = (status) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url, init) => { calls.push([init.method, url]); return { status, ok: status < 400, text: async () => "", headers: { get: () => null } }; } });
+    assert.equal(await mk(202).stop("a b"), true);
+    assert.deepEqual(calls[0], ["POST", "https://pool.example/v1/sandboxes/a%20b/stop"]);
+    assert.equal(await mk(404).stop("x"), false);
+    assert.equal(await mk(405).stop("x"), false);
+    await assert.rejects(mk(500).stop("x"), /stop: HTTP 500/);
+  });
+
+  it("provider.stop and providerLanes.stopLane signal the transport, never destroy; no transport stop = false", async () => {
+    const root = tmpDir("remote-stop-");
+    const batch = openBatch(root, "bt");
+    const { provider, transport } = remoteFixture(batch);
+    const stopped = [];
+    const lanes = providerLanes({ provider, laneArgs: () => [], ledgerFor: () => null });
+    assert.equal(await lanes.stopLane(1, { handle: { id: "sbx" } }), false, "the transport has no stop");
+    transport.stop = async (id) => { stopped.push(id); return true; };
+    assert.equal(await lanes.stopLane(1, { handle: { id: "sbx" } }), true);
+    assert.equal(await lanes.stopLane(1, {}), false, "no handle");
+    transport.stop = async () => { throw new Error("boom"); };
+    assert.equal(await provider.stop({ id: "sbx" }), false);
+    assert.deepEqual(stopped, ["sbx"]);
+    assert.deepEqual(transport.destroyed, []);
+  });
+});

package/bin/lean/sandbox-remote.mjs

@@ -23,7 +23,6 @@ import { scopedEnv } from "./execution.mjs";
 import { resolveCredential } from "./credentials.mjs";
 
 const REDACTED = "[redacted]";
-const STATUS_ERRORS = 5;
 /** Replace every value of `secrets` (4+ chars, longest first) in `text`. */
 export function redact(text, secrets = []) {
   let s = String(text ?? "");
@@ -44,6 +43,7 @@ export function isActionsEndpoint(endpoint) {
  * GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
  * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
  * 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
+ * Optional `POST /sandboxes/:id/stop` (#4612) asks the pool to SIGTERM the lane so its teardown checkpoints and it reports `deferred run.interrupted`; 2xx = sent, 404/405 = unsupported or gone (never a DELETE fallback).
  * Optional `GET /status -> {free: N}` is the pool's free slots (#4540); anything else (404, error, malformed) is unknown (null).
  */
 export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
@@ -57,6 +57,8 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
   };
   const fail = (what, r) => {
     const e = new Error(`sandbox ${what}: HTTP ${r.status}`);
+    e.status = r.status;
+    if ([502, 503, 504].includes(r.status)) e.transient = true; // #4672: a router restart or gateway blip, retried by the lane poll
     if (r.status === 429 || r.status === 503) { e.retryable = true; const n = Number(r.retryAfter); if (Number.isFinite(n) && n > 0) e.retryInSeconds = n; }
     return e;
   };
@@ -73,6 +75,7 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
     async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
     async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
+    async stop(id) { const r = await call("POST", `${p(id)}/stop`); if (r.ok) return true; if (r.status === 404 || r.status === 405) return false; throw fail("stop", r); },
     async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
     async capacity() {
       // `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
@@ -111,7 +114,6 @@ export function gitIdentityEnv(identity, laneEnv = {}) {
 export function remoteProvider({ batch, repo, settings = {}, name = "remote", transport, env = process.env, credentials = (name) => resolveCredential(name, { env }), now = () => Date.now(), gitIdentity = hostGitIdentity }) {
   if (!transport) throw new Error("remote provider needs a transport");
   const idFor = (issue) => `fdrsbx-${safeSegment(batch.id)}-${issue}`;
-  const errs = new Map(); // sandbox id -> consecutive status errors
   const known = new Set(); // secret values seen this process, for redaction
   const remember = (v) => { if (typeof v === "string" && v.length >= 4) known.add(v); };
   for (const k of settings.secrets ?? []) remember(credentials(k));
@@ -154,8 +156,8 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
     },
     async status(h) {
       let s;
-      // A transient transport error is not an exit; STATUS_ERRORS in a row is (the lane then ends without a result).
-      try { s = await transport.status(h.id); errs.delete(h.id); } catch { const k = (errs.get(h.id) ?? 0) + 1; errs.set(h.id, k); return k >= STATUS_ERRORS ? "gone" : "running"; }
+      // #4672: a transport error is not an exit; it propagates and providerLanes retries it for execution.unreachable_window_min.
+      s = await transport.status(h.id);
       await pullLogs(h);
       return s === "running" ? "running" : s === "gone" ? "gone" : "exited";
     },
@@ -167,6 +169,11 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
     async capacity() {
       try { return (await transport.capacity?.()) ?? null; } catch { return null; }
     },
+    /** #4612: a graceful stop at the lane's checkpoint; false when unsupported or failed. Never destroys the sandbox. */
+    async stop(h) {
+      if (typeof transport.stop !== "function") return false;
+      try { return (await transport.stop(h.id)) === true; } catch { return false; }
+    },
     async teardown(h) {
       await minutesOf(h);
       try { return (await transport.destroy(h.id)) !== false; } catch { return false; }

package/bin/lean/schedd.mjs

@@ -11,9 +11,10 @@
  *   {op:"attach", id, resume?}    → (`resume` also clears a pending cancel, #4592) log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
  *                                   ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
  *   {op:"set-parallel", id, k}    → {type:"ok", parallel}
+ *   {op:"requeue", id, issues, reason, by} → {type:"ok", requeued, refused, hosted, reopened} (#4550): see batch.requeue; a finished batch is hosted again
  *   {op:"add", id, issues}        → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
  *                                   hosted runner reads on its next pass and an unhosted batch on its next resume
- *   {op:"cancel", id}             → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
+ *   {op:"cancel", id, stopRemote?}           → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
  *                                   admits nothing more, its running local lanes get SIGTERM (their teardown saves the work)
  *                                   and the rest end held; no other batch and not the daemon. A finished, unhosted batch:
  *                                   {type:"ok", cancelled:false, finished:true}, nothing written
@@ -34,6 +35,7 @@
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
 import { createConnection, createServer } from "node:net";
 import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { schedDir, pidAlive } from "./sched.mjs";
 import { batchRoot, listBatches, openBatch } from "./batch.mjs";
@@ -41,22 +43,63 @@ import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 export const IDLE_MS = 10 * 60 * 1000;
 export const STOP_MS = 10 * 1000;
 export const LOCK_CHECK_MS = 5 * 1000;
+export const DUE_MS = 15 * 60 * 1000; // #4627: how often the due callback (windowed post-deploy monitoring) runs while the daemon is up
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
 }
 
-// Env keys a batch keeps for its lanes (account and backend selectors, tool paths); persisted in plan.json.
-const ENV_KEEP = /^(PATH|HOME|USER|SHELL|LANG|LC_ALL|TMPDIR|FORGEDOCK_.*|CLAUDE_.*|ANTHROPIC_.*|GH_.*|GITHUB_.*)$/;
+// Env keys a batch keeps for its lanes (account selectors, tool paths); persisted in plan.json (#4691). Session-scoped
+// `CLAUDE_CODE_*` / `CLAUDE_PID` and `ANTHROPIC_*` are not kept, except the non-secret provider/endpoint/model selectors
+// (`CLAUDE_CODE_USE_*`, `ANTHROPIC_BASE_URL`, `ANTHROPIC_MODEL`, ...; #4734); more names come from `orchestration.env_passthrough`.
+const ENV_KEEP = /^(PATH|HOME|USER|SHELL|LANG|LC_.*|TMPDIR|CLAUDE_CONFIG_DIR|CLAUDE_CODE_USE_.*|ANTHROPIC_(BASE_URL|MODEL|SMALL_FAST_MODEL|DEFAULT_.*_MODEL)|FORGEDOCK_.*|GH_.*|GITHUB_.*)$/;
+// What the 1.36.0 pin carried (plus FORGEDOCK_*): a key of these families not pinned now is named at submit (#4734).
+const LEGACY_PIN = /^(CLAUDE_.*|ANTHROPIC_.*|GH_.*|GITHUB_.*|FORGEDOCK_.*)$/;
+const SESSION_SCOPED = /^(CLAUDE_PID|CLAUDE_CODE_(SESSION|MESSAGING|ENTRYPOINT|EXECPATH|SSE).*)$/;
+// A secret never rides the allowlist: it resolves at lane start (env, keychain) unless the operator names it as a passthrough.
+export const SECRET_ENV = /(TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|API_?KEY|PRIVATE_?KEY)/i;
+const SESSION_DIR = /(^|\/)(arg0|claude[^/]*)(\/|$)/;
 
-/** The allowlisted part of `env`. */
-export function batchEnv(env = process.env) {
-  return Object.fromEntries(Object.entries(env).filter(([k, v]) => ENV_KEEP.test(k) && typeof v === "string"));
+/** A forge.yaml glob (`*` within a segment, `**` across segments, `?`) as an anchored RegExp. */
+export function globToRegExp(glob) {
+  const re = String(glob).replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "\u0000").replace(/\*/g, "[^/]*").replace(/\u0000/g, ".*").replace(/\?/g, "[^/]");
+  return new RegExp(`^${re}$`);
 }
 
-/** `base` without its allowlisted keys, plus `kept`: an account selector the client did not set never leaks in from the daemon. */
+/** `path` without non-existent dirs, per-session temp dirs (an `arg0`/`claude*` dir under a temp root) and `exclude` glob matches. */
+export function sanitizePath(path, { exclude = [], exists = existsSync, tmpRoots = ["/tmp", "/var/tmp", tmpdir(), process.env.TMPDIR] } = {}) {
+  const globs = exclude.map(globToRegExp);
+  const roots = tmpRoots.filter(Boolean).map((r) => r.replace(/\/+$/, ""));
+  const keep = (d) => d && exists(d) && !globs.some((g) => g.test(d.replace(/\/+$/, "")))
+    && !(roots.some((r) => d === r || d.startsWith(`${r}/`)) && SESSION_DIR.test(d));
+  const seen = new Set();
+  return String(path).split(":").filter((d) => keep(d) && !seen.has(d) && seen.add(d)).join(":");
+}
+
+/**
+ * The allowlisted part of `env`, with PATH sanitised. `passthrough` names extra keys to keep (`orchestration.env_passthrough`,
+ * the configured `execution.token_env`); `pathExclude` is `orchestration.path_exclude`. Secret-looking names are dropped unless named.
+ */
+export function batchEnv(env = process.env, { passthrough = [], pathExclude = [], exists = existsSync, tmpRoots } = {}) {
+  const extra = new Set(passthrough);
+  const out = Object.fromEntries(Object.entries(env).filter(([k, v]) => typeof v === "string" && (extra.has(k) || (ENV_KEEP.test(k) && !SECRET_ENV.test(k)))));
+  if (typeof out.PATH === "string") out.PATH = sanitizePath(out.PATH, { exclude: pathExclude, exists, ...(tmpRoots ? { tmpRoots } : {}) });
+  return out;
+}
+
+/** Sorted names (never values) of `env` keys the pre-allowlist pin carried that `kept` does not: the lane gets the daemon's value or resolves it at lane start (#4734). */
+export function droppedEnv(env = process.env, kept = {}) {
+  return Object.keys(env).filter((k) => typeof env[k] === "string" && LEGACY_PIN.test(k) && !SESSION_SCOPED.test(k) && !(k in kept)).sort();
+}
+
+/** `env` with secret-looking values redacted (`--show-env`). */
+export function redactEnv(env) {
+  return Object.fromEntries(Object.entries(env ?? {}).map(([k, v]) => [k, SECRET_ENV.test(k) ? "<redacted>" : v]));
+}
+
+/** `base` without its allowlisted non-secret keys (secret-named ones, e.g. GH_TOKEN, stay from the daemon: never pinned), plus `kept`: an account selector the client did not set never leaks in from the daemon. */
 export function mergeEnv(base, kept) {
-  const out = Object.fromEntries(Object.entries(base).filter(([k]) => !ENV_KEEP.test(k)));
+  const out = Object.fromEntries(Object.entries(base).filter(([k]) => k in kept || !ENV_KEEP.test(k) || SECRET_ENV.test(k)));
   return { ...out, ...kept };
 }
 
@@ -83,7 +126,7 @@ function takeLock(lock, pid, alive) {
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
   root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS,
-  lockCheckMs = LOCK_CHECK_MS } = {}) {
+  lockCheckMs = LOCK_CHECK_MS, due = null, dueMs = Number(process.env.FORGEDOCK_SCHEDD_DUE_MS) || DUE_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
   const lock = takeLock(p.lock, pid, alive);
@@ -139,8 +182,20 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (req.op === "add") {
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
-      const { added, skipped } = b.addIssues(req.issues ?? []);
-      return send({ type: "ok", added, skipped, hosted: hosted.has(req.id) }, true);
+      const { added, skipped, requeued = [] } = b.addIssues(req.issues ?? []);
+      // #4687: a re-queued skipped issue reopens a finished, unhosted batch like --requeue does
+      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
+      if (reopen) await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} });
+      return send({ type: "ok", added, skipped, requeued, hosted: hosted.has(req.id), reopened: reopen }, true);
+    }
+    if (req.op === "requeue") {
+      // #4550: the runner applies the entries on its next pass; a finished, unhosted batch is reopened (hosted as a resume).
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      const { requeued, refused } = b.requeue(req.issues ?? [], { reason: req.reason, by: req.by });
+      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
+      if (reopen) await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} });
+      return send({ type: "ok", requeued, refused, hosted: hosted.has(req.id), reopened: reopen }, true);
     }
     if (req.op === "hold" || req.op === "release") {
       const b = openBatch(root, req.id);
@@ -152,7 +207,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       if (!hosted.has(req.id) && b.readReport()) return send({ type: "ok", cancelled: false, finished: true }, true);
-      b.requestCancel({ reason: "operator" });
+      b.requestCancel({ reason: "operator", stopRemote: req.stopRemote === true });
       return send({ type: "ok", cancelled: true, hosted: hosted.has(req.id) }, true);
     }
     const sub = (m) => send(m, m.type === "done" || m.type === "error");
@@ -173,7 +228,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const rep = b.readReport();
-      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length) {
+      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length && !b.unrunRequeues().length) {
         return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
       }
       const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
@@ -212,6 +267,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (closing) return closing;
     clearInterval(idleTimer);
     clearInterval(lockTimer);
+    if (dueTimer) clearInterval(dueTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
     for (const ac of runs.values()) ac.abort();
@@ -241,6 +297,14 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     else if (!existsSync(p.sock)) { log(`schedd: lost socket ${p.sock}, exiting`); close(); }
   }, lockCheckMs);
   lockTimer.unref?.();
+  // #4627: windowed post-deploy checks. One run at a time; a failure is logged and never stops the daemon.
+  let dueRunning = false;
+  const dueTimer = due ? setInterval(async () => {
+    if (closing || dueRunning) return;
+    dueRunning = true;
+    try { await due(); } catch (e) { log(`schedd: due run failed: ${e?.message ?? e}`); } finally { dueRunning = false; }
+  }, dueMs) : null;
+  dueTimer?.unref?.();
 
   // Daemon restart: resume every daemon-hosted batch that never wrote its report; live lanes are re-adopted.
   if (resumeOnStart) {

package/bin/lean/schedd.test.mjs

@@ -3,12 +3,12 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { spawn } from "node:child_process";
-import { closeSync, existsSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches } from "./batch.mjs";
-import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv, exitOnSignals } from "./schedd.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv, droppedEnv, sanitizePath, redactEnv, exitOnSignals } from "./schedd.mjs";
 import { schedDir } from "./sched.mjs";
 import { formatTable, orchestrate } from "./orchestrate.mjs";
 import { orchestrateFromCli } from "../engine-cli.mjs";
@@ -208,6 +208,32 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#4550: --requeue on a finished batch with a report reopens it through the daemon and runs the issue again; --add still skips it", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        writeFileSync(h.release, "");
+        const first = await orchestrateFromCli(["1", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
+        assert.deepEqual(first.rows.map((r) => [r.issue, r.outcome]), [[1, "merged"]]);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        // The lane ended needs-human and the operator decided it: the finished batch holds that terminal row.
+        b.store.save({ ...b.store.load(), results: { 1: { issue: 1, outcome: "needs-human", pr: null, cost_usd: 0, minutes: 0 } } });
+        const rep = b.readReport();
+        b.writeReport({ ...rep, rows: [{ issue: 1, outcome: "needs-human" }] });
+        assert.deepEqual((await orchestrateFromCli(["--add", id, "1"], client)).skipped.map((x) => x.reason), ["already in batch"]);
+        const res = await orchestrateFromCli(["--requeue", id, "1", "--reason", "decided"], client);
+        assert.deepEqual(res.requeued, [1]);
+        await until(() => h.launched().length === 2);
+        await until(() => b.store.load()?.results?.[1]?.outcome === "merged" && b.readReport()?.rows?.[0]?.outcome === "merged");
+        assert.deepEqual(listBatches(h.root), [id], "no second batch");
+        assert.equal(b.readReport().rows[0].requeued.reason, "decided");
+      }));
+    } finally { await h.cleanup(); }
+  });
+
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
     const h = host();
     try {
@@ -481,9 +507,109 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
   });
 
   it("lane env: allowlisted keys come from the client only, the rest from the daemon", () => {
-    const kept = batchEnv({ PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", SECRET_X: "no", FORGEDOCK_SCHED_DIR: "/s" });
-    assert.deepEqual(kept, { PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });
+    const kept = batchEnv({ PATH: "/", CLAUDE_CONFIG_DIR: "/acct-b", SECRET_X: "no", FORGEDOCK_SCHED_DIR: "/s", LC_CTYPE: "C" });
+    assert.deepEqual(kept, { PATH: "/", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s", LC_CTYPE: "C" });
     const env = mergeEnv({ PATH: "/d", ANTHROPIC_API_KEY: "daemon-key", NODE_OPTIONS: "x" }, kept);
-    assert.deepEqual(env, { NODE_OPTIONS: "x", PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });
+    assert.equal(mergeEnv({ GH_TOKEN: "d" }, kept).GH_TOKEN, "d", "a secret-named daemon key (never pinned) stays for the lane");
+    assert.deepEqual(env, { NODE_OPTIONS: "x", ANTHROPIC_API_KEY: "daemon-key", PATH: "/", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s", LC_CTYPE: "C" });
+  });
+
+  it("#4734: the pin keeps non-secret provider selectors, drops secrets and session ids, and droppedEnv names what was left out", () => {
+    const client = { PATH: "/", GH_TOKEN: "fresh", CLAUDE_CODE_USE_BEDROCK: "1", ANTHROPIC_BASE_URL: "http://proxy", ANTHROPIC_MODEL: "m", ANTHROPIC_API_KEY: "k",
+      FORGEDOCK_POOL_TOKEN: "p", CLAUDE_CONFIG_DIR: "/acct", CLAUDE_CODE_SESSION_ID: "s", CLAUDE_PID: "1" };
+    const kept = batchEnv(client);
+    assert.deepEqual(Object.keys(kept).sort(), ["ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CONFIG_DIR", "PATH"]);
+    assert.deepEqual(droppedEnv(client, kept), ["ANTHROPIC_API_KEY", "FORGEDOCK_POOL_TOKEN", "GH_TOKEN"]);
+    assert.deepEqual(droppedEnv(client, batchEnv(client, { passthrough: ["GH_TOKEN"] })), ["ANTHROPIC_API_KEY", "FORGEDOCK_POOL_TOKEN"]);
+    assert.deepEqual(droppedEnv({ PATH: "/", CLAUDE_CODE_SESSION_ID: "s" }, {}), [], "nothing to name: no warning");
+    assert.equal(mergeEnv({ ANTHROPIC_BASE_URL: "daemon", GH_TOKEN: "d" }, kept).ANTHROPIC_BASE_URL, "http://proxy");
+  });
+
+  it("#4691: the pinned env drops session tokens and ids, session temp PATH dirs and path_exclude dirs", () => {
+    const work = tmpDir("schedd-env-");
+    const keep = join(work, "bin"), session = join(work, "claude-1", "arg0"), wrapper = join(work, "wrap", "shims");
+    for (const d of [keep, session, wrapper]) mkdirSync(d, { recursive: true });
+    const session_env = { PATH: [keep, session, wrapper, join(work, "gone")].join(":"), HOME: "/h", CLAUDE_CODE_MESSAGING_TOKEN: "tok", CLAUDE_CODE_SESSION_ID: "s", CLAUDE_PID: "1",
+      CLAUDE_CODE_SESSION_ATTENDED: "1", ANTHROPIC_API_KEY: "k", FORGEDOCK_SANDBOX_TOKEN: "t", MY_FLAG: "1", MY_SECRET_PASS: "p", GH_HOST: "h" };
+    const pin = batchEnv(session_env, { passthrough: ["MY_FLAG", "MY_SECRET_PASS"], pathExclude: [`${work}/wrap/*`], tmpRoots: [work] });
+    assert.equal(pin.PATH, keep, "only the live, non-session, non-excluded dir stays");
+    assert.deepEqual(Object.keys(pin).sort(), ["GH_HOST", "HOME", "MY_FLAG", "MY_SECRET_PASS", "PATH"]);
+    assert.equal(sanitizePath(`${session}:${keep}`, { tmpRoots: [work] }), keep);
+    assert.equal(sanitizePath(`${session}:${keep}`, { tmpRoots: ["/nowhere"] }), `${session}:${keep}`, "an arg0 dir outside a temp root is not session-scoped");
+    assert.deepEqual(redactEnv({ A: "1", GH_TOKEN: "x" }), { A: "1", GH_TOKEN: "<redacted>" });
+  });
+
+  it("#4691: plan.json holds none of the session token, temp PATH dir or excluded wrapper dir; --show/--set/--unset-env edit it", async () => {
+    const h = host();
+    const work = tmpDir("schedd-pin-");
+    const keep = join(work, "bin"), session = join(work, "claude-9", "arg0"), wrapper = join(work, "wrap", "shims");
+    for (const d of [keep, session, wrapper]) mkdirSync(d, { recursive: true });
+    writeFileSync(join(work, "forge.yaml"), `orchestration:\n  path_exclude: ${work}/wrap/*\n`);
+    const saved = { cwd: process.cwd(), path: process.env.PATH, tmp: process.env.TMPDIR };
+    let a;
+    try {
+      await h.daemon();
+      process.chdir(work);
+      process.env.PATH = [keep, session, wrapper].join(":");
+      process.env.TMPDIR = work;
+      process.env.CLAUDE_CODE_MESSAGING_TOKEN = "msg-token";
+      process.env.CLAUDE_CODE_SESSION_ID = "sess";
+      await withRoot(h, async () => {
+        a = capture(() => orchestrateFromCli(["1", "--repo", "o/r", "--no-prefilter"], { ...h.client, reconnectMs: 0 })).catch((e) => e);
+        await until(() => h.stubPids().length === 1 && h.lanePids().length === 1);
+      });
+      const [id] = listBatches(h.root);
+      const raw = readFileSync(join(h.root, id, "plan.json"), "utf8");
+      const env = JSON.parse(raw).env;
+      assert.equal(env.PATH, keep);
+      for (const bad of ["msg-token", "sess", "CLAUDE_CODE_", session, wrapper]) assert.equal(raw.includes(bad), false, `${bad} not pinned`);
+      await withRoot(h, async () => {
+        await capture(() => orchestrateFromCli(["--set-env", id, "MY_VAR=1"], h.client));
+        await capture(() => orchestrateFromCli(["--set-env", id, "GH_TOKEN=abc"], h.client));
+        assert.equal(openBatch(h.root, id).readPlan().env.MY_VAR, "1");
+        const { lines } = await capture(() => orchestrateFromCli(["--show-env", id], h.client));
+        assert.match(lines.join("\n"), /^GH_TOKEN=<redacted>$/m);
+        assert.match(lines.join("\n"), /^MY_VAR=1$/m);
+        assert.equal(lines.join("\n").includes("abc"), false);
+        await capture(() => orchestrateFromCli(["--unset-env", id, "MY_VAR"], h.client));
+        assert.equal("MY_VAR" in openBatch(h.root, id).readPlan().env, false);
+        await assert.rejects(() => orchestrateFromCli(["--set-env", id, "NOEQUALS"], h.client), /KEY=VAL/);
+      });
+      writeFileSync(h.release, "");
+    } finally {
+      process.chdir(saved.cwd); process.env.PATH = saved.path;
+      if (saved.tmp === undefined) delete process.env.TMPDIR; else process.env.TMPDIR = saved.tmp;
+      delete process.env.CLAUDE_CODE_MESSAGING_TOKEN; delete process.env.CLAUDE_CODE_SESSION_ID;
+      await h.cleanup();
+    }
+    await a;
+  });
+});
+
+describe("schedd due timer (#4627)", () => {
+  const boot = async (opts) => startDaemon({ dir: tmpDir("schedd-due-"), root: tmpDir("schedd-due-b-"), runBatch: async () => ({}), resumeOnStart: false, log: () => {}, ...opts });
+
+  it("runs the due callback on its interval, never overlapped, and survives a throwing callback", async () => {
+    let calls = 0, active = 0, overlap = false;
+    const logs = [];
+    const d = await boot({ dueMs: 15, log: (m) => logs.push(m), due: async () => {
+      calls++; active++; if (active > 1) overlap = true;
+      await new Promise((r) => setTimeout(r, 40));
+      active--; if (calls === 1) throw new Error("boom");
+    } });
+    try {
+      await until(() => calls >= 3);
+      assert.equal(overlap, false);
+      assert.ok(logs.some((m) => /due run failed: boom/.test(m)));
+      assert.ok(existsSync(d.sock));
+    } finally { await d.close(); }
+    const n = calls;
+    await new Promise((r) => setTimeout(r, 80));
+    assert.equal(calls, n, "the timer is cleared on close");
+  });
+
+  it("without a due callback there is no timer", async () => {
+    const d = await boot({ dueMs: 10 });
+    await d.close();
   });
 });

package/bin/lean/sequence.mjs

@@ -457,3 +457,27 @@ export async function generatorProblems({ git, base, ref = "HEAD" }) {
   }
   return problems;
 }
+
+/**
+ * #4661: journal `when` ordering. Newest-only migrators (drizzle `migrate()`) skip an entry stamped older than one already
+ * applied, so every entry the lane adds to a drizzle-style journal (`<dir>/meta/_journal.json`) must have `when` above the
+ * base journal's max, and `when` must increase within the lane journal. No journal or base journal: []. @returns {Promise<string[]>}
+ */
+export async function journalOrderProblems({ git, base, ref = "HEAD" }) {
+  const g = out(git), problems = [];
+  const changed = lines(await g(["-c", "core.quotepath=off", "diff", "--name-only", `${base}...${ref}`])).filter((f) => /(?:^|\/)meta\/_journal\.json$/.test(f) && !SKIP_RE.test(f));
+  const entriesAt = async (r, jp) => { try { const e = JSON.parse(await g(["show", `${r}:${jp}`])).entries; return Array.isArray(e) ? e : null; } catch { return null; } };
+  for (const jp of changed) {
+    const now = await entriesAt(ref, jp);
+    if (!now) continue;
+    const was = (await entriesAt(base, jp)) ?? [], wasTags = new Set(was.map((e) => e?.tag));
+    const max = Math.max(-Infinity, ...was.map((e) => Number(e?.when)).filter(Number.isFinite));
+    for (let i = 0; i < now.length; i++) {
+      const e = now[i] ?? {}, w = Number(e.when);
+      if (wasTags.has(e.tag)) continue;
+      if (Number.isFinite(max) && !(w > max)) problems.push(`${jp}: ${e.tag} when ${e.when} is not above the base max when ${max} (stale stamp: a newest-only migrator skips it on a persistent database)`);
+      else if (i > 0 && Number.isFinite(Number(now[i - 1]?.when)) && !(w > Number(now[i - 1].when))) problems.push(`${jp}: ${e.tag} when ${e.when} is not after ${now[i - 1].tag} (${now[i - 1].when})`);
+    }
+  }
+  return problems;
+}

package/bin/lean/sequence.test.mjs

@@ -8,7 +8,7 @@ import { execFileSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { sequenceCollisions, journalProblems, snapshotLane, verifyRenumber, numberOnlyChange, headerOnlyChange, renumberLane, resolveSequenceConflicts, generatorProblems } from "./sequence.mjs";
+import { sequenceCollisions, journalProblems, snapshotLane, verifyRenumber, numberOnlyChange, headerOnlyChange, renumberLane, resolveSequenceConflicts, generatorProblems, journalOrderProblems } from "./sequence.mjs";
 import { makeSequenceLeaseIo } from "./seqlease.mjs";
 import { sequencePrompt, sequenceSyncNote, migrationGeneratorPrompt, builderImplementPrompt } from "./prompts.mjs";
 
@@ -492,3 +492,33 @@ describe("hand-named migrations (#4591)", () => {
     assert.match(builderImplementPrompt("local"), /Never name or number migration files by hand/);
   });
 });
+
+describe("journal when ordering across parallel milestone branches (#4661)", () => {
+  const J = (...es) => `${JSON.stringify({ version: "7", entries: es.map(([tag, when], idx) => ({ idx, tag, when })) }, null, 2)}\n`;
+  function repo(laneEntries) {
+    const d = tmpDir("when-");
+    sh(d, "init", "-q", "-b", "main");
+    put(d, { "db/meta/_journal.json": J(["0_a", 5000], ["1_b", 9000]) });
+    commit(d, "init");
+    sh(d, "checkout", "-qb", "lane");
+    if (laneEntries) { put(d, { "db/meta/_journal.json": J(["0_a", 5000], ["1_b", 9000], ...laneEntries) }); commit(d, "lane"); }
+    return d;
+  }
+  it("flags an added entry stamped at or below the base max when", async () => {
+    const p = await journalOrderProblems({ git: gitIn(repo([["2_c", 7000]])), base: "main" });
+    assert.equal(p.length, 1);
+    assert.match(p[0], /db\/meta\/_journal\.json: 2_c when 7000 is not above the base max when 9000/);
+  });
+  it("passes entries above the base max, an unchanged journal, and no journal", async () => {
+    assert.deepEqual(await journalOrderProblems({ git: gitIn(repo([["2_c", 9001]])), base: "main" }), []);
+    assert.deepEqual(await journalOrderProblems({ git: gitIn(repo(null)), base: "main" }), []);
+    const d = tmpDir("when-none-");
+    sh(d, "init", "-q", "-b", "main"); put(d, { "x.txt": "x" }); commit(d, "i"); sh(d, "checkout", "-qb", "lane");
+    assert.deepEqual(await journalOrderProblems({ git: gitIn(d), base: "main" }), []);
+  });
+  it("flags non-increasing when inside the lane's added entries; the prompt asks for a re-stamp", async () => {
+    const p = await journalOrderProblems({ git: gitIn(repo([["2_c", 9100], ["3_d", 9050]])), base: "main" });
+    assert.match(p[0], /3_d when 9050 is not after 2_c/);
+    assert.match(migrationGeneratorPrompt(p), /re-stamp/);
+  });
+});

package/bin/lean/signoff.mjs

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * DCO sign-off for a lane's own unpushed commits (#4651). A resumed lane keeps the commits of an earlier run; one from a
+ * pool agent that predates the identity fix is authored `*@localhost` with no `Signed-off-by`, and DCO fails on the release
+ * PR. Before a push, the commits the push would add (reachable from HEAD, not from origin/<branch> nor origin/<lane>) are
+ * checked, and the offenders are rewritten with the lane identity and --signoff. Commits already on either remote ref are
+ * never in the range, so they are never rewritten. Deterministic, 0 LLM calls.
+ */
+const SEP = "\x1f", END = "\x1e";
+const PLACEHOLDER_RE = /@localhost$/i;
+
+/** True when the commit must be rewritten: placeholder author or committer, or no sign-off naming its author. */
+export function needsSignoff({ authorEmail, committerEmail, signedOff }) {
+  if (PLACEHOLDER_RE.test(authorEmail ?? "") || PLACEHOLDER_RE.test(committerEmail ?? "")) return true;
+  const a = String(authorEmail ?? "").toLowerCase();
+  return !signedOff.some((s) => s.toLowerCase().includes(`<${a}>`));
+}
+
+/** Parse `git log` output of FORMAT into the commits that need a rewrite, oldest first (the log is newest first). */
+export function offendersOf(out) {
+  return String(out ?? "").split(END).map((r) => r.replace(/^\n/, "")).filter(Boolean).map((r) => {
+    const [sha, authorEmail, committerEmail, trailers = "", parents = ""] = r.split(SEP);
+    return { sha, authorEmail, committerEmail, signedOff: trailers.split("\n").map((l) => l.trim()).filter(Boolean), merge: parents.trim().split(/\s+/).filter(Boolean).length > 1 };
+  }).filter((c) => !c.merge && needsSignoff(c)).reverse();
+}
+
+const FORMAT = `%H${SEP}%ae${SEP}%ce${SEP}%(trailers:key=Signed-off-by,valueonly,separator=%x0a)${SEP}%P${END}`;
+
+// filter-branch rewrites only the commits in the range (the excluded remote refs stay as they are), and only the listed ones change.
+const IS_BAD = 'case " $FD_BAD " in *" $GIT_COMMIT "*)';
+const ENV_FILTER = `${IS_BAD} export GIT_AUTHOR_NAME="$FD_NAME" GIT_AUTHOR_EMAIL="$FD_EMAIL" GIT_COMMITTER_NAME="$FD_NAME" GIT_COMMITTER_EMAIL="$FD_EMAIL";; esac`;
+const MSG_FILTER = `${IS_BAD} git interpret-trailers --trailer "Signed-off-by: $FD_NAME <$FD_EMAIL>";; *) cat;; esac`;
+
+/**
+ * @param {object} o
+ * @param {(args: string[], opts?: {env?: object}) => Promise<string>} o.git runs git in the lane worktree
+ * @param {string} o.lane the lane (base) branch
+ * @param {string} o.branch the lane's own branch
+ * @param {object} [o.env] where GIT_AUTHOR_NAME/EMAIL are read from (default process.env)
+ * @returns {Promise<{rewritten: number}>}
+ * @throws Error with `.laneCode` unset: callers type it. `.noIdentity` when a rewrite is needed but no usable identity exists.
+ */
+export async function ensureSignedOff({ git, lane, branch, env = process.env }) {
+  const ok = async (args) => { try { return String(await git(args)).trim(); } catch { return ""; } };
+  try { await git(["fetch", "origin", branch]); } catch { /* not pushed yet, or offline: the tracking refs we have decide */ }
+  const exclude = [];
+  for (const ref of [`origin/${branch}`, `origin/${lane}`]) if (await ok(["rev-parse", "--verify", "-q", `refs/remotes/${ref}`])) exclude.push(`^${ref}`);
+  const log = await ok(["log", `--format=${FORMAT}`, "HEAD", ...exclude]);
+  const bad = offendersOf(log);
+  if (!bad.length) return { rewritten: 0 };
+  // the lane identity: GIT_AUTHOR_* (what the sandbox sets and what a commit would use), else the worktree config
+  let name = env.GIT_AUTHOR_NAME || await ok(["config", "user.name"]), email = env.GIT_AUTHOR_EMAIL || await ok(["config", "user.email"]);
+  if (!name || !email) { // the runner's own git env (what a commit here would use), e.g. GIT_AUTHOR_* set only on the git runner
+    const m = /^(.*?)\s*<([^>]*)>/.exec(await ok(["var", "GIT_AUTHOR_IDENT"]));
+    if (m) { name ||= m[1]; email ||= m[2]; }
+  }
+  if (!name || !email || PLACEHOLDER_RE.test(email)) {
+    throw Object.assign(new Error(`${bad.length} unpushed commit(s) fail DCO (placeholder author or no sign-off: ${bad.slice(0, 5).map((c) => c.sha.slice(0, 8)).join(", ")}) and the worktree has no usable git identity to rewrite them`), { noIdentity: true });
+  }
+  const fenv = { FD_BAD: bad.map((c) => c.sha).join(" "), FD_NAME: name, FD_EMAIL: email, FILTER_BRANCH_SQUELCH_WARNING: "1" };
+  try { await git(["filter-branch", "-f", "--env-filter", ENV_FILTER, "--msg-filter", MSG_FILTER, "--", "HEAD", ...exclude], { env: fenv }); }
+  catch (e) { throw Object.assign(new Error(`could not rewrite ${bad.length} unpushed commit(s) for DCO: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`), { noIdentity: false }); }
+  await ok(["for-each-ref", "--format=delete %(refname)", "refs/original/"]).then(async (l) => { for (const r of l.split("\n").map((x) => x.replace(/^delete /, "")).filter(Boolean)) await ok(["update-ref", "-d", r]); });
+  return { rewritten: bad.length };
+}