@sema-agent/server 1.323.0 → 2.0.1

package/dist/leader/wire.js

@@ -1,3 +1,16 @@
+/**
+ * v2 MVP M4 — leader endpoint wiring (V2-MVP-PLAN.md). `createLeaderRunner` turns a LeaderRequestBody into a
+ * `runLeaderTask` invocation with REAL deps: per-worker E2B envs (leader-owned, static so they outlive the
+ * task for diff-out) + a real ephemeral E2B integration sandbox + the Coordinator's credentialed
+ * `git push --force-with-lease`. This is the productization of the joint dogfood (which validated the exact
+ * logic on real infra) — gated behind LEADER_ENABLED (default off).
+ *
+ * 🔴 DEPLOY REQUIREMENTS (document — not satisfied by the default bun-slim image): the control plane (service
+ * host) needs **git** on PATH for the Coordinator clone/apply/push, plus a reachable durable remote +
+ * write creds (LEADER_GIT_TOKEN). E2B workers need outbound net only if the base repo is cloned over the
+ * network; the MVP seeds the base from the Coordinator (no worker→remote net). MVP Planner = caller-provided
+ * `subtasks` (an LLM DAG-planner is a follow-up).
+ */
 import { execFileSync } from "node:child_process";
 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -11,6 +24,12 @@ import { pullDiff, safeRev } from "./diffout.js";
 import { runLeaderTask } from "./leader.js";
 import { attachRepairLoopDeps } from "./repair-wire.js";
 import { routePlanWithFallback, validateSubtasks } from "./planner.js";
+/** The single-agent `runRepairLoop` is now WIRED LIVE (the canary-enable slice): when `LEADER_REPAIR_LOOP=true`
+ *  the wire provisions a DEDICATED oracle-seeded grader sandbox per solo worker and attaches a real async
+ *  `resolve` to `attachRepairLoopDeps` (below). With this `true`, flipping `LEADER_REPAIR_LOOP` engages BOTH the
+ *  repair loop AND the §10.5 resource-suspend mutual exclusion (a repair-owned worker never resource-suspends).
+ *  Still OFF-by-default — with the flag off, `resolve` is undefined → `attachRepairLoopDeps` is a no-op → the
+ *  default single-agent path is byte-identical to today. */
 const REPAIR_LOOP_WIRED = true;
 const sh = (env) => async (cmd) => {
     const r = await env.exec(`bash -lc ${JSON.stringify(cmd)}`);
@@ -28,30 +47,77 @@ const asIntegrationEnv = (env) => ({
     writeFile: async (p, c) => { const r = await env.writeFile(p, c); if (!r.ok)
         throw r.error; },
 });
+/**
+ * Resume-safe worker staging (the design/68 F2(a) factory-env path). Runs adapter-side before the agent's
+ * first workspace-touching call: seed the base repo at the worker branch + plant the upload script (so the
+ * presigned URL never passes through the model).
+ *
+ * 🔴 BL-1 fix (design/74 Slice 6): on a RESOURCE-SUSPEND RESUME core re-invokes the `executionEnvFactory`,
+ * producing a FRESH `withStaging` wrapper (`staged=undefined`) whose `stage()` runs AGAIN — AFTER core's
+ * `resumeVM()` has already untar'd the snapshot onto the fresh pod (`resumeVM` is not a STAGED_METHOD, so it
+ * is NOT intercepted by the staging proxy; the agent's first `exec` is what triggers `stage()`, lazily, and
+ * by then the snapshot is restored). The old `stage()` ran the seed UNCONDITIONALLY:
+ *     `${seedCmd}; cd ${repo}; git checkout -q -b ${branch}`
+ * and `seedCmd` begins with `rm -rf /workspace/repo` — so it WIPED the just-restored snapshot and re-seeded at
+ * `baseSha`, then created the worker branch at base. The forensic proof (leaderRunId 019ecda7): the suspend
+ * snapshot object DID contain the worker's commit (`git log` in the S3 tarball = the real progress), yet every
+ * resumed slice's `git log` showed only the base commit — the worker rebuilt destroyed work for ~47 min. THIS
+ * (a staging re-seed clobbering a correctly-restored snapshot) is the dominant cause of failed make-real runs.
+ *
+ * The fix = a RESTORED-SNAPSHOT SENTINEL that no-ops staging on resume. The robust signal is the upload-script
+ * MARKER FILE (`uploadScriptPath`): staging itself plants it, so it is present in EVERY restored snapshot
+ * (verified: it survives in both a base-state in-progress snapshot AND a post-commit one) and ABSENT on a
+ * genuinely fresh pod — and it is INDEPENDENT of git/commit state. That last point matters: a worker can
+ * suspend with work in progress but no commit yet (HEAD still == baseSha), so a "HEAD advanced past baseSha"
+ * probe alone would WRONGLY re-seed+clobber that valid snapshot. The marker captures "staging already ran here"
+ * regardless of whether a commit landed. We additionally accept the worker branch already existing as a
+ * secondary restored-snapshot signal (belt-and-suspenders for the same git-state-independent reason).
+ *
+ * A FRESH pod has neither marker nor branch → full staging runs. A probe transport failure falls through to
+ * full staging too (fails honestly via `sh()`, never silently skips). `branch`/paths are `validateSubtasks`-
+ * clean / deployment config and are JSON-quoted under `bash -lc` regardless. Exported for the regression test.
+ */
 export async function stageWorkerEnv(env, opts) {
     const { seedCmd, repo, branch, uploadScriptPath, uploadScript } = opts;
+    // Restored-snapshot sentinel (non-throwing exec — we need the exit code, not an exception). Two independent
+    // signals, EITHER of which proves "staging already ran here, this is a resumed snapshot, do NOT re-seed":
+    //   (1) the upload-script marker file is present  — the definitive signal (planted by staging, git-state
+    //       independent, so it covers an in-progress-but-uncommitted snapshot where HEAD is still at base); and
+    //   (2) the worker branch already exists          — secondary, also git-state independent.
+    // Both probed in ONE round-trip; exit 0 ⇒ restored snapshot ⇒ NO-OP (never `rm -rf` the restored workspace).
+    // A transport failure ({ok:false}) falls through to full staging (honest fail via sh()), never silent-skips.
     const probe = await env.exec(`bash -lc ${JSON.stringify(`if [ -f ${uploadScriptPath} ]; then exit 0; fi; ` +
         `cd ${repo} 2>/dev/null && git rev-parse --verify --quiet refs/heads/${branch} >/dev/null && exit 0; ` +
         `exit 1`)}`);
     if (probe.ok && probe.value.exitCode === 0)
-        return;
+        return; // resume — repo/branch/upload-script already in the snapshot
     await sh(env)(`${seedCmd}; cd ${repo}; git checkout -q -b ${branch}`);
     const wr = await env.writeFile(uploadScriptPath, uploadScript);
     if (!wr.ok)
         throw new Error(`upload-script staging failed: ${wr.error.message}`);
 }
+/** Build the `runLeader(body)` the HTTP endpoint invokes. Constructs real per-task deps + runs the leader. */
+/**
+ * Per-leader-run resource bounds derived from env (2026-06-14). Exported for unit tests. The worker sandbox
+ * lifetime (BUG2), the worker's diff-upload presigned-URL TTL (BUG1), and the worker spec's triple bound
+ * (maxTurns/timeoutSec/maxCostUsd, #15) ALL track one `leaderTimeoutMs` — a hardcoded 10min (sandbox) or 1h
+ * (URL) silently killed long autonomous builds or expired the diff upload mid-run. core's turn cap is now a
+ * high net (10000), so the leader gives each worker an explicit triple bound; a sub-task's own limits override.
+ */
 export function leaderResourceConfig(env = process.env) {
-    const leaderTimeoutMs = Math.max(600_000, Math.floor(Number(env.LEADER_TIMEOUT_MS ?? "0")) || 86_400_000);
+    const leaderTimeoutMs = Math.max(600_000, Math.floor(Number(env.LEADER_TIMEOUT_MS ?? "0")) || 86_400_000); // default 24h (was 10min)
     const workerMaxTurns = Math.floor(Number(env.LEADER_WORKER_MAX_TURNS ?? "0")) || 10_000;
     const workerBudgetUsd = Number(env.LEADER_WORKER_BUDGET_USD ?? "0") || 100;
     const resourceSuspendOn = String(env.LEADER_RESOURCE_SUSPEND ?? "").toLowerCase() === "true";
+    // Per-slice window: a fraction of the total so a worker suspends+resumes several times across its budget
+    // (default = total/4, min $1). Smaller window ⇒ more, shorter slices ⇒ progress saved more often.
     const sliceMaxCostUsd = Number(env.LEADER_WORKER_SLICE_BUDGET_USD ?? "0") || Math.max(1, workerBudgetUsd / 4);
     return {
         leaderTimeoutMs,
         workerMaxTurns,
         workerBudgetUsd,
         workerLimits: { limits: { maxTurns: workerMaxTurns, timeoutSec: Math.floor(leaderTimeoutMs / 1000) }, maxCostUsd: workerBudgetUsd },
-        presignTtlSec: Math.ceil(leaderTimeoutMs / 1000) + 600,
+        presignTtlSec: Math.ceil(leaderTimeoutMs / 1000) + 600, // URL/sandbox must outlive the worker (BUG1/BUG2)
         ...(resourceSuspendOn
             ? {
                 resourceSuspend: {
@@ -71,23 +137,42 @@ export function createLeaderRunner(cfg) {
     const repairRounds = Math.max(0, Math.floor(Number(process.env.LEADER_REPAIR_ROUNDS ?? cfg.repairRounds ?? 0)) || 0);
     const repairBudgetUsd = Number(process.env.LEADER_REPAIR_BUDGET_USD ?? "4") || 4;
     const conflictRounds = Math.max(0, Math.floor(Number(process.env.LEADER_CONFLICT_ROUNDS ?? cfg.conflictRounds ?? 0)) || 0);
+    // ── LEADER-REPAIRLOOP-INTEGRATION §10 env knobs (FRESH hunk, §10.8 — mirrors LEADER_REPAIR_ROUNDS above) ──
+    // Every flag default-OFF so the whole feature is inert until flipped on a canary (the default path is
+    // byte-identical to today). `LEADER_REPAIR_LOOP` is the MASTER (gates the single-agent `runRepairLoop` dep);
+    // `LEADER_MEASURE_GATES` gates the merge push-hold-on-signal + measure-drives-repair (§10.3/§10.4);
+    // `LEADER_REPAIR_LOOP_ATTEMPTS` is the in-loop attempt ceiling (validated 2-3, default 2 — design/78 §5);
+    // `LEADER_ORACLE_FLAKY_K` is the flaky-settle re-isolation count (default 2 — §10.1 / repair-oracle.ts).
     const repairLoopOn = String(process.env.LEADER_REPAIR_LOOP ?? "").toLowerCase() === "true";
     const measureGatesOn = String(process.env.LEADER_MEASURE_GATES ?? "").toLowerCase() === "true";
     const repairLoopAttempts = Math.min(3, Math.max(2, Math.floor(Number(process.env.LEADER_REPAIR_LOOP_ATTEMPTS ?? "2")) || 2));
     const oracleFlakyK = Math.max(1, Math.floor(Number(process.env.LEADER_ORACLE_FLAKY_K ?? "2")) || 2);
+    // worker resource bounds (limits/maxCostUsd) + presign ttl are derived in leaderResourceConfig(), above.
+    // Bounded integration-repair (search 2026-06-14): when the merged tree compiles-clean-but-fails, run a strong
+    // agent WITH HANDS in the live merged sandbox to fix the cross-worker integration, then mergeBranches re-runs
+    // testCmd. OFF unless LEADER_REPAIR_ROUNDS>0 — closes over the RAW env (hands need the real RemoteExecutionEnv,
+    // not the exec/writeFile-only IntegrationEnv wrapper). Strong model (default role); no durable wiring (short).
     const mkRepair = (rawEnv, repoDir, oracleFiles) => {
         if (repairRounds <= 0)
             return undefined;
         return {
             maxRounds: repairRounds,
             run: async ({ round, errorText, testCmd }) => {
+                // 🔴 Measurement integrity (council blocker + partner-Codex gotcha): HIDE the hidden oracle so the repair
+                // agent (rooted at the repo) can't `cat` the test it's graded on and cheat; RE-INJECT it in `finally` so
+                // merge's re-test gate verifies the fix against the real oracle (and it's hidden again next round).
                 for (const f of oracleFiles) {
-                    await rawEnv.remove(f.path).catch(() => { });
+                    await rawEnv.remove(f.path).catch(() => { }); // best-effort delete (may be absent)
+                    // FAIL-CLOSED (partner-Codex): remove() returns a Result that does NOT throw on failure, so a silent
+                    // {ok:false} would leave the oracle readable and the repair model could `cat` it. VERIFY it's gone.
                     const ex = await rawEnv.exists(f.path);
                     if (ex.ok && ex.value)
                         throw new Error(`oracle still present after remove (${f.path}) — refusing repair (measurement integrity)`);
                 }
                 try {
+                    // rootPath: repoDir (Codex review #4) — without it core roots the file tools at executionEnv.cwd
+                    // (/workspace on Kata), one level above the repo (/workspace/repo). bash can cd, but the dedicated
+                    // read/edit tools would start too high + lose containment.
                     const runner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, executionEnv: rawEnv, rootPath: repoDir });
                     const objective = [
                         `The integrated project at ${repoDir} fails its build/test. Make the MINIMAL change to the working tree so this command exits 0 (cd into the repo and run it yourself to confirm):`,
@@ -95,6 +180,8 @@ export function createLeaderRunner(cfg) {
                         `Several agents each ported a DISJOINT module group; the failure is almost certainly a cross-module integration mismatch (a shared interface, a missing/renamed symbol, a type/signature disagreement between two groups). Read the failing files, fix the smallest set of files, and re-run the command to verify it passes.`,
                         `HARD RULES: do NOT git commit or git push; do NOT modify build configuration (gradle.properties / AGP / dependency versions / settings); do NOT add build-tooling files; do NOT call AskUserQuestion — decide autonomously. Preserve the ported semantics — fix the integration, never delete behavior just to make it pass.`,
                         ``,
+                        // BL-32 — the failing output is worker/build-authored; delimit it so an injected instruction in a stack
+                        // trace / test message can't steer the repair agent (design/53 trifecta).
                         delimitUntrusted("FAILING OUTPUT (untrusted)", errorText),
                     ].join("\n");
                     const res = await runner
@@ -105,11 +192,16 @@ export function createLeaderRunner(cfg) {
                         limits: { timeoutSec: Math.max(300, Math.floor(leaderTimeoutMs / 1000 / 2)) },
                     })
                         .result();
+                    // A non-completed repair (timeout/abort) may leave a process alive in the SAME Kata pod that would race
+                    // the merge re-test (Codex review #1) — surface it as a throw so mergeBranches does NOT re-test: it keeps
+                    // the original failure and the `finally` destroys the sandbox, killing any orphan.
                     if (res.status !== "completed")
                         throw new Error(`repair round ${round} did not complete (status=${res.status})`);
                     return { costUsd: res.stats?.costMicroUsd !== undefined ? res.stats.costMicroUsd / 1e6 : undefined, note: res.result?.slice(0, 200) };
                 }
                 finally {
+                    // Re-inject the oracle so merge's re-test gate (run right after) verifies the fix against the REAL hidden
+                    // test — even if the repair model threw. It's hidden again at the top of the next round.
                     for (const f of oracleFiles) {
                         const wr = await rawEnv.writeFile(f.path, f.content);
                         if (!wr.ok)
@@ -119,6 +211,12 @@ export function createLeaderRunner(cfg) {
             },
         };
     };
+    // Bounded merge-conflict resolution (search 2026-06-16): a `git apply --3way` conflict means two workers touched
+    // overlapping lines. Instead of the leader discarding ALL fan-out work + redoing single (collapseToSingle), run a
+    // strong agent WITH HANDS in the conflicted sandbox to integrate the conflicting worker's patch (the others
+    // already applied clean). OFF unless LEADER_CONFLICT_ROUNDS>0. Mirrors mkRepair: raw env for hands; hide the
+    // hidden oracle so the resolver can't grade-peek; restore in `finally` so the downstream test gate sees the real
+    // oracle. mergeBranches re-checks for leftover markers/.rej and degrades to collapse if the resolution is unclean.
     const mkConflictResolver = (rawEnv, repoDir, oracleFiles) => {
         if (conflictRounds <= 0)
             return undefined;
@@ -139,6 +237,8 @@ export function createLeaderRunner(cfg) {
                         `When done, NO conflict markers and NO *.rej files may remain under ${repoDir}. Verify yourself before finishing: grep -rIlE '^(<<<<<<<|>>>>>>>)' . ; find . -name '*.rej'  — BOTH must print nothing.`,
                         `HARD RULES: do NOT git commit / git add / git push (the coordinator squashes at the end); do NOT modify build configuration; do NOT call AskUserQuestion — decide autonomously. Preserve both sides' ported semantics.`,
                         ``,
+                        // BL-32 — the worker patch + conflict text are untrusted worker-adjacent content; delimit both so an
+                        // injected instruction can't steer the conflict-resolver agent (design/53 trifecta).
                         delimitUntrusted("CONFLICTING WORKER PATCH (untrusted)", patch.slice(0, 6000)),
                         delimitUntrusted("CONFLICT DETAILS (untrusted)", conflictText),
                     ].join("\n");
@@ -169,12 +269,22 @@ export function createLeaderRunner(cfg) {
         if (!body.objective || !body.durableRemote || !body.testCmd || !body.seedCmd || !body.baseSha) {
             return { ok: false, reports: [], cancelled: false, error: "request must provide objective, durableRemote, baseSha, testCmd, seedCmd" };
         }
+        // `durableRemote` is handed to `git clone` on the CONTROL PLANE (with the Coordinator's creds). git executes
+        // transport-helper URLs (`ext::sh -c '…'`, `file://…`) as commands, so allow only a normal http(s)/git/ssh
+        // URL or an scp-style user@host:path — reject the transport-helper class even though the caller is trusted.
         if (!/^(?:https?|git|ssh):\/\/[^\s]+$/i.test(body.durableRemote) && !/^[\w.-]+@[\w.-]+:[^\s]+$/.test(body.durableRemote)) {
             return { ok: false, reports: [], cancelled: false, error: "durableRemote must be an http(s)/git/ssh URL or user@host:path (no ext::/file:// transport helpers)" };
         }
+        // baseSha is interpolated into worker-shell lines on BOTH env paths (static: format-patch via pullDiff,
+        // which re-validates; factory: the staged upload script) — gate it here so the factory path is covered too.
         if (!safeRev(body.baseSha)) {
             return { ok: false, reports: [], cancelled: false, error: "baseSha must be a plain git rev (sha/branch token)" };
         }
+        // B-2: write the hidden oracle into the integration sandbox (workers never see it) + local-exclude it so the
+        // squash `git add -A` does NOT commit/push it into the durable integration branch (Codex review #1: else the
+        // hidden test leaks into the repo, visible to any future worker seeded from it). gradle compiles src/test
+        // regardless of git status, so the correctness gate still runs. NOTE: if integration-repair is enabled, the
+        // repair agent (rooted at the repo) could still READ the on-disk oracle — delete it before repair runs then.
         const injectOracles = async (env) => {
             const ofs = body.oracleFiles ?? [];
             if (ofs.length === 0)
@@ -184,6 +294,9 @@ export function createLeaderRunner(cfg) {
                 if (!wr.ok)
                     throw wr.error;
             }
+            // Local-exclude each oracle so `git add -A` never commits it (B-2 sandbox-only). Root-anchor with a leading
+            // "/" + escape a leading "#"/"!" so the path is a literal exclude, not a gitignore comment/negation (council
+            // #4). APPEND to (don't clobber) any seed-configured excludes — read the current file first (council #3).
             const anchor = (p) => "/" + (/^[#!]/.test(p) ? "\\" + p : p);
             const rel = ofs.map((f) => anchor(f.path.startsWith(`${repo}/`) ? f.path.slice(repo.length + 1) : f.path)).join("\n");
             const cur = await env.exec(`bash -lc ${JSON.stringify(`cat ${repo}/.git/info/exclude 2>/dev/null || true`)}`);
@@ -192,8 +305,18 @@ export function createLeaderRunner(cfg) {
             if (!ex.ok)
                 throw ex.error;
         };
+        // design/158 A13: the four LeaderDeps methods below (plan/provisionWorker/provisionIntegrationSandbox/push)
+        // are named local functions — same closure scope, same capture, same execution order as when they were
+        // inline object-literal properties; only the "anonymous inline closure" form changed (behavior identical).
+        // caller-provided sub-tasks = an explicit FAN-OUT override; else the difficulty ROUTER decides (one model
+        // completion): default single (a 1-subtask plan) and only fan out when clearly+economically decomposable
+        // Either way → VALIDATED disjoint-file sub-tasks (planner.ts enforces the Coordinator's invariant).
         const plan = async () => {
             if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
+                // Validate caller-supplied sub-tasks through the SAME deterministic checks as an LLM plan — the
+                // branch is interpolated into a worker-container shell command (planner.validateSubtasks rejects a
+                // shell-unsafe branch / leading "-" / "..") and the disjoint-file + unique-id invariants are what
+                // the Coordinator's parallel merge relies on. Was a straight map() that skipped all of it.
                 return validateSubtasks(body.subtasks);
             }
             const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
@@ -201,10 +324,14 @@ export function createLeaderRunner(cfg) {
                 const res = await planRunner.runTaskStream({
                     objective: prompt,
                     sessionId: `leader-route-${Date.now()}`,
+                    // Pin the route to a cheap classifier model (see `routerModel` doc): the heavy reasoning model
+                    // returns an empty completion ~2/3 of the time on this prompt, collapsing every fan-out to single.
                     ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
                 }).result();
                 return res.result ?? "";
             };
+            // §5 quality gate: a rejected/unparseable plan retries the LLM ≤1, then collapses to single (B1-B3) —
+            // never plan-fails the task; the collapse reason is logged as a router-quality signal.
             const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
                 ...(body.planContext ? { context: body.planContext } : {}),
                 ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
@@ -215,6 +342,9 @@ export function createLeaderRunner(cfg) {
         };
         const provisionWorker = async (sub) => {
             const sessionId = `leader-${sub.workerId}-${Date.now()}`;
+            // design/68 C4: durable suspend bits per worker task (mirrors main.ts's per-task wiring) — a gated
+            // ask/AskUserQuestion suspends the worker; ctx is seeded so the surfaced token resumes on the
+            // single-agent path (the resume body re-runs the sub-objective on this worker).
             let durableSpec = {};
             if (cfg.durable) {
                 const d = cfg.durable;
@@ -228,7 +358,27 @@ export function createLeaderRunner(cfg) {
                     cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
                 });
             }
+            // design/74 Slice 6: opt-in resource-slice suspend. A worker that overruns its per-slice budget window
+            // SUSPENDS (resumable) instead of failing; the leader's runWorker auto-resumes until the TOTAL is spent.
+            // Reuses the durable checkpoint store (so requires `cfg.durable` for the store + durable tool-results +
+            // session — the same durable-resume prerequisites). The per-slice window OVERRIDES workerLimits.maxCostUsd
+            // (spread last); the total + cumulative spend ride on the checkpoint's resource ledger (core debits it).
             let resourceSpec = {};
+            // Slice-6 review BLOCKER: core's resource-suspend needs the FULL durable-resume prerequisites —
+            // a checkpoint store (cfg.durable), a durable tool-result store (cfg.toolResultStore; an in-memory
+            // offload makes core REFUSE to suspend), AND a durable session store (cfg.sessionStore; without it a
+            // resumed slice dies with SessionError "Entry <leafId> not found" — the worker's progress is lost,
+            // wire.ts:52-55). Require ALL THREE before applying the per-slice window: otherwise the worker's
+            // maxCostUsd would be cut to a quarter WITHOUT the ability to suspend/resume → it fails at 1/4 its
+            // budget with no warning. When opted in (resourceCfg) but a prerequisite is missing, leave the budget
+            // intact + warn (which one).
+            // §10.5 ENFORCED mutual exclusion: when the single-agent repair loop actually OWNS a worker, that worker
+            // must NOT also resource-suspend — the repair loop owns its attempt budget, and a resource-suspend mid-loop
+            // risks `attemptCount` corruption + budget overrun (§10.5: a doc aspiration is not enough, this drops
+            // `resourceSpec` by construction so the two are NEVER on the same spec). Gated on the loop being WIRED
+            // (`REPAIR_LOOP_WIRED=true` — the canary-enable slice attached a LIVE `resolveRepair`), so flipping
+            // `LEADER_REPAIR_LOOP` now engages BOTH the single-agent repair loop AND this resource-suspend exclusion for
+            // the worker the loop owns. (`REPAIR_LOOP_WIRED` is the live/inert switch — it shipped false pre-activation.)
             const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
             if (repairLoopActive && resourceCfg) {
                 cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
@@ -251,6 +401,12 @@ export function createLeaderRunner(cfg) {
                             : "no durable session store",
                 });
             }
+            // Keep the resume ctx warm for the WHOLE first leg (found live: task02b run1). putCtx stamps
+            // `updated_at` once at provision, but reapCtx GC's any ctx older than runStaleSec that has no pending
+            // checkpoint and no task_active claim — sub-workers hold no task_active row (that lock belongs to the
+            // HTTP submit path), so a leg longer than the reap cutoff lost its ctx and the post-suspend /decide
+            // died 409 "resume context missing". Refresh inside the run; once suspended, the pending checkpoint
+            // itself shields the ctx from the reaper, and after a terminal result the ctx ages out as designed.
             const keepCtxWarm = (runner) => {
                 const d = cfg.durable;
                 if (!d)
@@ -267,29 +423,53 @@ export function createLeaderRunner(cfg) {
                 };
                 return runner;
             };
+            // design/68 §8 F2(a) — factory-env worker + self-upload diff-out (preferred; static mode = compat below).
             if (cfg.s3 && cfg.envFactory) {
                 const s3 = cfg.s3;
                 const envFactory = cfg.envFactory;
-                const baseSha = body.baseSha;
+                const baseSha = body.baseSha; // guarded non-empty at entry (narrowing is lost inside this closure)
                 const seedCmd = body.seedCmd;
+                // workerId is validateSubtasks-clean ([A-Za-z0-9._-]+) → safe as a key segment; the presigner
+                // backstop-rejects traversal anyway. One object per worker per leader run; a repair re-run overwrites.
                 const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
+                // The presigned URL must outlive the worker: it self-uploads its diff at the END of its run, which
+                // can be the full leader deadline (24h). The old `Math.min(3600, …)` clamp expired mid-run for any
+                // worker > 1h → the upload 403'd and the diff (progress) was lost. Track the worker deadline + slack.
+                // (Codex: instant-signing at upload time, from the control plane, is the cleaner long-term fix.)
                 const ttl = presignTtlSec;
                 const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
                 const putUrl = presignS3Url({ ...base, method: "PUT" });
                 const getUrl = presignS3Url({ ...base, method: "GET" });
                 const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
+                // Staging runs adapter-side before the agent's first workspace call: seed the base repo at the worker
+                // branch + plant the upload script (the presigned URL never passes through the model).
+                // Resume-safe (BL-1): stageWorkerEnv no-ops on a resumed snapshot (worker branch already present)
+                // instead of re-running `git checkout -b` (which would fail "branch exists" and kill the resumed
+                // worker). Now load-bearing because the prod auto-resume landed (leader.ts non-verify path, 9a91c60).
                 const stage = async (env) => {
                     await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
+                    // #16 tail (search 2026-06-17): INCREMENTAL progress upload. The upload script is READ-ONLY git
+                    // (format-patch base..HEAD — no add/commit/index write), so a background loop re-running it keeps the
+                    // worker's fixed diff key at its latest COMMITTED progress. If the env HARD-CRASHES before its final /
+                    // belt upload, salvage (fetchUploadedDiff) then recovers up to the last commit instead of losing all
+                    // (run 019eca71's "4 commits vaporized" was exactly a failed worker that never uploaded). No git
+                    // concurrency (read-only); the teardown belt upload stays authoritative (same key, complete diff).
+                    // Fire-and-forget + NEVER awaited: the loop never exits, so staging must not block on it — whether the
+                    // env's exec detaches or blocks, the loop RUNS either way and the dangling promise is reaped when the
+                    // pod dies. Best-effort: a launch failure (no nohup, etc.) just falls back to today's end-only upload.
                     void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
                 };
                 return {
                     workerId: sub.workerId, sessionId, branch: sub.branch,
-                    baseSha,
+                    baseSha, // seedCmd reproduces the durable base — same sha for every worker
                     runner: keepCtxWarm(new Runner({
                         brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
                         ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
                         ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
                         executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
+                            // Belt (task02b run6): workers that did the work but never `git commit`ed upload an EMPTY diff
+                            // (diff-out is committed-state vs base) — auto-commit any uncommitted work first. No-op for a
+                            // worker that committed properly; the commit identity marks belt-captured work for the reviewer.
                             await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
                         }),
                     })),
@@ -313,6 +493,8 @@ export function createLeaderRunner(cfg) {
             };
         };
         const provisionIntegrationSandbox = async () => {
+            // Factory lane (k8s or e2b): one fresh env from the service factory; lazy-connect — first exec
+            // provisions the pod/VM, destroy() reaps it. Static lane: the original explicit E2B env.
             if (cfg.envFactory) {
                 const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
                 try {
@@ -328,6 +510,8 @@ export function createLeaderRunner(cfg) {
             if (!cfg.e2bApiKey)
                 throw new Error("leader wire: no envFactory and no e2bApiKey");
             const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
+            // Once connected, the VM is billable — if seeding throws, destroy it HERE before propagating, otherwise
+            // mergeBranches never receives the `destroy` closure and the sandbox leaks until E2B's timeout reaps it.
             if (!(await env.connect()).ok) {
                 await env.destroy().catch(() => { });
                 throw new Error("integ connect failed");
@@ -343,7 +527,11 @@ export function createLeaderRunner(cfg) {
             return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
         };
         const push = async (integratedPatch, baseSha) => {
+            // Coordinator (control plane, creds): clone durable remote, apply integrated series, force-with-lease push.
+            // 🔴 requires `git` on the service host PATH (deploy req).
             const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
+            // Force C locale so the `raced` regex below matches git's (English) push-rejection messages regardless of
+            // the host locale — otherwise a translated message → regex miss → a real race mislabeled non-retryable (council #5).
             const gitEnv = { ...process.env, LC_ALL: "C" };
             try {
                 execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
@@ -353,12 +541,26 @@ export function createLeaderRunner(cfg) {
                 execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
                 rmSync(join(dir, ".leader.patch"));
                 const ref = body.targetRef ?? "refs/heads/main";
+                // CAS against the TARGET ref's ACTUAL current tip (not baseSha). The old `=ref:baseSha` lease assumed
+                // the target ref already pointed at baseSha — but a FRESH ref (or one at any other commit) is never at
+                // baseSha, so the lease rejected EVERY such push and mislabeled it "raced" (found live: auto1 reached
+                // push after a clean 6-worker fan-out + green gradle, and failed only here). ls-remote gives the ref's
+                // current oid (empty = it doesn't exist yet) → lease against THAT, which still detects a real concurrent
+                // merge race (the ref moved since we read it) but lets a normal create/update through.
+                // Does the target ref already exist? (exact full-ref match; --refs drops peeled annotated-tag lines.)
                 const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
                 const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
                 if (exists) {
+                    // Existing ref → keep the original CAS against baseSha (detects a concurrent merge that moved the ref
+                    // off the base this run started from — council#2). NOT against a just-read end-of-run tip: that would
+                    // let leader B force-push over leader A's concurrent merge (Codex review #1). A ref that exists but
+                    // isn't at baseSha → refuse (use a fresh per-run targetRef to avoid that — the proving-ground does).
                     execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
                 }
                 else {
+                    // Absent → create with a "must not exist" lease (empty expected oid): a concurrent create races safely
+                    // (rejected as stale info) instead of silently fast-forwarding. The original `:baseSha` lease wrongly
+                    // rejected a fresh ref (it's never at baseSha) — that was auto1's only failure after a green build.
                     execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
                 }
                 return { ok: true, ref };
@@ -378,14 +580,39 @@ export function createLeaderRunner(cfg) {
             push,
             testCmd: body.testCmd,
             repoDir: repo, workerRepoDir: repo,
-            timeoutMs: leaderTimeoutMs, maxConcurrency: 4,
+            // Per-wave fan-out deadline. LEADER_TIMEOUT_MS raises it (raise-only, parity with TASK_TIMEOUT_SEC —
+            // found live: task02b run5, porting workers + collapse-solo all 'run aborted' at the 600s default while
+            // mid-gradle; the per-worker presign ttl below tracks the same knob).
+            timeoutMs: leaderTimeoutMs, maxConcurrency: 4, // Semaphore cap; the planner emits ≤6 sub-tasks
+            // replan-lite (design/68 §6) always on in the wired leader; LEADER_BUDGET_USD bounds the fan-out spend.
             replan: { ...(process.env.LEADER_BUDGET_USD ? { budgetUsd: Number(process.env.LEADER_BUDGET_USD) } : {}) },
+            // LEADER-REPAIRLOOP-INTEGRATION §10.3 — the merge push-hold-on-signal (OFF unless LEADER_MEASURE_GATES). It
+            // needs NO extra sandbox: `strongOracleSeeded` keys on whether a hidden held-out oracle was injected into the
+            // integration sandbox (`injectOracles` runs `body.oracleFiles`), and the §10.3 hold combines it with the
+            // existing observe-only `measureCmd` result inside `mergeBranches`. §10.4's out-of-process measure grader
+            // (`measureDrivesRepair`) is a separate re-seedable sandbox — the canary-enable add-on (§6 step 5); it is
+            // intentionally NOT wired here in v1 (a half-mint would be a stopgap), so `measureGates` ships the §10.3
+            // signal-hold only. With LEADER_MEASURE_GATES off this whole block is absent → byte-identical merge path.
             ...(measureGatesOn
                 ? { mergeGates: { measureGates: true, ...((body.oracleFiles?.length ?? 0) > 0 ? { strongOracleSeeded: true } : {}) } }
                 : {}),
             ...(cfg.logger ? { logger: cfg.logger } : {}),
         };
+        // §10.8 — the SINGLE wire-call helper for the opt-in single-agent repair loop (LEADER_REPAIR_LOOP). The
+        // canary-enable slice attaches a LIVE async `resolve(solo, sub)` that, per solo worker, provisions a DEDICATED
+        // oracle-seeded GRADER sandbox (a SEPARATE sandbox from the worker + the integration sandbox — distinct object/
+        // sandboxId by construction, §10.2 structural isolation) and binds the per-worker diff source + the concrete
+        // Runner. OFF-by-default: `resolve` is attached ONLY when `repairLoopOn` (LEADER_REPAIR_LOOP=true); with the
+        // flag off `resolve` is undefined → `attachRepairLoopDeps` is a NO-OP → `deps.repairLoop` stays unset → the
+        // single-agent default path is byte-identical to today. The §10.6 push chokepoint (merge.ts) is LIVE
+        // regardless: once any report carries a `repairTerminal`, it holds.
+        //
+        // The grader is provisioned the SAME way as the integration sandbox (provisionIntegrationSandbox: raw env →
+        // seed → injectOracles → integBase), then exposed as a `GraderHandle` (the raw RemoteExecutionEnv IS an
+        // ExecutionEnv). `graderDestroy` reaps it in `runRepairLoopForLeader`'s `finally`. On a seed/inject throw the
+        // env is destroyed before propagating (mirrors provisionIntegrationSandbox :587-590 / :604-606).
         const resolveRepair = async (solo, _sub) => {
+            // Provision a DEDICATED grader: factory lane mints a fresh env (lazy-connect); static lane connects an E2B VM.
             let env;
             if (cfg.envFactory) {
                 env = cfg.envFactory({ sessionId: `leader-grader-${Date.now()}` });
@@ -400,6 +627,8 @@ export function createLeaderRunner(cfg) {
                 }
                 env = e2b;
             }
+            // Seed the base repo + inject the hidden oracle ONLY in this grader (never the worker env). On a throw,
+            // destroy the billable env before propagating (else it leaks until the adapter's timeout reaps it).
             let integBase;
             try {
                 await sh(env)(body.seedCmd);
@@ -417,20 +646,33 @@ export function createLeaderRunner(cfg) {
                 repoDir: repo,
                 integBase,
             };
+            // The worker diff source: factory lane = the worker's self-upload (`solo.fetchDiff`); static lane = pull the
+            // committed `base..HEAD` series straight out of the worker's still-alive env (the same merge `pullDiff` path).
             const workerRepoDir = repo;
             const pullWorkerDiff = solo.fetchDiff
-                ?
+                ? // 🔴 BL-33 factory-lane residual: the self-upload path captures only what the fix turn UPLOADED; core's
+                    // repairObjective carries no upload mandate, so an un-uploaded fix is missed. A deterministic belt needs
+                    // a worker-env exec handle the factory lane does not surface (env-recording factory = follow-up). The
+                    // fork2.1 stale-diff guard keeps this SAFE (no false candidate) until then.
                     solo.fetchDiff
                 : async () => {
+                    // 🔴 BL-33 — the STATIC lane EXPOSES the worker env (`solo.diffEnv`), so deterministically belt-commit
+                    // the worker's UNCOMMITTED fix-turn work BEFORE the committed-only `format-patch base..HEAD` pull —
+                    // else attempts ≥2 grade a STALE diff (the fix never observed). Best-effort + only when there is a
+                    // change to commit (no empty commits polluting the series). Mirrors the teardown belt, run per-grade.
                     const wenv = solo.diffEnv;
                     await wenv
                         .exec(`bash -lc ${JSON.stringify(`cd ${repo} && git add -A && (git diff --cached --quiet || git -c user.email=w@w.co -c user.name=worker commit -q -m repair-wip)`)}`)
                         .catch(() => { });
                     return pullDiff(solo.diffEnv, solo.baseSha, { ref: solo.branch, ...(workerRepoDir ? { cwd: workerRepoDir } : {}) });
                 };
+            // The trusted, spec-derived oracle steps (the integration testCmd) run in the grader; NEVER worker-authored.
             const oracleSteps = [{ command: body.testCmd, cwd: repo, label: "oracle-test" }];
+            // tier: a seeded hidden held-out oracle clears to `trusted_hidden`; a compile-only gate is `property_harness_weak`.
             const oracleFiles = body.oracleFiles ?? [];
             const passTier = oracleFiles.length > 0 ? "trusted_hidden" : "property_harness_weak";
+            // fork1.1 — make the INTENTIONAL factory-lane identity-check skip observable (distinct from an
+            // accidental workerEnv omission, which onWarn's generic MINOR-2 warning can't tell apart).
             if (solo.fetchDiff)
                 cfg.logger?.info?.("repair_loop_factory_lane_isolation", { workerId: solo.workerId, reason: "identity check intentionally skipped — isolation by independent grader sandbox construction (§10.2)" });
             return {
@@ -440,9 +682,14 @@ export function createLeaderRunner(cfg) {
                 pullWorkerDiff,
                 oracleSteps,
                 passTier,
+                // council 2026-06-17 — re-inject the seeded oracle after each hermetic reset (clean -fdx wipes it);
+                // fork1.2 — the paths a candidate patch must not touch (the oracle files' own paths).
                 ...(oracleFiles.length > 0 ? { oracleFiles, oraclePaths: oracleFiles.map((f) => f.path) } : {}),
                 maxAttempts: repairLoopAttempts,
                 flakyK: oracleFlakyK,
+                // §10.2 — on the factory lane the worker's RAW env is not surfaced (only fetchDiff), so OMIT workerEnv
+                // (runRepairLoop skips the identity check; the grader is a separate sandbox by construction). On the static
+                // lane the leader owns the worker's RemoteExecutionEnv (`solo.diffEnv` is that env) → pass it so the check runs.
                 ...(!solo.fetchDiff && solo.diffEnv ? { workerEnv: solo.diffEnv } : {}),
                 log: (event, x) => event.includes("fail") || event.includes("warn") || event.includes("threw")
                     ? cfg.logger?.warn?.(event, x)

package/dist/lsp/e2b-bridge.d.ts

@@ -1,8 +1,24 @@
+/**
+ * Bridges WS ⇄ a language server's stdio. Server command from `LSP_CMD`, port `LSP_PORT`, and a per-session
+ * **auth token** `LSP_TOKEN` (the endpoint is a public `wss://<port>-<id>.e2b.app` URL — the runner connects with
+ * `?token=<secret>`; without it, anyone with the URL could read the workspace via LSP, council #4). Frame parsing
+ * uses the unit-tested {@link FRAME_DECODER_JS} (council #1/#2/#5).
+ */
 export declare const BRIDGE_SOURCE: string;
+/** Language server command per languageId. P1 = typescript/javascript; dart/kotlin land with their template servers.
+ *  `--stdio` only where the server needs it to pick stdio over its other transports (the node-based servers);
+ *  gopls/jdtls/kotlin-language-server/sourcekit-lsp/`dart language-server` speak LSP over stdio by default. */
 export declare const SERVER_CMD: Readonly<Record<string, string>>;
+/** npm packages a non-baked sandbox needs (on-the-fly install). Languages absent here (go) REQUIRE a baked template. */
 export declare const SERVER_NPM: Readonly<Record<string, string>>;
+/** Bridge dir + port inside the sandbox. One bridge process per language → distinct ports. */
 export declare const BRIDGE_DIR = "/home/user/.lsp-bridge";
 export declare function portForLanguage(language: string): number;
+/** FOREGROUND install (run via `env.exec` — must finish before the start). A baked template makes both checks
+ *  skip instantly; a non-baked sandbox npm-installs the server + ws here (slow once per sandbox). */
 export declare function lspInstallCommand(language: string): string;
+/** Start the bridge — run as a TRUE background command (`env.startBackground`), NOT `&`-backgrounded inside a
+ *  foreground run: E2B reaps the foreground process group on completion, killing an in-shell `&`/nohup child.
+ *  So node is the FOREGROUND here and E2B keeps the background command alive. */
 export declare function lspStartCommand(language: string, port: number, token: string): string;
 //# sourceMappingURL=e2b-bridge.d.ts.map