@sema-agent/server 1.313.0 → 1.315.0

package/dist/leader/wire.js

@@ -63,7 +63,7 @@ export function leaderResourceConfig(env = process.env) {
             : {}),
     };
 }
-export function buildLeaderRunner(cfg) {
+export function createLeaderRunner(cfg) {
     const W = cfg.workspace ?? "/home/user";
     const repo = `${W}/repo`;
     const ident = cfg.git ?? { name: "leader", email: "leader@local" };
@@ -192,186 +192,190 @@ export function buildLeaderRunner(cfg) {
             if (!ex.ok)
                 throw ex.error;
         };
-        const deps = {
-            plan: async () => {
-                if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
-                    return validateSubtasks(body.subtasks);
-                }
-                const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
-                const runRoute = async (prompt) => {
-                    const res = await planRunner.runTaskStream({
-                        objective: prompt,
-                        sessionId: `leader-route-${Date.now()}`,
-                        ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
-                    }).result();
-                    return res.result ?? "";
+        const plan = async () => {
+            if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
+                return validateSubtasks(body.subtasks);
+            }
+            const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
+            const runRoute = async (prompt) => {
+                const res = await planRunner.runTaskStream({
+                    objective: prompt,
+                    sessionId: `leader-route-${Date.now()}`,
+                    ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
+                }).result();
+                return res.result ?? "";
+            };
+            const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
+                ...(body.planContext ? { context: body.planContext } : {}),
+                ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
+            });
+            if (collapsed)
+                cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
+            return subs;
+        };
+        const provisionWorker = async (sub) => {
+            const sessionId = `leader-${sub.workerId}-${Date.now()}`;
+            let durableSpec = {};
+            if (cfg.durable) {
+                const d = cfg.durable;
+                durableSpec = {
+                    toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
+                    checkpointStore: d.checkpointStore,
+                    durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
+                    onQuestion: QUESTION_AWAITS_RESUME,
                 };
-                const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
-                    ...(body.planContext ? { context: body.planContext } : {}),
-                    ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
+                await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
+                    cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
                 });
-                if (collapsed)
-                    cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
-                return subs;
-            },
-            provisionWorker: async (sub) => {
-                const sessionId = `leader-${sub.workerId}-${Date.now()}`;
-                let durableSpec = {};
-                if (cfg.durable) {
-                    const d = cfg.durable;
-                    durableSpec = {
-                        toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
-                        checkpointStore: d.checkpointStore,
-                        durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
-                        onQuestion: QUESTION_AWAITS_RESUME,
-                    };
-                    await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
-                        cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
-                    });
-                }
-                let resourceSpec = {};
-                const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
-                if (repairLoopActive && resourceCfg) {
-                    cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
-                }
-                if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
-                    resourceSpec = {
-                        checkpointStore: cfg.durable.checkpointStore,
-                        resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
-                        maxCostUsd: resourceCfg.sliceMaxCostUsd,
-                        maxSuspends: resourceCfg.maxSuspends,
-                    };
-                }
-                else if (resourceCfg && !repairLoopActive) {
-                    cfg.logger?.warn?.("leader_resource_suspend_inactive", {
-                        workerId: sub.workerId,
-                        reason: !cfg.durable
-                            ? "no durable checkpoint store"
-                            : !cfg.toolResultStore
-                                ? "no durable tool-result store"
-                                : "no durable session store",
-                    });
-                }
-                const keepCtxWarm = (runner) => {
-                    const d = cfg.durable;
-                    if (!d)
-                        return runner;
-                    const orig = runner.runTaskStream.bind(runner);
-                    runner.runTaskStream = (spec, resume) => {
-                        const stream = orig(spec, resume);
-                        const iv = setInterval(() => {
-                            void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
-                        }, 45_000);
-                        iv.unref?.();
-                        void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
-                        return stream;
-                    };
+            }
+            let resourceSpec = {};
+            const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
+            if (repairLoopActive && resourceCfg) {
+                cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
+            }
+            if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
+                resourceSpec = {
+                    checkpointStore: cfg.durable.checkpointStore,
+                    resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
+                    maxCostUsd: resourceCfg.sliceMaxCostUsd,
+                    maxSuspends: resourceCfg.maxSuspends,
+                };
+            }
+            else if (resourceCfg && !repairLoopActive) {
+                cfg.logger?.warn?.("leader_resource_suspend_inactive", {
+                    workerId: sub.workerId,
+                    reason: !cfg.durable
+                        ? "no durable checkpoint store"
+                        : !cfg.toolResultStore
+                            ? "no durable tool-result store"
+                            : "no durable session store",
+                });
+            }
+            const keepCtxWarm = (runner) => {
+                const d = cfg.durable;
+                if (!d)
                     return runner;
+                const orig = runner.runTaskStream.bind(runner);
+                runner.runTaskStream = (spec, resume) => {
+                    const stream = orig(spec, resume);
+                    const iv = setInterval(() => {
+                        void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
+                    }, 45_000);
+                    iv.unref?.();
+                    void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
+                    return stream;
+                };
+                return runner;
+            };
+            if (cfg.s3 && cfg.envFactory) {
+                const s3 = cfg.s3;
+                const envFactory = cfg.envFactory;
+                const baseSha = body.baseSha;
+                const seedCmd = body.seedCmd;
+                const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
+                const ttl = presignTtlSec;
+                const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
+                const putUrl = presignS3Url({ ...base, method: "PUT" });
+                const getUrl = presignS3Url({ ...base, method: "GET" });
+                const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
+                const stage = async (env) => {
+                    await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
+                    void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
                 };
-                if (cfg.s3 && cfg.envFactory) {
-                    const s3 = cfg.s3;
-                    const envFactory = cfg.envFactory;
-                    const baseSha = body.baseSha;
-                    const seedCmd = body.seedCmd;
-                    const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
-                    const ttl = presignTtlSec;
-                    const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
-                    const putUrl = presignS3Url({ ...base, method: "PUT" });
-                    const getUrl = presignS3Url({ ...base, method: "GET" });
-                    const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
-                    const stage = async (env) => {
-                        await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
-                        void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
-                    };
-                    return {
-                        workerId: sub.workerId, sessionId, branch: sub.branch,
-                        baseSha,
-                        runner: keepCtxWarm(new Runner({
-                            brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
-                            ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
-                            ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
-                            executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
-                                await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
-                            }),
-                        })),
-                        fetchDiff: () => fetchUploadedDiff(getUrl),
-                        spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
-                    };
-                }
-                if (!cfg.e2bApiKey)
-                    throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
-                const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
-                if (!(await env.connect()).ok)
-                    throw new Error(`connect failed for ${sub.workerId}`);
-                await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
-                const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
                 return {
-                    workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
-                    runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
-                    diffEnv: env,
-                    destroy: () => env.destroy().then(() => { }),
-                    spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
+                    workerId: sub.workerId, sessionId, branch: sub.branch,
+                    baseSha,
+                    runner: keepCtxWarm(new Runner({
+                        brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
+                        ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
+                        ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
+                        executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
+                            await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
+                        }),
+                    })),
+                    fetchDiff: () => fetchUploadedDiff(getUrl),
+                    spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
                 };
-            },
-            provisionIntegrationSandbox: async () => {
-                if (cfg.envFactory) {
-                    const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
-                    try {
-                        await sh(env)(body.seedCmd);
-                        await injectOracles(env);
-                    }
-                    catch (e) {
-                        await Promise.resolve(env.destroy()).catch(() => { });
-                        throw e;
-                    }
-                    return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
-                }
-                if (!cfg.e2bApiKey)
-                    throw new Error("leader wire: no envFactory and no e2bApiKey");
-                const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
-                if (!(await env.connect()).ok) {
-                    await env.destroy().catch(() => { });
-                    throw new Error("integ connect failed");
-                }
+            }
+            if (!cfg.e2bApiKey)
+                throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
+            const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
+            if (!(await env.connect()).ok)
+                throw new Error(`connect failed for ${sub.workerId}`);
+            await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
+            const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
+            return {
+                workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
+                runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
+                diffEnv: env,
+                destroy: () => env.destroy().then(() => { }),
+                spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
+            };
+        };
+        const provisionIntegrationSandbox = async () => {
+            if (cfg.envFactory) {
+                const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
                 try {
                     await sh(env)(body.seedCmd);
                     await injectOracles(env);
                 }
                 catch (e) {
-                    await env.destroy().catch(() => { });
+                    await Promise.resolve(env.destroy()).catch(() => { });
                     throw e;
                 }
-                return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
-            },
-            push: async (integratedPatch, baseSha) => {
-                const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
-                const gitEnv = { ...process.env, LC_ALL: "C" };
-                try {
-                    execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
-                    execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
-                    execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
-                    writeFileSync(join(dir, ".leader.patch"), integratedPatch);
-                    execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
-                    rmSync(join(dir, ".leader.patch"));
-                    const ref = body.targetRef ?? "refs/heads/main";
-                    const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
-                    const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
-                    if (exists) {
-                        execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
-                    }
-                    else {
-                        execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
-                    }
-                    return { ok: true, ref };
-                }
-                catch (e) {
-                    const msg = e instanceof Error ? e.message : String(e);
-                    return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
+                return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
+            }
+            if (!cfg.e2bApiKey)
+                throw new Error("leader wire: no envFactory and no e2bApiKey");
+            const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
+            if (!(await env.connect()).ok) {
+                await env.destroy().catch(() => { });
+                throw new Error("integ connect failed");
+            }
+            try {
+                await sh(env)(body.seedCmd);
+                await injectOracles(env);
+            }
+            catch (e) {
+                await env.destroy().catch(() => { });
+                throw e;
+            }
+            return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
+        };
+        const push = async (integratedPatch, baseSha) => {
+            const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
+            const gitEnv = { ...process.env, LC_ALL: "C" };
+            try {
+                execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
+                execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
+                execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
+                writeFileSync(join(dir, ".leader.patch"), integratedPatch);
+                execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
+                rmSync(join(dir, ".leader.patch"));
+                const ref = body.targetRef ?? "refs/heads/main";
+                const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
+                const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
+                if (exists) {
+                    execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
                 }
-                finally {
-                    rmSync(dir, { recursive: true, force: true });
+                else {
+                    execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
                 }
-            },
+                return { ok: true, ref };
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
+            }
+            finally {
+                rmSync(dir, { recursive: true, force: true });
+            }
+        };
+        const deps = {
+            plan,
+            provisionWorker,
+            provisionIntegrationSandbox,
+            push,
             testCmd: body.testCmd,
             repoDir: repo, workerRepoDir: repo,
             timeoutMs: leaderTimeoutMs, maxConcurrency: 4,