@sema-agent/server 1.205.0 → 1.207.0

package/dist/http/server.js

@@ -178,6 +178,7 @@ export function createHttpServer(deps) {
     const idemCache = new IdempotencyCache();
     const bakeSubmitLimiter = new RateLimiter(deps.config.imageBakes?.submitRateMax ?? 0, (deps.config.imageBakes?.submitRateWindowSec ?? 60) * 1000);
     const inflightRuns = new Map();
+    const cancelledViaVerb = new Set();
     const preemptableRuns = new Map();
     const steerableRuns = new Map();
     const wakeParkMints = new Map();
@@ -742,7 +743,7 @@ export function createHttpServer(deps) {
                             if (created.ok)
                                 deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
                             if (!created.ok) {
-                                const conflict = { error: "session already has an active run", activeTaskId: created.activeTaskId };
+                                const conflict = { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId };
                                 res.write(`data: ${JSON.stringify({ type: "done", result: { status: "failed", errorMessage: conflict.error, activeTaskId: conflict.activeTaskId } })}\n\n`);
                                 return { status: 409, body: conflict };
                             }
@@ -755,8 +756,21 @@ export function createHttpServer(deps) {
                             const rs = deps.runStore;
                             const tid = durableTaskId;
                             const owner = principal ?? null;
-                            durableHeartbeat = setInterval(() => void rs.heartbeat(tid, owner).catch(() => undefined), HEARTBEAT_MS);
+                            const abortFromVerb = () => {
+                                cancelledViaVerb.add(tid);
+                                ac.abort();
+                            };
+                            durableHeartbeat = setInterval(() => {
+                                void rs.heartbeat(tid, owner).catch(() => undefined);
+                                if (!ac.signal.aborted) {
+                                    void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
+                                        abortFromVerb(); }).catch(() => undefined);
+                                }
+                            }, HEARTBEAT_MS);
                             durableHeartbeat.unref?.();
+                            inflightRuns.set(tid, ac);
+                            void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !ac.signal.aborted)
+                                abortFromVerb(); }).catch(() => undefined);
                         }
                         const fleetPub = durableTaskId
                             ? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
@@ -841,6 +855,9 @@ export function createHttpServer(deps) {
                                             finalResult = stripCheckpointToken(ev.result);
                                             if (durableTaskId && finalResult)
                                                 finalResult = { ...finalResult, taskId: durableTaskId };
+                                            if (durableTaskId && cancelledViaVerb.has(durableTaskId) && finalResult?.status === "failed" && !finalResult.errorCode) {
+                                                finalResult = { ...finalResult, errorCode: "cancelled" };
+                                            }
                                             finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
                                             putRewindAnchor();
                                             settleFleet(finalResult?.status ?? "completed");
@@ -947,6 +964,10 @@ export function createHttpServer(deps) {
                                 clearInterval(durableHeartbeat);
                             if (durableTaskId && steerableRuns.get(durableTaskId) === liveStreamRef)
                                 steerableRuns.delete(durableTaskId);
+                            if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
+                                inflightRuns.delete(durableTaskId);
+                                cancelledViaVerb.delete(durableTaskId);
+                            }
                             for (const evict of subagentHandleEvictions)
                                 evict();
                             if (!runRowSettled && durableTaskId && deps.runStore) {
@@ -997,33 +1018,62 @@ export function createHttpServer(deps) {
                         const tid = uuidv7();
                         const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
                         if (!created.ok)
-                            return { status: 409, body: { error: "session already has an active run", activeTaskId: created.activeTaskId } };
+                            return { status: 409, body: { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId } };
                         deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
                         durableTaskId = tid;
                     }
                     if (deps.checkpointStore && prepared.spec.sessionId) {
                         await deps.checkpointStore.putCtx(prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
                     }
+                    const syncCancelCtrl = new AbortController();
+                    if (prepared.spec.sessionId)
+                        markChildrenStoppedByUserOnAbort(syncCancelCtrl.signal, prepared.spec.sessionId, prepared.auth?.principal);
                     if (durableTaskId && deps.runStore) {
                         const rs = deps.runStore;
                         const tid = durableTaskId;
                         const owner = principal ?? null;
-                        durableHeartbeat = setInterval(() => void rs.heartbeat(tid, owner).catch(() => undefined), HEARTBEAT_MS);
+                        const abortFromVerb = () => {
+                            cancelledViaVerb.add(tid);
+                            syncCancelCtrl.abort();
+                        };
+                        durableHeartbeat = setInterval(() => {
+                            void rs.heartbeat(tid, owner).catch(() => undefined);
+                            if (!syncCancelCtrl.signal.aborted) {
+                                void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
+                                    abortFromVerb(); }).catch(() => undefined);
+                            }
+                        }, HEARTBEAT_MS);
                         durableHeartbeat.unref?.();
+                        inflightRuns.set(tid, syncCancelCtrl);
+                        void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !syncCancelCtrl.signal.aborted)
+                            abortFromVerb(); }).catch(() => undefined);
+                    }
+                    else if (durableTaskId) {
+                        inflightRuns.set(durableTaskId, syncCancelCtrl);
                     }
+                    const specWithSignal = { ...prepared.spec, signal: syncCancelCtrl.signal };
                     let result;
+                    let cancelVerbLabel = false;
                     uncountedBillableInflight++;
                     try {
                         result = await withPrincipal(principal, () => prepared.verify
-                            ? runWithVerification(deps.runner, prepared.spec, { maxRounds: prepared.verify.maxRounds })
+                            ? runWithVerification(deps.runner, specWithSignal, { maxRounds: prepared.verify.maxRounds })
                             : prepared.cascade
-                                ? runCascade(deps.runner, prepared.spec, cascadeConfig(deps.config.cascadeLadder, prepared.spec.maxCostUsd))
-                                : deps.runner.runTask(prepared.spec));
+                                ? runCascade(deps.runner, specWithSignal, cascadeConfig(deps.config.cascadeLadder, prepared.spec.maxCostUsd))
+                                : deps.runner.runTask(specWithSignal));
                     }
                     finally {
                         uncountedBillableInflight--;
                         if (durableHeartbeat)
                             clearInterval(durableHeartbeat);
+                        if (durableTaskId && inflightRuns.get(durableTaskId) === syncCancelCtrl) {
+                            inflightRuns.delete(durableTaskId);
+                            cancelVerbLabel = cancelledViaVerb.has(durableTaskId);
+                            cancelledViaVerb.delete(durableTaskId);
+                        }
+                    }
+                    if (cancelVerbLabel && result.status === "failed" && !result.errorCode) {
+                        result = { ...result, errorCode: "cancelled" };
                     }
                     const resumeAtStatus = resumeAtHttpStatus(result);
                     if (resumeAtStatus) {
@@ -1099,7 +1149,7 @@ export function createHttpServer(deps) {
                     if (clientTaskId && created.activeTaskId === clientTaskId) {
                         return { status: 202, body: { taskId: clientTaskId, sessionId, status: "running" } };
                     }
-                    return { status: 409, body: { error: "session already has an active run", activeTaskId: created.activeTaskId } };
+                    return { status: 409, body: { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId } };
                 }
                 if (deps.checkpointStore) {
                     await deps.checkpointStore.putCtx(sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
@@ -1197,7 +1247,7 @@ export function createHttpServer(deps) {
             const taskId = cancelMatch[1];
             const run = await deps.runStore.getRun(taskId);
             if (!run) {
-                sendJson(res, 404, { error: "run not found" });
+                sendJson(res, 404, { error: "run not found — the id belongs to no run in this deployment's run store (a run from another server process, or an in-memory store that did not survive a restart, is not visible here)" });
                 return;
             }
             if (!runOwnerOk(req, res, run.owner))
@@ -1245,10 +1295,30 @@ export function createHttpServer(deps) {
                 const err = note;
                 try {
                     const result = { taskId, sessionId: run.sessionId, status: "failed", errorCode: "cancelled", errorMessage: err, stats: { turns: 0, tokens: 0 } };
-                    await rs.setTerminal(taskId, "failed", result, err);
+                    let lastErr;
+                    for (let attempt = 0;; attempt++) {
+                        try {
+                            await rs.setTerminal(taskId, "failed", result, err);
+                            lastErr = undefined;
+                            break;
+                        }
+                        catch (e) {
+                            lastErr = e;
+                            if (attempt >= 2)
+                                break;
+                            await new Promise((r) => setTimeout(r, 100 * (attempt + 1)));
+                        }
+                    }
+                    if (lastErr !== undefined)
+                        throw lastErr;
                 }
                 catch (e) {
-                    sendJson(res, 500, { taskId, error: `cancel progressed (${note}) but could not terminalize the run row — retry cancel; a stuck row is reaped after the stale window (${e instanceof Error ? e.message : String(e)})` });
+                    sendJson(res, 500, { taskId, error: `cancel progressed (${note}) but could not terminalize the run row (retried) — retry cancel; a stuck row is reaped after the stale window (${e instanceof Error ? e.message : String(e)})` });
+                    return;
+                }
+                const finalRow = await rs.getRun(taskId).catch(() => undefined);
+                if (finalRow && finalRow.errorCode !== "cancelled" && (finalRow.status === "failed" || finalRow.status === "completed" || finalRow.status === "blocked" || finalRow.status === "timeout")) {
+                    sendJson(res, 202, { taskId, status: finalRow.status, errorCode: finalRow.errorCode ?? null, note: `session unlocked; the run was terminalized concurrently (${finalRow.errorCode ?? finalRow.status}) before this cancel's write — reporting the actual ledger state` });
                     return;
                 }
                 sendJson(res, 202, { taskId, status: "failed", errorCode: "cancelled", note });
@@ -1265,7 +1335,10 @@ export function createHttpServer(deps) {
                     }
                     return;
                 }
-                inflightRuns.get(taskId)?.abort();
+                if (inflightRuns.has(taskId)) {
+                    cancelledViaVerb.add(taskId);
+                    inflightRuns.get(taskId).abort();
+                }
                 sendJson(res, 202, { taskId, status: "cancelling" });
             }
             else if (run.status === "suspended" || run.status === "needs_review") {
@@ -4323,8 +4396,10 @@ export function createHttpServer(deps) {
         }
         finally {
             if (taskId) {
-                inflightRuns.delete(taskId);
-                preemptableRuns.delete(taskId);
+                if (inflightRuns.get(taskId) === cancelCtrl)
+                    inflightRuns.delete(taskId);
+                if (preemptableRuns.get(taskId) === preemptCtrl)
+                    preemptableRuns.delete(taskId);
                 if (resumeStreamRef !== undefined && steerableRuns.get(taskId) === resumeStreamRef)
                     steerableRuns.delete(taskId);
                 for (const evict of subagentHandleEvictions)