@sema-agent/server 1.316.0 → 1.317.0

package/dist/http/routes/tasks.js

@@ -0,0 +1,632 @@
+import { runWithVerification, runCascade, uuidv7 } from "@sema-agent/core";
+import { markChildrenStoppedByUserOnAbort, resumeAtHttpStatus, stripCheckpointToken, HEARTBEAT_MS } from "../../runs.js";
+import { withPrincipal } from "../../observability/principal-context.js";
+import { fleetRunPublisher, fleetRunLabels, fleetRunResiduals } from "../../fleet/fleet-bus.js";
+import { defaultSubagentTailBus, projectTailFrame } from "../../fleet/subagent-tail-bus.js";
+import { emitPendingWorkflowCompletions, taskNotificationInboxEntry, taskNotificationStreamKey, NotifiedKeys } from "../../orchestration/workflow-completion-inbox.js";
+import { createLedgerSink } from "../../trace/ledger-sink.js";
+import { redactSecrets } from "../../trace/redact.js";
+import { turnEndEventData, contextUsageEventData, toolStartEventData, toolEndEventData, taskProgressEventData, taskNotificationEventData, compactedEventData, diagnosticsEventData, brainStatusEventData, steeringInjectedEventData, workspaceChangedEventData, appendModelUsageDelta, attachModelUsage } from "../../trace/project.js";
+import { cascadeConfig, runMeta } from "../run-meta.js";
+import { scopedIdempotencyKey } from "../idempotency.js";
+import { sendJson, sendError, sseHeaders } from "../send.js";
+import { headerStr, gatedPrincipal } from "../principal-gate.js";
+export async function handleTasks(req, res, url, ctx) {
+    const miss = { fell: false };
+    await handleTasksBody(req, res, url, ctx, miss);
+    return !miss.fell;
+}
+async function handleTasksBody(req, res, url, ctx, miss) {
+    const { deps } = ctx;
+    const { idemCache, inflightRuns, cancelledViaVerb, steerableRuns, counters } = ctx.registry;
+    const { rateLimited, quotaExceeded, leaseDenied } = ctx.helpers;
+    const { prepareSpec, finalizeTaskResult } = ctx.legs;
+    const reqState = ctx.req;
+    const source = ctx.req.source;
+    if (req.method === "POST" && (url === "/v1/tasks" || url === "/v1/tasks/stream")) {
+        const rawIdem = headerStr(req.headers["idempotency-key"]);
+        const idemKey = rawIdem ? scopedIdempotencyKey(rawIdem, source, gatedPrincipal(req, deps.config)) : undefined;
+        const cached = idemKey ? idemCache.peek(idemKey) : undefined;
+        if (cached) {
+            if (url === "/v1/tasks/stream") {
+                sseHeaders(res);
+                const hb = setInterval(() => {
+                    if (!res.writableEnded)
+                        res.write(`event: heartbeat\ndata: {}\n\n`);
+                }, 15_000);
+                try {
+                    const resp = await cached;
+                    if (!res.writableEnded)
+                        res.write(`data: ${JSON.stringify({ type: "done", result: resp.body, replay: true })}\n\n`);
+                }
+                finally {
+                    clearInterval(hb);
+                }
+                res.end();
+            }
+            else {
+                const resp = await cached;
+                sendJson(res, resp.status, resp.body);
+            }
+            return;
+        }
+        if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
+            return;
+        const prepared = await prepareSpec(req, res);
+        if (!prepared)
+            return;
+        const principal = prepared.auth?.principal;
+        if (url === "/v1/tasks/stream") {
+            if (prepared.verify || prepared.cascade) {
+                sendError(res, 400, "request.field_conflict", "verify / cascade are not supported on /v1/tasks/stream (both are multi-attempt, not a single stream) — use /v1/tasks or /v1/runs");
+                return;
+            }
+            const earlyDurableTid = deps.runStore && prepared.spec.sessionId ? uuidv7() : undefined;
+            const detachOnDisconnect = headerStr(req.headers["x-detach-on-disconnect"]) === "true";
+            if (detachOnDisconnect && earlyDurableTid === undefined) {
+                sendError(res, 400, "request.precondition_unmet", "x-detach-on-disconnect requires a durable run (a run store + sessionId) — without one the detached result would be unqueryable");
+                return;
+            }
+            reqState.streamTaskId = earlyDurableTid;
+            let usageKey;
+            reqState.streamDetached = detachOnDisconnect;
+            sseHeaders(res, earlyDurableTid ? { "x-task-id": earlyDurableTid } : undefined);
+            if (earlyDurableTid) {
+                res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", taskId: earlyDurableTid, ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}) })}\n\n`);
+            }
+            const ac = new AbortController();
+            if (prepared.spec.sessionId)
+                markChildrenStoppedByUserOnAbort(ac.signal, prepared.spec.sessionId, prepared.auth?.principal);
+            let closed = false;
+            let detachLogged = false;
+            const onDisconnect = () => {
+                if (detachOnDisconnect) {
+                    if (res.writableEnded)
+                        return;
+                    if (!detachLogged) {
+                        detachLogged = true;
+                        res.on("error", () => undefined);
+                        deps.logger?.info("stream_client_detached", { taskId: earlyDurableTid, sessionId: prepared.spec.sessionId, detached: true });
+                    }
+                    return;
+                }
+                closed = true;
+                ac.abort();
+            };
+            req.on("close", onDisconnect);
+            res.on("close", () => {
+                if (!res.writableEnded)
+                    onDisconnect();
+            });
+            const hb = setInterval(() => {
+                if (!res.writableEnded && !res.destroyed)
+                    res.write(`event: heartbeat\ndata: {}\n\n`);
+            }, 15_000);
+            let ranLive = false;
+            let resp;
+            const runStreamSubmitLeg = async () => {
+                ranLive = true;
+                let finalResult;
+                let runRowSettled = false;
+                let durableTaskId;
+                let durableHeartbeat;
+                let userMsgEntryId;
+                let anchorPut = false;
+                const putRewindAnchor = () => {
+                    if (anchorPut || !userMsgEntryId || !durableTaskId || !prepared.spec.sessionId || !deps.resumeAnchorStore)
+                        return;
+                    anchorPut = true;
+                    void deps.resumeAnchorStore
+                        .put(prepared.spec.sessionId, durableTaskId, userMsgEntryId, principal ?? null)
+                        .catch(() => deps.metrics?.inc("resume_anchor_capture_failed"));
+                };
+                if (deps.runStore && prepared.spec.sessionId) {
+                    const tid = earlyDurableTid ?? uuidv7();
+                    const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
+                    if (created.ok)
+                        deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
+                    if (!created.ok) {
+                        const conflict = { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", errorCode: "conflict.session_active_run", activeTaskId: created.activeTaskId };
+                        res.write(`data: ${JSON.stringify({ type: "done", result: { status: "failed", errorMessage: conflict.error, activeTaskId: conflict.activeTaskId } })}\n\n`);
+                        return { status: 409, body: conflict };
+                    }
+                    durableTaskId = tid;
+                }
+                let ledgerSink;
+                let ledgerTerminal = false;
+                if (detachOnDisconnect && durableTaskId && deps.runStore) {
+                    const rs = deps.runStore;
+                    const tid = durableTaskId;
+                    ledgerSink = createLedgerSink({ appendEvent: (seq, type, data) => rs.appendEvent(tid, seq, type, data), persistThinking: deps.config.traceThinking });
+                    usageKey = prepared.spec.sessionId;
+                    if (usageKey)
+                        deps.modelUsage?.register(usageKey);
+                }
+                if (deps.checkpointStore && prepared.spec.sessionId) {
+                    await deps.checkpointStore.putCtx(prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
+                }
+                if (durableTaskId && deps.runStore) {
+                    const rs = deps.runStore;
+                    const tid = durableTaskId;
+                    const owner = principal ?? null;
+                    const abortFromVerb = () => {
+                        cancelledViaVerb.add(tid);
+                        ac.abort();
+                    };
+                    durableHeartbeat = setInterval(() => {
+                        void rs.heartbeat(tid, owner).catch(() => undefined);
+                        if (!ac.signal.aborted) {
+                            void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
+                                abortFromVerb(); }).catch(() => undefined);
+                        }
+                    }, HEARTBEAT_MS);
+                    durableHeartbeat.unref?.();
+                    inflightRuns.set(tid, ac);
+                    void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !ac.signal.aborted)
+                        abortFromVerb(); }).catch(() => undefined);
+                }
+                let fleetPub;
+                try {
+                    fleetPub = durableTaskId
+                        ? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
+                        : undefined;
+                }
+                catch (e) {
+                    if (durableHeartbeat)
+                        clearInterval(durableHeartbeat);
+                    if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
+                        inflightRuns.delete(durableTaskId);
+                        cancelledViaVerb.delete(durableTaskId);
+                    }
+                    throw e;
+                }
+                let fleetSettled = false;
+                const settleFleet = (status, residuals) => {
+                    if (fleetSettled)
+                        return;
+                    fleetSettled = true;
+                    fleetPub?.onTerminal(status, residuals);
+                };
+                let liveStreamRef;
+                const subagentHandleEvictions = [];
+                let syncLegLive = true;
+                const syncNotifiedKeys = new NotifiedKeys();
+                try {
+                    fleetPub?.onStart();
+                }
+                catch (e) {
+                    if (durableHeartbeat)
+                        clearInterval(durableHeartbeat);
+                    if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
+                        inflightRuns.delete(durableTaskId);
+                        cancelledViaVerb.delete(durableTaskId);
+                    }
+                    settleFleet("failed");
+                    throw e;
+                }
+                try {
+                    const streamBody = async () => {
+                        await withPrincipal(principal, async () => {
+                            const fwdInternals = {
+                                onForwardEvent: (e) => {
+                                    fleetPub?.onForwardEvent(e);
+                                    ledgerSink?.onForwardEvent(e);
+                                    {
+                                        const bg = e.bgAgentId;
+                                        if (bg !== undefined && defaultSubagentTailBus.hasSubscribers(bg)) {
+                                            const f = projectTailFrame(e);
+                                            if (f)
+                                                defaultSubagentTailBus.publish(bg, f);
+                                        }
+                                    }
+                                    if (res.writableEnded)
+                                        return;
+                                    const t = e.type;
+                                    if (t === "task_progress") {
+                                        res.write(`data: ${JSON.stringify({ type: "task_progress", ...taskProgressEventData(e) })}\n\n`);
+                                    }
+                                    else if (t === "tool_start") {
+                                        const tsv = e;
+                                        res.write(`data: ${JSON.stringify({ type: "tool_start", ...toolStartEventData(tsv) })}\n\n`);
+                                    }
+                                    else if (t === "tool_end") {
+                                        const te = e;
+                                        res.write(`data: ${JSON.stringify({ type: "tool_end", ...toolEndEventData(te) })}\n\n`);
+                                    }
+                                    else if (t === "text_delta") {
+                                        const td = e;
+                                        res.write(`data: ${JSON.stringify({ type: "text_delta", delta: td.delta, ...(td.eventId ? { eventId: td.eventId } : {}), ...(td.parentToolCallId ? { parentToolCallId: td.parentToolCallId } : {}) })}\n\n`);
+                                    }
+                                    else if (t === "reasoning_delta") {
+                                        const rd = e;
+                                        res.write(`data: ${JSON.stringify({ type: "reasoning_delta", delta: redactSecrets(rd.delta), ...(rd.eventId ? { eventId: rd.eventId } : {}), ...(rd.parentToolCallId ? { parentToolCallId: rd.parentToolCallId } : {}) })}\n\n`);
+                                    }
+                                },
+                                onTaskNotification: (n) => {
+                                    if (n.task_type === "workflow")
+                                        return;
+                                    if (defaultSubagentTailBus.hasSubscribers(n.task_id)) {
+                                        defaultSubagentTailBus.publish(n.task_id, { type: "task_settled", taskId: n.task_id, status: n.status, ...(typeof n.seq === "number" ? { seq: n.seq } : {}), ...(n.summary ? { summary: redactSecrets(n.summary) } : {}) });
+                                    }
+                                    const hadRow = fleetPub?.onChildTerminal(n.sessionId ?? n.task_id, n.status, n.task_id, n.toolUseId) ?? false;
+                                    const deliverable = syncLegLive && !res.writableEnded && !res.destroyed;
+                                    const parked = !deliverable && Boolean(deps.workflowCompletionInbox && prepared.spec.sessionId);
+                                    deps.logger?.info?.("task_notification_observed", { route: "sync-stream", taskId: n.task_id, taskType: n.task_type, status: n.status, hadFleetRow: hadRow, legLive: syncLegLive, parkedDurable: parked });
+                                    if (parked) {
+                                        void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(prepared.spec.sessionId, askOwner, n, Date.now(), durableTaskId)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "sync-stream", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
+                                    }
+                                    else if (deliverable) {
+                                        const key = taskNotificationStreamKey(n);
+                                        if (syncNotifiedKeys.get(key) === undefined) {
+                                            syncNotifiedKeys.set(key, Promise.resolve(true));
+                                            res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData({ notification: n }) })}\n\n`);
+                                        }
+                                    }
+                                },
+                                ...(deps.subagentSteerRegistry && durableTaskId
+                                    ? {
+                                        onSubagentSpawn: (handle) => {
+                                            subagentHandleEvictions.push(deps.subagentSteerRegistry.register(durableTaskId, handle));
+                                        },
+                                    }
+                                    : {}),
+                            };
+                            const liveStream = deps.runner.runTaskStream({ ...prepared.spec, signal: ac.signal }, undefined, fwdInternals);
+                            liveStreamRef = liveStream;
+                            if (durableTaskId)
+                                steerableRuns.set(durableTaskId, liveStream);
+                            for await (const ev of liveStream) {
+                                if (closed)
+                                    break;
+                                if (ev.type === "message_committed" && ev.role === "user")
+                                    userMsgEntryId = ev.entryId;
+                                if (ev.type === "done") {
+                                    finalResult = stripCheckpointToken(ev.result);
+                                    if (durableTaskId && finalResult)
+                                        finalResult = { ...finalResult, taskId: durableTaskId };
+                                    if (durableTaskId && cancelledViaVerb.has(durableTaskId) && finalResult?.status === "failed" && !finalResult.errorCode) {
+                                        finalResult = { ...finalResult, errorCode: "cancelled" };
+                                    }
+                                    finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
+                                    putRewindAnchor();
+                                    settleFleet(finalResult?.status ?? "completed", fleetRunResiduals(finalResult));
+                                    if (ledgerSink && finalResult) {
+                                        if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
+                                            const rs2 = deps.runStore;
+                                            const tid2 = durableTaskId;
+                                            const fr = finalResult;
+                                            finalResult = await attachModelUsage(fr, { append: ledgerSink.append, getEvents: (_id, after) => rs2.getEvents(tid2, after), modelUsage: deps.modelUsage, taskId: usageKey }).catch(() => fr);
+                                        }
+                                        await ledgerSink.onDone(finalResult);
+                                        ledgerTerminal = true;
+                                    }
+                                    res.write(`data: ${JSON.stringify({ ...ev, result: finalResult })}\n\n`);
+                                    continue;
+                                }
+                                if (ledgerSink) {
+                                    await ledgerSink.onEvent(ev);
+                                    if (ev.type === "turn_end" && usageKey)
+                                        await appendModelUsageDelta(ledgerSink.append, deps.modelUsage, usageKey);
+                                }
+                                fleetPub?.onEvent(ev);
+                                if (ev.type === "status")
+                                    deps.metrics?.inc("brain_retry_total", { phase: String(ev.phase) });
+                                if (ev.type === "status") {
+                                    res.write(`data: ${JSON.stringify({ type: "status", ...brainStatusEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "tool_end") {
+                                    const te = ev;
+                                    res.write(`data: ${JSON.stringify({ type: "tool_end", ...toolEndEventData(te) })}\n\n`);
+                                }
+                                else if (ev.type === "tool_start") {
+                                    const tsv = ev;
+                                    res.write(`data: ${JSON.stringify({ type: "tool_start", ...toolStartEventData(tsv) })}\n\n`);
+                                }
+                                else if (ev.type === "task_notification") {
+                                    const tnv = ev.notification;
+                                    const tnvKey = tnv?.task_id ? taskNotificationStreamKey({ task_type: tnv.task_type, task_id: tnv.task_id, status: String(tnv.status), seq: tnv.seq }) : undefined;
+                                    const tnvPrior = tnvKey !== undefined ? syncNotifiedKeys.get(tnvKey) : undefined;
+                                    if (tnvPrior === undefined || !(await tnvPrior)) {
+                                        if (tnvKey !== undefined)
+                                            syncNotifiedKeys.set(tnvKey, Promise.resolve(true));
+                                        res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData(ev) })}\n\n`);
+                                    }
+                                }
+                                else if (ev.type === "reasoning_delta" && typeof ev.delta === "string") {
+                                    res.write(`data: ${JSON.stringify({ type: "reasoning_delta", delta: redactSecrets(ev.delta), ...(ev.eventId ? { eventId: ev.eventId } : {}), ...(ev.parentToolCallId ? { parentToolCallId: ev.parentToolCallId } : {}) })}\n\n`);
+                                }
+                                else if (ev.type === "task_progress") {
+                                    res.write(`data: ${JSON.stringify({ type: "task_progress", ...taskProgressEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "compacted") {
+                                    res.write(`data: ${JSON.stringify({ type: "compacted", ...compactedEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "diagnostics") {
+                                    res.write(`data: ${JSON.stringify({ type: "diagnostics", ...diagnosticsEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "steering_injected") {
+                                    res.write(`data: ${JSON.stringify({ type: "steering_injected", ...steeringInjectedEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "workspace_changed") {
+                                    res.write(`data: ${JSON.stringify({ type: "workspace_changed", ...workspaceChangedEventData(ev) })}\n\n`);
+                                }
+                                else if (ev.type === "text_delta" || ev.type === "turn_end" || ev.type === "message_committed" || ev.type === "context_usage") {
+                                    const e = ev;
+                                    const ident = {
+                                        ...(e.eventId !== undefined ? { eventId: e.eventId } : {}),
+                                        ...(e.parentToolCallId !== undefined ? { parentToolCallId: e.parentToolCallId } : {}),
+                                        ...(e.sourceTaskId !== undefined ? { sourceTaskId: e.sourceTaskId } : {}),
+                                        ...(e.bgAgentId !== undefined ? { bgAgentId: e.bgAgentId } : {}),
+                                    };
+                                    const arm = ev.type === "text_delta"
+                                        ? { type: "text_delta", delta: e.delta, ...ident }
+                                        : ev.type === "turn_end"
+                                            ? { type: "turn_end", ...(e.usage !== undefined ? { usage: e.usage } : {}), ...(e.usageMissing !== undefined ? { usageMissing: e.usageMissing } : {}), ...(e.stopReason !== undefined ? { stopReason: e.stopReason } : {}), ...ident }
+                                            : ev.type === "message_committed"
+                                                ? { type: "message_committed", entryId: e.entryId, role: e.role, ...(e.toolCallId !== undefined ? { toolCallId: e.toolCallId } : {}), ...ident }
+                                                : { type: "context_usage", ...contextUsageEventData(ev), ...ident };
+                                    res.write(`data: ${JSON.stringify(arm)}\n\n`);
+                                }
+                                else {
+                                    res.write(`data: ${JSON.stringify(ev)}\n\n`);
+                                }
+                            }
+                        });
+                    };
+                    const askTaskId = durableTaskId ?? uuidv7();
+                    const askOwner = gatedPrincipal(req, deps.config) ?? null;
+                    const emitAsk = (frame) => {
+                        if (!res.writableEnded)
+                            res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
+                    };
+                    await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, prepared.spec.sessionId, askOwner, (frame) => {
+                        if (closed || res.writableEnded || res.destroyed)
+                            throw new Error("stream ended before the completion frame was written");
+                        const f = frame;
+                        if (f.type === "task_notification" && f.task_id)
+                            syncNotifiedKeys.set(taskNotificationStreamKey({ task_type: f.task_type, task_id: f.task_id, status: String(f.status), seq: f.seq }), Promise.resolve(true));
+                        res.write(`data: ${JSON.stringify(frame)}\n\n`);
+                    }, { route: "sync-stream-open", connection: "live-sse", log: (m, x) => deps.logger?.info?.(m, x) });
+                    const withElicit = deps.elicitation
+                        ? () => deps.elicitation.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitAsk, abortSignal: ac.signal }, streamBody)
+                        : streamBody;
+                    const withQuestion = () => deps.question
+                        ? deps.question.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitAsk, abortSignal: ac.signal }, withElicit)
+                        : withElicit();
+                    const emitApproval = (frame) => {
+                        if (res.writableEnded || res.destroyed)
+                            throw new Error("live stream ended — approval card undeliverable");
+                        emitAsk(frame);
+                    };
+                    const approvalCtx = deps.toolApproval
+                        ? { taskId: askTaskId, owner: askOwner, emit: emitApproval, abortSignal: ac.signal, ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}) }
+                        : undefined;
+                    if (approvalCtx && deps.toolApproval) {
+                        prepared.spec.onAsk = deps.toolApproval.boundAsk({
+                            owner: askOwner,
+                            taskId: askTaskId,
+                            ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}),
+                        });
+                    }
+                    const withApproval = () => approvalCtx && deps.toolApproval ? deps.toolApproval.runWithContext(approvalCtx, withQuestion) : withQuestion();
+                    await (deps.sendUserFile
+                        ? deps.sendUserFile.runWithContext({
+                            taskId: askTaskId,
+                            emit: async (f) => {
+                                void (await emitAsk(f));
+                                if (ledgerSink) {
+                                    const { type, ...rest } = f;
+                                    await ledgerSink.append(type, rest).catch(() => undefined);
+                                }
+                            },
+                        }, withApproval)
+                        : withApproval());
+                    if (durableTaskId && deps.runStore) {
+                        if (finalResult?.status === "suspended" && deps.checkpointStore)
+                            await deps.runStore.setSuspended(durableTaskId);
+                        else if (finalResult?.status === "needs_review" && deps.checkpointStore)
+                            await deps.runStore.setNeedsReview(durableTaskId);
+                        else if (finalResult)
+                            await deps.runStore.setTerminal(durableTaskId, finalResult.status, finalResult, finalResult.errorMessage ?? null);
+                        else {
+                            putRewindAnchor();
+                            const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
+                            if (pending) {
+                                if (ledgerSink && !ledgerTerminal) {
+                                    ledgerTerminal = true;
+                                    await ledgerSink.flush().catch(() => undefined);
+                                    await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
+                                }
+                                await deps.runStore.setSuspended(durableTaskId);
+                                settleFleet("suspended");
+                            }
+                            else {
+                                const dcErr = "stream client disconnected before completion (run aborted)";
+                                const dc = { taskId: durableTaskId, sessionId: prepared.spec.sessionId ?? "", status: "failed", errorCode: "cancelled", errorMessage: dcErr, stats: { turns: 0, tokens: 0 } };
+                                if (ledgerSink && !ledgerTerminal) {
+                                    ledgerTerminal = true;
+                                    await ledgerSink.flush().catch(() => undefined);
+                                    await ledgerSink.append("failed", { errorMessage: dcErr, errorCode: "cancelled" }).catch(() => undefined);
+                                }
+                                await deps.runStore.setTerminal(durableTaskId, "failed", dc, dcErr);
+                                settleFleet("failed");
+                            }
+                        }
+                        runRowSettled = true;
+                    }
+                }
+                finally {
+                    syncLegLive = false;
+                    if (durableHeartbeat)
+                        clearInterval(durableHeartbeat);
+                    if (durableTaskId && steerableRuns.get(durableTaskId) === liveStreamRef)
+                        steerableRuns.delete(durableTaskId);
+                    if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
+                        inflightRuns.delete(durableTaskId);
+                        cancelledViaVerb.delete(durableTaskId);
+                    }
+                    for (const evict of subagentHandleEvictions)
+                        evict();
+                    if (!runRowSettled && durableTaskId && deps.runStore) {
+                        try {
+                            const row = await deps.runStore.getRun(durableTaskId);
+                            if (row?.status === "running") {
+                                const pending = prepared.spec.sessionId ? await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId) : undefined;
+                                if (pending) {
+                                    if (ledgerSink && !ledgerTerminal) {
+                                        ledgerTerminal = true;
+                                        await ledgerSink.flush().catch(() => undefined);
+                                        await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
+                                    }
+                                    await deps.runStore.setSuspended(durableTaskId);
+                                    settleFleet("suspended");
+                                }
+                                else if (finalResult) {
+                                    if (ledgerSink && !ledgerTerminal) {
+                                        ledgerTerminal = true;
+                                        await ledgerSink.onDone(finalResult).catch(() => undefined);
+                                    }
+                                    await deps.runStore.setTerminal(durableTaskId, finalResult.status, finalResult, finalResult.errorMessage ?? null);
+                                    settleFleet(finalResult.status, fleetRunResiduals(finalResult));
+                                }
+                                else if (ac.signal.aborted) {
+                                    const err = "stream aborted before completion (run cancelled)";
+                                    const c = { taskId: durableTaskId, sessionId: prepared.spec.sessionId ?? "", status: "failed", errorCode: "cancelled", errorMessage: err, stats: { turns: 0, tokens: 0 } };
+                                    if (ledgerSink && !ledgerTerminal) {
+                                        ledgerTerminal = true;
+                                        await ledgerSink.flush().catch(() => undefined);
+                                        await ledgerSink.append("failed", { errorMessage: err, errorCode: "cancelled" }).catch(() => undefined);
+                                    }
+                                    await deps.runStore.setTerminal(durableTaskId, "failed", c, err);
+                                }
+                                else {
+                                    const err = "stream leg threw before a terminal event";
+                                    if (ledgerSink && !ledgerTerminal) {
+                                        ledgerTerminal = true;
+                                        await ledgerSink.flush().catch(() => undefined);
+                                        await ledgerSink.append("failed", { errorMessage: err }).catch(() => undefined);
+                                    }
+                                    await deps.runStore.setTerminal(durableTaskId, "failed", null, err);
+                                }
+                            }
+                        }
+                        catch {
+                        }
+                    }
+                    settleFleet("failed");
+                }
+                return { status: 200, body: finalResult };
+            };
+            counters.uncountedBillableInflight++;
+            try {
+                resp = await idemCache.run(idemKey, runStreamSubmitLeg, (r) => r.body?.status === "completed");
+            }
+            catch (e) {
+                deps.logger?.error("stream_error", { closed, err: e instanceof Error ? e.message : String(e) });
+                if (!closed)
+                    throw e;
+            }
+            finally {
+                counters.uncountedBillableInflight--;
+                clearInterval(hb);
+                if (usageKey)
+                    deps.modelUsage?.clear(usageKey);
+            }
+            if (!ranLive && resp && !res.writableEnded) {
+                res.write(`data: ${JSON.stringify({ type: "done", result: resp.body, replay: true })}\n\n`);
+            }
+            res.end();
+        }
+        else {
+            const runSyncSubmitLeg = async () => {
+                let durableTaskId;
+                let durableHeartbeat;
+                if (deps.runStore && prepared.spec.sessionId) {
+                    const tid = uuidv7();
+                    const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
+                    if (!created.ok)
+                        return { status: 409, body: { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", errorCode: "conflict.session_active_run", activeTaskId: created.activeTaskId } };
+                    deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
+                    durableTaskId = tid;
+                }
+                if (deps.checkpointStore && prepared.spec.sessionId) {
+                    await deps.checkpointStore.putCtx(prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
+                }
+                const syncCancelCtrl = new AbortController();
+                if (prepared.spec.sessionId)
+                    markChildrenStoppedByUserOnAbort(syncCancelCtrl.signal, prepared.spec.sessionId, prepared.auth?.principal);
+                if (durableTaskId && deps.runStore) {
+                    const rs = deps.runStore;
+                    const tid = durableTaskId;
+                    const owner = principal ?? null;
+                    const abortFromVerb = () => {
+                        cancelledViaVerb.add(tid);
+                        syncCancelCtrl.abort();
+                    };
+                    durableHeartbeat = setInterval(() => {
+                        void rs.heartbeat(tid, owner).catch(() => undefined);
+                        if (!syncCancelCtrl.signal.aborted) {
+                            void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
+                                abortFromVerb(); }).catch(() => undefined);
+                        }
+                    }, HEARTBEAT_MS);
+                    durableHeartbeat.unref?.();
+                    inflightRuns.set(tid, syncCancelCtrl);
+                    void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !syncCancelCtrl.signal.aborted)
+                        abortFromVerb(); }).catch(() => undefined);
+                }
+                else if (durableTaskId) {
+                    inflightRuns.set(durableTaskId, syncCancelCtrl);
+                }
+                const specWithSignal = { ...prepared.spec, signal: syncCancelCtrl.signal };
+                let result;
+                let cancelVerbLabel = false;
+                counters.uncountedBillableInflight++;
+                try {
+                    const verifyLeg = (v) => runWithVerification(deps.runner, specWithSignal, v);
+                    const cascadeLeg = () => runCascade(deps.runner, specWithSignal, cascadeConfig(deps.config.cascadeLadder, prepared.spec.maxCostUsd));
+                    const plainLeg = () => deps.runner.runTask(specWithSignal);
+                    result = await withPrincipal(principal, () => prepared.verify ? verifyLeg(prepared.verify) : prepared.cascade ? cascadeLeg() : plainLeg());
+                }
+                finally {
+                    counters.uncountedBillableInflight--;
+                    if (durableHeartbeat)
+                        clearInterval(durableHeartbeat);
+                    if (durableTaskId && inflightRuns.get(durableTaskId) === syncCancelCtrl) {
+                        inflightRuns.delete(durableTaskId);
+                        cancelVerbLabel = cancelledViaVerb.has(durableTaskId);
+                        cancelledViaVerb.delete(durableTaskId);
+                    }
+                }
+                if (cancelVerbLabel && result.status === "failed" && !result.errorCode) {
+                    result = { ...result, errorCode: "cancelled" };
+                }
+                const resumeAtStatus = resumeAtHttpStatus(result);
+                if (resumeAtStatus) {
+                    if (durableTaskId && deps.runStore)
+                        await deps.runStore.setTerminal(durableTaskId, "failed", stripCheckpointToken(result), result.errorMessage ?? null);
+                    return { status: resumeAtStatus, body: { errorCode: result.errorCode, error: result.errorMessage ?? "resume-at failed" } };
+                }
+                if (result.status === "suspended" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
+                    if (durableTaskId)
+                        await deps.runStore.setSuspended(durableTaskId);
+                    return { status: 200, body: { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "suspended" } };
+                }
+                if (result.status === "needs_review" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
+                    if (durableTaskId)
+                        await deps.runStore.setNeedsReview(durableTaskId);
+                    return { status: 200, body: { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "needs_review" } };
+                }
+                if (durableTaskId && deps.runStore)
+                    await deps.runStore.setTerminal(durableTaskId, result.status, stripCheckpointToken(result), result.errorMessage ?? null);
+                finalizeTaskResult(result, principal, prepared.spec.objective, prepared.spec.sessionId);
+                return { status: 200, body: stripCheckpointToken(result) };
+            };
+            const resp = await idemCache.run(idemKey, runSyncSubmitLeg, (r) => r.status < 400);
+            sendJson(res, resp.status, resp.body);
+        }
+        return;
+    }
+    miss.fell = true;
+}
+//# sourceMappingURL=tasks.js.map

package/dist/http/routes/trace-usage.d.ts

@@ -0,0 +1,4 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteCtx } from "../route-ctx.js";
+export declare function handleTraceUsage(req: IncomingMessage, res: ServerResponse, url: string, ctx: RouteCtx): Promise<boolean>;
+//# sourceMappingURL=trace-usage.d.ts.map