@sema-agent/server 7.147.0 → 7.149.0

package/dist/leader/endpoint.d.ts

@@ -1,6 +1,9 @@
 import type { LeaderResult } from "./leader.js";
 import { type LeaderRunRecord, type LeaderRunStatus, type NewLeaderRun, type LeaderRunTerminal } from "../plugins/leader-run-store-sql.js";
 import { type IdemCaller } from "../http/idempotency.js";
+import type { Subject } from "../owner-brands.js";
+import type { RunIdentity } from "../identity-source.js";
+import type { LeaderParkExpiry } from "./wire.js";
 /**
  * endpoint 消费的 store 面(结构型,不是 nominal union):真实现是 `SqlLeaderRunStore` 的两个方言孪生,
  * 测试用进程内假件。窄到只有 endpoint 真调的三个方法 —— 端点不该看见店的其余口。
@@ -14,6 +17,8 @@ export interface LeaderRunStore {
     /** 按提交幂等键(已 scope 的定宽 hash)读行 —— `GET /v1/leader?idempotencyKey=` 的读口。 */
     getByIdem(idemKey: string): Promise<LeaderRunRecord | undefined>;
     finishRun(id: string, patch: LeaderRunTerminal): Promise<boolean>;
+    /** 翻耐久取消旗(只翻 `running` 行;回执 = 翻没翻)。驱动副本经 `getRun` 读它(一次读拿到状态 + 旗)。 */
+    requestCancel(id: string): Promise<boolean>;
 }
 export interface LeaderRequestBody {
     /** The task to decompose + execute (the Planner turns it into N disjoint sub-tasks). */
@@ -36,6 +41,10 @@ export interface LeaderRun {
     /** BL-4: the principal that submitted this run (null = anonymous/single-tenant). The GET is owner-checked so
      *  an owned run is visible ONLY to its owner (404 to others — no existence oracle), like GET /v1/runs/:id. */
     owner?: string | null;
+    /** 降级臂的取消旗(店在场时以库上的旗为准,本地这一格只服务降级臂的读面)。 */
+    cancelRequested: boolean;
+    /** 取消口停下的终局带 `cancelled`。 */
+    errorCode?: LeaderRunTerminal["errorCode"];
 }
 export interface LeaderEndpoint {
     /** Returns a {status,body} to send, or null if the (method,url) is not a leader route. Kicks the background
@@ -45,7 +54,9 @@ export interface LeaderEndpoint {
      *  header; `GET /v1/leader`: the `idempotencyKey` query — the read port "which leader run did this key create"; scoped by the
      *  caller's identity bit `principal ?? source` before it reaches the store — the one key minter `idemKeyOf(raw, caller, "leader")`).
      *  ASYNC because the durable arm reads/writes SQL. */
-    handle(method: string, url: string, body: unknown, caller?: IdemCaller, idempotencyKey?: string | null): Promise<{
+    handle(method: string, url: string, body: unknown, caller?: IdemCaller, idempotencyKey?: string | null, 
+    /** POST 受理时的身份元组(路由经 `runIdentityOf` 铸;缺席 = `null` = 不合格)。只写进行,不上 wire。 */
+    identity?: RunIdentity | null): Promise<{
         status: number;
         body: object;
     } | null>;
@@ -59,7 +70,9 @@ export interface LeaderEndpoint {
      *  With a store in place this Map holds only the runs THIS replica is driving. Exposed for tests. */
     runs: Map<string, LeaderRun>;
 }
-export declare function createLeaderEndpoint(runLeader: (body: LeaderRequestBody) => Promise<LeaderResult>, opts: {
+/** 取消终局的 `error` 句(机读位是 `errorCode: "cancelled"`)。 */
+export declare const LEADER_RUN_CANCELLED_ERROR = "leader run cancelled (POST /v1/leader/:id/cancel): it stopped at an abort checkpoint before any push";
+export declare function createLeaderEndpoint(runLeader: (body: LeaderRequestBody, signal: AbortSignal) => Promise<LeaderResult>, opts: {
     logger?: {
         info?: (m: string, x?: Record<string, unknown>) => void;
         error?: (m: string, x?: Record<string, unknown>) => void;
@@ -73,8 +86,16 @@ export declare function createLeaderEndpoint(runLeader: (body: LeaderRequestBody
     requiredFields?: readonly string[];
     /** [ref] 车4 件1:durable 登记表。缺席 ⇒ 显式的单副本内存降级臂(见文件头注)。 */
     store?: LeaderRunStore;
-    /** [ref] RH-3:「这个调用方读得到这一行 leader run 吗」—— 属主存取器的判官形(`http/owner-access.ts` 的 `createLeaderRunVisibility`,boot 装一次)。
-     *  必填:本模块不自判属主(修前手抄 `owner == null || owner === requester`,缺 [ref] C8 窄互认臂而注称 mirrors runOwnerOk),读面与幂等重放同读它。 */
-    ownerVisible: (principal: string | null | undefined, owner: string | null | undefined) => boolean;
+    /** [ref] RH-3:「这个调用方读得到这一行 leader run 吗」—— 属主存取器的判官形(`http/owner-access.ts` 的 `seesRunAs("run.leader")`,boot 装一次)。
+     *  必填:本模块不自判属主(修前手抄 `owner == null || owner === requester`,缺 [ref] C8 窄互认臂而注称 mirrors admitRunRow),读面与幂等重放同读它。 */
+    ownerVisible: (subject: Subject | null | undefined, owner: string | null | undefined) => boolean;
+    /** 过期一组 worker 会话的待决停驻(`wire.ts` 的 `createLeaderParkExpirer`,boot 在有 checkpoint 店时递)。缺席 = 本部署没有 checkpoint 店 ⇒
+     *  worker 由构造即不可能停驻(durable spec 要那只店),没有可过期的东西。 */
+    expireWorkerParks?: (workers: ReadonlyArray<{
+        readonly workerId: string;
+        readonly sessionId: string;
+    }>) => Promise<LeaderParkExpiry>;
+    /** 旗读拍距(ms)。缺席 = `LIVENESS_HEARTBEAT_MS`(与 run 心跳同一个数);测试注入短拍。 */
+    cancelPollMs?: number;
 }): LeaderEndpoint;
 //# sourceMappingURL=endpoint.d.ts.map

package/dist/leader/endpoint.js

@@ -1,9 +1,11 @@
 import { uuidv7 } from "@sema-agent/core";
 import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal.js";
-import { withPrincipal } from "../observability/principal-context.js";
+import { withSubject } from "../observability/principal-context.js";
 import {} from "../plugins/leader-run-store-sql.js";
 import { idemKeyOf } from "../http/idempotency.js";
 import { refusalBody } from "../http/refusal-body.js";
+import { LIVENESS_HEARTBEAT_MS } from "../config-invariants.js";
+import { seatText } from "../trace/exit-seat.js";
 function statusForLeaderResult(result) {
     if (result.conflict)
         return "needs_human";
@@ -20,6 +22,9 @@ function statusForLeaderResult(result) {
     }
 }
 const LEADER_ID_RE = /^\/v1\/leader\/([^/]+)$/;
+const LEADER_CANCEL_RE = /^\/v1\/leader\/([^/]+)\/cancel$/;
+export const LEADER_RUN_CANCELLED_ERROR = "leader run cancelled (POST /v1/leader/:id/cancel): it stopped at an abort checkpoint before any push";
+const LEADER_RUN_ROW_SETTLED_ELSEWHERE = "leader run stopped: its row was settled elsewhere (stale sweep or external delete) — this replica stopped driving it";
 const RUN_RETENTION_MS = 60 * 60_000;
 const TERMINAL_WRITE_ATTEMPTS = 3;
 const TERMINAL_WRITE_RETRY_MS = 200;
@@ -31,6 +36,12 @@ function idemShapeError(raw) {
 }
 export function createLeaderEndpoint(runLeader, opts) {
     const runs = new Map();
+    const controllers = new Map();
+    const abortLocal = (id, cause) => {
+        const ac = controllers.get(id);
+        if (ac && !ac.signal.aborted)
+            ac.abort(cause);
+    };
     const memIdem = new Map();
     function reap() {
         const cutoff = Date.now() - RUN_RETENTION_MS;
@@ -44,24 +55,63 @@ export function createLeaderEndpoint(runLeader, opts) {
         }
     }
     function drive(id, body, owner) {
+        const ac = new AbortController();
+        controllers.set(id, ac);
+        let pollWarned = false;
+        const poll = opts.store
+            ? setInterval(() => {
+                if (ac.signal.aborted)
+                    return;
+                void readRow(id).then((row) => {
+                    if (row === undefined || row.status !== "running")
+                        abortLocal(id, "stall_verdict");
+                    else if (row.cancelRequested)
+                        abortLocal(id, "cancel_verb");
+                }, (e) => {
+                    if (pollWarned)
+                        return;
+                    pollWarned = true;
+                    opts.logger?.error?.("leader_cancel_poll_failed", { id, err: opErr(e), effect: "a cancel landed on another replica is not seen until a later tick reads the row" });
+                });
+            }, opts.cancelPollMs ?? LIVENESS_HEARTBEAT_MS)
+            : undefined;
+        poll?.unref?.();
         void (async () => {
             let terminal;
+            let result;
             try {
-                const result = redactLeaderResult(await withPrincipal(owner ?? undefined, () => runLeader(body)));
-                const status = statusForLeaderResult(result);
-                terminal = { status: status, result };
-                opts.logger?.info?.("leader_run_done", { id, ok: result.ok, status, ...(result.repairTerminal ? { repairTerminal: result.repairTerminal } : {}), merged: result.merge && result.merge.ok ? result.merge.merged : undefined });
+                result = redactLeaderResult(await withSubject(owner ?? undefined, () => runLeader(body, ac.signal)));
+                const cause = ac.signal.aborted ? ac.signal.reason : undefined;
+                if (result.stopped === true && cause === "cancel_verb") {
+                    terminal = { status: "failed", result, error: LEADER_RUN_CANCELLED_ERROR, errorCode: "cancelled" };
+                }
+                else if (result.stopped === true && cause === "stall_verdict") {
+                    terminal = { status: "failed", result, error: LEADER_RUN_ROW_SETTLED_ELSEWHERE };
+                }
+                else {
+                    const status = statusForLeaderResult(result);
+                    terminal = { status: status, result };
+                }
+                opts.logger?.info?.("leader_run_done", { id, ok: result.ok, status: terminal.status, ...(terminal.errorCode ? { errorCode: terminal.errorCode } : {}), ...(cause ? { stopCause: cause } : {}), ...(result.repairTerminal ? { repairTerminal: result.repairTerminal } : {}), merged: result.merge && result.merge.ok ? result.merge.merged : undefined });
             }
             catch (e) {
                 terminal = { status: "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
                 opts.logger?.error?.("leader_run_error", { id, err: terminal.error });
             }
+            finally {
+                if (poll)
+                    clearInterval(poll);
+                controllers.delete(id);
+            }
+            let settledElsewhere = ac.signal.reason === "stall_verdict";
             if (opts.store) {
                 for (let attempt = 1;; attempt += 1) {
                     try {
                         const moved = await opts.store.finishRun(id, terminal);
-                        if (!moved)
+                        if (!moved) {
+                            settledElsewhere = true;
                             opts.logger?.error?.("leader_run_terminal_not_applied", { id, status: terminal.status, why: "row missing or already terminal — another writer settled it (e.g. the stale-running sweep), or the row was deleted" });
+                        }
                         break;
                     }
                     catch (e) {
@@ -74,16 +124,68 @@ export function createLeaderEndpoint(runLeader, opts) {
                 }
             }
             const local = runs.get(id);
+            const residueEnds = settledElsewhere || ac.signal.reason === "cancel_verb" || (opts.store ? await cancelFlagAfterSettle(id) : local?.cancelRequested === true);
+            if (residueEnds && result)
+                await expireResidue(id, result, "drive");
             if (local) {
                 local.status = terminal.status;
                 if (terminal.result !== undefined)
                     local.result = terminal.result;
                 if (terminal.error !== undefined)
                     local.error = terminal.error;
+                if (terminal.errorCode !== undefined)
+                    local.errorCode = terminal.errorCode;
                 local.finishedAt = Date.now();
             }
         })();
     }
+    function readRow(id) {
+        return opts.store.getRun(id);
+    }
+    const opErr = (e) => seatText("operator-log-error", e instanceof Error ? e.message : String(e));
+    async function cancelFlagAfterSettle(id) {
+        try {
+            return (await readRow(id))?.cancelRequested === true;
+        }
+        catch (e) {
+            opts.logger?.error?.("leader_cancel_flag_reread_failed", { id, err: opErr(e), effect: "if a cancel landed on this run just before its terminal write, its worker parks are expired only when the cancel is re-sent" });
+            return false;
+        }
+    }
+    function workerSessionsOf(result) {
+        const out = new Map();
+        const r = result && typeof result === "object" ? result : {};
+        for (const list of [r.reports, r.suspended]) {
+            if (!Array.isArray(list))
+                continue;
+            for (const x of list) {
+                const w = x;
+                if (w && typeof w.workerId === "string" && typeof w.sessionId === "string")
+                    out.set(`${w.workerId}\u0000${w.sessionId}`, { workerId: w.workerId, sessionId: w.sessionId });
+            }
+        }
+        return [...out.values()];
+    }
+    async function expireResidue(id, result, site) {
+        const none = { expired: 0, lost: 0 };
+        if (!opts.expireWorkerParks)
+            return none;
+        const sessions = workerSessionsOf(result);
+        if (sessions.length === 0)
+            return none;
+        try {
+            const r = await opts.expireWorkerParks(sessions);
+            if (r.expired > 0 || r.lost > 0)
+                opts.logger?.info?.("leader_cancel_parks_expired", { id, site, expired: r.expired, lost: r.lost });
+            return r;
+        }
+        catch (e) {
+            if (site === "cancel")
+                throw e;
+            opts.logger?.error?.("leader_cancel_parks_expire_failed", { id, err: opErr(e), effect: "the worker parks of this cancelled run stay pending until their own deadline or until the cancel is re-sent" });
+            return none;
+        }
+    }
     const CLEAN_MEMO_CAP = 4096;
     const cleanIds = new Set();
     const DIRTY_MEMO_CAP = 256;
@@ -125,12 +227,12 @@ export function createLeaderEndpoint(runLeader, opts) {
         return view;
     }
     function getBody(run) {
-        return { status: 200, body: { id: run.id, status: run.status, ...redactedView(run) } };
+        return { status: 200, body: { id: run.id, status: run.status, cancelRequested: run.cancelRequested, ...(run.errorCode ? { errorCode: run.errorCode } : {}), ...redactedView(run) } };
     }
     const NOT_FOUND = { status: 404, body: refusalBody({ errorCode: "not_found.leader_run", message: "leader run not found" }) };
-    async function handle(method, url, body, caller, idempotencyKey) {
-        const requester = caller?.principal ?? null;
-        const idemCaller = caller ?? { principal: null, source: null };
+    async function handle(method, url, body, caller, idempotencyKey, identity) {
+        const requester = caller?.subject ?? null;
+        const idemCaller = caller ?? { subject: null, source: null };
         if (method === "POST" && url === "/v1/leader") {
             const b = body;
             if (!b || typeof b.objective !== "string" || b.objective.trim() === "") {
@@ -152,14 +254,14 @@ export function createLeaderEndpoint(runLeader, opts) {
             }
             reap();
             if (opts.store) {
-                const { created, run } = await opts.store.createRun({ objective: b.objective, owner, idemKey: scopedIdem });
+                const { created, run } = await opts.store.createRun({ objective: b.objective, owner, idemKey: scopedIdem, identity: identity ?? null });
                 if (!created) {
                     if (!opts.ownerVisible(requester, run.owner)) {
                         return { status: 409, body: refusalBody({ errorCode: "conflict.leader_run_exists", message: "idempotency key already used by another principal" }) };
                     }
                     return { status: 202, body: { leaderRunId: run.id, status: run.status } };
                 }
-                runs.set(run.id, { id: run.id, status: "running", startedAt: run.startedAtMs, owner });
+                runs.set(run.id, { id: run.id, status: "running", startedAt: run.startedAtMs, owner, cancelRequested: false });
                 drive(run.id, b, owner);
                 return { status: 202, body: { leaderRunId: run.id, status: "running" } };
             }
@@ -170,7 +272,7 @@ export function createLeaderEndpoint(runLeader, opts) {
                     return { status: 202, body: { leaderRunId: priorRun.id, status: priorRun.status } };
             }
             const id = uuidv7();
-            runs.set(id, { id, status: "running", startedAt: Date.now(), owner });
+            runs.set(id, { id, status: "running", startedAt: Date.now(), owner, cancelRequested: false });
             if (scopedIdem)
                 memIdem.set(scopedIdem, id);
             drive(id, b, owner);
@@ -196,11 +298,14 @@ export function createLeaderEndpoint(runLeader, opts) {
                 return NOT_FOUND;
             return getBody(run);
         }
+        const c = method === "POST" ? LEADER_CANCEL_RE.exec(url) : null;
+        if (c)
+            return cancel(c[1], ownerOk);
         const m = method === "GET" ? LEADER_ID_RE.exec(url) : null;
         if (m) {
             const id = m[1];
             if (opts.store) {
-                const run = await opts.store.getRun(id);
+                const run = await readRow(id);
                 if (!run || !ownerOk(run.owner))
                     return NOT_FOUND;
                 return getBody(run);
@@ -212,6 +317,41 @@ export function createLeaderEndpoint(runLeader, opts) {
         }
         return null;
     }
+    async function answerTerminal(id, row) {
+        abortLocal(id, "cancel_verb");
+        const r = await expireResidue(id, row.result, "cancel");
+        if (r.lost > 0) {
+            return { status: 409, body: refusalBody({ errorCode: "conflict.approval_settled", message: `this run is already ${row.status}; ${r.lost} of its worker approvals were settled concurrently (decided or expired) while this cancel tried to expire them — a decided approval continues as its own run (stop it with POST /v1/runs/:id/cancel); retry cancel to expire what is still pending` }) };
+        }
+        return { status: 202, body: { leaderRunId: id, status: row.status, cancelRequested: row.cancelRequested, note: r.expired > 0 ? `already ${row.status} — ${r.expired} pending worker approval(s) it left behind were expired` : `already ${row.status} — nothing left to stop` } };
+    }
+    async function cancel(id, ownerOk) {
+        if (opts.store) {
+            const row = await readRow(id);
+            if (!row || !ownerOk(row.owner))
+                return NOT_FOUND;
+            if (row.status === "running") {
+                if (await opts.store.requestCancel(id)) {
+                    abortLocal(id, "cancel_verb");
+                    return { status: 202, body: { leaderRunId: id, status: "cancelling", cancelRequested: true } };
+                }
+                const now = await readRow(id);
+                if (!now)
+                    return NOT_FOUND;
+                return answerTerminal(id, now);
+            }
+            return answerTerminal(id, row);
+        }
+        const run = runs.get(id);
+        if (!run || !ownerOk(run.owner))
+            return NOT_FOUND;
+        if (run.status === "running") {
+            run.cancelRequested = true;
+            abortLocal(id, "cancel_verb");
+            return { status: 202, body: { leaderRunId: id, status: "cancelling", cancelRequested: true } };
+        }
+        return answerTerminal(id, run);
+    }
     return { handle, inflight: () => [...runs.values()].filter((r) => r.status === "running").length, runs };
 }
 //# sourceMappingURL=endpoint.js.map

package/dist/leader/fanout.d.ts

@@ -100,6 +100,9 @@ export interface FanOutOptions {
      *  suspended worker itself has already settled (its stream is deregistered), so the cancel sweep cannot
      *  touch its checkpoint/paused env (the destroy-exemption falls out of the registration lifecycle). */
     cancelOnSuspend?: boolean;
+    /** 这条 leader run 的中止信号(取消口 / 陈旧裁决;与 core `TaskSpec.signal` 同一约定:缺席 = 不可外部取消)。abort ⇒ 走同一只
+     *  {@link fanOut} 内的 `cancelAll`(先 abort 共享信号、再 destroy 已登记的流)—— 父中止与总超时是同一个动作的两个触发,不是两套收法。 */
+    signal?: AbortSignal;
 }
 export interface FanOutResult {
     reports: WorkerReport[];

package/dist/leader/fanout.js

@@ -37,76 +37,91 @@ export async function fanOut(subtasks, opts) {
     };
     const sem = new Semaphore(opts.maxConcurrency);
     let pendingCancel;
-    const maybeCancelOnSuspend = (r) => {
-        if (opts.cancelOnSuspend && r.status === "suspended" && !cancelled) {
-            opts.logger?.info?.("fanout_suspend_cancel", { suspendedWorker: r.workerId });
-            pendingCancel ??= cancelAll();
-        }
-        return r;
-    };
-    const runOne = async (st) => {
-        const base = { workerId: st.workerId, sessionId: st.sessionId, branch: st.branch, baseSha: st.baseSha };
-        return sem.run(async () => {
-            if (sharedAC.signal.aborted)
-                return { ...base, status: "timeout", error: "cancelled before start" };
-            if (opts.runWorker) {
+    let parentStop;
+    let deadlineTimer;
+    let graceTimer;
+    const parentStopped = new Promise((resolve) => { parentStop = () => resolve("stopped"); });
+    const onParentAbort = () => { pendingCancel ??= cancelAll(); parentStop(); };
+    if (opts.signal?.aborted)
+        onParentAbort();
+    else
+        opts.signal?.addEventListener("abort", onParentAbort, { once: true });
+    try {
+        const maybeCancelOnSuspend = (r) => {
+            if (opts.cancelOnSuspend && r.status === "suspended" && !cancelled) {
+                opts.logger?.info?.("fanout_suspend_cancel", { suspendedWorker: r.workerId });
+                pendingCancel ??= cancelAll();
+            }
+            return r;
+        };
+        const runOne = async (st) => {
+            const base = { workerId: st.workerId, sessionId: st.sessionId, branch: st.branch, baseSha: st.baseSha };
+            return sem.run(async () => {
+                if (sharedAC.signal.aborted)
+                    return { ...base, status: "timeout", error: "cancelled before start" };
+                if (opts.runWorker) {
+                    try {
+                        return maybeCancelOnSuspend(redactWorkerReportError(await opts.runWorker(st, sharedAC.signal)));
+                    }
+                    catch (e) {
+                        return { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
+                    }
+                }
+                let stream;
                 try {
-                    return maybeCancelOnSuspend(redactWorkerReportError(await opts.runWorker(st, sharedAC.signal)));
+                    stream = opts.runner.runTaskStream({ ...st.spec, sessionId: st.sessionId, signal: sharedAC.signal });
                 }
                 catch (e) {
-                    return { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
+                    return { ...base, status: "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
                 }
-            }
-            let stream;
-            try {
-                stream = opts.runner.runTaskStream({ ...st.spec, sessionId: st.sessionId, signal: sharedAC.signal });
-            }
-            catch (e) {
-                return { ...base, status: "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
-            }
-            streams.set(st.workerId, stream);
-            let report;
-            try {
-                const result = await stream.result();
-                const plane = outwardPlane(result.terminal);
-                report = { ...base, status: mapStatus(result.terminal), stats: result.stats, ...(plane.errorMessage ? { error: redactErrorMessage(plane.errorMessage) ?? "" } : {}) };
-            }
-            catch (e) {
-                report = { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
-            }
-            finally {
-                streams.delete(st.workerId);
-            }
-            return maybeCancelOnSuspend(report);
-        });
-    };
-    const results = new Map();
-    const work = Promise.all(subtasks.map((st) => runOne(st).then((r) => { results.set(st.workerId, r); return r; })));
-    const timer = new Promise((resolve) => {
-        const id = setTimeout(() => resolve("timeout"), opts.timeoutMs);
-        void work.then(() => clearTimeout(id));
-    });
-    const raced = await Promise.race([work.then(() => "done"), timer]);
-    if (raced === "timeout") {
-        opts.logger?.warn?.("fanout_timeout", { timeoutMs: opts.timeoutMs, workers: subtasks.length });
-        await cancelAll();
-        const graceMs = opts.cancelGraceMs ?? CANCEL_GRACE_MS;
-        const grace = new Promise((resolve) => {
-            const id = setTimeout(() => resolve("grace"), graceMs);
-            void work.then(() => clearTimeout(id));
+                streams.set(st.workerId, stream);
+                let report;
+                try {
+                    const result = await stream.result();
+                    const plane = outwardPlane(result.terminal);
+                    report = { ...base, status: mapStatus(result.terminal), stats: result.stats, ...(plane.errorMessage ? { error: redactErrorMessage(plane.errorMessage) ?? "" } : {}) };
+                }
+                catch (e) {
+                    report = { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
+                }
+                finally {
+                    streams.delete(st.workerId);
+                }
+                return maybeCancelOnSuspend(report);
+            });
+        };
+        const results = new Map();
+        const work = Promise.all(subtasks.map((st) => runOne(st).then((r) => { results.set(st.workerId, r); return r; })));
+        const timer = new Promise((resolve) => {
+            deadlineTimer = setTimeout(() => resolve("timeout"), opts.timeoutMs);
         });
-        if ((await Promise.race([work.then(() => "done"), grace])) === "grace") {
-            opts.logger?.warn?.("fanout_cancel_grace_expired", { graceMs, settled: results.size, of: subtasks.length });
-            return {
-                reports: subtasks.map((st) => results.get(st.workerId) ?? { workerId: st.workerId, sessionId: st.sessionId, branch: st.branch, baseSha: st.baseSha, status: "timeout", error: "did not settle within cancel grace" }),
-                cancelled,
-            };
+        const raced = await Promise.race([work.then(() => "done"), timer, parentStopped]);
+        if (raced === "timeout") {
+            opts.logger?.warn?.("fanout_timeout", { timeoutMs: opts.timeoutMs, workers: subtasks.length });
+            pendingCancel ??= cancelAll();
         }
+        if (pendingCancel) {
+            const sweep = pendingCancel.catch(() => { });
+            const graceMs = opts.cancelGraceMs ?? CANCEL_GRACE_MS;
+            const grace = new Promise((resolve) => {
+                graceTimer = setTimeout(() => resolve("grace"), graceMs);
+            });
+            if ((await Promise.race([Promise.all([sweep, work]).then(() => "done"), grace])) === "grace") {
+                opts.logger?.warn?.("fanout_cancel_grace_expired", { graceMs, settled: results.size, of: subtasks.length });
+                return {
+                    reports: subtasks.map((st) => results.get(st.workerId) ?? { workerId: st.workerId, sessionId: st.sessionId, branch: st.branch, baseSha: st.baseSha, status: "timeout", error: "did not settle within cancel grace" }),
+                    cancelled,
+                };
+            }
+        }
+        const reports = await work;
+        return { reports, cancelled };
+    }
+    finally {
+        opts.signal?.removeEventListener("abort", onParentAbort);
+        clearTimeout(deadlineTimer);
+        clearTimeout(graceTimer);
     }
-    const reports = await work;
-    if (pendingCancel)
-        await pendingCancel.catch(() => { });
-    return { reports, cancelled };
 }
 const CANCEL_GRACE_MS = 30_000;
 //# sourceMappingURL=fanout.js.map

package/dist/leader/leader.d.ts

@@ -84,6 +84,12 @@ export interface LeaderRunEntry {
  */
 export declare function withLeaderLeg<S extends Partial<TaskSpec>>(spec: S, entry: LeaderRunEntry): S & UnattendedLegSpec;
 export interface LeaderDeps {
+    /**
+     * 本 run 的中止信号(取消口 / 陈旧裁决;HTTP 驱动腿恒给,`endpoint.ts` 的 `drive`)。与 core `TaskSpec.signal` 同一约定:缺席 = 不可外部取消(单测 / bench 形)。
+     * 读法只有一条:段界核点(规划后、每次建 worker 前、每波扇出后、模块门、collapse / redispatch、合并前)看见 abort ⇒ 交 {@link stoppedResult} 的那一份,
+     * 不再起任何新的计费工作;在跑的腿经 spec 的同一根信号中止(fan-out 共享信号 / 修复腿 / 单代理修复环);合并在 push 咽喉前再核一次(`merge.ts`)。
+     */
+    signal?: AbortSignal;
     /** 本 run 的入口体({@link LeaderRunEntry}:运维地板在 run 出生时盖一次)。必填:没有地板的装配写 `{}` —— 缺席不是「没有地板」的合法拼法。
      *  模块门修复与单代理修复环这两个出生点从这里取词(经 {@link withLeaderLeg})。 */
     entry: LeaderRunEntry;
@@ -175,6 +181,9 @@ export interface LeaderResult {
     reports: WorkerReport[];
     merge?: MergeResult;
     cancelled: boolean;
+    /** 只在这条 run 在某个中止核点停下时在场(恒 `true`):核点之后没有起新的工作、**零 push**(push 咽喉的核点在 `deps.push` 正上方)。
+     *  缺席 = run 走完了自己的路(含「取消晚到、push 已发出」—— 那一趟的结局照实)。HTTP 驱动腿据它判取消终局(`failed` + `errorCode: "cancelled"`)。 */
+    stopped?: true;
     /** Which route the difficulty router picked (search [43]): "single" = a 1-subtask plan, "fanout" = N. */
     route?: "single" | "fanout";
     /** Set when planning/provisioning failed before fan-out. */
@@ -243,6 +252,13 @@ export interface LeaderResult {
         gradedHash?: string;
     };
 }
+/** 中止核点停下的那一句(人读;机读位是 {@link LeaderResult.stopped})。 */
+export declare const LEADER_RUN_STOPPED = "leader run stopped at an abort checkpoint (cancel): no further work was started and nothing was pushed after the stop";
+/**
+ * 中止核点的**唯一**结果形(`runLeaderTask` 每一处段界核点、合并停下的那一形都交这一只):`stopped: true` + 已知的报告 +
+ * 其中停驻着的 worker(`suspended[]`,与 C4 的形同 —— 取消落定时 HTTP 驱动腿按它和 `reports[]` 过期这条 run 的残留停驻)。不打捞(取消 = 立刻停)。
+ */
+export declare function stoppedResult(reports: WorkerReport[], route: LeaderResult["route"]): LeaderResult;
 /**
  * Run a task to an integrated, pushed result. Never throws — failures (plan, provision, fan-out, merge) are
  * captured in LeaderResult. ALWAYS destroys every provisioned worker env (leader owns the lifecycle).