@sema-agent/server 7.93.4 → 7.93.6

package/dist/bake-runner/main.d.ts

@@ -10,16 +10,43 @@ interface RunnerEnv {
     heartbeatMs: number;
     idlePollMs: number;
     minFreeGb: number;
+    shutdownGraceMs: number;
 }
 declare function loadEnv(): RunnerEnv;
 /** The real image-api HTTP client. Authenticates claim/heartbeat with the runner bearer; ingest additionally
  *  carries the per-bake ingest secret header (so image-api can reject a stale/rogue runner — §P2.4e). */
 declare function makeApiClient(env: RunnerEnv, log: BakeRunnerLogger): ImageApiClient;
+/**
+ * S-567 —— **bake-runner 自己的在飞进程组簿**。
+ *
+ * 为什么不复用 host 前台簿(`src/plugins/host-platform.ts` 的 `adoptHostForegroundChild`)—— **属主语义不同**
+ * (车LD §3(a) 判定)：那本簿的契约逐字是「被一个**在飞 turn** 拥有的 host lane 前台 exec」，收割姿势是
+ * 无条件整组硬杀、不等宽限；而 build 组有自己的属主(一条 30–45min 的租约 + cancel/deadline 梯子 +
+ * image-api 的 reaper)。两者合进一本簿 = 把两条不同的属主语义压成一个词，规则集变大。而且 bake-runner
+ * 是**另一个进程**，`boot/exit-boundary.ts` 那只装配器的三条腿有两条在这里无对象。
+ *
+ * 本簿的契约就一句：**这个进程 spawn 出来、现在还活着的进程组领袖 pgid**。让它承重的是
+ * {@link killLiveGroupsSync} —— 挂在 bake-runner 自己的 `process.once("exit")` 上的**同步**收割，盖住信号腿
+ * 之外的那一族：崩溃退 / `main().catch(… process.exit(1))` / 梯子的 ④。(SIGKILL 对 `exit` 不触发 ——
+ * 这是内核语义，任何写法都救不了；关机梯子的意义恰恰是让运维不再被逼到 `kill -9`。)
+ */
+declare const liveGroupPgids: Set<number>;
+declare function killLiveGroupsSync(): void;
 /** The real child spawner: `setsid`-style detached process GROUP so a cancel/deadline kill signals the whole tree
  *  (channel-close / a plain kill of the immediate child does NOT stop a detached build — §P2.11). */
 declare function makeSpawner(): ChildSpawner;
+/**
+ * 有界、整组击杀、进簿的 `sh -c` —— **本进程唯一的 host exec 面**(C5)。
+ *
+ * 语义与被它取代的 `promisify(exec)` 对齐：非零退出/信号 ⇒ reject(调用方的 try/catch 与重试环照旧成立)，
+ * 成功 ⇒ `{ stdout }`。多出来的两件：① `timeoutMs` 到点整组 SIGKILL 并 reject；② 它的组在 {@link liveGroupPgids}
+ * 里，于是关机梯子的 ④(`process.exit`)经退出边界收得到它 —— 旧形下一只挂死的 `docker push` 是看不见也杀不死的。
+ */
+export declare function execGroup(cmd: string, timeoutMs: number): Promise<{
+    stdout: string;
+}>;
 /** The real host ops: fs manifest read, df /data, recipe checkout, and the thin docker-push big-blob retry. */
 declare function makeHostOps(env: RunnerEnv): HostOps;
 declare function main(): Promise<void>;
-export { main, loadEnv, makeApiClient, makeSpawner, makeHostOps };
+export { main, loadEnv, makeApiClient, makeSpawner, makeHostOps, liveGroupPgids, killLiveGroupsSync };
 //# sourceMappingURL=main.d.ts.map

package/dist/bake-runner/main.js

@@ -1,13 +1,15 @@
-import { spawn, exec as execCb } from "node:child_process";
+import { spawn } from "node:child_process";
 import { readFile } from "node:fs/promises";
-import { promisify } from "node:util";
 import { createLogger } from "../observability/logger.js";
 import { parseNumOrFailNonNegative } from "../config.js";
 import { shellQuote as shellSafe } from "../plugins/remote-shell.js";
 import { BakeRunner, } from "./runner.js";
-const exec = promisify(execCb);
 const BAKE_CLAIM_INGEST_FETCH_TIMEOUT_MS = 30_000;
 const BAKE_HEARTBEAT_FETCH_TIMEOUT_MS = 10_000;
+const BAKE_HOST_PROBE_TIMEOUT_MS = 10_000;
+const BAKE_RECIPE_CHECKOUT_TIMEOUT_MS = 600_000;
+const BAKE_DOCKER_PUSH_TIMEOUT_MS = 1_800_000;
+const BAKE_HOST_EXEC_OUTPUT_CAP = 1024 * 1024;
 function loadEnv() {
     const need = (k) => {
         const v = process.env[k];
@@ -26,6 +28,7 @@ function loadEnv() {
         heartbeatMs: parseNumOrFailNonNegative("BAKE_HEARTBEAT_MS", process.env.BAKE_HEARTBEAT_MS || "30000"),
         idlePollMs: parseNumOrFailNonNegative("BAKE_IDLE_POLL_MS", process.env.BAKE_IDLE_POLL_MS || "5000"),
         minFreeGb: parseNumOrFailNonNegative("IMAGE_BAKE_MIN_FREE_GB", process.env.IMAGE_BAKE_MIN_FREE_GB || "20"),
+        shutdownGraceMs: parseNumOrFailNonNegative("BAKE_SHUTDOWN_GRACE_MS", process.env.BAKE_SHUTDOWN_GRACE_MS || "5000"),
     };
 }
 function makeApiClient(env, log) {
@@ -90,34 +93,104 @@ function makeApiClient(env, log) {
         },
     };
 }
-function makeSpawner() {
+const liveGroupPgids = new Set();
+function signalGroupQuietly(pgid, signal) {
+    if (pgid <= 0)
+        return;
+    try {
+        process.kill(-pgid, signal);
+    }
+    catch {
+    }
+}
+function pruneEmptyGroups() {
+    for (const pgid of [...liveGroupPgids]) {
+        if (!groupHasMembers(pgid))
+            liveGroupPgids.delete(pgid);
+    }
+}
+function killLiveGroupsSync() {
+    for (const pgid of liveGroupPgids)
+        signalGroupQuietly(pgid, "SIGKILL");
+    liveGroupPgids.clear();
+}
+function spawnGroupChild(opts) {
+    const child = spawn(opts.argv[0], opts.argv.slice(1), {
+        cwd: opts.cwd,
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe"],
+        env: process.env,
+    });
+    const pgid = child.pid ?? -1;
+    pruneEmptyGroups();
+    if (pgid > 0)
+        liveGroupPgids.add(pgid);
+    bufferLines(child.stdout, opts.onStdoutLine);
+    bufferLines(child.stderr, opts.onStderrLine);
+    const exited = new Promise((resolve) => {
+        child.on("exit", (code, signal) => {
+            pruneEmptyGroups();
+            resolve({ code, signal });
+        });
+        child.on("error", () => {
+            liveGroupPgids.delete(pgid);
+            resolve({ code: 127, signal: null });
+        });
+    });
     return {
-        spawn(opts) {
-            const child = spawn(opts.argv[0], opts.argv.slice(1), {
-                cwd: opts.cwd,
-                detached: true,
-                stdio: ["ignore", "pipe", "pipe"],
-                env: process.env,
-            });
-            const pgid = child.pid ?? -1;
-            bufferLines(child.stdout, opts.onStdoutLine);
-            bufferLines(child.stderr, opts.onStderrLine);
-            const exited = new Promise((resolve) => {
-                child.on("exit", (code, signal) => resolve({ code, signal }));
-                child.on("error", () => resolve({ code: 127, signal: null }));
-            });
-            const handle = {
-                pgid,
-                exited,
-                killGroup(signal) {
-                    if (pgid > 0)
-                        process.kill(-pgid, signal);
-                },
-            };
-            return handle;
+        pgid,
+        exited,
+        killGroup(signal) {
+            if (pgid > 0)
+                process.kill(-pgid, signal);
         },
+        groupAlive: () => groupHasMembers(pgid),
     };
 }
+function groupHasMembers(pgid) {
+    if (pgid <= 0)
+        return false;
+    try {
+        process.kill(-pgid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === "EPERM";
+    }
+}
+function makeSpawner() {
+    return { spawn: (opts) => spawnGroupChild(opts) };
+}
+export async function execGroup(cmd, timeoutMs) {
+    let stdout = "";
+    let stderr = "";
+    const append = (buf, line) => (buf.length >= BAKE_HOST_EXEC_OUTPUT_CAP ? buf : `${buf}${line}\n`);
+    const child = spawnGroupChild({
+        argv: ["/bin/sh", "-c", cmd],
+        onStdoutLine: (l) => {
+            stdout = append(stdout, l);
+        },
+        onStderrLine: (l) => {
+            stderr = append(stderr, l);
+        },
+    });
+    let timedOut = false;
+    const timer = setTimeout(() => {
+        timedOut = true;
+        signalGroupQuietly(child.pgid, "SIGKILL");
+    }, timeoutMs);
+    try {
+        const { code, signal } = await child.exited;
+        if (timedOut)
+            throw new Error(`host exec timed out after ${timeoutMs}ms (group killed): ${cmd.slice(0, 120)}`);
+        if (code !== 0)
+            throw new Error(`host exec exited ${code ?? signal}: ${cmd.slice(0, 120)}${stderr.trim() ? ` — ${stderr.trim().slice(-400)}` : ""}`);
+        return { stdout };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 function bufferLines(stream, onLine) {
     if (!stream)
         return;
@@ -143,18 +216,18 @@ function makeHostOps(env) {
             return JSON.parse(await readFile(path, "utf8"));
         },
         async dfData() {
-            const { stdout } = await exec(`df -P -BG ${shellSafe(env.dataDir)}`);
+            const { stdout } = await execGroup(`df -P -BG ${shellSafe(env.dataDir)}`, BAKE_HOST_PROBE_TIMEOUT_MS);
             return stdout;
         },
         async checkoutRecipe(ref) {
             if (ref && ref !== "HEAD") {
                 const dir = shellSafe(env.recipeDir);
                 try {
-                    await exec(`git -C ${dir} fetch --depth 1 origin ${shellSafe(ref)} && ` +
-                        `git -C ${dir} checkout --quiet --detach FETCH_HEAD`);
+                    await execGroup(`git -C ${dir} fetch --depth 1 origin ${shellSafe(ref)} && ` +
+                        `git -C ${dir} checkout --quiet --detach FETCH_HEAD`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS);
                 }
                 catch {
-                    await exec(`git -C ${dir} fetch origin && git -C ${dir} checkout --quiet --detach ${shellSafe(ref)}`);
+                    await execGroup(`git -C ${dir} fetch origin && git -C ${dir} checkout --quiet --detach ${shellSafe(ref)}`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS);
                 }
             }
             return env.recipeDir;
@@ -162,8 +235,8 @@ function makeHostOps(env) {
         async dockerPushRetry(tag, attempts) {
             for (let i = 0; i < attempts; i++) {
                 try {
-                    await exec(`docker push ${shellSafe(tag)}`);
-                    const { stdout } = await exec(`docker image inspect ${shellSafe(tag)} --format '{{index .RepoDigests 0}}'`);
+                    await execGroup(`docker push ${shellSafe(tag)}`, BAKE_DOCKER_PUSH_TIMEOUT_MS);
+                    const { stdout } = await execGroup(`docker image inspect ${shellSafe(tag)} --format '{{index .RepoDigests 0}}'`, BAKE_HOST_PROBE_TIMEOUT_MS);
                     const repoDigest = stdout.trim();
                     const digest = repoDigest.includes("@") ? repoDigest.slice(repoDigest.indexOf("@") + 1) : null;
                     return { ok: true, digest };
@@ -204,13 +277,24 @@ async function main() {
         heartbeatMs: env.heartbeatMs,
         idlePollMs: env.idlePollMs,
         minFreeGb: env.minFreeGb,
+        shutdownGraceMs: env.shutdownGraceMs,
     });
-    const shutdown = (sig) => {
+    process.once("exit", killLiveGroupsSync);
+    let laddering = false;
+    const onSignal = (sig, signo) => {
+        if (laddering) {
+            log.warn("bake_runner_stop_forced", { sig });
+            process.exit(128 + signo);
+        }
+        laddering = true;
         log.info("bake_runner_stop", { sig });
-        runner.stop();
+        void runner
+            .shutdown(sig)
+            .catch((err) => log.error("bake_runner_shutdown_error", { sig, err: String(err) }))
+            .finally(() => process.exit(128 + signo));
     };
-    process.on("SIGTERM", () => shutdown("SIGTERM"));
-    process.on("SIGINT", () => shutdown("SIGINT"));
+    process.on("SIGTERM", () => onSignal("SIGTERM", 15));
+    process.on("SIGINT", () => onSignal("SIGINT", 2));
     await runner.loop();
 }
 const invokedDirectly = process.argv[1] !== undefined && /bake-runner[/\\]main\.(ts|js)$/.test(process.argv[1]);
@@ -220,5 +304,5 @@ if (invokedDirectly) {
         process.exit(1);
     });
 }
-export { main, loadEnv, makeApiClient, makeSpawner, makeHostOps };
+export { main, loadEnv, makeApiClient, makeSpawner, makeHostOps, liveGroupPgids, killLiveGroupsSync };
 //# sourceMappingURL=main.js.map

package/dist/bake-runner/runner.d.ts

@@ -31,6 +31,10 @@ export interface ChildHandle {
     }>;
     /** Send a signal to the process GROUP (SIGTERM then SIGKILL on the cancel/deadline path). */
     killGroup(signal: NodeJS.Signals): void;
+    /** Does the process GROUP still have ANY member? (`kill(-pgid, 0)`.) 🔴 S-567 codex ①: this — not {@link exited} — is
+     *  the group's lifetime. `exited` resolves when the LEADER exits; a leader that dies on SIGTERM can leave a
+     *  TERM-immune descendant in the same group, and treating leader-exit as "the group is gone" lets it escape. */
+    groupAlive(): boolean;
 }
 /** Spawn `build.sh <argv> --json` detached in its own process group; stdout/stderr are delivered LINE-BY-LINE via
  *  the callbacks (the impl splits on newlines). The runner feeds ONLY stdout lines through the JSON parser; stderr
@@ -96,22 +100,75 @@ export interface BakeRunnerOpts {
     /** Minimum free GiB on /data at claim; below this the bake fails fast (507-class) rather than dying mid-build and
      *  leaving a half-layer (k3s + docker containerd both on /data — a fill triggers kubelet GC eviction — §P2.7). */
     minFreeGb?: number;
+    /** S-567 —— **一只注定要死的 build 进程组的「体面赴死窗」**(ms,`BAKE_SHUTDOWN_GRACE_MS`,默认 5000)。
+     *
+     *  两条路读同一个数,因为它们问的是**同一个问题**:
+     *   · cancel / deadline 击杀({@link BakeRunner.runOne} 的 supervisor):SIGTERM → 本窗 → SIGKILL;
+     *   · 关机梯子({@link BakeRunner.shutdown}):本窗同时是**整条梯子的总预算** —— 第②步(软杀→硬杀)花掉多少,
+     *     第③步(等 `runOne` 把终局 done 送出去)就只剩多少。于是对外只有一句话的契约:
+     *     **收到 SIGTERM/SIGINT 后,runner 在约 `BAKE_SHUTDOWN_GRACE_MS` 内退出**。
+     *
+     *  `0` = 不给窗,SIGTERM 之后立刻 SIGKILL(且梯子第③步没有预算 —— 终局 done 交给 image-api 的 lease reaper
+     *  兜底:reaper 记的是**失败**,不是把已完成的工作找回来)。 */
+    shutdownGraceMs?: number;
 }
 export declare class BakeRunner {
     private readonly o;
     private readonly heartbeatMs;
     private readonly idlePollMs;
     private readonly minFreeGb;
-    private stopped;
+    private readonly shutdownGraceMs;
+    /** ① of the ladder — the claim gate. Read at the loop head AND immediately above `spawner.spawn` (see `runOne`). */
+    private stopping;
+    /** The ladder itself, memoized ⇒ idempotent: N signals, one ladder, one awaitable. */
+    private ladder;
+    private inFlight;
+    /** The in-flight `runOne` (terminal fallback included) — what the ladder's ③ awaits so the bake lands. */
+    private inFlightRun;
+    /**
+     * 🔴 **在飞组杀梯子的属主**（修复验证轮）。三条路（cancel / deadline / 关机）起的梯子全登记在这里，
+     * 因为每一只都以一只 `clock.sleep` 作为升级的承重点 —— 而梯子的 ④ 是 `process.exit`，退出会把一切
+     * 未到点的定时器**取消**。只把关机自己发起的那只 await 住是不够的：取消 / deadline 发完 SIGTERM
+     * 正等升级时，领袖一退 `runOne` 就收车、`inFlight` 被清，关机于是看到「没有在飞的 build」立刻收敛，
+     * 那只待发的 SIGKILL 就死在退出上。这是 C2 同类，且是**本批引入的回归**：v7.93.5 的关机根本不退（只置
+     * flag），那只定时器总会到点。现在「在飞的组杀」有了属主，关机在预算内等它们走完。
+     */
+    private readonly liveKills;
     constructor(opts: BakeRunnerOpts);
-    /** Cooperative shutdown — finishes the in-flight bake's current frame, then exits the claim loop. */
-    stop(): void;
+    /**
+     * S-567 —— **关机梯子**：幂等、可 await、四步，一处承重。
+     *
+     * ── 病（车LD §2.3，28 次真进程×两套运行时、零反例）───────────────────────────────
+     * 旧 `stop()` 只置一个 flag，而 `loop()` 此刻正 `await this.runOne(...)`、`runOne` 正 `await child.exited`
+     * ⇒ 进程挂到在飞构建自然结束（30–45min 量级）。实测：SIGINT/SIGTERM 后 t+6s runner 与 build 组
+     * **全 alive**。运维看到的是「Ctrl-C 按了没反应」，于是只剩 `kill -9` —— 而 SIGKILL 恰好是**唯一留孤儿**
+     * 的那一形（build 组 `ppid=1`）。两只什么也不做的处理器，把人**引到了**病的入口。
+     *
+     * ── 梯子四步（判点坐标写在各步旁）────────────────────────────────────────
+     *  ① **停 claim**：`stopping = true`。在飞的那一次 claim 跟完，不再取下一条。
+     *  ② **在飞 build 组：软杀 → 宽限 → 硬杀**（{@link killGroupLadder}，与 cancel/deadline 同一只梯子）。
+     *  ③ **等 `runOne` 落地**：终局 `done{CANCELLED}` 送达 image-api（不当成功、也不当失败重试风暴 ——
+     *     CANCELLED 在 image-api 侧落成 coarse `failed` + errorCode `cancelled`，是既有闭集里的词，本批不新增词）。
+     *  ④ **退**：`process.exit(128+signo)` —— 这一步在 `main.ts`（进程面归入口文件，本文件不碰 `process`）。
+     *
+     * 幂等 = 记住梯子本人（不是一个 bool）：第二个信号 await 到**同一只** promise，梯子只跑一遍。
+     * 「第二个信号 = 直接硬退」那一臂在 `main.ts`（它才有 `process.exit`）。
+     */
+    shutdown(reason: string): Promise<void>;
+    private runLadder;
+    /**
+     * 起一只组杀梯子并**登记**它 —— cancel / deadline / 关机 三条路的**唯一**入口（一处承重）。
+     * 调用方可以 await 它（supervisor 就这么做），也可以不 await（关机的 ②）—— 不论哪种，
+     * 关机的 ③ 都能在 {@link liveKills} 里找到它并把它等完。
+     */
+    private startKill;
     /** The long-poll loop: claim → run → release → poll next. Single-flight is enforced DB-side (the claim CAS), so a
-     *  second runner replica simply never wins a claim. Runs until `stop()`. */
+     *  second runner replica simply never wins a claim. Runs until {@link shutdown} flips the claim gate. */
     loop(): Promise<void>;
     /** Run a single CLAIMED bake end-to-end. Owns the lineOrd counter, the heartbeat/cancel/deadline timers, the
      *  child process group, the stdout→ingest pump, and the terminal done synthesis. */
     runOne(bake: ClaimedBake): Promise<void>;
+    private runOneInner;
     /** Forward build.sh's own `done` line — but FIRST read the manifest BODY off disk and attach it so image-api can
      *  auto-register (build.sh's done carries only a build-host path, NOT the body — §P2.10). A manifest read failure is
      *  non-fatal: the done still forwards (the digest is on the line); image-api then registers indexId:null partial.
@@ -122,9 +179,30 @@ export declare class BakeRunner {
      *  with the recovered post-push digest (no manifest on disk — build.sh dies before the manifest write — so image-
      *  api registers the built-and-pushed-but-unregistered partial, which the P3 reconciliation tick finalizes). */
     private forwardDone;
-    /** Kill the process GROUP: SIGTERM, then SIGKILL shortly after (a SIGKILL mid-docker-build can leave dangling
-     *  layers; the dup-check + lockhash prevent a poisoned re-bake, and a startup workdir sweep reclaims them — §P2.11). */
-    private killGroup;
+    /**
+     * 组杀**梯子**：SIGTERM → 最多 `budgetMs` 等组死 → SIGKILL。**可 await，且在调用方的 await 链里跑完**。
+     *
+     * 🔴 为什么不能再挂定时器（车LD/codex C2，亲核旧码）：旧写法把 SIGKILL 升级挂在
+     * `void clock.sleep(5_000).then(...)` 上并立即返回 void。cancel/deadline 那条路上它碰巧能走到（进程还活着），
+     * 而关机路上 `process.exit()` 一到就把那只定时器**取消**了 ⇒ 一只 `trap '' TERM` 的后代照旧存活，
+     * 而读数是「已收割」。改成可 await 之后，升级在**退出之前**就已经发出去了。
+     *
+     * 两条路共用同一只梯子（cancel/deadline 与关机）—— 它们问的是同一个问题：“一只注定要死的组，
+     * 给它多久体面赴死”。`budgetMs <= 0` = 不给窗，SIGTERM 后立刻 SIGKILL。
+     *
+     * 射程如实记（本仓任何改法都够不着）：build.sh 的 `docker build` 把真正的构建提交给**宿主 dockerd**
+     * （挂载的 `/var/run/docker.sock`，在 pod cgroup 之外）—— 组杀收的是 docker **客户端**，宿主侧的构建
+     * 不受本进程信号支配（既有射程，非本批新开）。一次 SIGKILL 死在 docker build 中间会留悬挂层；
+     * dup-check + lockhash 保证重烤不被毒化，悬挂层本身由 docker 自己的 prune / 运维回收。
+     * （旧注声称“一条启动期的工作目录清扫会把它们收回去”—— **假补偿**：全 `src/` 无此实现，build.sh 侧也只有
+     * per-job workdir + `mktemp -d` 回退，没有任何启动期清扫 —— 车LD §3(3) 亲核，S-567 删句：过期的“已被兜住”
+     * 声明比欠账更危险。真要做清扫，那是带自己设计的另一台车。）
+     */
+    private killGroupLadder;
+    private signalGroup;
+    /** `p` 赢了 ⇒ true；`budgetMs` 先到（或预算已空）⇒ false。`p` 的异常不往外抛（梯子只关心「落地了没」）。 */
+    private raceBudget;
+    private remainingMs;
     private stopSupervisor;
     /** Pre-flight free-GiB read; null (fail-open) on a df parse/exec miss — a df hiccup must not block a bake (the
      *  reaper + a mid-build fill guard still protect the host). */

package/dist/bake-runner/runner.js

@@ -3,24 +3,68 @@ import { lineToFrame, dockerBuildTickFrame, synthDoneFrame, exitDoneFrame, deadl
 const DEFAULT_HEARTBEAT_MS = 30_000;
 const DEFAULT_IDLE_POLL_MS = 5_000;
 const DEFAULT_MIN_FREE_GB = 20;
+const DEFAULT_SHUTDOWN_GRACE_MS = 5_000;
+const GROUP_REAP_POLL_MS = 50;
 const TERMINAL_FALLBACK_ORD = 2_000_000_000;
 export class BakeRunner {
     o;
     heartbeatMs;
     idlePollMs;
     minFreeGb;
-    stopped = false;
+    shutdownGraceMs;
+    stopping = false;
+    ladder = null;
+    inFlight = null;
+    inFlightRun = null;
+    liveKills = [];
     constructor(opts) {
         this.o = opts;
         this.heartbeatMs = opts.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
         this.idlePollMs = opts.idlePollMs ?? DEFAULT_IDLE_POLL_MS;
         this.minFreeGb = opts.minFreeGb ?? DEFAULT_MIN_FREE_GB;
+        this.shutdownGraceMs = opts.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;
     }
-    stop() {
-        this.stopped = true;
+    shutdown(reason) {
+        this.ladder ??= this.runLadder(reason);
+        return this.ladder;
+    }
+    async runLadder(reason) {
+        const deadline = this.o.clock.now() + this.shutdownGraceMs;
+        this.stopping = true;
+        this.o.log.info("bake_runner_shutdown", { reason, graceMs: this.shutdownGraceMs, hasInFlight: this.inFlight !== null });
+        const inFlight = this.inFlight;
+        if (inFlight !== null && inFlight.killed === null) {
+            inFlight.killed = { kind: "cancelled", error: `bake aborted: bake-runner shutting down (${reason})` };
+            if (inFlight.child !== null)
+                this.startKill(inFlight.child, this.remainingMs(deadline));
+        }
+        const pending = [...this.liveKills];
+        if (this.inFlightRun !== null)
+            pending.push(this.inFlightRun);
+        if (pending.length > 0) {
+            const landed = await this.raceBudget(Promise.allSettled(pending).then(() => true), this.remainingMs(deadline));
+            if (!landed) {
+                this.o.log.warn("bake_shutdown_budget_exhausted", {
+                    reason,
+                    graceMs: this.shutdownGraceMs,
+                    liveKills: this.liveKills.length,
+                    runPending: this.inFlightRun !== null,
+                });
+            }
+        }
+    }
+    startKill(child, budgetMs) {
+        const p = this.killGroupLadder(child, budgetMs);
+        this.liveKills.push(p);
+        void p.finally(() => {
+            const i = this.liveKills.indexOf(p);
+            if (i >= 0)
+                this.liveKills.splice(i, 1);
+        });
+        return p;
     }
     async loop() {
-        while (!this.stopped) {
+        while (!this.stopping) {
             let claimed = null;
             try {
                 claimed = await this.o.api.claimNext();
@@ -29,21 +73,43 @@ export class BakeRunner {
                 this.o.log.warn("bake_claim_error", { err: String(err) });
             }
             if (!claimed) {
+                if (this.stopping)
+                    return;
                 await this.o.clock.sleep(this.idlePollMs);
                 continue;
             }
+            const bake = claimed;
+            const run = (async () => {
+                try {
+                    await this.runOne(bake);
+                }
+                catch (err) {
+                    this.o.log.error("bake_run_error", { bakeId: bake.bakeId, err: String(err) });
+                    await this.safeIngest(bake, exitDoneFrame(TERMINAL_FALLBACK_ORD, 1, String(err)));
+                }
+            })();
+            this.inFlightRun = run;
             try {
-                await this.runOne(claimed);
+                await run;
             }
-            catch (err) {
-                this.o.log.error("bake_run_error", { bakeId: claimed.bakeId, err: String(err) });
-                await this.safeIngest(claimed, exitDoneFrame(TERMINAL_FALLBACK_ORD, 1, String(err)));
+            finally {
+                this.inFlightRun = null;
             }
         }
     }
     async runOne(bake) {
         const ord = { n: 0 };
         this.o.log.info("bake_claimed", { bakeId: bake.bakeId, push: bake.push, dryRun: bake.dryRun, logs: bake.logs });
+        const inFlight = { child: null, killed: null };
+        this.inFlight = inFlight;
+        try {
+            await this.runOneInner(bake, ord, inFlight);
+        }
+        finally {
+            this.inFlight = null;
+        }
+    }
+    async runOneInner(bake, ord, inFlight) {
         let pump = Promise.resolve();
         const enqueueIngest = (frame) => {
             pump = pump.then(() => this.safeIngest(bake, frame));
@@ -77,6 +143,11 @@ export class BakeRunner {
         let sawDone = false;
         const buildShAbs = resolvePath(cwd, this.o.buildShPath);
         const buildArgv = [buildShAbs, ...bake.argv, "--json"];
+        if (this.stopping) {
+            this.o.log.info("bake_shutdown_before_spawn", { bakeId: bake.bakeId });
+            await this.safeIngest(bake, synthDoneFrame(nextOrd(ord), "cancelled", { error: "bake aborted: bake-runner shut down before the build started" }));
+            return;
+        }
         const child = this.o.spawner.spawn({
             argv: buildArgv,
             cwd,
@@ -100,20 +171,20 @@ export class BakeRunner {
                     stderrTail.push(raw);
             },
         });
+        inFlight.child = child;
         const deadlineMs = deadlineMsForProfile(bake.profile ?? "custom", bake.bands ?? null);
         const startedAt = this.o.clock.now();
-        let killedFor = null;
         let childExited = false;
         const supervisor = (async () => {
             while (true) {
                 await this.o.clock.sleep(this.heartbeatMs);
-                if (childExited || killedFor)
+                if (childExited || inFlight.killed)
                     return;
                 const elapsed = this.o.clock.now() - startedAt;
                 if (elapsed >= deadlineMs) {
-                    killedFor = "timeout";
+                    inFlight.killed = { kind: "timeout" };
                     this.o.log.warn("bake_deadline_kill", { bakeId: bake.bakeId, elapsedMs: elapsed, deadlineMs });
-                    this.killGroup(child);
+                    await this.startKill(child, this.shutdownGraceMs);
                     return;
                 }
                 let ack;
@@ -125,15 +196,15 @@ export class BakeRunner {
                     continue;
                 }
                 if (ack.cancelRequested) {
-                    killedFor = "cancelled";
+                    inFlight.killed = { kind: "cancelled" };
                     this.o.log.info("bake_cancel_kill", { bakeId: bake.bakeId });
-                    this.killGroup(child);
+                    await this.startKill(child, this.shutdownGraceMs);
                     return;
                 }
                 if (ack.leaseValid === false) {
-                    killedFor = "timeout";
+                    inFlight.killed = { kind: "timeout", error: "bake-runner lost its lease (reaped as stale) — build killed" };
                     this.o.log.warn("bake_lease_lost", { bakeId: bake.bakeId });
-                    this.killGroup(child);
+                    await this.startKill(child, this.shutdownGraceMs);
                     return;
                 }
                 enqueueIngest(dockerBuildTickFrame(nextOrd(ord), Math.floor(elapsed / 1000)));
@@ -150,8 +221,9 @@ export class BakeRunner {
         childExited = true;
         this.stopSupervisor(supervisor);
         await pump.catch(() => undefined);
-        if (killedFor) {
-            await this.safeIngest(bake, synthDoneFrame(nextOrd(ord), killedFor));
+        const killed = inFlight.killed;
+        if (killed) {
+            await this.safeIngest(bake, synthDoneFrame(nextOrd(ord), killed.kind, killed.error === undefined ? undefined : { error: killed.error }));
             return;
         }
         if (sawDone) {
@@ -193,19 +265,36 @@ export class BakeRunner {
         }
         await this.safeIngest(bake, frame);
     }
-    killGroup(child) {
+    async killGroupLadder(child, budgetMs) {
+        this.signalGroup(child, "SIGTERM");
+        const deadline = this.o.clock.now() + budgetMs;
+        while (child.groupAlive()) {
+            const left = deadline - this.o.clock.now();
+            if (left <= 0) {
+                this.signalGroup(child, "SIGKILL");
+                return;
+            }
+            await this.o.clock.sleep(Math.min(left, GROUP_REAP_POLL_MS));
+        }
+    }
+    signalGroup(child, signal) {
         try {
-            child.killGroup("SIGTERM");
+            child.killGroup(signal);
         }
         catch {
         }
-        void this.o.clock.sleep(5_000).then(() => {
-            try {
-                child.killGroup("SIGKILL");
-            }
-            catch {
-            }
-        });
+    }
+    async raceBudget(p, budgetMs) {
+        let landed = false;
+        const settled = p.then(() => (landed = true), () => (landed = true));
+        if (budgetMs <= 0) {
+            await Promise.resolve();
+            return landed;
+        }
+        return Promise.race([settled.then(() => true), this.o.clock.sleep(budgetMs).then(() => false)]);
+    }
+    remainingMs(deadline) {
+        return Math.max(0, deadline - this.o.clock.now());
     }
     stopSupervisor(p) {
         void p.catch(() => undefined);

package/dist/boot/exit-boundary.d.ts

@@ -0,0 +1,22 @@
+import type { Logger } from "../observability/logger.js";
+/** {@link installProcessExitBoundary} 的把手。 */
+export interface ProcessExitBoundary {
+    /**
+     * **优化点**,不承担不变量:受控退的收尾链在它自己的终点早按一次子进程腿(理由逐字写在
+     * `boot/shutdown.ts` 的调用点上 —— 那之后进程还活十秒量级,后台腿的软→硬梯子跑得完)。
+     * 工作区腿**刻意不给**这条早按:停机中段 `rm -rf` 会把在飞 turn 正在写的目录从它脚下抽掉。
+     */
+    reapChildProcessesEarly(): void;
+    /** 摘掉本次装配的 `exit` 钩。幂等(同进程内反复装配的调用方=测试必须用它收尾)。 */
+    dispose(): void;
+}
+/**
+ * 装上本进程的退出边界(`process.once("exit")`)。返回 {@link ProcessExitBoundary}。
+ *
+ * **进程级注册表是全局单例** ⇒ 装两次就是两只钩(两个写者)。生产里每条车道只装一次、装到死;
+ * 同进程内反复装配的调用方(测试)必须 `dispose()`。
+ */
+export declare function installProcessExitBoundary({ logger }: {
+    logger: Logger;
+}): ProcessExitBoundary;
+//# sourceMappingURL=exit-boundary.d.ts.map