@sema-agent/server 7.147.0 → 7.149.0

package/dist/plugins/local-docker-capacity.d.ts

@@ -0,0 +1,93 @@
+/** 帽配置(`config.remoteExec.capacity`,只在 local-docker 车道、且 `DOCKER_MAX_CONTAINERS` > 0 时在场)。 */
+export interface LocalDockerCapacityConfig {
+    /** `DOCKER_MAX_CONTAINERS`(> 0)。 */
+    readonly maxContainers: number;
+}
+/**
+ * 本 worker 的稳定身份 = 数据根绝对路径的 sha256 前 16 位 hex(标签值;不把路径本身写进 docker 元数据)。同一数据根上重起 ⇒ 同一身份;数据根是这份身份的唯一源,不另存 id。
+ * 数据根缺席 ⇒ 抛(装配错:local-docker 车道说不出这只 worker 是谁,失败关闭 —— 真实起服恒铸 `localDataRoot`)。
+ */
+export declare function localDockerWorkerOf(localDataRoot: string | undefined): string;
+/** 本 worker 的身份标签(`docker run --label` 的值、读数的过滤值)= `sema.local-docker=<worker>`。 */
+export declare function localDockerWorkerLabelOf(localDataRoot: string | undefined): string;
+/** daemon 列表里的一行(本 worker 标签下的一只容器)。 */
+export interface RegistryRow {
+    readonly name: string;
+    readonly state: string;
+}
+/** 一次读数:读出 ⇒ 行;读不出 ⇒ 原因(进拒句与运维行)。 */
+export type RegistryReading = {
+    readonly ok: true;
+    readonly rows: readonly RegistryRow[];
+} | {
+    readonly ok: false;
+    readonly reason: string;
+};
+/** 等待中的重读间隔(外部删除 / 别的写者只能靠重读看见)。 */
+export declare const REREAD_MS = 1000;
+/** 一次起动的结局:`present` = `docker run` 退 0;`absent` = 请求没到 daemon(二进制没起来);`unknown` = 其余一切(非零退出、被信号杀、超时、中止)。 */
+export type StartOutcome = "present" | "absent" | "unknown";
+/** 放行交回的票:起动落定后恰调一次 `settle`。 */
+export interface CapacityTicket {
+    settle(outcome: StartOutcome): void;
+}
+/** 拒:`full` = 读数证明满;`unconfirmed` = 读不出;`aborted` = 调用方 / env 的信号断了。 */
+export interface CapacityRefusal {
+    readonly verdict: "full" | "unconfirmed" | "aborted";
+    readonly message: string;
+}
+export type AdmitResult = {
+    readonly ok: true;
+    readonly ticket: CapacityTicket;
+} | {
+    readonly ok: false;
+    readonly refusal: CapacityRefusal;
+};
+/** `/health.sandboxCapacity` 的格(键集属主;`self-minted-wire.ts` 的 `healthSandboxCapacityKey` 按它登记)。 */
+export interface HealthSandboxCapacitySection {
+    readonly lane: "local-docker";
+    /** 上限(`DOCKER_MAX_CONTAINERS`)。 */
+    readonly limit: number;
+    /** 最近一次读数算出的在用数;还没读过 / 最近一次读不出 ⇒ `null`。 */
+    readonly inUse: number | null;
+    /** 最近一次读(读出或读不出)的时刻(ms epoch);还没读过 ⇒ `null`。 */
+    readonly readAt: number | null;
+    /** 此刻在等空位的起动数(提交门读它)。 */
+    readonly waiting: number;
+}
+/** 闸的只读面:提交门与 `/health` 读它。 */
+export interface SandboxCapacityView {
+    /** 此刻在等空位的起动数(看过一次「没空位」、还没起成也还没被拒的)。 */
+    waiting(): number;
+    health(): HealthSandboxCapacitySection;
+}
+/** 读 daemon 一次:界由闸给,信号是这一次准入的信号(env 销毁 / 连接被取消 ⇒ 读本身也断)。 */
+export type RegistryRead = (budgetMs: number, signal: AbortSignal | undefined) => Promise<RegistryReading>;
+export interface LocalDockerCapacity extends SandboxCapacityView {
+    /** 读的那只 worker 标签(装配时与适配器的 `workerLabel` 同一个值;拒句里给运维的过滤式用它)。 */
+    readonly label: string;
+    /**
+     * 一次起动的准入。`read(budgetMs)` = 调用方经自己的 docker 通道读一次 daemon(读的界由闸给);`deadline` = 调用方这只 env 的等待预算的终点(ms epoch);
+     * 预算已花完 ⇒ 只读一次,有空位就放、没有立拒。
+     */
+    admit(req: {
+        readonly name: string;
+        readonly read: RegistryRead;
+        readonly deadline: number;
+        readonly readBoundMs: number;
+        readonly signal?: AbortSignal;
+    }): Promise<AdmitResult>;
+    /** env 的 `docker rm -f` 被证实(退 0 / daemon 答 no such container):这只名出未定集、唤醒等待者。 */
+    removed(name: string): void;
+}
+interface CapacityLog {
+    warn(event: string, fields: Record<string, unknown>): void;
+}
+/** 建一只闸(一进程一只;帽关着 ⇒ 调用方不建)。`label` = {@link localDockerWorkerLabelOf};`now` 是钉的时钟缝。 */
+export declare function createLocalDockerCapacity(cfg: LocalDockerCapacityConfig, opts: {
+    readonly label: string;
+    readonly logger?: CapacityLog;
+    readonly now?: () => number;
+}): LocalDockerCapacity;
+export {};
+//# sourceMappingURL=local-docker-capacity.d.ts.map

package/dist/plugins/local-docker-capacity.js

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+export function localDockerWorkerOf(localDataRoot) {
+    if (localDataRoot === undefined || localDataRoot === "")
+        throw new Error("the local-docker lane needs the server's data root to name this worker (every container carries the worker label) — config.localDataRoot is absent");
+    return createHash("sha256").update(resolve(localDataRoot)).digest("hex").slice(0, 16);
+}
+export function localDockerWorkerLabelOf(localDataRoot) {
+    return `sema.local-docker=${localDockerWorkerOf(localDataRoot)}`;
+}
+const LIVE_STATES = new Set(["running", "paused", "restarting", "removing"]);
+export const REREAD_MS = 1000;
+export function createLocalDockerCapacity(cfg, opts) {
+    const now = opts.now ?? Date.now;
+    const label = opts.label;
+    const limit = cfg.maxContainers;
+    const filter = `label=${label}`;
+    const entries = new Map();
+    let seq = 0;
+    let last;
+    let waiting = 0;
+    let tail = Promise.resolve();
+    const wakers = new Set();
+    const wake = () => {
+        for (const w of [...wakers])
+            w();
+    };
+    async function measure(read, budgetMs, signal) {
+        const snapshot = new Map(entries);
+        const readSeq = ++seq;
+        let r;
+        try {
+            r = await read(budgetMs, signal);
+        }
+        catch (e) {
+            r = { ok: false, reason: e instanceof Error ? e.message : String(e) };
+        }
+        if (!r.ok) {
+            last = { inUse: null, at: now() };
+            return { unread: r.reason };
+        }
+        const listed = new Set(r.rows.map((row) => row.name));
+        const counted = new Set(r.rows.filter((row) => LIVE_STATES.has(row.state)).map((row) => row.name));
+        for (const [name, e] of snapshot) {
+            if (e.kind === "inflight" || listed.has(name) || e.settledSeq >= readSeq)
+                counted.add(name);
+        }
+        for (const [name, e] of [...entries]) {
+            if (e.kind === "unknown" && e.settledSeq < readSeq && !listed.has(name))
+                entries.delete(name);
+        }
+        last = { inUse: counted.size, at: now() };
+        return { inUse: counted.size };
+    }
+    function refusalOf(verdict, inUse, reason, waitedMs) {
+        const waited = `${Math.round(waitedMs / 100) / 10}s`;
+        const message = verdict === "full"
+            ? `local-docker sandbox capacity reached: ${inUse ?? limit} of ${limit} containers carrying this server's worker label are running (containers left by an earlier run on the same data root count too) — waited ${waited} for one to end; nothing was started (limit.rate_exceeded · sandbox_capacity). ` +
+                `Retry later, raise DOCKER_MAX_CONTAINERS, or remove containers left behind (docker ps -a --filter ${filter}).`
+            : `local-docker sandbox capacity could not be confirmed: the docker daemon did not list this worker's containers (${reason ?? "no answer"}) — waited ${waited}; nothing was started (limit.rate_exceeded · sandbox_capacity). Retry later.`;
+        opts.logger?.warn("local_docker_capacity_refused", { verdict, ...(inUse !== undefined ? { inUse } : {}), limit, waitedMs });
+        return { verdict, message };
+    }
+    function pause(ms, signal) {
+        return new Promise((resolve) => {
+            const done = (why) => {
+                clearTimeout(timer);
+                wakers.delete(onWake);
+                signal?.removeEventListener("abort", onAbort);
+                resolve(why);
+            };
+            const onWake = () => done("wake");
+            const onAbort = () => done("aborted");
+            const timer = setTimeout(() => done("timer"), Math.max(0, ms));
+            wakers.add(onWake);
+            if (signal?.aborted)
+                return done("aborted");
+            signal?.addEventListener("abort", onAbort, { once: true });
+        });
+    }
+    function queue(prev, deadline, signal) {
+        return new Promise((resolve) => {
+            let settled = false;
+            const done = (why) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                signal?.removeEventListener("abort", onAbort);
+                resolve(why);
+            };
+            const onAbort = () => done("aborted");
+            const timer = setTimeout(() => done("deadline"), Math.max(0, deadline - now()));
+            if (signal?.aborted)
+                return done("aborted");
+            signal?.addEventListener("abort", onAbort, { once: true });
+            void prev.then(() => done("turn"));
+        });
+    }
+    async function admit(req) {
+        const { name, read, deadline, readBoundMs, signal } = req;
+        const startedAt = now();
+        let release;
+        const prev = tail;
+        tail = new Promise((r) => (release = r));
+        let starved = false;
+        let lastInUse;
+        let lastUnread;
+        let turn = "deadline";
+        try {
+            const spent = deadline <= startedAt;
+            turn = await queue(prev, spent ? startedAt + readBoundMs : deadline, signal);
+            if (turn === "aborted")
+                return { ok: false, refusal: { verdict: "aborted", message: "local-docker container start aborted while waiting for sandbox capacity" } };
+            if (turn === "deadline")
+                return { ok: false, refusal: refusalOf(last?.inUse === null ? "unconfirmed" : "full", last?.inUse ?? undefined, last?.inUse === null ? "no reading" : undefined, now() - startedAt) };
+            for (let reads = 0;; reads++) {
+                const remaining = deadline - now();
+                if (reads > 0 && remaining <= 0)
+                    return { ok: false, refusal: refusalOf(lastUnread !== undefined ? "unconfirmed" : "full", lastInUse, lastUnread, now() - startedAt) };
+                const m = await measure(read, remaining > 0 ? Math.min(remaining, readBoundMs) : readBoundMs, signal);
+                if (signal?.aborted)
+                    return { ok: false, refusal: { verdict: "aborted", message: "local-docker container start aborted while waiting for sandbox capacity" } };
+                if ("inUse" in m) {
+                    lastInUse = m.inUse;
+                    lastUnread = undefined;
+                    if (m.inUse + 1 <= limit) {
+                        const entry = { kind: "inflight" };
+                        entries.set(name, entry);
+                        let once = false;
+                        return {
+                            ok: true,
+                            ticket: {
+                                settle: (outcome) => {
+                                    if (once || entries.get(name) !== entry)
+                                        return;
+                                    once = true;
+                                    if (outcome === "unknown")
+                                        entries.set(name, { kind: "unknown", settledSeq: seq });
+                                    else {
+                                        entries.delete(name);
+                                        if (outcome === "absent")
+                                            wake();
+                                    }
+                                },
+                            },
+                        };
+                    }
+                }
+                else {
+                    lastUnread = m.unread;
+                }
+                if (!starved) {
+                    starved = true;
+                    waiting++;
+                }
+                const left = deadline - now();
+                if (left <= 0)
+                    return { ok: false, refusal: refusalOf(lastUnread !== undefined ? "unconfirmed" : "full", lastInUse, lastUnread, now() - startedAt) };
+                const slice = Math.min(REREAD_MS, left);
+                const woke = await pause(slice, signal);
+                if (woke === "aborted")
+                    return { ok: false, refusal: { verdict: "aborted", message: "local-docker container start aborted while waiting for sandbox capacity" } };
+                if (woke === "timer" && slice === left)
+                    return { ok: false, refusal: refusalOf(lastUnread !== undefined ? "unconfirmed" : "full", lastInUse, lastUnread, now() - startedAt) };
+            }
+        }
+        finally {
+            if (starved)
+                waiting--;
+            if (turn === "turn")
+                release();
+            else
+                void prev.then(release);
+        }
+    }
+    return {
+        label,
+        admit,
+        removed(name) {
+            entries.delete(name);
+            wake();
+        },
+        waiting: () => waiting,
+        health: () => ({
+            lane: "local-docker",
+            limit,
+            inUse: last?.inUse ?? null,
+            readAt: last?.at ?? null,
+            waiting,
+        }),
+    };
+}
+//# sourceMappingURL=local-docker-capacity.js.map

package/dist/plugins/local-docker-reaper.d.ts

@@ -0,0 +1,92 @@
+/**
+ * local-docker 车道的无主容器回收器 —— 本 worker 起的容器、它的 run 已经终局而没人删的,由它删(停驻中取消 / 停驻过期 / 重起之后才结清的停驻:引擎不销毁停驻腿的环境,
+ * 本服务的取消与过期走店层结清、不经引擎仲裁器,于是那只容器一直在跑、一直占车道容器帽的一格;进程崩溃遗留的同理)。
+ *
+ * ## 律(一句)
+ * 容器的主人 = 它那条 run:run 行是活的(跑着 / 停驻着,还能被批准或续跑)⇒ 容器是它的手,不动;run 行**终局了、并且已经过了一段保持窗**({@link LOCAL_DOCKER_REAP_HOLD_MS})⇒ 无主,删。
+ * 拿不准恒不删(销毁不可逆;误删的代价是一个活 run 的工作区,漏删的代价只是一格容器帽):名读不回身份 / 查不到 run 行 / 读抛 / 词表外 / 终局时刻读不出 / 还在保持窗内 ⇒ 都不删。
+ *
+ * ## 为什么终局之后还要再等一段(而不是终局即删)
+ * 引擎在同一条腿里先把 `done` 帧交出去(本服务就在这一帧上把 run 行落成终局),**之后**才跑它的收尾:文件历史的边界捕获(从执行环境里读文件,自带 30 s 上限)、后台 shell 清扫、
+ * 子代清退、销毁执行环境本身(每条拆场腿至多 `teardownLegBudget()`)。终局行因此比这些用环境的收尾早 —— 终局即删会在捕获读到一半时把容器删了。保持窗远大于这一串收尾的上界;
+ * 窗过了容器还在 = 引擎自己的销毁没有发生(停驻中被取消 / 过期的环境没有销毁者、进程崩了、销毁没证实)—— 这才是无主。时刻读 run 行转终态那一拍(行上的 `updatedAt`,与 k8s pod 回收同一个钟)。
+ *
+ * ## 判据只有一个读法,删只有一个状态机
+ *  · 身份 = 容器名里读回的那只 run 号({@link LocalDockerReapPort.identityOf}:`localDockerContainerNameOf` 的逆,用正向函数验过摘要才认);
+ *    run 行 = 唯一属主 `runStatusClassOf`(词 → 类,本模块只写「类 → 删不删」那一句,与 E2B 孤儿回收 / k8s Pod 回收同一形:两向都是编译期问题)。
+ *  · 删 = {@link LocalDockerReapPort.settle}:交给适配器里**同一只**欠着机(`remote-env-local-docker.ts` 的 `LocalDockerCustody`:先记成欠着、按实例 id 删、证实才清);
+ *    本模块不碰 docker、不碰容器名以外的任何东西。「没证实」= 这一只仍欠着,留到下一轮(欠着的删除由工厂交给同一身份的下一只环境,谁都不附着将被删的那只)。
+ *  · 列举 = 车道容器帽读的那一只读数口(本 worker 标签下的全部容器):别的 worker 的容器不在列;列出之后名下被换成别人的 ⇒ 欠着机认领时按标签判「不是本 worker 的」,不动。
+ *
+ * ## 不覆盖的(读得出的边界)
+ *  · 委派子代的容器(身份 = 它自己的会话号,没有 run 行)与本版之前起的容器(名带随机尾,读不回身份):`run_unread` / `identity_unreadable` ⇒ 不动 ——
+ *    它们的主人不在 run 店里,要另一只读口才判得了;运维按标签过滤式手删。
+ *
+ * 每轮一行 `local_docker_reap_round`(`listed` / `scanned` / 按原因分的 `deleted` 与 `skipped` / `unconfirmed`),每删一只一行 `local_docker_reaped`,计数
+ * `local_docker_containers_reaped_total{reason}`;**列举失败**这一轮一只都不删 —— F 类兜底口 `server.local-docker-reaper.degraded`(`why=list_failed`)+ 一行 warn;
+ * **某一只删除证实不了**不挡其余 —— 同一只口(`why=delete_unconfirmed`)+ 一行 warn,下一轮再判再删。节拍 = 维护 tick 上的沙箱回收腿({@link SANDBOX_REAP_PERIOD_MS},起服后第一拍即跑)。
+ */
+import type { Logger } from "../observability/logger.js";
+import type { Metrics } from "../observability/metrics.js";
+import type { RegistryReading } from "./local-docker-capacity.js";
+import type { SandboxReaper, SandboxReaperRunReader } from "./sandbox-reaper.js";
+/**
+ * run 行转终态之后、回收器才认「无主」之前要再等的一段(毫秒)。非行为旋钮,常量:它只需要「远大于终局之后引擎那串收尾的上界(边界捕获 30 s + 每条拆场腿至多 `teardownLegBudget()` 15 s)、
+ * 远小于一次容器泄漏的代价」—— 5 min 对两端都有量级余量,错设它的运维事故面比它护住的窗口更大(与 E2B 孤儿回收的 create → 行落库窗同一条理由)。
+ */
+export declare const LOCAL_DOCKER_REAP_HOLD_MS: number;
+/** 删的原因(闭集;现只有一个:run 行已终局)。 */
+export type LocalDockerReapDeleteReason = "run_ended";
+/** 不动的原因(闭集):前八个是判决(没去删);`nothing_to_remove` 是判了删、收场时名下已没有我们删的(引擎自己先收了 / 名下是别人的)。 */
+export type LocalDockerReapSkipReason = "identity_unreadable" | "run_store_absent" | "run_unread" | "run_read_failed" | "run_status_unknown" | "run_live" | "run_end_unreadable" | "within_run_end_hold" | "nothing_to_remove";
+export type LocalDockerReapVerdict = {
+    readonly delete: LocalDockerReapDeleteReason;
+} | {
+    readonly skip: Exclude<LocalDockerReapSkipReason, "identity_unreadable" | "nothing_to_remove">;
+};
+/** 一只身份的 run 读数(闭集):读到行 / 店答没有这一行 / 读抛 / 本部署没有 run 店可问。 */
+export type LocalDockerRunReading = {
+    readonly found: {
+        readonly status: string;
+        readonly updatedAt: string;
+    };
+} | {
+    readonly absent: true;
+} | {
+    readonly failed: true;
+} | {
+    readonly storeAbsent: true;
+};
+/** 一只身份的容器删不删(纯判官;律见文件头):只读它那条 run 的读数。判决链的缺省臂是**不动作**:删必须被显式选中。名读不回身份的容器(`identity_unreadable`)连 run 都没得读,由调用方先挡下。 */
+export declare function judgeLocalDockerRun(run: LocalDockerRunReading, nowMs: number): LocalDockerReapVerdict;
+/** 回收器对 docker 一侧的窄面(真实装配 = `remote-env-local-docker.ts` 的 `localDockerExecutionEnvFactory(...).reapPort`;测试注入假面)。 */
+export interface LocalDockerReapPort {
+    /** 本 worker 的身份标签(`sema.local-docker=<worker>`):运维行里点名这一只 worker。 */
+    readonly label: string;
+    /** 本 worker 标签下的全部容器(与车道容器帽同一只读数口;读不出 ⇒ 原因)。 */
+    list(): Promise<RegistryReading>;
+    /** 容器名 → 它那只身份;不是本 worker 按律铸的名(旧版随机尾 / 可读形被截断 / 摘要不符)⇒ `undefined`。 */
+    identityOf(name: string): string | undefined;
+    /** 收这只身份的无主容器:交给同一只欠着机按实例 id 删。`removed` = 真删掉了一只;`nothing` = 删除证实了、但没有我们删的(名下已没有本 worker 的容器 / 是别人的);`owed` = 没证实,仍欠着(下一轮再删)。 */
+    settle(identity: string): Promise<"removed" | "nothing" | "owed">;
+}
+/** 一轮的读数(轮次行与计数的来源)。`deleted` / `skipped` 只列出现过的原因。 */
+export interface LocalDockerReapRound {
+    readonly listed: boolean;
+    readonly scanned: number;
+    readonly deleted: Partial<Record<LocalDockerReapDeleteReason, number>>;
+    readonly skipped: Partial<Record<LocalDockerReapSkipReason, number>>;
+    /** 判了删、删除没证实(仍欠着)的只数。 */
+    readonly unconfirmed: number;
+}
+export interface LocalDockerReaper extends SandboxReaper {
+    reapOnce(runs: SandboxReaperRunReader | undefined): Promise<LocalDockerReapRound>;
+}
+export declare function createLocalDockerReaper(deps: {
+    readonly port: LocalDockerReapPort;
+    readonly logger?: Pick<Logger, "info" | "warn">;
+    readonly metrics?: Pick<Metrics, "inc">;
+    /** 钟缝(测试;缺省 `Date.now`)。 */
+    readonly now?: () => number;
+}): LocalDockerReaper;
+//# sourceMappingURL=local-docker-reaper.d.ts.map

package/dist/plugins/local-docker-reaper.js

@@ -0,0 +1,102 @@
+import { recordFailOpen } from "../observability/fail-open.js";
+import { runStatusClassOf } from "./store-contracts.js";
+export const LOCAL_DOCKER_REAP_HOLD_MS = 5 * 60_000;
+const CLASS_VERDICT = {
+    live: "live",
+    parked: "live",
+    succeeded: "terminal",
+    unsuccessful: "terminal",
+};
+export function judgeLocalDockerRun(run, nowMs) {
+    if ("storeAbsent" in run)
+        return { skip: "run_store_absent" };
+    if ("failed" in run)
+        return { skip: "run_read_failed" };
+    if ("absent" in run)
+        return { skip: "run_unread" };
+    const cls = runStatusClassOf(run.found.status);
+    if (cls === undefined)
+        return { skip: "run_status_unknown" };
+    if (CLASS_VERDICT[cls] === "live")
+        return { skip: "run_live" };
+    const endedAt = Date.parse(run.found.updatedAt);
+    if (!Number.isFinite(endedAt))
+        return { skip: "run_end_unreadable" };
+    if (nowMs - endedAt <= LOCAL_DOCKER_REAP_HOLD_MS)
+        return { skip: "within_run_end_hold" };
+    return { delete: "run_ended" };
+}
+export function createLocalDockerReaper(deps) {
+    const { port } = deps;
+    const now = deps.now ?? Date.now;
+    return {
+        async reapOnce(runs) {
+            const deleted = {};
+            const skipped = {};
+            let unconfirmed = 0;
+            let listing;
+            try {
+                listing = await port.list();
+            }
+            catch (e) {
+                listing = { ok: false, reason: e instanceof Error ? e.message : String(e) };
+            }
+            if (!listing.ok) {
+                recordFailOpen("server.local-docker-reaper.degraded", `why=list_failed worker=${port.label} ${listing.reason}`);
+                deps.logger?.warn("local_docker_reap_list_failed", {
+                    worker: port.label,
+                    error: listing.reason,
+                    effect: "containers of ended runs were not deleted this round — they keep holding a slot of DOCKER_MAX_CONTAINERS until a later round lists them (one period later)",
+                });
+                return { listed: false, scanned: 0, deleted, skipped, unconfirmed };
+            }
+            const at = now();
+            const reads = new Map();
+            const readRun = (identity) => {
+                if (runs === undefined)
+                    return Promise.resolve({ storeAbsent: true });
+                let r = reads.get(identity);
+                if (r === undefined) {
+                    r = runs.getRun(identity).then((row) => (row === undefined || row === null ? { absent: true } : { found: row }), () => ({ failed: true }));
+                    reads.set(identity, r);
+                }
+                return r;
+            };
+            for (const row of listing.rows) {
+                const identity = port.identityOf(row.name);
+                if (identity === undefined) {
+                    skipped.identity_unreadable = (skipped.identity_unreadable ?? 0) + 1;
+                    continue;
+                }
+                const verdict = judgeLocalDockerRun(await readRun(identity), at);
+                if ("skip" in verdict) {
+                    skipped[verdict.skip] = (skipped[verdict.skip] ?? 0) + 1;
+                    continue;
+                }
+                const settled = await port.settle(identity).catch(() => "owed");
+                if (settled === "nothing") {
+                    skipped.nothing_to_remove = (skipped.nothing_to_remove ?? 0) + 1;
+                    continue;
+                }
+                if (settled === "owed") {
+                    unconfirmed++;
+                    recordFailOpen("server.local-docker-reaper.degraded", `why=delete_unconfirmed worker=${port.label} container=${row.name}`);
+                    deps.logger?.warn("local_docker_reap_unconfirmed", {
+                        worker: port.label,
+                        container: row.name,
+                        reason: verdict.delete,
+                        effect: "docker did not confirm the removal — the container stays (and keeps its DOCKER_MAX_CONTAINERS slot); it is owed, never attached again, and the next round deletes it first",
+                    });
+                    continue;
+                }
+                deleted[verdict.delete] = (deleted[verdict.delete] ?? 0) + 1;
+                deps.metrics?.inc("local_docker_containers_reaped_total", { reason: verdict.delete });
+                deps.logger?.info("local_docker_reaped", { worker: port.label, container: row.name, reason: verdict.delete });
+            }
+            const round = { listed: true, scanned: listing.rows.length, deleted, skipped, unconfirmed };
+            deps.logger?.info("local_docker_reap_round", { worker: port.label, ...round });
+            return round;
+        },
+    };
+}
+//# sourceMappingURL=local-docker-reaper.js.map

package/dist/plugins/local-engine-high-water.d.ts

@@ -1,11 +1,34 @@
+import { type DataRootFileText } from "./local-session-store.js";
+import { type ReleaseTriple } from "./release-version.js";
 /** marker 的文件名(数据根顶层;公开契约)。 */
 export declare const ENGINE_HIGH_WATER_FILE = "engine-high-water.json";
 /**
- * **判与抬**(唯一的座;唯一调用点 `openLocalBackend`,数据根锁在手、本仓任何一只店建起之前):
- *  ① 本构建版本 `engine` 按发布线读不出({@link buildReleaseTriple})⇒ 抛 {@link StoreBootRefusalError}(marker 不读、不写);
- *  ② 读 `<root>/engine-high-water.json`:读不出 / 高水位高于本构建 ⇒ 抛 {@link LocalDataRootSchemaMismatchError}(`boot` 形;文件原地不动,不抬);
- *  ③ 缺席或低于本构建 ⇒ 原子写 `{"highWater": <本构建的纯三段>}`(`required`);相等 ⇒ 不写,只过持久化屏障(读到的可能是上一次没证完的那份)。写不进 / 证不了 ⇒ 抛(点名路径与原因)。
- * 同步:它在后端打开的同步段里跑(建店之前),拒在这里就不会有一只店被建起。
+ * 准入判出的高水位(**读数**,不是活物):{@link raiseEngineHighWater} 按它抬。唯一的铸点 = {@link judgeEngineHighWater} / 空根那一格的 {@link ownReleaseLine}
+ * (都在数据根准入 `local-data-root-admission.ts` 的 `admitLocalDataRoot` 里)。
  */
-export declare function holdEngineHighWater(root: string, engine: string): void;
+export interface EngineHighWaterReading {
+    /** 本构建的发布线(纯三段;判与抬都按它)。 */
+    readonly own: ReleaseTriple;
+    /** marker 记的高水位(缺席 = `undefined`);跳版拒启臂读它。 */
+    readonly prior: ReleaseTriple | undefined;
+}
+/**
+ * 本构建的发布线(判与抬都按它):`engine` 按发布线读不出({@link buildReleaseTriple})⇒ 抛 {@link StoreBootRefusalError} —— 判不了「不比打开过这只根的引擎旧」,
+ * 也写不出自己的水位(坏包,不服务这只根;数据根上有没有东西都一样拒)。
+ */
+export declare function ownReleaseLine(root: string, engine: string): ReleaseTriple;
+/** marker 在一只(物理)数据根下的路径。 */
+export declare function engineHighWaterPath(root: string): string;
+/**
+ * **判**(纯;唯一调用点 = 数据根准入:判的是准入那一次观察里 marker 的字节 `raw`,不自己读盘):
+ *  ① 本构建版本按发布线读不出 ⇒ 抛({@link ownReleaseLine});
+ *  ② marker 读不出 / 高水位高于本构建 ⇒ 抛 {@link LocalDataRootSchemaMismatchError}(`boot` 形);
+ *  ③ 否则交回读数(缺席 / 低于 / 等于)。
+ */
+export declare function judgeEngineHighWater(root: string, raw: DataRootFileText, engine: string): EngineHighWaterReading;
+/**
+ * **抬**(锁内;紧随准入的「数据根没动」核对与跳版臂的记账之后、本仓任何一只店建起之前):缺席或低于本构建 ⇒ 原子写 `{"highWater": <本构建的纯三段>}`(`required`);
+ * 相等 ⇒ 不写,只过持久化屏障(读到的可能是上一次没证完的那份)。写不进 / 证不了 ⇒ 抛(点名路径与原因)。
+ */
+export declare function raiseEngineHighWater(physicalRoot: string, admitted: EngineHighWaterReading): void;
 //# sourceMappingURL=local-engine-high-water.d.ts.map

package/dist/plugins/local-engine-high-water.js

@@ -2,7 +2,7 @@ import { join, resolve } from "node:path";
 import { z } from "zod";
 import { atomicWriteFileSync, proveDurableSync } from "./owner-only-file-mode.js";
 import { createZodColumnReader, decodeJsonColumn } from "./sql-json-column.js";
-import { LocalDataRootSchemaMismatchError, dataRootMismatchOf, readDataRootFileTextSync } from "./local-session-store.js";
+import { LocalDataRootSchemaMismatchError, dataRootMismatchOf } from "./local-session-store.js";
 import { buildReleaseTriple, compareReleaseTriples, releaseText, releaseTriple } from "./release-version.js";
 import { StoreBootRefusalError } from "./store-contracts.js";
 export const ENGINE_HIGH_WATER_FILE = "engine-high-water.json";
@@ -19,37 +19,45 @@ const ENGINE_HIGH_WATER_SHAPE = z
     return { highWater: v.highWater, triple };
 });
 const readEngineHighWater = createZodColumnReader(ENGINE_HIGH_WATER_SHAPE);
-export function holdEngineHighWater(root, engine) {
+export function ownReleaseLine(root, engine) {
     const own = buildReleaseTriple(engine);
     if (own === undefined) {
         throw new StoreBootRefusalError(`DB_BACKEND=local: this build's own version (package.json "version": ${JSON.stringify(engine.slice(0, 64))}) is not a release version (x.y.z, optionally with a prerelease or build suffix) — refusing to open the data root ${root}: ` +
             "the engine high-water fence needs this build's release line to show it is not older than the engines that opened this data root, and to record its own. " +
             "Run a build whose package.json carries a release version.");
     }
-    const path = resolve(join(root, ENGINE_HIGH_WATER_FILE));
-    const raw = readDataRootFileTextSync(path);
-    let equal = false;
+    return own;
+}
+export function engineHighWaterPath(root) {
+    return resolve(join(root, ENGINE_HIGH_WATER_FILE));
+}
+export function judgeEngineHighWater(root, raw, engine) {
+    const own = ownReleaseLine(root, engine);
+    const path = engineHighWaterPath(root);
     if (raw.state === "unreadable")
         throw new LocalDataRootSchemaMismatchError([dataRootMismatchOf(path, ENGINE_HIGH_WATER_LABEL, raw.fault)], "boot");
-    if (raw.state === "text") {
-        let marker;
-        try {
-            marker = decodeJsonColumn(raw.text, readEngineHighWater, ENGINE_HIGH_WATER_LABEL);
-        }
-        catch (e) {
-            throw new LocalDataRootSchemaMismatchError([dataRootMismatchOf(path, ENGINE_HIGH_WATER_LABEL, e)], "boot");
-        }
-        const order = compareReleaseTriples(marker.triple, own);
-        if (order > 0) {
-            throw new LocalDataRootSchemaMismatchError([{ path, label: ENGINE_HIGH_WATER_LABEL, problem: "newer-engine", unknownFields: [], engineHighWater: { highWater: marker.highWater, engine } }], "boot");
-        }
-        equal = order === 0;
+    if (raw.state === "absent")
+        return { own, prior: undefined };
+    let marker;
+    try {
+        marker = decodeJsonColumn(raw.text, readEngineHighWater, ENGINE_HIGH_WATER_LABEL);
     }
+    catch (e) {
+        throw new LocalDataRootSchemaMismatchError([dataRootMismatchOf(path, ENGINE_HIGH_WATER_LABEL, e)], "boot");
+    }
+    if (compareReleaseTriples(marker.triple, own) > 0) {
+        throw new LocalDataRootSchemaMismatchError([{ path, label: ENGINE_HIGH_WATER_LABEL, problem: "newer-engine", unknownFields: [], engineHighWater: { highWater: marker.highWater, engine } }], "boot");
+    }
+    return { own, prior: marker.triple };
+}
+export function raiseEngineHighWater(physicalRoot, admitted) {
+    const path = engineHighWaterPath(physicalRoot);
+    const equal = admitted.prior !== undefined && compareReleaseTriples(admitted.prior, admitted.own) === 0;
     try {
         if (equal)
             proveDurableSync(path);
         else
-            atomicWriteFileSync(path, `${JSON.stringify({ highWater: releaseText(own) })}\n`, { durability: "required" });
+            atomicWriteFileSync(path, `${JSON.stringify({ highWater: releaseText(admitted.own) })}\n`, { durability: "required" });
     }
     catch (e) {
         throw new Error(`DB_BACKEND=local cannot record the engine high-water ${path} durably: ${e instanceof Error ? e.message : String(e)} — refusing to open the data root: ` +