@sema-agent/server 7.131.0 → 7.133.0

package/dist/plugins/remote-env-k8s.js

@@ -11,13 +11,14 @@ import { spaceCommitAlive } from "../workspace-space-commit.js";
 import { incarnationLogPrefix, redactIncarnations } from "../session-incarnation.js";
 import { WORKSPACE_CACHE_SPACE_LABEL, workspaceCacheClaimOf, workspaceCacheEnv } from "../workspace-cache.js";
 import { recordFailOpen } from "../observability/fail-open.js";
-import { commandBoundsOf, laneLivenessOf, livenessCutMessage, LIVENESS_MS_DEFAULT } from "../exec-bounds.js";
+import { buildLivenessCut, commandBoundsOf, laneLivenessOf, LIVENESS_MS_DEFAULT } from "../exec-bounds.js";
 import { SANDBOX_IPV6_DEFAULT, sandboxEgressGaps, sandboxPodSysctls } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
 import { fileErrorFromExec, classifyFsStderr } from "./remote-env-file-error.js";
 import { createPosixShellFs } from "./posix-shell-fs.js";
 import { randomUUID } from "node:crypto";
 import { BackgroundShellManager, seedMemStream, feedMemStream, drainMemStream } from "./background-shell-support.js";
+import { LANE_HOME_PROBE_COMMAND, LaneHomeSeat } from "../lane-home-probe.js";
 const PROVIDER = "k8s";
 const K8S_BG_DIR_ROOT = "/tmp/.sema-bg";
 const K8S_BG_MAX_CONCURRENT = numEnvOr("K8S_BG_MAX_CONCURRENT", 8, 1);
@@ -26,8 +27,8 @@ const K8S_BG_MAX_TIMEOUT_SEC = numEnvOr("K8S_BG_MAX_TIMEOUT_SEC", 1800, K8S_BG_D
 const K8S_BG_READ_CAP = numEnvOr("K8S_BG_READ_CAP", 1024 * 1024, 64 * 1024);
 const K8S_BG_FILE_CAP = numEnvOr("K8S_BG_FILE_CAP", 64 * 1024 * 1024, K8S_BG_READ_CAP);
 const K8S_BG_MEM_CAP = numEnvOr("K8S_BG_MEM_CAP", 8 * 1024 * 1024, 64 * 1024);
-import { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, parseBgPollOutput, } from "./k8s-bg-scripts.js";
-export { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, parseBgPollOutput, } from "./k8s-bg-scripts.js";
+import { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, buildKillAllScript, KILL_ALL_RECHECK_MS, parseBgPollOutput, } from "./k8s-bg-scripts.js";
+export { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, buildKillAllScript, parseBgPollOutput, } from "./k8s-bg-scripts.js";
 const ok = (value) => ({ ok: true, value });
 const unsupported = (op) => ({
     ok: false,
@@ -116,6 +117,45 @@ class FenceCallAborted extends Error {
 function dataTimeoutMsOf(config) {
     return config.dataTimeoutMs ?? WORKSPACE_SNAPSHOT_TIMEOUT_MS_DEFAULT;
 }
+const RPC_TIMEOUT_MS_DEFAULT = 30_000;
+export function rpcTimeoutMsOf(config) {
+    return config.rpcTimeoutMs ?? RPC_TIMEOUT_MS_DEFAULT;
+}
+export function boundedStoreCall(op, bound) {
+    const signal = bound.abort?.signal;
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return reject(bound.abort.error());
+        const settle = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", onAbort);
+        };
+        const onAbort = () => {
+            settle();
+            reject(bound.abort.error());
+        };
+        const timer = setTimeout(() => {
+            settle();
+            reject(bound.onTimeout());
+        }, bound.ms);
+        signal?.addEventListener("abort", onAbort, { once: true });
+        let pending;
+        try {
+            pending = op();
+        }
+        catch (e) {
+            settle();
+            return reject(e);
+        }
+        pending.then((v) => {
+            settle();
+            resolve(v);
+        }, (e) => {
+            settle();
+            reject(e);
+        });
+    });
+}
 import { execSocketTlsOptions, exitCodeFromStatus } from "./k8s-exec-protocol.js";
 export { execSocketTlsOptions } from "./k8s-exec-protocol.js";
 export function errorMessageOf(e) {
@@ -134,10 +174,14 @@ export class RemoteK8sExecutionEnv {
     podIP;
     destroyed = false;
     handleSnapshotId;
-    inFlightExecs = 0;
-    connectPromise;
-    restoreRefusal;
-    homeDir;
+    inFlight = new Set();
+    idleWaiters = [];
+    connectInFlight;
+    connectRefusal;
+    get homeDir() {
+        return this.homeSeat.home;
+    }
+    homeSeat;
     cdPath = "unknown";
     cfg;
     deps;
@@ -150,6 +194,7 @@ export class RemoteK8sExecutionEnv {
     cacheUnavailableSaid = false;
     logger;
     onWorkspaceSnapshot;
+    cacheMisses;
     constructor(config, deps) {
         this.cfg = {
             apiUrl: config.apiUrl,
@@ -162,7 +207,7 @@ export class RemoteK8sExecutionEnv {
             mountPath: config.mountPath ?? K8S_WORKSPACE_MOUNT_DEFAULT,
             timeoutMs: config.timeoutMs ?? 30 * 60_000,
             readyTimeoutMs: config.readyTimeoutMs ?? 120_000,
-            rpcTimeoutMs: config.rpcTimeoutMs ?? 30_000,
+            rpcTimeoutMs: rpcTimeoutMsOf(config),
             livenessMs: config.livenessMs ?? LIVENESS_MS_DEFAULT,
             dataTimeoutMs: dataTimeoutMsOf(config),
             cpu: config.cpu ?? "1",
@@ -176,7 +221,7 @@ export class RemoteK8sExecutionEnv {
             ...(config.snapshotExclude && config.snapshotExclude.length > 0 ? { snapshotExclude: config.snapshotExclude } : {}),
         };
         this.cwd = this.cfg.mountPath;
-        this.homeDir = config.homeDir;
+        this.homeSeat = new LaneHomeSeat({ lane: PROVIDER, declared: config.homeDir, verifiesDeclared: true, onAbsent: config.onHomeAbsent });
         this.deps = deps ?? {};
         this.laneLiveness = laneLivenessOf(this.cfg.livenessMs, "K8S_LIVENESS_MS");
         const occupant = occupantKeyOf(config.spaceOccupant);
@@ -185,6 +230,7 @@ export class RemoteK8sExecutionEnv {
         this.cacheLane = cacheLaneOf(config.cache, config.spaceOccupant);
         this.logger = config.logger;
         this.onWorkspaceSnapshot = config.onWorkspaceSnapshot;
+        this.cacheMisses = config.cacheMisses;
         this.capabilities = { isolation: true, suspendable: !!this.cfg.s3Snapshot, canonicalPathAuthoritative: true };
         const bgMax = Math.max(1, Math.min(K8S_BG_MAX_TIMEOUT_SEC, Math.floor(this.cfg.timeoutMs / 1000)));
         this.backgroundCapabilities = {
@@ -283,7 +329,15 @@ export class RemoteK8sExecutionEnv {
             ...(url.startsWith("wss:") ? execSocketTlsOptions(ca, this.cfg.insecureTls) : {}),
         });
     }
-    async connect(config) {
+    connect(config) {
+        this.connectInFlight ??= this.establish(config).finally(() => {
+            this.connectInFlight = undefined;
+        });
+        return this.connectInFlight;
+    }
+    async establish(config) {
+        if (this.connectRefusal)
+            return this.connectRefusal;
         const signal = config?.abortSignal;
         const lane = this.spaceLane;
         switch (lane.state) {
@@ -295,8 +349,6 @@ export class RemoteK8sExecutionEnv {
             case "unreadable":
                 return this.provision(signal);
             case "open": {
-                if (this.restoreRefusal)
-                    return this.restoreRefusal;
                 const settled = await this.awaitSpaceCommit(lane.space, signal);
                 if (!settled.ok)
                     return settled;
@@ -322,27 +374,10 @@ export class RemoteK8sExecutionEnv {
         await this.fenceCall(() => space.fence.end(), "delete").catch(() => undefined);
     }
     fenceCall(op, what, signal) {
-        return new Promise((resolve, reject) => {
-            if (signal?.aborted)
-                return reject(new FenceCallAborted());
-            const onAbort = () => {
-                clearTimeout(timer);
-                reject(new FenceCallAborted());
-            };
-            const timer = setTimeout(() => {
-                signal?.removeEventListener("abort", onAbort);
-                reject(new Error(`workspace space commit state ${what} did not answer within ${this.cfg.rpcTimeoutMs}ms`));
-            }, this.cfg.rpcTimeoutMs);
-            signal?.addEventListener("abort", onAbort, { once: true });
-            op().then((v) => {
-                clearTimeout(timer);
-                signal?.removeEventListener("abort", onAbort);
-                resolve(v);
-            }, (e) => {
-                clearTimeout(timer);
-                signal?.removeEventListener("abort", onAbort);
-                reject(e);
-            });
+        return boundedStoreCall(op, {
+            ms: this.cfg.rpcTimeoutMs,
+            onTimeout: () => new Error(`workspace space commit state ${what} did not answer within ${this.cfg.rpcTimeoutMs}ms`),
+            ...(signal ? { abort: { signal, error: () => new FenceCallAborted() } } : {}),
         });
     }
     sayUnavailable(reason) {
@@ -355,6 +390,21 @@ export class RemoteK8sExecutionEnv {
             effect: "WORKSPACE_PERSIST=space, but the session store has no occupant (incarnation) for this run's session — its /workspace is neither restored nor saved (it is never keyed by the session id instead)",
         });
     }
+    holdRefusal(error) {
+        this.connectRefusal = { ok: false, error };
+        return this.connectRefusal;
+    }
+    refusePod(pod, phase, reason, message) {
+        this.logger?.warn("sandbox_pod_refused", {
+            pod,
+            phase,
+            reason: reason ?? "?",
+            ...(message ? { detail: message.slice(0, 300) } : {}),
+            ...(this.cfg.sessionId ? { sessionId: this.cfg.sessionId } : {}),
+            effect: "the cluster created this run's sandbox pod and it ended before it ran — this run's operations answer the same refusal without creating another pod; a retry of the run builds a fresh sandbox",
+        });
+        return this.holdRefusal(new RemoteExecutionError("connect_failed", `pod entered ${phase} before Running (${reason ?? "?"}) — the cluster refused this run's sandbox pod; the refusal stands for the run (no pod is created again), a retry of the run asks the cluster afresh`));
+    }
     refuseUnreadableOccupant(code) {
         const ways = [...(this.spaceLane.state === "unreadable" ? ["WORKSPACE_PERSIST=off"] : []), ...(this.cacheLane.state === "unreadable" ? ["WORKSPACE_CACHE=off"] : [])].join(" and ");
         if (!this.unreadableOccupantSaid) {
@@ -374,6 +424,8 @@ export class RemoteK8sExecutionEnv {
     provision(signal) {
         const space = this.spaceLane;
         const cache = this.cacheLane;
+        if (this.connectRefusal)
+            return Promise.resolve(this.connectRefusal);
         if (space.state === "unreadable")
             return Promise.resolve(this.refuseUnreadableOccupant(space.code));
         if (cache.state === "unreadable")
@@ -384,19 +436,23 @@ export class RemoteK8sExecutionEnv {
             backoffMs: (failed) => this.cfg.retry.backoffMs * failed,
         };
         const once = this.deps.connectOnce ?? ((s) => this.connectOnce(cache, s));
-        return withRetry(() => once(signal), policy, { signal });
+        const refused = new AbortController();
+        const halt = signal ? AbortSignal.any([signal, refused.signal]) : refused.signal;
+        return withRetry(async () => {
+            const r = await once(signal);
+            if (!r.ok && this.connectRefusal)
+                refused.abort();
+            return r;
+        }, policy, { signal: halt });
     }
     async ensureConnected() {
-        if (this.podName)
-            return ok(undefined);
         if (this.destroyed)
             return { ok: false, error: new ExecutionError("shell_unavailable", "k8s sandbox destroyed") };
-        this.connectPromise ??= this.connect();
-        const r = await this.connectPromise;
-        if (!r.ok) {
-            this.connectPromise = undefined;
+        if (this.podName && this.connectInFlight === undefined)
+            return ok(undefined);
+        const r = await this.connect();
+        if (!r.ok)
             return { ok: false, error: new ExecutionError("shell_unavailable", `k8s sandbox connect failed: ${r.error.message}`, r.error) };
-        }
         return ok(undefined);
     }
     async connectOnce(cacheLane, signal) {
@@ -409,6 +465,8 @@ export class RemoteK8sExecutionEnv {
         if (!("cacheMiss" in first))
             return first;
         this.cacheUnavailable(first.cacheMiss, first.detail, cache);
+        if (cache)
+            this.cacheMisses?.remember(cache.spaceKey, { reason: first.cacheMiss, detail: first.detail }, this.cfg.dataTimeoutMs);
         const second = await this.createPod(signal, undefined);
         return "cacheMiss" in second ? { ok: false, error: new RemoteExecutionError("connect_failed", "unreachable: a pod without the cache volume was given up for it") } : second;
     }
@@ -420,6 +478,11 @@ export class RemoteK8sExecutionEnv {
                 this.cacheUnavailable(lane.reason, "the session store has no occupant (incarnation) for this run's session — no space key to find its cache volume by (never keyed by the session id instead)");
                 return undefined;
             case "open": {
+                const remembered = this.cacheMisses?.recall(lane.cache.spaceKey);
+                if (remembered !== undefined) {
+                    this.cacheUnavailable(remembered.reason, `${remembered.detail} — remembered from an earlier run of this space; not asked for again within ${this.cfg.dataTimeoutMs}ms`, lane.cache);
+                    return undefined;
+                }
                 const ensured = await this.ensureCacheClaim(lane.cache, signal);
                 if (!ensured.ok) {
                     this.cacheUnavailable("claim_failed", ensured.detail, lane.cache);
@@ -586,7 +649,7 @@ export class RemoteK8sExecutionEnv {
             }
             if (phase === "Failed" || phase === "Succeeded") {
                 await this.deletePod(name).catch(() => { });
-                return { ok: false, error: new RemoteExecutionError("connect_failed", `pod entered ${phase} before Running (${status?.reason ?? "?"})`) };
+                return this.refusePod(name, phase, status?.reason, status?.message);
             }
             const waiting = status?.containerStatuses?.[0]?.state?.waiting;
             if (waiting?.reason === "ImagePullBackOff" || waiting?.reason === "ErrImagePull") {
@@ -616,13 +679,24 @@ export class RemoteK8sExecutionEnv {
         }
         this.podName = name;
         this.podIP = podIP;
-        const mk = await this.runExec(`mkdir -p ${shellQuote(this.cfg.mountPath)}`);
+        const askHome = this.homeSeat.asks;
+        const mk = await this.runExec(`mkdir -p ${shellQuote(this.cfg.mountPath)}${askHome ? ` && { ${LANE_HOME_PROBE_COMMAND} || true; }` : ""}`);
         if (!mk.ok || mk.value.exitCode !== 0) {
             await this.deletePod(name).catch(() => { });
             this.podName = undefined;
             this.podIP = undefined;
             return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace mkdir failed: ${mk.ok ? mk.value.stderr : mk.error.message}`) };
         }
+        if (askHome) {
+            const verdict = this.homeSeat.settle({ ok: true, stdout: mk.value.stdout });
+            if (verdict !== undefined) {
+                await this.deletePod(name).catch(() => { });
+                this.podName = undefined;
+                this.podIP = undefined;
+                const error = new RemoteExecutionError("connect_failed", verdict.refusal);
+                return verdict.answered ? this.holdRefusal(error) : { ok: false, error };
+            }
+        }
         if (cache?.fresh)
             this.logger?.warn("workspace_cache_fresh", { space: incarnationLogPrefix(cache.spaceKey), claim: cache.claim.name, size: cache.size });
         return ok(this.workspaceHandle());
@@ -864,12 +938,15 @@ export class RemoteK8sExecutionEnv {
             await this.endSpace(this.spaceLane.space);
         return r;
     }
-    async uploadSnapshot(target) {
-        if (this.inFlightExecs > 0)
+    async uploadSnapshot(target, boundMs = this.cfg.dataTimeoutMs) {
+        if (target.lane === "park" && this.inFlight.size > 0)
             return { ok: false, error: new RemoteExecutionError("command_in_flight", "cannot suspend while a command is in flight (would snapshot a torn workspace)") };
         const snapshotId = mintSnapshotId(this.occupantIncarnation);
-        if (!this.podName)
-            return ok(snapshotId);
+        if (!this.podName) {
+            return target.lane === "space"
+                ? { ok: false, error: new RemoteExecutionError("connect_failed", "the sandbox pod was gone when the run-end snapshot reached the upload — nothing was saved (the workspace could not be read), and `latest` keeps naming the previous snapshot") }
+                : ok(snapshotId);
+        }
         const presigned = this.presignSnapshot({ ...target, method: "PUT", snapshotId });
         if (!presigned.ok)
             return presigned;
@@ -878,7 +955,7 @@ export class RemoteK8sExecutionEnv {
         const excludes = (this.cfg.snapshotExclude ?? []).map((item) => ` --exclude=${shellQuote(`./${item}`)}`).join("");
         const cmd = `tar czf ${tmp}${excludes} -C ${shellQuote(this.cfg.mountPath)} . && curl -fsS -X PUT -T ${tmp} ${shellQuote(url)}; ` +
             `rc=$?; rm -f ${tmp}; exit $rc`;
-        const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs });
+        const r = await this.runExec(cmd, { timeoutMs: boundMs });
         if (!r.ok)
             return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot upload failed: ${this.redactUrl(r.error.message, url)}`, r.error) };
         if (r.value.exitCode !== 0)
@@ -941,13 +1018,9 @@ export class RemoteK8sExecutionEnv {
                 case "space": {
                     const tries = attempt === 1 ? "1 attempt" : `${attempt} attempts`;
                     this.spaceLine("workspace_restore_failed", target.space, { snapshotId, class: failure.cls, attempts: attempt, boundMs: this.cfg.dataTimeoutMs, error: failure.detail });
-                    this.restoreRefusal = {
-                        ok: false,
-                        error: new RemoteExecutionError("connect_failed", `${failure.detail} — the workspace's latest snapshot (${snapshotId}) is in the store but did not come back (${failure.cls}, ${tries}, bound ${this.cfg.dataTimeoutMs}ms each), ` +
-                            "so this run does not start on an empty /workspace. Ways out: raise WORKSPACE_SNAPSHOT_TIMEOUT_MS if the archive is large, " +
-                            "or set WORKSPACE_PERSIST=off for one run (it starts from an empty /workspace and deletes no snapshot)", failure.cause),
-                    };
-                    return this.restoreRefusal;
+                    return this.holdRefusal(new RemoteExecutionError("connect_failed", `${failure.detail} — the workspace's latest snapshot (${snapshotId}) is in the store but did not come back (${failure.cls}, ${tries}, bound ${this.cfg.dataTimeoutMs}ms each), ` +
+                        "so this run does not start on an empty /workspace. Ways out: raise WORKSPACE_SNAPSHOT_TIMEOUT_MS if the archive is large, " +
+                        "or set WORKSPACE_PERSIST=off for one run (it starts from an empty /workspace and deletes no snapshot)", failure.cause));
                 }
                 default: {
                     const unreachable = target;
@@ -987,6 +1060,8 @@ export class RemoteK8sExecutionEnv {
         return ok(undefined);
     }
     async reconnect(_sessionToken) {
+        if (this.connectInFlight)
+            return this.connectInFlight;
         if (this.podName) {
             const alive = await this.request("GET", `/api/v1/namespaces/${this.cfg.namespace}/pods/${this.podName}`).catch(() => undefined);
             const phase = alive?.json?.status?.phase;
@@ -1032,6 +1107,63 @@ export class RemoteK8sExecutionEnv {
     async cleanup() {
         return this.destroy();
     }
+    beginExec() {
+        const ac = new AbortController();
+        this.inFlight.add(ac);
+        return ac;
+    }
+    endExec(ac) {
+        this.inFlight.delete(ac);
+        if (this.inFlight.size > 0)
+            return;
+        for (const wake of this.idleWaiters.splice(0))
+            wake();
+    }
+    whenIdle(ms) {
+        if (this.inFlight.size === 0)
+            return Promise.resolve(true);
+        return new Promise((resolve) => {
+            const idle = () => {
+                clearTimeout(timer);
+                resolve(true);
+            };
+            const timer = setTimeout(() => {
+                this.idleWaiters = this.idleWaiters.filter((w) => w !== idle);
+                resolve(false);
+            }, ms);
+            this.idleWaiters.push(idle);
+        });
+    }
+    async quiesce(space) {
+        const inFlight = this.inFlight.size;
+        if (inFlight === 0 && !this.podName)
+            return { ok: true, waitedMs: 0 };
+        const startedAt = Date.now();
+        const ended = await this.whenIdle(this.cfg.rpcTimeoutMs);
+        const killed = await this.killPodProcesses();
+        for (const ac of [...this.inFlight])
+            ac.abort();
+        const waitedMs = Date.now() - startedAt;
+        if (!ended) {
+            this.spaceLine("workspace_snapshot_commands_cut", space, {
+                inFlight,
+                waitedMs,
+                killed,
+                effect: killed
+                    ? "the run is over but its commands were still running after the wait — their processes were killed (the sandbox is deleted right after) so the run-end snapshot reads a still tree; the workspace is saved as it stood"
+                    : "the run is over but its commands were still running after the wait and the pod's processes could not be confirmed stopped — the workspace is NOT saved (it may still be changing); `latest` keeps naming the previous snapshot",
+            });
+        }
+        return killed
+            ? { ok: true, waitedMs }
+            : { ok: false, error: new RemoteExecutionError("command_in_flight", "the pod's processes could not be confirmed stopped when the run ended — not saving a workspace that may still be changing") };
+    }
+    async killPodProcesses() {
+        if (!this.podName)
+            return false;
+        const r = await this.runExec(buildKillAllScript(), { timeoutMs: this.cfg.rpcTimeoutMs + KILL_ALL_RECHECK_MS });
+        return r.ok && r.value.exitCode === 0;
+    }
     async persistWorkspace(space) {
         const failed = (stage, error) => {
             this.spaceLine("workspace_snapshot_failed", space, { stage, code: error.code, error: error.message });
@@ -1052,7 +1184,10 @@ export class RemoteK8sExecutionEnv {
         try {
             let snap;
             try {
-                snap = await this.uploadSnapshot({ lane: "space", space });
+                const quiet = await this.quiesce(space);
+                if (!quiet.ok)
+                    return failed("upload", quiet.error);
+                snap = await this.uploadSnapshot({ lane: "space", space }, Math.max(1, this.cfg.dataTimeoutMs - quiet.waitedMs));
             }
             catch (e) {
                 return failed("upload", new RemoteExecutionError("unknown", errorMessageOf(e)));
@@ -1142,7 +1277,8 @@ export class RemoteK8sExecutionEnv {
                 clearTimeout(livenessTimer);
                 const idle = Date.now() - lastProgress;
                 if (idle >= liveness.ms) {
-                    finish({ ok: false, error: new ExecutionError("timeout", livenessCutMessage(liveness)) });
+                    const cut = buildLivenessCut(liveness);
+                    finish({ ok: false, error: new ExecutionError(cut.code, cut.message, undefined, undefined, cut.noOutputSec) });
                     return;
                 }
                 livenessTimer = setTimeout(armLiveness, liveness.ms - idle);
@@ -1276,11 +1412,12 @@ export class RemoteK8sExecutionEnv {
         if (!c.ok)
             return c;
         const detachSignal = options?.detachSignal;
-        this.inFlightExecs++;
+        const inflight = this.beginExec();
+        const abortSignal = options?.abortSignal ? AbortSignal.any([options.abortSignal, inflight.signal]) : inflight.signal;
         try {
             if (!detachSignal) {
                 return call.settle(await this.runExec(call.command, {
-                    signal: options?.abortSignal,
+                    signal: abortSignal,
                     timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined,
                     boundOutput: true,
                     onStdout: options?.onStdout,
@@ -1292,7 +1429,7 @@ export class RemoteK8sExecutionEnv {
             const dir = `${K8S_BG_DIR_ROOT}/exec-${adoptId}`;
             const tee = { out: seedMemStream(""), err: seedMemStream("") };
             return call.settle(await this.runExec(buildDetachCapableExec(dir, call.command), {
-                signal: options?.abortSignal,
+                signal: abortSignal,
                 timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined,
                 boundOutput: true,
                 onStdout: options?.onStdout,
@@ -1316,7 +1453,7 @@ export class RemoteK8sExecutionEnv {
             }));
         }
         finally {
-            this.inFlightExecs--;
+            this.endExec(inflight);
         }
     }
     async getHost(port) {
@@ -1337,7 +1474,7 @@ export class RemoteK8sExecutionEnv {
         const c = await this.ensureConnected();
         if (!c.ok)
             throw new RemoteExecutionError("connect_failed", c.error.message, c.error);
-        this.inFlightExecs++;
+        const ac = this.beginExec();
         const queue = [];
         let finished = false;
         let failure;
@@ -1350,7 +1487,6 @@ export class RemoteK8sExecutionEnv {
         };
         const cap = options?.maxOutputBytes;
         let bytes = 0;
-        const ac = new AbortController();
         const signal = options?.signal ? AbortSignal.any([options.signal, ac.signal]) : ac.signal;
         const done = this.runExec(buildCwdEnvCommand(command, this.workingDirOf(options?.cwd), options?.env, "raw").command, {
             signal,
@@ -1375,9 +1511,11 @@ export class RemoteK8sExecutionEnv {
                         ? "transport_lost"
                         : r.error.code === "timeout"
                             ? "timeout"
-                            : r.error.code === "spawn_error" || r.error.code === "shell_unavailable"
-                                ? "unknown"
-                                : "connect_failed";
+                            : r.error.code === "liveness_no_output"
+                                ? "liveness_no_output"
+                                : r.error.code === "spawn_error" || r.error.code === "shell_unavailable"
+                                    ? "unknown"
+                                    : "connect_failed";
                 failure = new RemoteExecutionError(code, r.error.message, r.error);
             }
             finished = true;
@@ -1409,7 +1547,7 @@ export class RemoteK8sExecutionEnv {
         finally {
             ac.abort();
             await done.catch(() => { });
-            this.inFlightExecs--;
+            this.endExec(ac);
         }
     }
     backgroundCapabilities;
@@ -1626,42 +1764,97 @@ export class RemoteK8sExecutionEnv {
         return dir ? this.resolve(dir) : undefined;
     }
 }
+const SPACE_RECLAIM_CONCURRENCY = 4;
+function createGate(limit) {
+    let active = 0;
+    const waiting = [];
+    const release = () => {
+        const next = waiting.shift();
+        if (next)
+            next();
+        else
+            active--;
+    };
+    return async (task) => {
+        if (active < limit)
+            active++;
+        else
+            await new Promise((resolve) => waiting.push(resolve));
+        try {
+            return await task();
+        }
+        finally {
+            release();
+        }
+    };
+}
+function tallyFailure(classes, key, at, pos, sample) {
+    const seen = classes.get(key);
+    if (seen === undefined)
+        classes.set(key, { count: 1, at, pos, sample });
+    else {
+        seen.count++;
+        if (at < seen.at || (at === seen.at && pos < seen.pos)) {
+            seen.at = at;
+            seen.pos = pos;
+            seen.sample = sample;
+        }
+    }
+}
+function failureClassesInOrder(classes) {
+    return [...classes.values()].sort((a, b) => a.at - b.at || a.pos - b.pos);
+}
 export function createSpaceReclaimer(config, deps) {
+    const gate = createGate(SPACE_RECLAIM_CONCURRENCY);
     return {
         async reclaim(spaces) {
             const deadline = AbortSignal.timeout(dataTimeoutMsOf(config));
-            for (const { sessionId, spaceKey } of spaces) {
-                if (spaceKey === "")
-                    continue;
-                const env = new RemoteK8sExecutionEnv({ ...config, sessionId }, deps);
-                if (config.cache !== undefined) {
-                    const r = await env.reclaimSpaceCache(spaceKey, deadline).catch((e) => ({ ok: false, reason: "delete_failed", detail: errorMessageOf(e) }));
-                    if (!r.ok) {
-                        recordFailOpen("server.workspace-cache.reclaim-failed", `reason=${r.reason} label=${workspaceCacheClaimOf(spaceKey).label}`);
-                        config.logger?.warn("workspace_cache_reclaim_failed", {
-                            reason: r.reason,
-                            space: incarnationLogPrefix(spaceKey),
-                            label: workspaceCacheClaimOf(spaceKey).label,
-                            error: redactIncarnations(r.detail),
-                            effect: `the ended space's cache volume stays — delete it by label ${WORKSPACE_CACHE_SPACE_LABEL}=<label> (the deletion it belongs to went ahead)`,
-                        });
+            const volumeFailures = new Map();
+            const objectFailures = new Map();
+            const settled = await Promise.allSettled(spaces.map(({ sessionId, spaceKey }, at) => spaceKey === ""
+                ? Promise.resolve()
+                : gate(async () => {
+                    const env = new RemoteK8sExecutionEnv({ ...config, sessionId }, deps);
+                    if (config.cache !== undefined) {
+                        const r = await env.reclaimSpaceCache(spaceKey, deadline).catch((e) => ({ ok: false, reason: "delete_failed", detail: errorMessageOf(e) }));
+                        if (!r.ok) {
+                            recordFailOpen("server.workspace-cache.reclaim-failed", `reason=${r.reason} label=${workspaceCacheClaimOf(spaceKey).label}`);
+                            tallyFailure(volumeFailures, r.reason, at, 0, { spaceKey, failure: r });
+                        }
                     }
-                }
-                if (config.s3Snapshot !== undefined) {
-                    const failures = await env.reclaimSpaceSnapshots(spaceKey, deadline).catch((e) => [{ lane: "space", stage: "delete", detail: errorMessageOf(e) }]);
-                    for (const f of failures) {
-                        recordFailOpen("server.workspace-space.reclaim-failed", `lane=${f.lane} stage=${f.stage}`);
-                        config.logger?.warn("workspace_space_reclaim_failed", {
-                            space: incarnationLogPrefix(spaceKey),
-                            sessionId,
-                            lane: f.lane,
-                            stage: f.stage,
-                            error: redactIncarnations(f.detail),
-                            effect: `the ended space's ${f.lane === "space" ? "workspace snapshots" : "park snapshots"} may stay in the bucket (unreadable through the API; the bucket's lifecycle rule is the fallback) — the deletion it belongs to went ahead`,
+                    if (config.s3Snapshot !== undefined) {
+                        const failures = await env.reclaimSpaceSnapshots(spaceKey, deadline).catch((e) => [{ lane: "space", stage: "delete", detail: errorMessageOf(e) }]);
+                        failures.forEach((f, pos) => {
+                            recordFailOpen("server.workspace-space.reclaim-failed", `lane=${f.lane} stage=${f.stage}`);
+                            tallyFailure(objectFailures, `${f.lane}/${f.stage}`, at, pos, { sessionId, spaceKey, failure: f });
                         });
                     }
-                }
+                })));
+            for (const { count, sample } of failureClassesInOrder(volumeFailures)) {
+                config.logger?.warn("workspace_cache_reclaim_failed", {
+                    reason: sample.failure.reason,
+                    count,
+                    space: incarnationLogPrefix(sample.spaceKey),
+                    label: workspaceCacheClaimOf(sample.spaceKey).label,
+                    error: redactIncarnations(sample.failure.detail),
+                    effect: `the ended space's cache volume stays — delete it by label ${WORKSPACE_CACHE_SPACE_LABEL}=<label> (the deletion it belongs to went ahead)${count > 1 ? `; ${count} spaces of this call met the same failure, the line names the first` : ""}`,
+                });
+            }
+            for (const { count, sample } of failureClassesInOrder(objectFailures)) {
+                const { sessionId, spaceKey, failure: f } = sample;
+                config.logger?.warn("workspace_space_reclaim_failed", {
+                    space: incarnationLogPrefix(spaceKey),
+                    sessionId,
+                    lane: f.lane,
+                    stage: f.stage,
+                    count,
+                    error: redactIncarnations(f.detail),
+                    effect: `the ended space's ${f.lane === "space" ? "workspace snapshots" : "park snapshots"} may stay in the bucket (unreadable through the API; the bucket's lifecycle rule is the fallback) — the deletion it belongs to went ahead${count > 1 ? `; ${count} spaces of this call met the same failure, the line names the first` : ""}`,
+                });
             }
+            const threw = settled.find((r) => r.status === "rejected");
+            if (threw)
+                throw threw.reason;
         },
     };
 }