@sema-agent/server 7.125.1 → 7.127.0

package/dist/plugins/remote-env-k8s.js

@@ -6,7 +6,12 @@ import { StringDecoder } from "node:string_decoder";
 import WebSocket from "ws";
 import { buildCwdEnvCommand, createCwdGuardedCall, shellQuote, kindFromMode, numEnvOr } from "./remote-shell.js";
 import { presignS3Url, presignS3ListUrl } from "./s3-presign.js";
+import { K8S_WORKSPACE_MOUNT_DEFAULT, WORKSPACE_SNAPSHOT_TIMEOUT_MS_DEFAULT, mintSnapshotId, snapshotNameOwnedBy } from "../workspace-space.js";
+import { spaceCommitAlive } from "../workspace-space-commit.js";
 import { incarnationLogPrefix, redactIncarnations } from "../session-incarnation.js";
+import { WORKSPACE_CACHE_SPACE_LABEL, workspaceCacheClaimOf, workspaceCacheEnv } from "../workspace-cache.js";
+import { recordFailOpen } from "../observability/fail-open.js";
+import { SANDBOX_IPV6_DEFAULT, sandboxEgressGaps, sandboxPodSysctls } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
 import { fileErrorFromExec, classifyFsStderr } from "./remote-env-file-error.js";
 import { createPosixShellFs } from "./posix-shell-fs.js";
@@ -32,21 +37,84 @@ const EXEC_SUBPROTOCOLS = ["v5.channel.k8s.io", "v4.channel.k8s.io"];
 const WRITE_CHUNK_BYTES = 64 * 1024;
 import { applyPodSpecPatch } from "./k8s-exec-protocol.js";
 export { applyPodSpecPatch } from "./k8s-exec-protocol.js";
-function spaceLaneOf(policy, incarnation) {
+const CACHE_VOLUME = "ws-cache";
+function occupantKeyOf(occupant) {
+    if (occupant === undefined)
+        return { state: "unavailable", reason: "incarnation_unknown" };
+    if ("incarnation" in occupant)
+        return occupant;
+    if ("unread" in occupant)
+        return { state: "unavailable", reason: occupant.unread };
+    return { state: "unreadable", code: occupant.fault };
+}
+function cacheLaneOf(policy, occupant) {
+    if (policy === undefined)
+        return { state: "off" };
+    const key = occupantKeyOf(occupant);
+    if (!("incarnation" in key))
+        return key;
+    return { state: "open", cache: { mount: policy.mount, size: policy.size, claim: workspaceCacheClaimOf(key.incarnation), spaceKey: key.incarnation } };
+}
+function spaceLaneOf(policy, occupant) {
     if (policy === undefined)
         return { state: "off" };
     switch (policy.persist) {
         case "off":
             return { state: "off" };
-        case "space":
-            return incarnation ? { state: "open", space: { incarnation, keep: policy.keep } } : { state: "unavailable" };
+        case "space": {
+            const key = occupantKeyOf(occupant);
+            return "incarnation" in key ? { state: "open", space: { incarnation: key.incarnation, keep: policy.keep, fence: key.fence } } : key;
+        }
         default: {
             const unreachable = policy.persist;
             throw new Error(`unhandled WORKSPACE_PERSIST word: ${String(unreachable)}`);
         }
     }
 }
+function spaceReadFailed(what, detail, cause) {
+    return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} read failed${detail}`, cause instanceof Error ? cause : undefined) };
+}
+function restoreFailureIsTransient(cls) {
+    switch (cls) {
+        case "timeout":
+        case "transport":
+        case "store_busy":
+            return true;
+        case "download":
+        case "untar":
+            return false;
+        default: {
+            const unreachable = cls;
+            throw new Error(`unhandled restore failure class: ${String(unreachable)}`);
+        }
+    }
+}
+const RESTORE_MAX_ATTEMPTS = { park: 1, space: 2 };
+const RESTORE_DOWNLOAD_EXIT = 22;
+const RESTORE_TRANSFER_EXIT = 75;
+const RESTORE_STORE_BUSY_STATUSES = [408, 429, 500, 502, 503, 504];
+const RESTORE_STORE_BUSY_EXIT = 69;
+function restoreClassOfExit(exitCode) {
+    switch (exitCode) {
+        case RESTORE_STORE_BUSY_EXIT:
+            return "store_busy";
+        case RESTORE_DOWNLOAD_EXIT:
+            return "download";
+        case RESTORE_TRANSFER_EXIT:
+            return "transport";
+        default:
+            return "untar";
+    }
+}
 const SPACE_COMMIT_POLL_MS = 1000;
+class FenceCallAborted extends Error {
+    constructor() {
+        super("aborted");
+    }
+}
+function dataTimeoutMsOf(config) {
+    return config.dataTimeoutMs ?? WORKSPACE_SNAPSHOT_TIMEOUT_MS_DEFAULT;
+}
 import { execSocketTlsOptions, exitCodeFromStatus } from "./k8s-exec-protocol.js";
 export { execSocketTlsOptions } from "./k8s-exec-protocol.js";
 export function errorMessageOf(e) {
@@ -56,6 +124,8 @@ export function errorMessageOf(e) {
     return typeof m === "string" && m ? m : String(e);
 }
 export { exitCodeFromStatus } from "./k8s-exec-protocol.js";
+export const K8S_SANDBOX_NAMESPACE_DEFAULT = "sandboxes";
+const SANDBOX_POD_LABELS = { app: "sema-sandbox", "managed-by": "sema-server" };
 export class RemoteK8sExecutionEnv {
     capabilities;
     cwd;
@@ -65,11 +135,17 @@ export class RemoteK8sExecutionEnv {
     handleSnapshotId;
     inFlightExecs = 0;
     connectPromise;
+    restoreRefusal;
     homeDir;
+    cdPath = "unknown";
     cfg;
     deps;
+    occupantIncarnation;
     spaceLane;
     spaceUnavailableSaid = false;
+    unreadableOccupantSaid = false;
+    cacheLane;
+    cacheUnavailableSaid = false;
     logger;
     onWorkspaceSnapshot;
     constructor(config, deps) {
@@ -78,27 +154,32 @@ export class RemoteK8sExecutionEnv {
             token: config.token,
             caCert: config.caCert,
             insecureTls: config.insecureTls ?? false,
-            namespace: config.namespace ?? "sandboxes",
+            namespace: config.namespace ?? K8S_SANDBOX_NAMESPACE_DEFAULT,
             image: config.image,
             runtimeClass: config.runtimeClass ?? "kata-qemu",
-            mountPath: config.mountPath ?? "/workspace",
+            mountPath: config.mountPath ?? K8S_WORKSPACE_MOUNT_DEFAULT,
             timeoutMs: config.timeoutMs ?? 30 * 60_000,
             readyTimeoutMs: config.readyTimeoutMs ?? 120_000,
             rpcTimeoutMs: config.rpcTimeoutMs ?? 30_000,
             livenessMs: config.livenessMs ?? 120_000,
-            dataTimeoutMs: config.dataTimeoutMs ?? 5 * 60_000,
+            dataTimeoutMs: dataTimeoutMsOf(config),
             cpu: config.cpu ?? "1",
             memory: config.memory ?? "2Gi",
+            ipv6: config.ipv6 ?? SANDBOX_IPV6_DEFAULT,
             retry: { maxAttempts: config.retry?.maxAttempts ?? 3, backoffMs: config.retry?.backoffMs ?? 1000 },
             ...(config.s3Snapshot ? { s3Snapshot: config.s3Snapshot } : {}),
             ...(config.sessionId ? { sessionId: config.sessionId } : {}),
             ...(config.podSpecPatch ? { podSpecPatch: config.podSpecPatch } : {}),
             ...(config.podEnv ? { podEnv: config.podEnv } : {}),
+            ...(config.snapshotExclude && config.snapshotExclude.length > 0 ? { snapshotExclude: config.snapshotExclude } : {}),
         };
         this.cwd = this.cfg.mountPath;
         this.homeDir = config.homeDir;
         this.deps = deps ?? {};
-        this.spaceLane = spaceLaneOf(config.s3Snapshot?.space, config.spaceIncarnation);
+        const occupant = occupantKeyOf(config.spaceOccupant);
+        this.occupantIncarnation = "incarnation" in occupant ? occupant.incarnation : undefined;
+        this.spaceLane = spaceLaneOf(config.s3Snapshot?.space, config.spaceOccupant);
+        this.cacheLane = cacheLaneOf(config.cache, config.spaceOccupant);
         this.logger = config.logger;
         this.onWorkspaceSnapshot = config.onWorkspaceSnapshot;
         this.capabilities = { isolation: true, suspendable: !!this.cfg.s3Snapshot, canonicalPathAuthoritative: true };
@@ -206,18 +287,23 @@ export class RemoteK8sExecutionEnv {
             case "off":
                 return this.provision(signal);
             case "unavailable":
-                this.sayUnavailable();
+                this.sayUnavailable(lane.reason);
+                return this.provision(signal);
+            case "unreadable":
                 return this.provision(signal);
             case "open": {
+                if (this.restoreRefusal)
+                    return this.restoreRefusal;
                 const settled = await this.awaitSpaceCommit(lane.space, signal);
                 if (!settled.ok)
                     return settled;
                 const latest = await this.readLatest(lane.space, signal);
                 if (!latest.ok)
                     return latest;
-                if (latest.value === undefined)
-                    return this.provision(signal);
-                return this.restoreSnapshot({ lane: "space", space: lane.space }, latest.value, { missing: "refuse", ...(signal ? { abortSignal: signal } : {}) });
+                const c = latest.value === undefined ? await this.provision(signal) : await this.restoreSnapshot({ lane: "space", space: lane.space }, latest.value, signal);
+                if (c.ok)
+                    this.armSpace();
+                return c;
             }
             default: {
                 const unreachable = lane;
@@ -225,27 +311,76 @@ export class RemoteK8sExecutionEnv {
             }
         }
     }
-    sayUnavailable() {
+    armSpace() {
+        if (this.spaceLane.state === "open")
+            this.spaceLane.space.fence.arm(this.cfg.dataTimeoutMs);
+    }
+    async endSpace(space) {
+        await this.fenceCall(() => space.fence.end(), "delete").catch(() => undefined);
+    }
+    fenceCall(op, what, signal) {
+        return new Promise((resolve, reject) => {
+            if (signal?.aborted)
+                return reject(new FenceCallAborted());
+            const onAbort = () => {
+                clearTimeout(timer);
+                reject(new FenceCallAborted());
+            };
+            const timer = setTimeout(() => {
+                signal?.removeEventListener("abort", onAbort);
+                reject(new Error(`workspace space commit state ${what} did not answer within ${this.cfg.rpcTimeoutMs}ms`));
+            }, this.cfg.rpcTimeoutMs);
+            signal?.addEventListener("abort", onAbort, { once: true });
+            op().then((v) => {
+                clearTimeout(timer);
+                signal?.removeEventListener("abort", onAbort);
+                resolve(v);
+            }, (e) => {
+                clearTimeout(timer);
+                signal?.removeEventListener("abort", onAbort);
+                reject(e);
+            });
+        });
+    }
+    sayUnavailable(reason) {
         if (this.spaceUnavailableSaid)
             return;
         this.spaceUnavailableSaid = true;
         this.logger?.warn("workspace_space_unavailable", {
-            reason: "incarnation_unknown",
+            reason,
             sessionId: this.cfg.sessionId,
-            effect: "WORKSPACE_PERSIST=space, but this run's session occupant (incarnation) was not read on this replica — its /workspace is neither restored nor saved (it is never keyed by the session id instead)",
+            effect: "WORKSPACE_PERSIST=space, but the session store has no occupant (incarnation) for this run's session — its /workspace is neither restored nor saved (it is never keyed by the session id instead)",
         });
     }
+    refuseUnreadableOccupant(code) {
+        const ways = [...(this.spaceLane.state === "unreadable" ? ["WORKSPACE_PERSIST=off"] : []), ...(this.cacheLane.state === "unreadable" ? ["WORKSPACE_CACHE=off"] : [])].join(" and ");
+        if (!this.unreadableOccupantSaid) {
+            this.unreadableOccupantSaid = true;
+            this.logger?.warn("workspace_space_unreadable_occupant", {
+                sessionId: this.cfg.sessionId,
+                code,
+                effect: `the session store did not answer for this run's session occupant (incarnation) — this run is refused rather than started on a workspace that is neither restored nor saved; ${ways} runs without it`,
+            });
+        }
+        return spaceReadFailed("occupant", ` (${code}) — refusing to start this run on a workspace that could be neither restored nor saved; retry the run once the session store answers, or set ${ways}`);
+    }
     spaceLine(event, space, fields) {
         const safe = Object.fromEntries(Object.entries(fields).map(([k, v]) => [k, typeof v === "string" ? redactIncarnations(v) : v]));
         this.logger?.warn(event, { space: incarnationLogPrefix(space.incarnation), ...safe });
     }
     provision(signal) {
+        const space = this.spaceLane;
+        const cache = this.cacheLane;
+        if (space.state === "unreadable")
+            return Promise.resolve(this.refuseUnreadableOccupant(space.code));
+        if (cache.state === "unreadable")
+            return Promise.resolve(this.refuseUnreadableOccupant(cache.code));
         const policy = {
             retryableCodes: ["connect_failed", "timeout", "transport_lost"],
             maxAttempts: this.cfg.retry.maxAttempts,
             backoffMs: (failed) => this.cfg.retry.backoffMs * failed,
         };
-        const once = this.deps.connectOnce ?? ((s) => this.connectOnce(s));
+        const once = this.deps.connectOnce ?? ((s) => this.connectOnce(cache, s));
         return withRetry(() => once(signal), policy, { signal });
     }
     async ensureConnected() {
@@ -261,21 +396,133 @@ export class RemoteK8sExecutionEnv {
         }
         return ok(undefined);
     }
-    async connectOnce(signal) {
+    async connectOnce(cacheLane, signal) {
         if (signal?.aborted)
             return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted") };
         if (this.destroyed)
             return { ok: false, error: new RemoteExecutionError("connect_failed", "env already destroyed") };
+        const cache = await this.attachCache(cacheLane, signal);
+        const first = await this.createPod(signal, cache);
+        if (!("cacheMiss" in first))
+            return first;
+        this.cacheUnavailable(first.cacheMiss, first.detail, cache);
+        const second = await this.createPod(signal, undefined);
+        return "cacheMiss" in second ? { ok: false, error: new RemoteExecutionError("connect_failed", "unreachable: a pod without the cache volume was given up for it") } : second;
+    }
+    async attachCache(lane, signal) {
+        switch (lane.state) {
+            case "off":
+                return undefined;
+            case "unavailable":
+                this.cacheUnavailable(lane.reason, "the session store has no occupant (incarnation) for this run's session — no space key to find its cache volume by (never keyed by the session id instead)");
+                return undefined;
+            case "open": {
+                const ensured = await this.ensureCacheClaim(lane.cache, signal);
+                if (!ensured.ok) {
+                    this.cacheUnavailable("claim_failed", ensured.detail, lane.cache);
+                    return undefined;
+                }
+                return { ...lane.cache, fresh: ensured.fresh, bindDeadline: Date.now() + this.cfg.dataTimeoutMs };
+            }
+            default: {
+                const unreachable = lane;
+                throw new Error(`unhandled cache lane state: ${String(unreachable.state)}`);
+            }
+        }
+    }
+    async ensureCacheClaim(cache, signal) {
+        const base = `/api/v1/namespaces/${this.cfg.namespace}/persistentvolumeclaims`;
+        try {
+            const got = await this.request("GET", `${base}/${cache.claim.name}`, undefined, { signal });
+            if (got.status === 200)
+                return { ok: true, fresh: false };
+            if (got.status !== 404)
+                return { ok: false, detail: `claim read answered HTTP ${got.status}: ${JSON.stringify(got.json).slice(0, 200)}` };
+            const claim = {
+                apiVersion: "v1",
+                kind: "PersistentVolumeClaim",
+                metadata: { name: cache.claim.name, labels: { [WORKSPACE_CACHE_SPACE_LABEL]: cache.claim.label, "managed-by": "sema-server" } },
+                spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: cache.size } } },
+            };
+            const made = await this.request("POST", base, claim, { signal });
+            if (made.status === 200 || made.status === 201)
+                return { ok: true, fresh: true };
+            if (made.status === 409)
+                return { ok: true, fresh: false };
+            return { ok: false, detail: `claim create answered HTTP ${made.status}: ${JSON.stringify(made.json).slice(0, 200)}` };
+        }
+        catch (e) {
+            return { ok: false, detail: `claim request failed: ${errorMessageOf(e)}` };
+        }
+    }
+    async cacheClaimBound(cache, signal) {
+        const got = await this.request("GET", `/api/v1/namespaces/${this.cfg.namespace}/persistentvolumeclaims/${cache.claim.name}`, undefined, { signal }).catch(() => undefined);
+        return got?.status === 200 && got.json?.status?.phase === "Bound";
+    }
+    async reclaimSpaceCache(spaceKey, deadline) {
+        if (deadline.aborted)
+            return { ok: false, reason: "budget", detail: this.reclaimDeadlinePassed("the claim delete-by-label was not sent") };
+        const { label } = workspaceCacheClaimOf(spaceKey);
+        const selector = encodeURIComponent(`${WORKSPACE_CACHE_SPACE_LABEL}=${label}`);
+        try {
+            const r = await this.request("DELETE", `/api/v1/namespaces/${this.cfg.namespace}/persistentvolumeclaims?labelSelector=${selector}`, undefined, { timeoutMs: this.cfg.rpcTimeoutMs, signal: deadline });
+            if (r.status < 200 || r.status >= 300)
+                return { ok: false, reason: "delete_failed", detail: `claim delete-by-label answered HTTP ${r.status}: ${JSON.stringify(r.json).slice(0, 200)}` };
+            const items = r.json?.items;
+            return { ok: true, deleted: Array.isArray(items) ? items.length : 0 };
+        }
+        catch (e) {
+            if (deadline.aborted)
+                return { ok: false, reason: "budget", detail: this.reclaimDeadlinePassed("the claim delete-by-label was cut in flight") };
+            return { ok: false, reason: "delete_failed", detail: `claim delete-by-label failed: ${errorMessageOf(e)}` };
+        }
+    }
+    async egressPolicyGaps() {
+        const read = async (apiPath) => {
+            try {
+                const r = await this.request("GET", apiPath);
+                return r.status >= 200 && r.status < 300 ? { ok: true, json: r.json } : { ok: false, detail: `HTTP ${r.status}` };
+            }
+            catch (e) {
+                return { ok: false, detail: errorMessageOf(e) };
+            }
+        };
+        const nodes = await read("/api/v1/nodes");
+        const policies = await read(`/apis/networking.k8s.io/v1/namespaces/${this.cfg.namespace}/networkpolicies`);
+        const probe = { metadata: { labels: { ...SANDBOX_POD_LABELS } }, spec: {} };
+        if (this.cfg.podSpecPatch?.labels)
+            applyPodSpecPatch(probe, { labels: this.cfg.podSpecPatch.labels });
+        return sandboxEgressGaps({ nodes, policies, podLabels: probe.metadata.labels, ipv6: this.cfg.ipv6 });
+    }
+    reclaimDeadlinePassed(what) {
+        return `reclaim deadline (${this.cfg.dataTimeoutMs} ms, the snapshot time bound, one per reclaim call) passed — ${what}`;
+    }
+    cacheUnavailable(reason, detail, cache) {
+        if (this.cacheUnavailableSaid)
+            return;
+        this.cacheUnavailableSaid = true;
+        recordFailOpen("server.workspace-cache.unavailable", `reason=${reason}`);
+        this.logger?.warn("workspace_cache_unavailable", {
+            ...(cache ? { space: incarnationLogPrefix(cache.spaceKey), claim: cache.claim.name } : { sessionId: this.cfg.sessionId }),
+            reason,
+            error: redactIncarnations(detail),
+            effect: "this run starts without the build-cache volume and without the cache env (tools use the image's own cache directories; builds re-download)",
+        });
+    }
+    async createPod(signal, cache) {
         const ns = this.cfg.namespace;
+        const podEnv = { ...(cache ? workspaceCacheEnv(cache.mount) : {}), ...(this.cfg.podEnv ?? {}) };
+        const sysctls = sandboxPodSysctls(this.cfg.ipv6);
         const pod = {
             apiVersion: "v1",
             kind: "Pod",
             metadata: {
                 generateName: "sbx-",
-                labels: { app: "sema-sandbox", "managed-by": "sema-server" },
+                labels: { ...SANDBOX_POD_LABELS },
             },
             spec: {
                 ...(this.cfg.runtimeClass ? { runtimeClassName: this.cfg.runtimeClass } : {}),
+                ...(sysctls ? { securityContext: { sysctls: sysctls.map((x) => ({ ...x })) } } : {}),
                 restartPolicy: "Never",
                 automountServiceAccountToken: false,
                 activeDeadlineSeconds: Math.ceil(this.cfg.timeoutMs / 1000),
@@ -287,12 +534,12 @@ export class RemoteK8sExecutionEnv {
                         command: this.cfg.podEnv?.SEMA_PKG_SOURCE
                             ? ["sh", "-c", "[ -f /opt/sema/pkg-source.sh ] && sh /opt/sema/pkg-source.sh || true; sleep infinity"]
                             : ["sh", "-c", "sleep infinity"],
-                        ...(this.cfg.podEnv && Object.keys(this.cfg.podEnv).length > 0
-                            ? { env: Object.entries(this.cfg.podEnv).map(([name, value]) => ({ name, value: String(value) })) }
-                            : {}),
+                        ...(Object.keys(podEnv).length > 0 ? { env: Object.entries(podEnv).map(([name, value]) => ({ name, value: String(value) })) } : {}),
+                        ...(cache ? { volumeMounts: [{ name: CACHE_VOLUME, mountPath: cache.mount }] } : {}),
                         resources: { limits: { cpu: this.cfg.cpu, memory: this.cfg.memory } },
                     },
                 ],
+                ...(cache ? { volumes: [{ name: CACHE_VOLUME, persistentVolumeClaim: { claimName: cache.claim.name } }] } : {}),
             },
         };
         if (this.cfg.podSpecPatch)
@@ -344,6 +591,17 @@ export class RemoteK8sExecutionEnv {
                 return { ok: false, error: new RemoteExecutionError("connect_failed", `image pull failed for ${this.cfg.image}: ${waiting.message ?? waiting.reason}`) };
             }
             if (Date.now() > deadline) {
+                if (cache) {
+                    const bound = await this.cacheClaimBound(cache, signal);
+                    if (!bound && Date.now() < cache.bindDeadline) {
+                        await new Promise((r) => setTimeout(r, 1000));
+                        continue;
+                    }
+                    await this.deletePod(name).catch(() => { });
+                    return bound
+                        ? { cacheMiss: "claim_unmountable", detail: `the pod carrying the bound claim ${cache.claim.name} was not Running within ${this.cfg.readyTimeoutMs}ms (phase=${phase ?? "?"}${waiting?.reason ? `, ${waiting.reason}` : ""})` }
+                        : { cacheMiss: "claim_unbound", detail: `the claim ${cache.claim.name} was not bound within ${this.cfg.dataTimeoutMs}ms (no default StorageClass, or its provisioner is down)` };
+                }
                 await this.deletePod(name).catch(() => { });
                 return { ok: false, error: new RemoteExecutionError("timeout", `pod not Running within ${this.cfg.readyTimeoutMs}ms (phase=${phase ?? "?"})`) };
             }
@@ -362,6 +620,8 @@ export class RemoteK8sExecutionEnv {
             this.podIP = undefined;
             return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace mkdir failed: ${mk.ok ? mk.value.stderr : mk.error.message}`) };
         }
+        if (cache?.fresh)
+            this.logger?.warn("workspace_cache_fresh", { space: incarnationLogPrefix(cache.spaceKey), claim: cache.claim.name, size: cache.size });
         return ok(this.workspaceHandle());
     }
     presignSnapshot(req) {
@@ -369,8 +629,14 @@ export class RemoteK8sExecutionEnv {
         const prefix = s3.keyPrefix ?? "kata";
         const [label, partition] = req.lane === "space" ? ["spaceIncarnation", req.space.incarnation] : ["sessionId", this.cfg.sessionId ?? "_"];
         const snapshotId = "snapshotId" in req ? req.snapshotId : undefined;
+        const objectName = "object" in req ? req.object : undefined;
         const safe = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
-        const segments = [[label, partition], ...(snapshotId !== undefined ? [["snapshotId", snapshotId]] : []), ["keyPrefix", prefix]];
+        const segments = [
+            [label, partition],
+            ...(snapshotId !== undefined ? [["snapshotId", snapshotId]] : []),
+            ...(objectName !== undefined ? [["object", objectName]] : []),
+            ["keyPrefix", prefix],
+        ];
         for (const [label, seg] of segments) {
             if (!safe.test(seg))
                 return { ok: false, error: new RemoteExecutionError("connect_failed", `unsafe snapshot key segment (${label}): refusing to build a traversing object key`) };
@@ -383,12 +649,86 @@ export class RemoteK8sExecutionEnv {
             ...(s3.region ? { region: s3.region } : {}),
             expiresSec: Math.min(s3.presignTtlSec ?? 600, 3600),
         };
-        const base = req.lane === "space" ? `${prefix}/${partition}/space/` : `${prefix}/${partition}/`;
+        const base = this.snapshotBase(req);
         if (req.method === "LIST")
             return { ok: true, value: presignS3ListUrl({ ...common, prefix: base, maxKeys: 1000 }) };
-        const object = "pointer" in req ? req.pointer : `${req.snapshotId}.tar.gz`;
+        const object = "pointer" in req ? req.pointer : "object" in req ? req.object : `${req.snapshotId}.tar.gz`;
         return { ok: true, value: presignS3Url({ ...common, key: `${base}${object}`, method: req.method }) };
     }
+    snapshotBase(req) {
+        const prefix = this.cfg.s3Snapshot.keyPrefix ?? "kata";
+        return req.lane === "space" ? `${prefix}/${req.space.incarnation}/space/` : `${prefix}/${this.cfg.sessionId ?? "_"}/`;
+    }
+    async reclaimSpaceSnapshots(spaceKey, deadline) {
+        if (!this.cfg.s3Snapshot)
+            return [];
+        const failures = [];
+        const deadlinePassed = (lane) => [
+            ...failures,
+            { lane, stage: "budget", detail: this.reclaimDeadlinePassed("stopped; the objects not yet deleted stay for the bucket's lifecycle rule") },
+        ];
+        const targets = [{ lane: "space", req: { lane: "space", space: { incarnation: spaceKey }, method: "LIST" } }];
+        if (this.cfg.sessionId !== undefined)
+            targets.push({ lane: "park", req: { lane: "park", method: "LIST" } });
+        for (const { lane, req } of targets) {
+            if (deadline.aborted)
+                return deadlinePassed(lane);
+            const list = this.presignSnapshot(req);
+            if (!list.ok)
+                continue;
+            let xml;
+            try {
+                const res = await this.storeFetch(list.value, { method: "GET", signal: deadline });
+                xml = await res.text();
+                if (!res.ok) {
+                    failures.push({ lane, stage: "list", detail: `list HTTP ${res.status}` });
+                    continue;
+                }
+            }
+            catch (e) {
+                if (deadline.aborted)
+                    return deadlinePassed(lane);
+                failures.push({ lane, stage: "list", detail: `list: ${this.redactUrl(errorMessageOf(e), list.value)}` });
+                continue;
+            }
+            if (/<IsTruncated>true<\/IsTruncated>/.test(xml))
+                failures.push({ lane, stage: "list", detail: "listing truncated (more than 1000 objects in this partition) — deleting the objects seen; the rest stay" });
+            const base = this.snapshotBase(req);
+            const names = [...xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)]
+                .map((m) => /<Key>([^<]*)<\/Key>/.exec(m[1])?.[1] ?? "")
+                .filter((key) => key.startsWith(base) && !key.slice(base.length).includes("/"))
+                .map((key) => key.slice(base.length))
+                .filter((name) => lane === "space" || snapshotNameOwnedBy(name, spaceKey));
+            const del = [];
+            let outOfTime = false;
+            for (const name of names) {
+                if (deadline.aborted) {
+                    outOfTime = true;
+                    break;
+                }
+                const target = req.lane === "space" ? this.presignSnapshot({ lane: "space", space: req.space, method: "DELETE", object: name }) : this.presignSnapshot({ lane: "park", method: "DELETE", object: name });
+                if (!target.ok)
+                    continue;
+                try {
+                    const res = await this.storeFetch(target.value, { method: "DELETE", signal: deadline });
+                    if (!res.ok && res.status !== 404)
+                        del.push(`${name}: HTTP ${res.status}`);
+                }
+                catch (e) {
+                    if (deadline.aborted) {
+                        outOfTime = true;
+                        break;
+                    }
+                    del.push(`${name}: ${this.redactUrl(errorMessageOf(e), target.value)}`);
+                }
+            }
+            if (del.length > 0)
+                failures.push({ lane, stage: "delete", detail: `${del.length} of ${names.length} objects not deleted — first: ${del[0]}` });
+            if (outOfTime)
+                return deadlinePassed(lane);
+        }
+        return failures;
+    }
     storeFetch(url, init) {
         const bound = AbortSignal.timeout(Math.max(1, Math.min(this.cfg.rpcTimeoutMs, init.timeoutMs ?? this.cfg.rpcTimeoutMs)));
         const signal = init.signal ? AbortSignal.any([init.signal, bound]) : bound;
@@ -398,7 +738,7 @@ export class RemoteK8sExecutionEnv {
         const url = this.presignSnapshot({ lane: "space", space, method: "GET", pointer });
         if (!url.ok)
             return url;
-        const what = pointer === "latest" ? "pointer" : "commit marker";
+        const what = "pointer";
         let res;
         let body;
         try {
@@ -406,12 +746,12 @@ export class RemoteK8sExecutionEnv {
             body = await res.text();
         }
         catch (e) {
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} read failed: ${this.redactUrl(errorMessageOf(e), url.value)}`, e instanceof Error ? e : undefined) };
+            return spaceReadFailed(what, `: ${this.redactUrl(errorMessageOf(e), url.value)}`, e);
         }
         if (res.status === 404)
             return ok(undefined);
         if (!res.ok)
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} read failed (HTTP ${res.status})`) };
+            return spaceReadFailed(what, ` (HTTP ${res.status})`);
         return ok(body.trim());
     }
     async readLatest(space, signal) {
@@ -423,7 +763,7 @@ export class RemoteK8sExecutionEnv {
         const url = this.presignSnapshot({ lane: "space", space, method, pointer });
         if (!url.ok)
             return url;
-        const what = pointer === "latest" ? "pointer" : "commit marker";
+        const what = "pointer";
         try {
             const res = await this.storeFetch(url.value, { method, ...(body === null ? {} : { body }), ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
             if (!res.ok && !(method === "DELETE" && res.status === 404))
@@ -436,17 +776,30 @@ export class RemoteK8sExecutionEnv {
     }
     async awaitSpaceCommit(space, signal) {
         for (;;) {
-            const marker = await this.readPointer(space, "committing", signal);
-            if (!marker.ok)
-                return marker;
-            if (marker.value === undefined)
+            let row;
+            try {
+                row = await this.fenceCall(() => space.fence.read(), "read", signal);
+            }
+            catch (e) {
+                if (e instanceof FenceCallAborted)
+                    return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted while waiting for the previous run's workspace commit") };
+                return spaceReadFailed("commit state", `: ${redactIncarnations(errorMessageOf(e))} — refusing to guess whether a commit is in flight`, e);
+            }
+            if (row === undefined)
                 return ok(undefined);
-            const startedAt = /^\d+$/.test(marker.value) ? Number(marker.value) : NaN;
-            if (!Number.isFinite(startedAt))
-                return { ok: false, error: new RemoteExecutionError("connect_failed", "workspace space commit marker is malformed (its body is not an epoch-ms timestamp) — refusing to guess whether a commit is in flight") };
-            const ageMs = Date.now() - startedAt;
-            if (ageMs >= this.cfg.dataTimeoutMs) {
-                this.spaceLine("workspace_commit_abandoned", space, { ageMs });
+            if (!spaceCommitAlive(row)) {
+                let gone;
+                try {
+                    gone = await this.fenceCall(() => space.fence.abandon(row.taskId), "delete", signal);
+                }
+                catch (e) {
+                    if (e instanceof FenceCallAborted)
+                        return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted while waiting for the previous run's workspace commit") };
+                    return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space commit state write failed: ${redactIncarnations(errorMessageOf(e))} — refusing to guess whether the previous commit is gone`, e instanceof Error ? e : undefined) };
+                }
+                if (!gone)
+                    continue;
+                this.spaceLine("workspace_commit_abandoned", space, { remainingMs: row.remainingMs, state: row.state });
                 return ok(undefined);
             }
             await new Promise((resolveWait) => {
@@ -503,12 +856,15 @@ export class RemoteK8sExecutionEnv {
     async suspendVM(_options) {
         if (!this.cfg.s3Snapshot)
             return unsupported("suspendVM");
-        return this.uploadSnapshot({ lane: "park" });
+        const r = await this.uploadSnapshot({ lane: "park" });
+        if (r.ok && this.spaceLane.state === "open")
+            await this.endSpace(this.spaceLane.space);
+        return r;
     }
     async uploadSnapshot(target) {
         if (this.inFlightExecs > 0)
             return { ok: false, error: new RemoteExecutionError("command_in_flight", "cannot suspend while a command is in flight (would snapshot a torn workspace)") };
-        const snapshotId = `kata-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+        const snapshotId = mintSnapshotId(this.occupantIncarnation);
         if (!this.podName)
             return ok(snapshotId);
         const presigned = this.presignSnapshot({ ...target, method: "PUT", snapshotId });
@@ -516,7 +872,8 @@ export class RemoteK8sExecutionEnv {
             return presigned;
         const url = presigned.value;
         const tmp = "/tmp/ws-snap.tgz";
-        const cmd = `tar czf ${tmp} -C ${shellQuote(this.cfg.mountPath)} . && curl -fsS -X PUT -T ${tmp} ${shellQuote(url)}; ` +
+        const excludes = (this.cfg.snapshotExclude ?? []).map((item) => ` --exclude=${shellQuote(`./${item}`)}`).join("");
+        const cmd = `tar czf ${tmp}${excludes} -C ${shellQuote(this.cfg.mountPath)} . && curl -fsS -X PUT -T ${tmp} ${shellQuote(url)}; ` +
             `rc=$?; rm -f ${tmp}; exit $rc`;
         const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs, livenessMs: 0 });
         if (!r.ok)
@@ -532,40 +889,94 @@ export class RemoteK8sExecutionEnv {
     async resumeVM(snapshotId, options) {
         if (!this.cfg.s3Snapshot)
             return unsupported("resumeVM");
-        const r = await this.restoreSnapshot({ lane: "park" }, snapshotId, { missing: "empty", ...(options?.abortSignal ? { abortSignal: options.abortSignal } : {}) });
-        if (r.ok)
-            this.handleSnapshotId = snapshotId;
-        return r.ok ? ok(this.workspaceHandle()) : r;
+        const r = await this.restoreSnapshot({ lane: "park" }, snapshotId, options?.abortSignal);
+        if (!r.ok)
+            return r;
+        this.handleSnapshotId = snapshotId;
+        this.armSpace();
+        return ok(this.workspaceHandle());
     }
-    async restoreSnapshot(target, snapshotId, opts) {
+    async restoreSnapshot(target, snapshotId, abortSignal) {
         const presigned = this.presignSnapshot({ ...target, method: "GET", snapshotId });
         if (!presigned.ok)
             return presigned;
-        const c = await this.provision(opts.abortSignal);
-        if (!c.ok)
-            return c;
         const url = presigned.value;
-        const cmd = `code=$(curl -s -o /tmp/ws.tgz -w '%{http_code}' ${shellQuote(url)}); ` +
+        const cmd = `code=$(curl -s -o /tmp/ws.tgz -w '%{http_code}' ${shellQuote(url)}); rc=$?; ` +
+            `if [ "$rc" != "0" ]; then echo "snapshot GET got no whole answer (curl exit $rc, HTTP $code)" >&2; exit ${RESTORE_TRANSFER_EXIT}; fi; ` +
             `if [ "$code" = "404" ]; then echo EMPTY_SNAPSHOT; exit 0; fi; ` +
-            `if [ "$code" != "200" ]; then echo "snapshot GET HTTP $code" >&2; exit 22; fi; ` +
+            `case "$code" in ${RESTORE_STORE_BUSY_STATUSES.join("|")}) echo "snapshot GET HTTP $code" >&2; exit ${RESTORE_STORE_BUSY_EXIT};; esac; ` +
+            `if [ "$code" != "200" ]; then echo "snapshot GET HTTP $code" >&2; exit ${RESTORE_DOWNLOAD_EXIT}; fi; ` +
             `tar xzf /tmp/ws.tgz -C ${shellQuote(this.cfg.mountPath)} && rm -f /tmp/ws.tgz`;
-        const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs, livenessMs: 0 });
-        const failed = !r.ok
-            ? new RemoteExecutionError("connect_failed", `workspace snapshot restore failed: ${this.redactUrl(r.error.message, url)}`, r.error)
-            : r.value.exitCode !== 0
-                ? new RemoteExecutionError("connect_failed", `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}`)
-                : opts.missing === "refuse" && r.value.stdout.includes("EMPTY_SNAPSHOT")
-                    ? new RemoteExecutionError("connect_failed", `workspace snapshot ${snapshotId} is named by the space pointer but its archive is missing (HTTP 404) — refusing to start on an empty workspace (delete the space's latest pointer, or WORKSPACE_PERSIST=off)`)
+        for (let attempt = 1;; attempt++) {
+            const c = await this.provision(abortSignal);
+            if (!c.ok)
+                return c;
+            const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs, livenessMs: 0 });
+            const failure = !r.ok
+                ? { cls: r.error.code === "timeout" ? "timeout" : "transport", detail: `workspace snapshot restore failed: ${this.redactUrl(r.error.message, url)}`, cause: r.error }
+                : r.value.exitCode !== 0
+                    ? { cls: restoreClassOfExit(r.value.exitCode), detail: `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}` }
                     : undefined;
-        if (failed) {
+            if (!failure) {
+                if (target.lane === "space" && r.ok && r.value.stdout.includes("EMPTY_SNAPSHOT"))
+                    await this.dropDanglingLatest(target.space, snapshotId);
+                return ok(this.workspaceHandle());
+            }
             const name = this.podName;
             this.podName = undefined;
             this.podIP = undefined;
             if (name)
                 await this.deletePod(name).catch(() => { });
-            return { ok: false, error: failed };
+            if (attempt < RESTORE_MAX_ATTEMPTS[target.lane] && restoreFailureIsTransient(failure.cls)) {
+                if (await this.restorePause(abortSignal))
+                    continue;
+                return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted while retrying the workspace restore") };
+            }
+            switch (target.lane) {
+                case "park":
+                    return { ok: false, error: new RemoteExecutionError("connect_failed", failure.detail, failure.cause) };
+                case "space": {
+                    const tries = attempt === 1 ? "1 attempt" : `${attempt} attempts`;
+                    this.spaceLine("workspace_restore_failed", target.space, { snapshotId, class: failure.cls, attempts: attempt, boundMs: this.cfg.dataTimeoutMs, error: failure.detail });
+                    this.restoreRefusal = {
+                        ok: false,
+                        error: new RemoteExecutionError("connect_failed", `${failure.detail} — the workspace's latest snapshot (${snapshotId}) is in the store but did not come back (${failure.cls}, ${tries}, bound ${this.cfg.dataTimeoutMs}ms each), ` +
+                            "so this run does not start on an empty /workspace. Ways out: raise WORKSPACE_SNAPSHOT_TIMEOUT_MS if the archive is large, " +
+                            "or set WORKSPACE_PERSIST=off for one run (it starts from an empty /workspace and deletes no snapshot)", failure.cause),
+                    };
+                    return this.restoreRefusal;
+                }
+                default: {
+                    const unreachable = target;
+                    throw new Error(`unhandled snapshot lane: ${String(unreachable.lane)}`);
+                }
+            }
         }
-        return ok(this.workspaceHandle());
+    }
+    restorePause(signal) {
+        return new Promise((resolve) => {
+            if (signal?.aborted)
+                return resolve(false);
+            const onAbort = () => {
+                clearTimeout(timer);
+                resolve(false);
+            };
+            const timer = setTimeout(() => {
+                signal?.removeEventListener("abort", onAbort);
+                resolve(true);
+            }, this.cfg.retry.backoffMs);
+            signal?.addEventListener("abort", onAbort, { once: true });
+        });
+    }
+    async dropDanglingLatest(space, snapshotId) {
+        const prefix = this.cfg.s3Snapshot?.keyPrefix ?? "kata";
+        const dropped = await this.writePointer(space, "latest", null);
+        this.spaceLine("workspace_latest_dangling", space, {
+            key: `${prefix}/${space.incarnation}/space/${snapshotId}.tar.gz`,
+            deleted: dropped.ok,
+            ...(dropped.ok ? {} : { error: dropped.error.message }),
+            effect: "the space's latest snapshot is gone (removed by a bucket lifecycle rule or by hand) — this run starts from an empty /workspace; its run-end snapshot becomes the new latest",
+        });
     }
     async postResumeInit() {
         if (!this.cfg.s3Snapshot)
@@ -590,20 +1001,23 @@ export class RemoteK8sExecutionEnv {
         await this.disposeBackgroundShells().catch(() => { });
         this.destroyed = true;
         const lane = this.spaceLane;
-        if (this.podName) {
-            switch (lane.state) {
-                case "open":
+        switch (lane.state) {
+            case "open":
+                if (this.podName)
                     await this.persistWorkspace(lane.space);
-                    break;
-                case "unavailable":
-                    this.sayUnavailable();
-                    break;
-                case "off":
-                    break;
-                default: {
-                    const unreachable = lane;
-                    void unreachable;
-                }
+                else
+                    await this.endSpace(lane.space);
+                break;
+            case "unavailable":
+                if (this.podName)
+                    this.sayUnavailable(lane.reason);
+                break;
+            case "off":
+            case "unreadable":
+                break;
+            default: {
+                const unreachable = lane;
+                void unreachable;
             }
         }
         const name = this.podName;
@@ -620,10 +1034,18 @@ export class RemoteK8sExecutionEnv {
             this.spaceLine("workspace_snapshot_failed", space, { stage, code: error.code, error: error.message });
             this.onWorkspaceSnapshot?.("failed");
         };
-        const startedAt = Date.now();
-        const marked = await this.writePointer(space, "committing", String(startedAt));
-        if (!marked.ok)
-            return failed("marker", marked.error);
+        let claimed;
+        try {
+            claimed = await this.fenceCall(() => space.fence.begin(), "write");
+        }
+        catch (e) {
+            await this.endSpace(space);
+            return failed("fence", new RemoteExecutionError("connect_failed", `workspace space commit state write failed: ${redactIncarnations(errorMessageOf(e))}`));
+        }
+        if (!claimed) {
+            await this.endSpace(space);
+            return failed("fence", new RemoteExecutionError("timeout", "the commit fence is lost (this run's commit row is past its deadline, or another run's live commit holds the space) — not saving (a next run may already have restored the previous snapshot)"));
+        }
         try {
             let snap;
             try {
@@ -634,9 +1056,16 @@ export class RemoteK8sExecutionEnv {
             }
             if (!snap.ok)
                 return failed("upload", snap.error);
-            const remaining = startedAt + this.cfg.dataTimeoutMs - Date.now();
+            let deadline;
+            try {
+                deadline = await this.fenceCall(() => space.fence.publish(), "write");
+            }
+            catch (e) {
+                return failed("fence", new RemoteExecutionError("connect_failed", `workspace space commit state write failed: ${redactIncarnations(errorMessageOf(e))}`));
+            }
+            const remaining = deadline === undefined ? 0 : deadline - Date.now();
             if (remaining <= 0)
-                return failed("pointer", new RemoteExecutionError("timeout", `the commit outlived its bound (${this.cfg.dataTimeoutMs}ms since its marker) — not publishing latest (a next run may already have restored the previous one)`));
+                return failed("fence", new RemoteExecutionError("timeout", "the commit lost its fence before publishing (its row is past its deadline, or another run judged it abandoned and took the space) — not publishing latest (a next run may already have restored the previous one)"));
             const moved = await this.writePointer(space, "latest", snap.value, remaining);
             if (!moved.ok)
                 return failed("pointer", moved.error);
@@ -644,9 +1073,7 @@ export class RemoteK8sExecutionEnv {
             await this.pruneSpace(space, snap.value).catch((e) => this.spaceLine("workspace_snapshot_prune_failed", space, { detail: errorMessageOf(e) }));
         }
         finally {
-            const cleared = await this.writePointer(space, "committing", null);
-            if (!cleared.ok)
-                this.spaceLine("workspace_commit_marker_stuck", space, { error: cleared.error.message });
+            await this.endSpace(space);
         }
     }
     execUrl(command) {
@@ -1208,6 +1635,48 @@ export class RemoteK8sExecutionEnv {
         return dir ? this.resolve(dir) : undefined;
     }
 }
+export function createSpaceReclaimer(config, deps) {
+    return {
+        async reclaim(spaces) {
+            const deadline = AbortSignal.timeout(dataTimeoutMsOf(config));
+            for (const { sessionId, spaceKey } of spaces) {
+                if (spaceKey === "")
+                    continue;
+                const env = new RemoteK8sExecutionEnv({ ...config, sessionId }, deps);
+                if (config.cache !== undefined) {
+                    const r = await env.reclaimSpaceCache(spaceKey, deadline).catch((e) => ({ ok: false, reason: "delete_failed", detail: errorMessageOf(e) }));
+                    if (!r.ok) {
+                        recordFailOpen("server.workspace-cache.reclaim-failed", `reason=${r.reason} label=${workspaceCacheClaimOf(spaceKey).label}`);
+                        config.logger?.warn("workspace_cache_reclaim_failed", {
+                            reason: r.reason,
+                            space: incarnationLogPrefix(spaceKey),
+                            label: workspaceCacheClaimOf(spaceKey).label,
+                            error: redactIncarnations(r.detail),
+                            effect: `the ended space's cache volume stays — delete it by label ${WORKSPACE_CACHE_SPACE_LABEL}=<label> (the deletion it belongs to went ahead)`,
+                        });
+                    }
+                }
+                if (config.s3Snapshot !== undefined) {
+                    const failures = await env.reclaimSpaceSnapshots(spaceKey, deadline).catch((e) => [{ lane: "space", stage: "delete", detail: errorMessageOf(e) }]);
+                    for (const f of failures) {
+                        recordFailOpen("server.workspace-space.reclaim-failed", `lane=${f.lane} stage=${f.stage}`);
+                        config.logger?.warn("workspace_space_reclaim_failed", {
+                            space: incarnationLogPrefix(spaceKey),
+                            sessionId,
+                            lane: f.lane,
+                            stage: f.stage,
+                            error: redactIncarnations(f.detail),
+                            effect: `the ended space's ${f.lane === "space" ? "workspace snapshots" : "park snapshots"} may stay in the bucket (unreadable through the API; the bucket's lifecycle rule is the fallback) — the deletion it belongs to went ahead`,
+                        });
+                    }
+                }
+            }
+        },
+    };
+}
+export async function checkSandboxEgressPolicy(config, deps) {
+    return new RemoteK8sExecutionEnv(config, deps).egressPolicyGaps();
+}
 export function k8sExecutionEnvFactory(config, deps) {
     return (ctx) => new RemoteK8sExecutionEnv({ ...config, sessionId: ctx.sessionId }, deps);
 }