@sema-agent/server 7.124.0 → 7.125.0

package/dist/plugins/remote-env-k8s.d.ts

@@ -1,3 +1,4 @@
+import type { WorkspaceSyncCapability } from "../workspace-space.js";
 import { FileError, ExecutionError, RemoteExecutionError, type ExecutionEnv, type RemoteExecutionEnv, type WorkspaceHandle, type FileInfo, type Result, type SymlinkChain, type OutputChunk, type ExecStreamOptions, type RemoteConnectConfig, type SnapshotId, type SessionToken, type VmLifecycleOptions, type ExecutionEnvFactory, type BackgroundShellCapability, type BackgroundShellId, type BackgroundShellError, type BackgroundSpawnOptions, type BackgroundPoll } from "@sema-agent/core";
 import type { PodSpecPatch } from "./k8s-exec-protocol.js";
 type ExecOpts = Parameters<ExecutionEnv["exec"]>[1];
@@ -27,7 +28,8 @@ export interface K8sEnvConfig {
     homeDir?: string;
     /** Workspace root inside the pod; created on connect. Default "/workspace". */
     mountPath?: string;
-    /** Pod lifetime (activeDeadlineSeconds) — the leak guard. Default 30min. */
+    /** Pod lifetime (`activeDeadlineSeconds`; env `K8S_TIMEOUT_MS`) — the whole-pod leak guard, and the ceiling background-shell
+     *  timeouts are clamped to. NOT a per-command timeout: each command's bound comes from its tool call. Default 30min. */
     timeoutMs?: number;
     /** Pod schedule+pull+start bound. Default 120s (image pull can be slow once per node). */
     readyTimeoutMs?: number;
@@ -55,6 +57,12 @@ export interface K8sEnvConfig {
      * (never the access/secret key) — a credential reaching code that runs untrusted work is the [ref] lethal
      * trifecta. The presigned URL is used only inside an adapter-issued exec at suspend/resume time, so it never
      * reaches an agent turn.
+     *
+     * WORKSPACE SPACE (`workspacePersist`, default `off` — opt-in `space`): the same store also carries the workspace ACROSS
+     * RUNS, in its own sub-prefix `<prefix>/<spaceIncarnation>/space/` — `space/latest` (body = the newest snapshotId) names the
+     * snapshot `connect()` restores; `destroy()` snapshots into `space/`, moves `latest`, keeps the newest `workspaceSnapshotKeep`
+     * (the same one upload / one download as suspend / resume, see `SnapshotTarget`). The pointer read / write, the listing and the deletes are
+     * WORKER-side requests to `endpoint` (the snapshot-BLOB offload dials the same `MINIO_ENDPOINT` from the worker).
      */
     s3Snapshot?: {
         endpoint: string;
@@ -64,9 +72,36 @@ export interface K8sEnvConfig {
         region?: string;
         keyPrefix?: string;
         presignTtlSec?: number;
+        /** The workspace-space policy (the capability fact `workspaceSpace`'s witness, handed in by the factory; the adapter
+         *  never resolves defaults itself). Absent or `persist: "off"` ⇒ no persistence across runs. */
+        space?: WorkspaceSyncCapability;
     };
-    /** Set by the factory from the task's ExecutionEnvFactoryContext — namespaces snapshot keys per session. */
+    /**
+     * The PARK lane's partition (`<prefix>/<sessionId>/<snapshotId>.tar.gz`), set by {@link k8sExecutionEnvFactory} from
+     * `ExecutionEnvFactoryContext.sessionId` — the key form parked runs have always used, so stored parks stay readable.
+     * Why the session id is right HERE: a park archive is only ever read back by its own parked run's checkpoint (the
+     * snapshotId lives in that checkpoint; deleting the session deletes its checkpoints) — the partition only namespaces,
+     * it never decides who reads. Segment shape is validated in exactly one place ({@link RemoteK8sExecutionEnv.presignSnapshot}).
+     */
     sessionId?: string;
+    /**
+     * The SPACE lane's partition (`<prefix>/<spaceIncarnation>/space/…`): the session OCCUPANT's incarnation token
+     * (`session-incarnation.ts`, the one owner of "is this the same occupant"), handed in per run by the factory binding
+     * (`boot/execution-env.ts`) from the same seat the memory engine reads. Why not the session id: the space lane is read
+     * by whoever runs next in the session, so its partition decides WHO reads — a session id deleted and registered again
+     * (by anyone) is a new occupant and starts from an empty workspace. Absent while persistence is on ⇒ the space lane
+     * does not open for this env (connect restores nothing, destroy saves nothing) and one `workspace_space_unavailable`
+     * line says so — never a fallback to the session id. Never logged whole: operator lines carry `incarnationLogPrefix`.
+     */
+    spaceIncarnation?: string;
+    /** Operator log seat (the e2b adapter's shape): `workspace_snapshot_failed` / `workspace_snapshot_prune_failed` /
+     *  `workspace_space_unavailable` / `workspace_commit_abandoned` / `workspace_commit_marker_stuck`. */
+    logger?: {
+        warn(event: string, fields?: Record<string, unknown>): void;
+    };
+    /** Run-end outcome seat — called once per `destroy()` that attempted a space snapshot (the factory's caller binds it
+     *  to the run, so the run's terminal frame can carry `workspaceSnapshot: "failed"`). */
+    onWorkspaceSnapshot?: (outcome: "saved" | "failed") => void;
     /**
      * 🔴 ADDITIVE pod-spec patch (a SERVICE-side adapter seam, NOT a core seam: the K8s pod shape is
      * provider-specific, so it stays in the adapter per [ref]). A deployment supplies extra volumes / volumeMounts
@@ -112,6 +147,8 @@ export interface K8sEnvDeps {
     }>;
     openExec?: (url: string, headers: Record<string, string>) => ExecSocket;
     connectOnce?: (signal?: AbortSignal) => Promise<ConnectResult>;
+    /** Worker-side object-store requests (the space pointer, listing, deletes). Default: global `fetch`. */
+    fetch?: typeof fetch;
 }
 export { execSocketTlsOptions } from "./k8s-exec-protocol.js";
 /** `Error`-or-event → message. Bun's ws shim emits browser-style ErrorEvents (NOT instanceof Error) whose
@@ -150,6 +187,12 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     readonly homeDir?: string | undefined;
     private readonly cfg;
     private readonly deps;
+    /** This env's space lane (resolved once from the policy + the occupant's incarnation; see {@link SpaceLane}). */
+    private readonly spaceLane;
+    /** `workspace_space_unavailable` is said once per env. */
+    private spaceUnavailableSaid;
+    private readonly logger?;
+    private readonly onWorkspaceSnapshot?;
     constructor(config: K8sEnvConfig, deps?: K8sEnvDeps);
     /** 166-T2 — ExecutionEnv 寿命声明:pod `activeDeadlineSeconds = ceil(timeoutMs/1000)` 是平台真硬死线
      *  (k8s 到点杀 pod,无 keepalive 续期机制)⇒ 恒声明,core 在死线−60s 处 suspend(基建合格)或响亮
@@ -163,7 +206,23 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     /** Control-plane REST call (pod create/get/delete), bounded by rpcTimeoutMs. Injectable for tests. */
     private request;
     private openExec;
+    /**
+     * Connect = the workspace SPACE's restore point. Space lane open ⇒ first wait out a commit still in flight
+     * ({@link awaitSpaceCommit}), then read `<prefix>/<incarnation>/space/latest`: present ⇒
+     * restore it through {@link restoreSnapshot} (the one tar download, shared with `resumeVM`; a missing archive is
+     * refused); absent (404) ⇒ a fresh pod; a read that FAILS ⇒ a loud
+     * `connect_failed` — never a silent fresh workspace (a run that believes it resumed while it did not is the worst form).
+     * Persistence off / no store ⇒ a fresh pod, exactly as before. Persistence on but the occupant unknown ⇒ a fresh pod
+     * and one `workspace_space_unavailable` line (no restore by any other key).
+     */
     connect(config?: RemoteConnectConfig): Promise<ConnectResult>;
+    /** The space lane is on but this env's occupant is unknown — said once per env (at connect, or at a destroy with a pod). */
+    private sayUnavailable;
+    /** Every space-lane operator line: the space is named by its incarnation's log prefix only, and free text (a curl
+     *  stderr, a fetch error) passes {@link redactIncarnations} — the full token never reaches a log body. */
+    private spaceLine;
+    /** A fresh pod (create → Running → workspace root), bounded retry. Shared by {@link connect} and {@link resumeVM}. */
+    private provision;
     /**
      * Lazy-connect on first use — 🔴 core NEVER calls `connect()` (verified: the Runner only calls `resumeVM` on
      * the resume path), so an adapter that demands an explicit connect can never work through the production
@@ -173,14 +232,45 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     private ensureConnected;
     private connectOnce;
     /**
-     * Build a validated, presigned URL for a snapshot object — `<prefix>/<sessionId>/<snapshotId>.tar.gz`.
-     * 🔴 sessionId comes from the client (`body.sessionId`, security.ts) and is NOT charset-validated upstream, so
-     * each path segment is allowlisted here (`^[A-Za-z0-9][A-Za-z0-9._-]*$`) before it enters the object key: a
-     * segment of `..` / `../x` / one containing `/` would otherwise let one tenant's snapshot key normalize into
-     * another tenant's prefix (cross-tenant read on resume / clobber on suspend). Returns a typed error instead of
-     * signing a traversing key. Folds the old snapshotKey()+s3Presign() pair so the two call sites can't diverge.
+     * Build a validated, presigned URL for a snapshot object of either lane (see {@link SnapshotTarget}) — the ONE place a
+     * snapshot object key is spelled, and the one place each lane's partition is chosen (park = the session id, space = the
+     * occupant's incarnation carried by the request).
+     * 🔴 the park partition is the session id, which comes from the client (`body.sessionId`, security.ts) and is NOT
+     * charset-validated upstream, so each path segment is allowlisted here (`^[A-Za-z0-9][A-Za-z0-9._-]*$`) before it enters
+     * the object key: a segment of `..` / `../x` / one containing `/` would otherwise let one tenant's snapshot key normalize into
+     * another tenant's prefix (cross-tenant read on resume / clobber on suspend). The space partition (a minted v4 uuid) goes
+     * through the same one allowlist. Returns a typed error instead of signing a traversing key. Folds the old
+     * snapshotKey()+s3Presign() pair so the two call sites can't diverge.
      */
     private presignSnapshot;
+    /** Worker-side object-store request, bounded by the control-plane RPC wall-clock — or a tighter caller deadline — and
+     *  the caller's abort. */
+    private storeFetch;
+    /** Read one space pointer object: its trimmed body, or `undefined` = absent (404); any other failure is a typed
+     *  `connect_failed` naming the pointer. */
+    private readPointer;
+    /** Read the space's `latest` pointer. The body's shape is checked where it is used — restoreSnapshot → presignSnapshot
+     *  (the one segment check). */
+    private readLatest;
+    /** PUT one space pointer object (a single-object PUT replaces the object atomically) or DELETE it; `timeoutMs` = a
+     *  caller deadline tighter than the RPC bound. */
+    private writePointer;
+    /**
+     * Wait out the previous run's space commit. `space/committing` present and younger than the upload's own wall-clock
+     * bound (`dataTimeoutMs` — the SAME number `uploadSnapshot` gives its tar + curl exec) ⇒ re-read every
+     * {@link SPACE_COMMIT_POLL_MS} until it is gone or has aged past that bound. A marker at / past the bound = a commit
+     * whose worker died inside it (a live upload fails at the bound and clears its marker) ⇒ one
+     * `workspace_commit_abandoned` line; `latest` still names the previous COMPLETE snapshot. A marker read that fails, or
+     * a body that is not an epoch-ms timestamp, is a loud `connect_failed` (the same form as a failed `latest` read).
+     */
+    private awaitSpaceCommit;
+    /**
+     * Keep the newest `keep` snapshots of this space (the one `latest` now names always among them); delete the rest.
+     * Works inside the `space/` sub-prefix only — park archives are never in its view. The bucket listing is the only
+     * record (no index). A failure loses nothing (the space just keeps more objects) —
+     * it is reported, never thrown.
+     */
+    private pruneSpace;
     /** Redact a presigned URL (and any lingering signature) from text before it enters an error/log — the URL is a
      *  short-lived bearer credential and must never reach a durable store (secret boundary). */
     private redactUrl;
@@ -200,10 +290,15 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
         ok: false;
         error: RemoteExecutionError;
     }>;
+    /**
+     * The ONE tar upload (both lanes): tar the workspace to a pod-local file, presigned PUT into the lane's namespace,
+     * release the pod. `suspendVM` = lane `park`; the run-end space snapshot = lane `space` (its open space).
+     */
+    private uploadSnapshot;
     /**
      * Resume a workspace snapshot onto a FRESH pod (resume = executionEnvFactory rebuild + resumeVM):
-     * provision a pod, then download + untar the snapshot into the workspace. A missing object (empty/plan-only
-     * snapshot) resumes as a clean workspace.
+     * provision a pod, then download + untar the snapshot into the workspace ({@link restoreSnapshot}, lane `park`). A
+     * missing object (empty/plan-only snapshot) resumes as a clean workspace.
      *
      * [ref]①(priorHandle 根保真)——k8s 走**诚实臂**而非绑定臂:pod 的 workspace volume 挂在**当前**
      * 配置根(cfg.mountPath),往 priorHandle.mountPath 绑=往容器 rootfs untar 大工作区,ephemeral-storage
@@ -212,6 +307,14 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
      * 变更窗的分歧由 core 侧 fail-closed reopen 兜底,审批不烧错根。
      */
     resumeVM(snapshotId: SnapshotId, options?: VmLifecycleOptions): Promise<ConnectResult>;
+    /**
+     * The ONE tar download (both lanes): provision a FRESH pod, download + untar the lane's archive into the workspace.
+     * `missing: "empty"` (park: a plan-only suspend has no object) tolerates a 404 as a clean workspace; `missing:
+     * "refuse"` (space: its pointer is only ever written after a successful upload ⇒ a 404 there is lost data) refuses it.
+     * A restore that does not complete releases the pod it provisioned: no tool and no run-end snapshot may ever see a
+     * torn or empty-by-accident workspace.
+     */
+    private restoreSnapshot;
     /**
      * Honest post-resume consistency ([ref] §5 council #8): the workspace FILES are restored byte-for-byte
      * from the snapshot, so read-before-edit hashing already lines up — there's nothing to re-fetch. In-VM
@@ -227,9 +330,21 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     }>;
     reconnect(_sessionToken: SessionToken): Promise<ConnectResult>;
     private deletePod;
-    /** Delete the sandbox pod — the workspace is DISPOSABLE (vs SSH/ADB disconnect-only). Idempotent, never throws. */
+    /**
+     * Delete the sandbox pod. Idempotent, never throws. Space lane open and a pod exists ⇒ FIRST the run-end space
+     * snapshot ({@link persistWorkspace}); its failure is loud and never blocks the delete (the pod lifetime is the
+     * leak guard). A run that never connected has no workspace to save — `latest` stays where it was. Space lane
+     * unavailable (occupant unknown) ⇒ nothing is saved (the once-per-env line says so).
+     */
     destroy(): Promise<void>;
     cleanup(): Promise<void>;
+    /**
+     * Run-end space snapshot: PUT the `committing` marker → the one tar upload ({@link uploadSnapshot}, lane `space` — it
+     * also releases the pod) → move `latest` → keep N → DELETE the marker (always, saved or failed).
+     * Upload or pointer failure ⇒ one `workspace_snapshot_failed` line + the `failed` outcome; `latest` keeps naming the
+     * previous snapshot, so the next run restores THAT — never an empty workspace dressed up as a restore.
+     */
+    private persistWorkspace;
     private execUrl;
     /**
      * Run a command over the exec WebSocket. Frames: byte0=channel (1 stdout / 2 stderr / 3 status JSON with the
@@ -347,5 +462,5 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
  * created lazily on connect and deleted by `destroy()` (Runner-owned lifetime; `activeDeadlineSeconds` is the
  * leak guard if destroy never runs).
  */
-export declare function k8sExecutionEnvFactory(config: K8sEnvConfig): ExecutionEnvFactory;
+export declare function k8sExecutionEnvFactory(config: K8sEnvConfig, deps?: K8sEnvDeps): ExecutionEnvFactory;
 //# sourceMappingURL=remote-env-k8s.d.ts.map

package/dist/plugins/remote-env-k8s.js

@@ -5,7 +5,8 @@ import http from "node:http";
 import { StringDecoder } from "node:string_decoder";
 import WebSocket from "ws";
 import { buildCwdEnvCommand, createCwdGuardedCall, shellQuote, kindFromMode, numEnvOr } from "./remote-shell.js";
-import { presignS3Url } from "./s3-presign.js";
+import { presignS3Url, presignS3ListUrl } from "./s3-presign.js";
+import { incarnationLogPrefix, redactIncarnations } from "../session-incarnation.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
 import { fileErrorFromExec, classifyFsStderr } from "./remote-env-file-error.js";
 import { createPosixShellFs } from "./posix-shell-fs.js";
@@ -31,6 +32,21 @@ const EXEC_SUBPROTOCOLS = ["v5.channel.k8s.io", "v4.channel.k8s.io"];
 const WRITE_CHUNK_BYTES = 64 * 1024;
 import { applyPodSpecPatch } from "./k8s-exec-protocol.js";
 export { applyPodSpecPatch } from "./k8s-exec-protocol.js";
+function spaceLaneOf(policy, incarnation) {
+    if (policy === undefined)
+        return { state: "off" };
+    switch (policy.persist) {
+        case "off":
+            return { state: "off" };
+        case "space":
+            return incarnation ? { state: "open", space: { incarnation, keep: policy.keep } } : { state: "unavailable" };
+        default: {
+            const unreachable = policy.persist;
+            throw new Error(`unhandled WORKSPACE_PERSIST word: ${String(unreachable)}`);
+        }
+    }
+}
+const SPACE_COMMIT_POLL_MS = 1000;
 import { execSocketTlsOptions, exitCodeFromStatus } from "./k8s-exec-protocol.js";
 export { execSocketTlsOptions } from "./k8s-exec-protocol.js";
 export function errorMessageOf(e) {
@@ -52,6 +68,10 @@ export class RemoteK8sExecutionEnv {
     homeDir;
     cfg;
     deps;
+    spaceLane;
+    spaceUnavailableSaid = false;
+    logger;
+    onWorkspaceSnapshot;
     constructor(config, deps) {
         this.cfg = {
             apiUrl: config.apiUrl,
@@ -78,6 +98,9 @@ export class RemoteK8sExecutionEnv {
         this.cwd = this.cfg.mountPath;
         this.homeDir = config.homeDir;
         this.deps = deps ?? {};
+        this.spaceLane = spaceLaneOf(config.s3Snapshot?.space, config.spaceIncarnation);
+        this.logger = config.logger;
+        this.onWorkspaceSnapshot = config.onWorkspaceSnapshot;
         this.capabilities = { isolation: true, suspendable: !!this.cfg.s3Snapshot, canonicalPathAuthoritative: true };
         const bgMax = Math.max(1, Math.min(K8S_BG_MAX_TIMEOUT_SEC, Math.floor(this.cfg.timeoutMs / 1000)));
         this.backgroundCapabilities = {
@@ -177,13 +200,53 @@ export class RemoteK8sExecutionEnv {
         });
     }
     async connect(config) {
+        const signal = config?.abortSignal;
+        const lane = this.spaceLane;
+        switch (lane.state) {
+            case "off":
+                return this.provision(signal);
+            case "unavailable":
+                this.sayUnavailable();
+                return this.provision(signal);
+            case "open": {
+                const settled = await this.awaitSpaceCommit(lane.space, signal);
+                if (!settled.ok)
+                    return settled;
+                const latest = await this.readLatest(lane.space, signal);
+                if (!latest.ok)
+                    return latest;
+                if (latest.value === undefined)
+                    return this.provision(signal);
+                return this.restoreSnapshot({ lane: "space", space: lane.space }, latest.value, { missing: "refuse", ...(signal ? { abortSignal: signal } : {}) });
+            }
+            default: {
+                const unreachable = lane;
+                throw new Error(`unhandled space lane state: ${String(unreachable.state)}`);
+            }
+        }
+    }
+    sayUnavailable() {
+        if (this.spaceUnavailableSaid)
+            return;
+        this.spaceUnavailableSaid = true;
+        this.logger?.warn("workspace_space_unavailable", {
+            reason: "incarnation_unknown",
+            sessionId: this.cfg.sessionId,
+            effect: "WORKSPACE_PERSIST=space, but this run's session occupant (incarnation) was not read on this replica — its /workspace is neither restored nor saved (it is never keyed by the session id instead)",
+        });
+    }
+    spaceLine(event, space, fields) {
+        const safe = Object.fromEntries(Object.entries(fields).map(([k, v]) => [k, typeof v === "string" ? redactIncarnations(v) : v]));
+        this.logger?.warn(event, { space: incarnationLogPrefix(space.incarnation), ...safe });
+    }
+    provision(signal) {
         const policy = {
             retryableCodes: ["connect_failed", "timeout", "transport_lost"],
             maxAttempts: this.cfg.retry.maxAttempts,
             backoffMs: (failed) => this.cfg.retry.backoffMs * failed,
         };
         const once = this.deps.connectOnce ?? ((s) => this.connectOnce(s));
-        return withRetry(() => once(config?.abortSignal), policy, { signal: config?.abortSignal });
+        return withRetry(() => once(signal), policy, { signal });
     }
     async ensureConnected() {
         if (this.podName)
@@ -301,28 +364,138 @@ export class RemoteK8sExecutionEnv {
         }
         return ok(this.workspaceHandle());
     }
-    presignSnapshot(method, snapshotId) {
+    presignSnapshot(req) {
         const s3 = this.cfg.s3Snapshot;
         const prefix = s3.keyPrefix ?? "kata";
-        const session = this.cfg.sessionId ?? "_";
+        const [label, partition] = req.lane === "space" ? ["spaceIncarnation", req.space.incarnation] : ["sessionId", this.cfg.sessionId ?? "_"];
+        const snapshotId = "snapshotId" in req ? req.snapshotId : undefined;
         const safe = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
-        for (const [label, seg] of [["sessionId", session], ["snapshotId", snapshotId], ["keyPrefix", prefix]]) {
+        const segments = [[label, partition], ...(snapshotId !== undefined ? [["snapshotId", snapshotId]] : []), ["keyPrefix", prefix]];
+        for (const [label, seg] of segments) {
             if (!safe.test(seg))
                 return { ok: false, error: new RemoteExecutionError("connect_failed", `unsafe snapshot key segment (${label}): refusing to build a traversing object key`) };
         }
-        return {
-            ok: true,
-            value: presignS3Url({
-                endpoint: s3.endpoint,
-                bucket: s3.bucket,
-                key: `${prefix}/${session}/${snapshotId}.tar.gz`,
-                method,
-                accessKey: s3.accessKey,
-                secretKey: s3.secretKey,
-                ...(s3.region ? { region: s3.region } : {}),
-                expiresSec: Math.min(s3.presignTtlSec ?? 600, 3600),
-            }),
+        const common = {
+            endpoint: s3.endpoint,
+            bucket: s3.bucket,
+            accessKey: s3.accessKey,
+            secretKey: s3.secretKey,
+            ...(s3.region ? { region: s3.region } : {}),
+            expiresSec: Math.min(s3.presignTtlSec ?? 600, 3600),
         };
+        const base = req.lane === "space" ? `${prefix}/${partition}/space/` : `${prefix}/${partition}/`;
+        if (req.method === "LIST")
+            return { ok: true, value: presignS3ListUrl({ ...common, prefix: base, maxKeys: 1000 }) };
+        const object = "pointer" in req ? req.pointer : `${req.snapshotId}.tar.gz`;
+        return { ok: true, value: presignS3Url({ ...common, key: `${base}${object}`, method: req.method }) };
+    }
+    storeFetch(url, init) {
+        const bound = AbortSignal.timeout(Math.max(1, Math.min(this.cfg.rpcTimeoutMs, init.timeoutMs ?? this.cfg.rpcTimeoutMs)));
+        const signal = init.signal ? AbortSignal.any([init.signal, bound]) : bound;
+        return (this.deps.fetch ?? fetch)(url, { method: init.method, ...(init.body !== undefined ? { body: init.body } : {}), signal });
+    }
+    async readPointer(space, pointer, signal) {
+        const url = this.presignSnapshot({ lane: "space", space, method: "GET", pointer });
+        if (!url.ok)
+            return url;
+        const what = pointer === "latest" ? "pointer" : "commit marker";
+        let res;
+        let body;
+        try {
+            res = await this.storeFetch(url.value, { method: "GET", ...(signal ? { signal } : {}) });
+            body = await res.text();
+        }
+        catch (e) {
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} read failed: ${this.redactUrl(errorMessageOf(e), url.value)}`, e instanceof Error ? e : undefined) };
+        }
+        if (res.status === 404)
+            return ok(undefined);
+        if (!res.ok)
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} read failed (HTTP ${res.status})`) };
+        return ok(body.trim());
+    }
+    async readLatest(space, signal) {
+        const r = await this.readPointer(space, "latest", signal);
+        return r.ok ? ok(r.value === undefined ? undefined : r.value) : r;
+    }
+    async writePointer(space, pointer, body, timeoutMs) {
+        const method = body === null ? "DELETE" : "PUT";
+        const url = this.presignSnapshot({ lane: "space", space, method, pointer });
+        if (!url.ok)
+            return url;
+        const what = pointer === "latest" ? "pointer" : "commit marker";
+        try {
+            const res = await this.storeFetch(url.value, { method, ...(body === null ? {} : { body }), ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
+            if (!res.ok && !(method === "DELETE" && res.status === 404))
+                return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} ${method === "PUT" ? "write" : "delete"} failed (HTTP ${res.status})`) };
+        }
+        catch (e) {
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} ${method === "PUT" ? "write" : "delete"} failed: ${this.redactUrl(errorMessageOf(e), url.value)}`, e instanceof Error ? e : undefined) };
+        }
+        return ok(undefined);
+    }
+    async awaitSpaceCommit(space, signal) {
+        for (;;) {
+            const marker = await this.readPointer(space, "committing", signal);
+            if (!marker.ok)
+                return marker;
+            if (marker.value === undefined)
+                return ok(undefined);
+            const startedAt = /^\d+$/.test(marker.value) ? Number(marker.value) : NaN;
+            if (!Number.isFinite(startedAt))
+                return { ok: false, error: new RemoteExecutionError("connect_failed", "workspace space commit marker is malformed (its body is not an epoch-ms timestamp) — refusing to guess whether a commit is in flight") };
+            const ageMs = Date.now() - startedAt;
+            if (ageMs >= this.cfg.dataTimeoutMs) {
+                this.spaceLine("workspace_commit_abandoned", space, { ageMs });
+                return ok(undefined);
+            }
+            await new Promise((resolveWait) => {
+                const t = setTimeout(resolveWait, SPACE_COMMIT_POLL_MS);
+                signal?.addEventListener("abort", () => { clearTimeout(t); resolveWait(); }, { once: true });
+            });
+            if (signal?.aborted)
+                return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted while waiting for the previous run's workspace commit") };
+        }
+    }
+    async pruneSpace(space, newest) {
+        const list = this.presignSnapshot({ lane: "space", space, method: "LIST" });
+        if (!list.ok)
+            return;
+        const failed = (detail) => this.spaceLine("workspace_snapshot_prune_failed", space, { detail });
+        let xml;
+        try {
+            const res = await this.storeFetch(list.value, { method: "GET" });
+            xml = await res.text();
+            if (!res.ok)
+                return failed(`list HTTP ${res.status}`);
+        }
+        catch (e) {
+            return failed(`list: ${this.redactUrl(errorMessageOf(e), list.value)}`);
+        }
+        if (/<IsTruncated>true<\/IsTruncated>/.test(xml))
+            return failed("listing truncated (more than 1000 objects under this space) — refusing to prune from a partial view");
+        const tail = ".tar.gz";
+        const archives = [...xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)]
+            .map((m) => ({ key: /<Key>([^<]*)<\/Key>/.exec(m[1])?.[1] ?? "", at: Date.parse(/<LastModified>([^<]*)<\/LastModified>/.exec(m[1])?.[1] ?? "") }))
+            .filter((o) => o.key.endsWith(tail))
+            .map((o) => ({ ...o, id: o.key.slice(o.key.lastIndexOf("/") + 1, -tail.length) }));
+        const mine = archives.find((o) => o.id === newest);
+        if (!mine || !Number.isFinite(mine.at))
+            return failed("the snapshot just written is not in the listing (or has no LastModified) — refusing to prune without an age anchor");
+        const older = archives.filter((o) => Number.isFinite(o.at) && o.at < mine.at).sort((a, b) => b.at - a.at || (a.key < b.key ? 1 : -1));
+        for (const o of older.slice(Math.max(0, space.keep - 1))) {
+            const del = this.presignSnapshot({ lane: "space", space, method: "DELETE", snapshotId: o.id });
+            if (!del.ok)
+                continue;
+            try {
+                const res = await this.storeFetch(del.value, { method: "DELETE" });
+                if (!res.ok && res.status !== 404)
+                    failed(`delete ${o.id}: HTTP ${res.status}`);
+            }
+            catch (e) {
+                failed(`delete ${o.id}: ${this.redactUrl(errorMessageOf(e), del.value)}`);
+            }
+        }
     }
     redactUrl(text, url) {
         return text.split(url).join("<presigned-url>").replace(/X-Amz-Signature=[0-9a-f]+/gi, "X-Amz-Signature=<redacted>");
@@ -330,12 +503,15 @@ export class RemoteK8sExecutionEnv {
     async suspendVM(_options) {
         if (!this.cfg.s3Snapshot)
             return unsupported("suspendVM");
+        return this.uploadSnapshot({ lane: "park" });
+    }
+    async uploadSnapshot(target) {
         if (this.inFlightExecs > 0)
             return { ok: false, error: new RemoteExecutionError("command_in_flight", "cannot suspend while a command is in flight (would snapshot a torn workspace)") };
         const snapshotId = `kata-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
         if (!this.podName)
             return ok(snapshotId);
-        const presigned = this.presignSnapshot("PUT", snapshotId);
+        const presigned = this.presignSnapshot({ ...target, method: "PUT", snapshotId });
         if (!presigned.ok)
             return presigned;
         const url = presigned.value;
@@ -356,10 +532,16 @@ export class RemoteK8sExecutionEnv {
     async resumeVM(snapshotId, options) {
         if (!this.cfg.s3Snapshot)
             return unsupported("resumeVM");
-        const presigned = this.presignSnapshot("GET", snapshotId);
+        const r = await this.restoreSnapshot({ lane: "park" }, snapshotId, { missing: "empty", ...(options?.abortSignal ? { abortSignal: options.abortSignal } : {}) });
+        if (r.ok)
+            this.handleSnapshotId = snapshotId;
+        return r.ok ? ok(this.workspaceHandle()) : r;
+    }
+    async restoreSnapshot(target, snapshotId, opts) {
+        const presigned = this.presignSnapshot({ ...target, method: "GET", snapshotId });
         if (!presigned.ok)
             return presigned;
-        const c = await this.connect({ abortSignal: options?.abortSignal });
+        const c = await this.provision(opts.abortSignal);
         if (!c.ok)
             return c;
         const url = presigned.value;
@@ -368,11 +550,21 @@ export class RemoteK8sExecutionEnv {
             `if [ "$code" != "200" ]; then echo "snapshot GET HTTP $code" >&2; exit 22; fi; ` +
             `tar xzf /tmp/ws.tgz -C ${shellQuote(this.cfg.mountPath)} && rm -f /tmp/ws.tgz`;
         const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs, livenessMs: 0 });
-        if (!r.ok)
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot restore failed: ${this.redactUrl(r.error.message, url)}`, r.error) };
-        if (r.value.exitCode !== 0)
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}`) };
-        this.handleSnapshotId = snapshotId;
+        const failed = !r.ok
+            ? new RemoteExecutionError("connect_failed", `workspace snapshot restore failed: ${this.redactUrl(r.error.message, url)}`, r.error)
+            : r.value.exitCode !== 0
+                ? new RemoteExecutionError("connect_failed", `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}`)
+                : opts.missing === "refuse" && r.value.stdout.includes("EMPTY_SNAPSHOT")
+                    ? new RemoteExecutionError("connect_failed", `workspace snapshot ${snapshotId} is named by the space pointer but its archive is missing (HTTP 404) — refusing to start on an empty workspace (delete the space's latest pointer, or WORKSPACE_PERSIST=off)`)
+                    : undefined;
+        if (failed) {
+            const name = this.podName;
+            this.podName = undefined;
+            this.podIP = undefined;
+            if (name)
+                await this.deletePod(name).catch(() => { });
+            return { ok: false, error: failed };
+        }
         return ok(this.workspaceHandle());
     }
     async postResumeInit() {
@@ -397,6 +589,23 @@ export class RemoteK8sExecutionEnv {
     async destroy() {
         await this.disposeBackgroundShells().catch(() => { });
         this.destroyed = true;
+        const lane = this.spaceLane;
+        if (this.podName) {
+            switch (lane.state) {
+                case "open":
+                    await this.persistWorkspace(lane.space);
+                    break;
+                case "unavailable":
+                    this.sayUnavailable();
+                    break;
+                case "off":
+                    break;
+                default: {
+                    const unreachable = lane;
+                    void unreachable;
+                }
+            }
+        }
         const name = this.podName;
         this.podName = undefined;
         this.podIP = undefined;
@@ -406,6 +615,40 @@ export class RemoteK8sExecutionEnv {
     async cleanup() {
         return this.destroy();
     }
+    async persistWorkspace(space) {
+        const failed = (stage, error) => {
+            this.spaceLine("workspace_snapshot_failed", space, { stage, code: error.code, error: error.message });
+            this.onWorkspaceSnapshot?.("failed");
+        };
+        const startedAt = Date.now();
+        const marked = await this.writePointer(space, "committing", String(startedAt));
+        if (!marked.ok)
+            return failed("marker", marked.error);
+        try {
+            let snap;
+            try {
+                snap = await this.uploadSnapshot({ lane: "space", space });
+            }
+            catch (e) {
+                return failed("upload", new RemoteExecutionError("unknown", errorMessageOf(e)));
+            }
+            if (!snap.ok)
+                return failed("upload", snap.error);
+            const remaining = startedAt + this.cfg.dataTimeoutMs - Date.now();
+            if (remaining <= 0)
+                return failed("pointer", new RemoteExecutionError("timeout", `the commit outlived its bound (${this.cfg.dataTimeoutMs}ms since its marker) — not publishing latest (a next run may already have restored the previous one)`));
+            const moved = await this.writePointer(space, "latest", snap.value, remaining);
+            if (!moved.ok)
+                return failed("pointer", moved.error);
+            this.onWorkspaceSnapshot?.("saved");
+            await this.pruneSpace(space, snap.value).catch((e) => this.spaceLine("workspace_snapshot_prune_failed", space, { detail: errorMessageOf(e) }));
+        }
+        finally {
+            const cleared = await this.writePointer(space, "committing", null);
+            if (!cleared.ok)
+                this.spaceLine("workspace_commit_marker_stuck", space, { error: cleared.error.message });
+        }
+    }
     execUrl(command) {
         const { apiUrl, token } = this.creds();
         const base = apiUrl.replace(/^http/, "ws");
@@ -965,7 +1208,7 @@ export class RemoteK8sExecutionEnv {
         return dir ? this.resolve(dir) : undefined;
     }
 }
-export function k8sExecutionEnvFactory(config) {
-    return (ctx) => new RemoteK8sExecutionEnv({ ...config, sessionId: ctx.sessionId });
+export function k8sExecutionEnvFactory(config, deps) {
+    return (ctx) => new RemoteK8sExecutionEnv({ ...config, sessionId: ctx.sessionId }, deps);
 }
 //# sourceMappingURL=remote-env-k8s.js.map