@sema-agent/server 7.140.0 → 7.142.0

package/dist/plugins/remote-env-k8s.js

@@ -19,6 +19,7 @@ import { buildCapturedOutput, createCutError } from "../exec-cut.js";
 import { SANDBOX_IPV6_DEFAULT, sandboxEgressGaps, sandboxPodSysctls } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
 import { capabilitiesFor } from "../execution-lane-caps.js";
+import { ADAPTER_START, BOOTSTRAP_START, buildFloorPodSeat, createSandboxReadFloor, FLOOR_K8S_CONFIGMAP_KEY, engineStart, floorArgv, } from "./sandbox-read-floor.js";
 import { fileErrorFromExec, classifyFsStderr } from "./remote-env-file-error.js";
 import { createPosixShellFs } from "./posix-shell-fs.js";
 import { randomUUID } from "node:crypto";
@@ -47,6 +48,9 @@ export { applyPodSpecPatch } from "./k8s-exec-protocol.js";
 import { sandboxRunEndHoldMs } from "./k8s-pod-reaper.js";
 import { BROKER_COMMAND, BROKER_CONTAINER_NAME, BROKER_LEASE_DIR, BROKER_LEASE_KEY, BROKER_PROBE_ARG, BROKER_VOLUME_NAME } from "../broker/constants.js";
 import { buildSandboxLeaseFile, placeholderLeaseEntries, sandboxGitRouteEnv } from "../sandbox-git-lease.js";
+function minted(c) {
+    return c.ok ? { ...c, origin: "minted" } : c;
+}
 const CACHE_VOLUME = ADAPTER_VOLUME_NAMES.cache;
 function occupantKeyOf(occupant) {
     if (occupant === undefined)
@@ -202,6 +206,7 @@ function brokerContainer(image) {
 }
 export class RemoteK8sExecutionEnv {
     capabilities;
+    floor;
     cwd;
     podName;
     podIP;
@@ -211,6 +216,7 @@ export class RemoteK8sExecutionEnv {
     inFlight = new Set();
     idleWaiters = [];
     connectInFlight;
+    resumeInFlight;
     connectRefusal;
     get homeDir() {
         return this.homeSeat.home;
@@ -229,6 +235,7 @@ export class RemoteK8sExecutionEnv {
     cacheUnavailableSaid = false;
     logger;
     onWorkspaceSnapshot;
+    workspaceFill;
     cacheMisses;
     constructor(config, deps) {
         this.cfg = {
@@ -268,8 +275,10 @@ export class RemoteK8sExecutionEnv {
         this.cacheLane = cacheLaneOf(config.cache, config.spaceOccupant);
         this.logger = config.logger;
         this.onWorkspaceSnapshot = config.onWorkspaceSnapshot;
+        this.workspaceFill = config.workspaceFill;
         this.cacheMisses = config.cacheMisses;
-        this.capabilities = capabilitiesFor({ remoteExec: { provider: PROVIDER, s3Snapshot: this.cfg.s3Snapshot } });
+        this.floor = this.deps.readFloor ?? createSandboxReadFloor({ lane: PROVIDER, knob: config.sandboxReadFloor, ...(config.logger !== undefined ? { logger: config.logger } : {}) });
+        this.capabilities = capabilitiesFor({ remoteExec: { provider: PROVIDER, s3Snapshot: this.cfg.s3Snapshot }, sandboxReadFloor: config.sandboxReadFloor });
         const bgMax = Math.max(1, Math.min(K8S_BG_MAX_TIMEOUT_SEC, Math.floor(this.cfg.timeoutMs / 1000)));
         this.backgroundCapabilities = {
             supported: true,
@@ -371,10 +380,14 @@ export class RemoteK8sExecutionEnv {
         });
     }
     connect(config) {
-        this.connectInFlight ??= this.establish(config).finally(() => {
-            this.connectInFlight = undefined;
+        if (this.connectInFlight)
+            return this.connectInFlight;
+        const walk = this.establish(config).finally(() => {
+            if (this.connectInFlight === walk)
+                this.connectInFlight = undefined;
         });
-        return this.connectInFlight;
+        this.connectInFlight = walk;
+        return walk;
     }
     async establish(config) {
         if (this.connectRefusal)
@@ -383,10 +396,10 @@ export class RemoteK8sExecutionEnv {
         const lane = this.spaceLane;
         switch (lane.state) {
             case "off":
-                return this.provision(signal);
+                return this.settle(minted(await this.provision(signal)));
             case "unavailable":
                 this.sayUnavailable(lane.reason);
-                return this.provision(signal);
+                return this.settle(minted(await this.provision(signal)));
             case "unreadable":
                 return this.provision(signal);
             case "open": {
@@ -396,7 +409,7 @@ export class RemoteK8sExecutionEnv {
                 const latest = await this.readLatest(lane.space, signal);
                 if (!latest.ok)
                     return latest;
-                const c = latest.value === undefined ? await this.provision(signal) : await this.restoreSnapshot({ lane: "space", space: lane.space }, latest.value, signal);
+                const c = await this.settle(latest.value === undefined ? minted(await this.provision(signal)) : await this.restoreSnapshot({ lane: "space", space: lane.space }, latest.value, signal));
                 if (c.ok)
                     this.armSpace();
                 return c;
@@ -407,6 +420,43 @@ export class RemoteK8sExecutionEnv {
             }
         }
     }
+    async settle(c) {
+        if (!c.ok)
+            return c;
+        const fill = this.workspaceFill;
+        if (fill !== undefined) {
+            const filled = await fill.fill(this.fillChannel(), this.cwd, c.origin);
+            if (!filled.ok) {
+                await this.releasePod();
+                return filled.standing ? this.holdRefusal(filled.error) : { ok: false, error: filled.error };
+            }
+        }
+        if (this.destroyed) {
+            await this.releasePod();
+            return { ok: false, error: new RemoteExecutionError("connect_failed", "env destroyed during connect") };
+        }
+        return { ok: true, value: c.value };
+    }
+    fillChannel() {
+        const exec = (command) => this.runExec(ADAPTER_START, command, { capMs: this.cfg.dataTimeoutMs });
+        return {
+            run: async (command) => {
+                const r = await exec(command);
+                return r.ok ? { ok: true, exitCode: r.value.exitCode, stdout: r.value.stdout, stderr: r.value.stderr } : { ok: false, error: r.error.message };
+            },
+            write: async (abs, bytes) => {
+                const w = await this.writeChunked(abs, bytes, true, undefined, exec);
+                return w.ok ? { ok: true } : { ok: false, error: w.error.message };
+            },
+        };
+    }
+    async releasePod() {
+        const name = this.podName;
+        this.podName = undefined;
+        this.podIP = undefined;
+        if (name)
+            await this.deletePod(name).catch(() => { });
+    }
     armSpace() {
         if (this.spaceLane.state === "open")
             this.spaceLane.space.fence.arm(this.cfg.dataTimeoutMs);
@@ -667,6 +717,8 @@ export class RemoteK8sExecutionEnv {
         ];
         const pkgSourceMarked = Boolean(this.cfg.podEnv?.SEMA_PKG_SOURCE || secretEnv.SEMA_PKG_SOURCE);
         const sysctls = sandboxPodSysctls(this.cfg.ipv6);
+        const floorSeat = await this.ensureFloorSeat(signal);
+        const mounts = [...(cache ? [{ name: CACHE_VOLUME, mountPath: cache.mount }] : []), ...(floorSeat?.mounts ?? [])];
         const pod = {
             apiVersion: "v1",
             kind: "Pod",
@@ -691,7 +743,7 @@ export class RemoteK8sExecutionEnv {
                             ? ["sh", "-c", "[ -f /opt/sema/pkg-source.sh ] && sh /opt/sema/pkg-source.sh || true; sleep infinity"]
                             : ["sh", "-c", "sleep infinity"],
                         ...(envEntries.length > 0 ? { env: envEntries } : {}),
-                        ...(cache ? { volumeMounts: [{ name: CACHE_VOLUME, mountPath: cache.mount }] } : {}),
+                        ...(mounts.length > 0 ? { volumeMounts: mounts } : {}),
                         resources: { limits: { cpu: this.cfg.cpu, memory: this.cfg.memory } },
                     },
                     ...(broker !== undefined ? [brokerContainer(broker.image)] : []),
@@ -699,6 +751,7 @@ export class RemoteK8sExecutionEnv {
                 ...((volumes) => (volumes.length > 0 ? { volumes } : {}))([
                     ...(cache ? [{ name: CACHE_VOLUME, persistentVolumeClaim: { claimName: cache.claim.name } }] : []),
                     ...(leaseName !== undefined ? [{ name: BROKER_VOLUME_NAME, secret: { secretName: leaseName, items: [{ key: BROKER_LEASE_KEY, path: BROKER_LEASE_KEY }], defaultMode: 0o400 } }] : []),
+                    ...(floorSeat?.volumes ?? []),
                 ]),
             },
         };
@@ -806,7 +859,7 @@ export class RemoteK8sExecutionEnv {
         this.podIP = podIP;
         this.leaseRef = leaseName !== undefined && broker !== undefined ? { namespace: ns, podName: name, leaseSecretName: leaseName, origins: [...broker.origins] } : undefined;
         const askHome = this.homeSeat.asks;
-        const mk = await this.runExec(`mkdir -p ${shellQuote(this.cfg.mountPath)}${askHome ? ` && { ${LANE_HOME_PROBE_COMMAND} || true; }` : ""}`);
+        const mk = await this.runExec(ADAPTER_START, `mkdir -p ${shellQuote(this.cfg.mountPath)}${askHome ? ` && { ${LANE_HOME_PROBE_COMMAND} || true; }` : ""}`);
         if (!mk.ok || mk.value.exitCode !== 0) {
             await this.deletePod(name).catch(() => { });
             this.podName = undefined;
@@ -825,6 +878,7 @@ export class RemoteK8sExecutionEnv {
         }
         if (cache?.fresh)
             this.logger?.warn("workspace_cache_fresh", { space: incarnationLogPrefix(cache.spaceKey), claim: cache.claim.name, size: cache.size });
+        await this.floor.attach(this.floorTransport(), { origin: "created", workspaceRoot: this.cfg.mountPath });
         return ok(this.workspaceHandle());
     }
     async createPodSecret(secretName, podName, podUid, values, signal) {
@@ -1184,7 +1238,7 @@ export class RemoteK8sExecutionEnv {
         const excludes = (this.cfg.snapshotExclude ?? []).map((item) => ` --exclude=${shellQuote(`./${item}`)}`).join("");
         const cmd = `tar czf ${tmp}${excludes} -C ${shellQuote(this.cfg.mountPath)} . && curl -fsS -X PUT -T ${tmp} ${shellQuote(url)}; ` +
             `rc=$?; rm -f ${tmp}; exit $rc`;
-        const r = await this.runExec(cmd, { timeoutMs: boundMs });
+        const r = await this.runExec(ADAPTER_START, cmd, { timeoutMs: boundMs });
         if (!r.ok)
             return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot upload failed: ${this.redactUrl(r.error.message, url)}`, r.error) };
         if (r.value.exitCode !== 0)
@@ -1195,10 +1249,31 @@ export class RemoteK8sExecutionEnv {
         await this.deletePod(name).catch(() => { });
         return ok(snapshotId);
     }
-    async resumeVM(snapshotId, options) {
+    resumeVM(snapshotId, options) {
         if (!this.cfg.s3Snapshot)
-            return unsupported("resumeVM");
-        const r = await this.restoreSnapshot({ lane: "park" }, snapshotId, options?.abortSignal);
+            return Promise.resolve(unsupported("resumeVM"));
+        const same = this.resumeInFlight;
+        if (same?.snapshotId === snapshotId)
+            return same.walk;
+        const prior = this.connectInFlight;
+        const walk = this.resumeWalk(prior, snapshotId, options).finally(() => {
+            if (this.connectInFlight === walk)
+                this.connectInFlight = undefined;
+            if (this.resumeInFlight?.walk === walk)
+                this.resumeInFlight = undefined;
+        });
+        this.connectInFlight = walk;
+        this.resumeInFlight = { snapshotId, walk };
+        return walk;
+    }
+    async resumeWalk(prior, snapshotId, options) {
+        if (prior)
+            await prior;
+        if (this.podName) {
+            this.logger?.warn("sandbox_pod_released_before_restore", { pod: this.podName, ...(this.cfg.sessionId ? { sessionId: this.cfg.sessionId } : {}), effect: "this env held a pod minted before the park snapshot's restore — it is deleted; the restore provisions the leg's one pod" });
+            await this.releasePod();
+        }
+        const r = await this.settle(await this.restoreSnapshot({ lane: "park" }, snapshotId, options?.abortSignal));
         if (!r.ok)
             return r;
         this.handleSnapshotId = snapshotId;
@@ -1220,22 +1295,21 @@ export class RemoteK8sExecutionEnv {
             const c = await this.provision(abortSignal);
             if (!c.ok)
                 return c;
-            const r = await this.runExec(cmd, { timeoutMs: this.cfg.dataTimeoutMs });
+            const r = await this.runExec(ADAPTER_START, cmd, { timeoutMs: this.cfg.dataTimeoutMs });
             const failure = !r.ok
                 ? { cls: r.error.code === "timeout" ? "timeout" : "transport", detail: `workspace snapshot restore failed: ${this.redactUrl(r.error.message, url)}`, cause: r.error }
                 : r.value.exitCode !== 0
                     ? { cls: restoreClassOfExit(r.value.exitCode), detail: `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}` }
                     : undefined;
             if (!failure) {
-                if (target.lane === "space" && r.ok && r.value.stdout.includes("EMPTY_SNAPSHOT"))
-                    await this.dropDanglingLatest(target.space, snapshotId);
-                return ok(this.workspaceHandle());
+                if (r.ok && r.value.stdout.includes("EMPTY_SNAPSHOT")) {
+                    if (target.lane === "space")
+                        await this.dropDanglingLatest(target.space, snapshotId);
+                    return { ok: true, value: this.workspaceHandle(), origin: "minted" };
+                }
+                return { ok: true, value: this.workspaceHandle(), origin: "restored" };
             }
-            const name = this.podName;
-            this.podName = undefined;
-            this.podIP = undefined;
-            if (name)
-                await this.deletePod(name).catch(() => { });
+            await this.releasePod();
             if (attempt < RESTORE_MAX_ATTEMPTS[target.lane] && restoreFailureIsTransient(failure.cls)) {
                 if (await this.restorePause(abortSignal))
                     continue;
@@ -1326,6 +1400,7 @@ export class RemoteK8sExecutionEnv {
     }
     async destroy() {
         await this.disposeBackgroundShells().catch(() => { });
+        this.floor.dispose();
         this.destroyed = true;
         const lane = this.spaceLane;
         switch (lane.state) {
@@ -1410,7 +1485,7 @@ export class RemoteK8sExecutionEnv {
     async killPodProcesses() {
         if (!this.podName)
             return false;
-        const r = await this.runExec(buildKillAllScript(), { timeoutMs: this.cfg.rpcTimeoutMs + KILL_ALL_RECHECK_MS });
+        const r = await this.runExec(ADAPTER_START, buildKillAllScript(), { timeoutMs: this.cfg.rpcTimeoutMs + KILL_ALL_RECHECK_MS });
         return r.ok && r.value.exitCode === 0;
     }
     async persistWorkspace(space) {
@@ -1463,23 +1538,124 @@ export class RemoteK8sExecutionEnv {
             await this.endSpace(space);
         }
     }
-    execUrl(command) {
+    async ensureFloorSeat(signal) {
+        const seat = buildFloorPodSeat(this.floor);
+        if (seat === undefined)
+            return undefined;
+        if (seat.configMaps.length === 0) {
+            this.floor.refusePlacement("this server carries no pinned supervisor build");
+            return undefined;
+        }
+        for (const cm of seat.configMaps) {
+            const body = { apiVersion: "v1", kind: "ConfigMap", metadata: { name: cm.name }, immutable: true, binaryData: { [FLOOR_K8S_CONFIGMAP_KEY]: cm.binaryDataB64 } };
+            let r;
+            try {
+                r = await this.request("POST", `/api/v1/namespaces/${this.cfg.namespace}/configmaps`, body, { signal });
+            }
+            catch (e) {
+                this.floor.refusePlacement(`creating the supervisor ConfigMap ${cm.name} failed: ${errorMessageOf(e)}`);
+                return undefined;
+            }
+            if (r.status !== 409 && (r.status < 200 || r.status >= 300)) {
+                this.floor.refusePlacement(`creating the supervisor ConfigMap ${cm.name} was answered ${r.status}${r.status === 401 || r.status === 403 ? " (the server's service account needs create on configmaps in this namespace)" : ""}`);
+                return undefined;
+            }
+        }
+        return seat;
+    }
+    floorTransport() {
+        return {
+            bootstrap: async (command) => {
+                const r = await this.runExec(BOOTSTRAP_START, command, { timeoutMs: this.cfg.rpcTimeoutMs });
+                return r.ok ? { ok: true, exitCode: r.value.exitCode, stdout: r.value.stdout, stderr: r.value.stderr } : { ok: false, error: r.error.message };
+            },
+            place: async () => ({ ok: true }),
+            openRelay: (identity, command, events) => {
+                let closed = false;
+                const close = (why) => {
+                    if (closed)
+                        return;
+                    closed = true;
+                    events.closed(why);
+                };
+                switch (identity) {
+                    case "root":
+                        queueMicrotask(() => close("a k8s exec cannot start the relay as root"));
+                        return { send: () => { }, close: () => { } };
+                    case "exec-user":
+                        break;
+                }
+                let sock;
+                try {
+                    const { url, headers } = this.execUrl(["sh", "-c", command], true);
+                    sock = this.openExec(url, headers);
+                }
+                catch (e) {
+                    queueMicrotask(() => close(`the relay exec could not be opened: ${errorMessageOf(e)}`));
+                    return { send: () => { }, close: () => { } };
+                }
+                const decoder = new StringDecoder("utf8");
+                let open = false;
+                const pending = [];
+                const write = (frame) => {
+                    if (sock.send === undefined)
+                        return close("the exec socket cannot write (no stdin channel)");
+                    try {
+                        sock.send(frame);
+                    }
+                    catch (e) {
+                        close(`writing to the relay failed: ${errorMessageOf(e)}`);
+                    }
+                };
+                sock.on("open", () => {
+                    open = true;
+                    for (const f of pending.splice(0))
+                        write(f);
+                });
+                sock.on("message", (...args) => {
+                    const data = args[0];
+                    const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
+                    if (buf.length > 1 && buf[0] === 1)
+                        events.data(decoder.write(buf.subarray(1)));
+                });
+                sock.on("close", () => close("the relay exec closed"));
+                sock.on("error", (e) => close(`the relay exec failed: ${errorMessageOf(e)}`));
+                return {
+                    send: (line) => {
+                        const frame = Buffer.concat([Buffer.from([0]), Buffer.from(line, "utf8")]);
+                        if (open)
+                            write(frame);
+                        else
+                            pending.push(frame);
+                    },
+                    close: () => (sock.terminate ?? sock.close).call(sock),
+                };
+            },
+        };
+    }
+    execUrl(argv, stdin = false) {
         const { apiUrl, token } = this.creds();
         const base = apiUrl.replace(/^http/, "ws");
         const q = new URLSearchParams();
         q.append("container", "sandbox");
+        if (stdin)
+            q.append("stdin", "true");
         q.append("stdout", "true");
         q.append("stderr", "true");
-        for (const part of ["sh", "-c", command])
+        for (const part of argv)
             q.append("command", part);
         return { url: `${base}/api/v1/namespaces/${this.cfg.namespace}/pods/${this.podName}/exec?${q.toString()}`, headers: { authorization: `Bearer ${token}` } };
     }
-    runExec(command, opts) {
+    runExec(start, command, opts) {
+        const launch = this.floor.wrap(start);
+        if (!launch.ok)
+            return Promise.resolve(launch);
+        opts?.onLaunch?.(launch.value);
         let sock;
         let headers;
         let url;
         try {
-            ({ url, headers } = this.execUrl(command));
+            ({ url, headers } = this.execUrl(floorArgv(launch.value, ["sh", "-c", command])));
             sock = this.openExec(url, headers);
         }
         catch (e) {
@@ -1673,12 +1849,12 @@ export class RemoteK8sExecutionEnv {
         });
     }
     async exec(command, options) {
-        return this.execCall(createCwdGuardedCall("k8s", command, this.workingDirOf(options?.cwd), options?.env), options);
+        return this.execCall(engineStart(options?.readFloor), createCwdGuardedCall("k8s", command, this.workingDirOf(options?.cwd), options?.env), options);
     }
     fsExec(command, abortSignal) {
-        return this.execCall(createCwdGuardedCall("k8s", command, undefined, undefined), { abortSignal });
+        return this.execCall(ADAPTER_START, createCwdGuardedCall("k8s", command, undefined, undefined), { abortSignal });
     }
-    async execCall(call, options) {
+    async execCall(start, call, options) {
         const c = await this.ensureConnected();
         if (!c.ok)
             return c;
@@ -1687,7 +1863,7 @@ export class RemoteK8sExecutionEnv {
         const abortSignal = options?.abortSignal ? AbortSignal.any([options.abortSignal, inflight.signal]) : inflight.signal;
         try {
             if (!detachSignal) {
-                return call.settle(await this.runExec(call.command, {
+                return call.settle(await this.runExec(start, call.command, {
                     signal: abortSignal,
                     timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined,
                     boundOutput: true,
@@ -1699,7 +1875,10 @@ export class RemoteK8sExecutionEnv {
             const adoptId = randomUUID();
             const dir = `${K8S_BG_DIR_ROOT}/exec-${adoptId}`;
             const tee = { out: seedMemStream(""), err: seedMemStream("") };
-            return call.settle(await this.runExec(buildDetachCapableExec(dir, call.command), {
+            let startId;
+            let adoptedHere = false;
+            const settled = call.settle(await this.runExec(start.kind === "engine" ? engineStart(start.readFloor, true) : start, buildDetachCapableExec(dir, call.command), {
+                onLaunch: (l) => void (startId = l.mode === "run" ? l.startId : undefined),
                 signal: abortSignal,
                 timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined,
                 boundOutput: true,
@@ -1716,12 +1895,17 @@ export class RemoteK8sExecutionEnv {
                                 data: (channel, payload) => feedMemStream(channel === 1 ? mem.out : mem.err, payload, K8S_BG_MEM_CAP),
                                 closed: (terminal) => ctx.onTerminal(terminal.failed, terminal.exitCode),
                             });
-                            return { jobId: adoptId, dir, stdoutCursor: 0, stderrCursor: 0, adopt: { mem, sock: io.sock } };
+                            return { jobId: adoptId, dir, stdoutCursor: 0, stderrCursor: 0, adopt: { mem, sock: io.sock }, ...(startId !== undefined ? { startId } : {}) };
                         });
+                        if (adopted !== undefined)
+                            adoptedHere = true;
                         return adopted?.shellId;
                     },
                 },
             }));
+            if (!adoptedHere)
+                this.floor.release(startId);
+            return settled;
         }
         finally {
             this.endExec(inflight);
@@ -1759,7 +1943,7 @@ export class RemoteK8sExecutionEnv {
         const cap = options?.maxOutputBytes;
         let bytes = 0;
         const signal = options?.signal ? AbortSignal.any([options.signal, ac.signal]) : ac.signal;
-        const done = this.runExec(buildCwdEnvCommand(command, this.workingDirOf(options?.cwd), options?.env, "raw").command, {
+        const done = this.runExec(engineStart(undefined), buildCwdEnvCommand(command, this.workingDirOf(options?.cwd), options?.env, "raw").command, {
             signal,
             timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined,
             readTimeoutMs: options?.readTimeoutMs,
@@ -1828,7 +2012,8 @@ export class RemoteK8sExecutionEnv {
     }
     makeBgDriver() {
         const self = this;
-        const bgExec = (script, boundOutput) => self.runExec(script, { timeoutMs: self.cfg.rpcTimeoutMs, boundOutput });
+        const bgExec = (start, script, boundOutput, onLaunch) => self.runExec(start, script, { timeoutMs: self.cfg.rpcTimeoutMs, boundOutput, ...(onLaunch !== undefined ? { onLaunch } : {}) });
+        const under = (state) => ({ kind: "under", startId: state.startId });
         return {
             async launch(command, options, ctx) {
                 const conn = await self.ensureConnected();
@@ -1839,14 +2024,17 @@ export class RemoteK8sExecutionEnv {
                 const inner = buildCwdEnvCommand(command, self.workingDirOf(options?.cwd), options?.env, "raw").command;
                 const cmdB64 = Buffer.from(inner, "utf8").toString("base64");
                 const runnerB64 = Buffer.from(buildBgRunnerScript(dir), "utf8").toString("base64");
-                const r = await bgExec(buildBgLauncherScript(dir, cmdB64, runnerB64), true);
+                let startId;
+                const r = await bgExec(engineStart(options?.readFloor, true), buildBgLauncherScript(dir, cmdB64, runnerB64), true, (l) => void (startId = l.mode === "run" ? l.startId : undefined));
+                if (!r.ok || r.value.exitCode !== 0 || !r.value.stdout.includes("SEMABG_LAUNCHED"))
+                    self.floor.release(startId);
                 if (!r.ok)
                     throw new Error(`background launch failed: ${r.error.message}`);
                 if (r.value.exitCode !== 0 || !r.value.stdout.includes("SEMABG_LAUNCHED")) {
                     throw new Error(`background launch exited ${r.value.exitCode}: ${r.value.stderr.slice(0, 200)}`);
                 }
                 void ctx;
-                return { jobId, dir, stdoutCursor: 0, stderrCursor: 0 };
+                return { jobId, dir, stdoutCursor: 0, stderrCursor: 0, ...(startId !== undefined ? { startId } : {}) };
             },
             async read(state) {
                 if (state.adopt) {
@@ -1859,7 +2047,7 @@ export class RemoteK8sExecutionEnv {
                         bytesDroppedBeforeCursor: o.dropped + e.dropped,
                     };
                 }
-                const r = await bgExec(buildBgPollScript(state.dir, state.stdoutCursor, state.stderrCursor, K8S_BG_READ_CAP), false);
+                const r = await bgExec(under(state), buildBgPollScript(state.dir, state.stdoutCursor, state.stderrCursor, K8S_BG_READ_CAP), false);
                 if (!r.ok)
                     throw new Error(`background poll failed: ${r.error.message}`);
                 const p = parseBgPollOutput(r.value.stdout);
@@ -1874,7 +2062,7 @@ export class RemoteK8sExecutionEnv {
                 state.stderrCursor = p.errTotal;
                 let terminal = p.exitCode !== undefined ? { failed: false, exitCode: p.exitCode } : !p.alive ? { failed: true } : undefined;
                 if (!terminal && (p.outTotal > K8S_BG_FILE_CAP || p.errTotal > K8S_BG_FILE_CAP)) {
-                    void bgExec(buildBgKillScript(state.dir), true).catch(() => { });
+                    void bgExec(under(state), buildBgKillScript(state.dir), true).catch(() => { });
                     terminal = { failed: true };
                 }
                 return {
@@ -1886,7 +2074,7 @@ export class RemoteK8sExecutionEnv {
                 };
             },
             async kill(state) {
-                await bgExec(buildBgKillScript(state.dir), true).catch(() => { });
+                await bgExec(under(state), buildBgKillScript(state.dir), true).catch(() => { });
             },
             async disposeOne(state) {
                 const closeAdopted = () => {
@@ -1899,10 +2087,12 @@ export class RemoteK8sExecutionEnv {
                 };
                 if (self.destroyed) {
                     closeAdopted();
+                    self.floor.release(state.startId);
                     return;
                 }
-                await bgExec(buildBgDisposeScript(state.dir), true).catch(() => { });
+                await bgExec(under(state), buildBgDisposeScript(state.dir), true).catch(() => { });
                 closeAdopted();
+                self.floor.release(state.startId);
             },
         };
     }
@@ -1958,12 +2148,12 @@ export class RemoteK8sExecutionEnv {
         const abs = this.resolve(p);
         return appendFileStaged(this.stagedLane(content), abs, abortSignal, (ctx) => ctx.inPlace((signal) => this.writeChunked(abs, content, false, signal)));
     }
-    async writeChunked(abs, content, truncateFirst, abortSignal) {
+    async writeChunked(abs, content, truncateFirst, abortSignal, exec = (command) => this.execData(command, abortSignal)) {
         const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : Buffer.from(content);
         for (let off = 0, first = true; off < bytes.length || first; off += WRITE_CHUNK_BYTES, first = false) {
             const b64 = bytes.subarray(off, off + WRITE_CHUNK_BYTES).toString("base64");
             const redirect = first && truncateFirst ? ">" : ">>";
-            const r = await this.execData(`printf '%s' ${shellQuote(b64)} | base64 -d ${redirect} ${shellQuote(abs)}`, abortSignal);
+            const r = await exec(`printf '%s' ${shellQuote(b64)} | base64 -d ${redirect} ${shellQuote(abs)}`);
             if (!r.ok)
                 return { ok: false, error: fileErrorFromExec(r.error, abs) };
             if (r.value.exitCode !== 0)
@@ -2013,7 +2203,7 @@ export class RemoteK8sExecutionEnv {
         const c = await this.ensureConnected();
         if (!c.ok)
             return c;
-        return this.runExec(command, { signal: abortSignal, capMs: this.cfg.dataTimeoutMs });
+        return this.runExec(ADAPTER_START, command, { signal: abortSignal, capMs: this.cfg.dataTimeoutMs });
     }
     workingDirOf(cwd) {
         const dir = cwd ?? this.cwd;