@sema-agent/server 7.147.0 → 7.149.0

package/dist/plugins/remote-env-k8s.js

@@ -19,6 +19,7 @@ import { buildCapturedOutput, createCutError } from "../exec-cut.js";
 import { SANDBOX_IPV6_DEFAULT, sandboxEgressGaps, sandboxPodSysctls } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
 import { capabilitiesFor } from "../execution-lane-caps.js";
+import { DEPLOYMENT_SHAPE_PREDICATES } from "../deployment-shape.js";
 import { ADAPTER_START, BOOTSTRAP_START, buildFloorPodSeat, createSandboxReadFloor, FLOOR_K8S_CONFIGMAP_KEY, engineStart, floorArgv, } from "./sandbox-read-floor.js";
 import { fileErrorFromExec, classifyFsStderr } from "./remote-env-file-error.js";
 import { createPosixShellFs } from "./posix-shell-fs.js";
@@ -48,6 +49,8 @@ export { applyPodSpecPatch } from "./k8s-exec-protocol.js";
 import { sandboxRunEndHoldMs } from "./k8s-pod-reaper.js";
 import { BROKER_COMMAND, BROKER_CONTAINER_NAME, BROKER_LEASE_DIR, BROKER_LEASE_KEY, BROKER_PROBE_ARG, BROKER_VOLUME_NAME } from "../broker/constants.js";
 import { buildSandboxLeaseFile, placeholderLeaseEntries, sandboxGitRouteEnv } from "../sandbox-git-lease.js";
+import { witnessEnv } from "../env-witness.js";
+import { createEgressFetch } from "../egress-redirect.js";
 function minted(c) {
     return c.ok ? { ...c, origin: "minted" } : c;
 }
@@ -108,6 +111,8 @@ const RESTORE_DOWNLOAD_EXIT = 22;
 const RESTORE_TRANSFER_EXIT = 75;
 const RESTORE_STORE_BUSY_STATUSES = [408, 429, 500, 502, 503, 504];
 const RESTORE_STORE_BUSY_EXIT = 69;
+const RESTORE_404_MARKER = "SNAPSHOT_GET_404";
+const RESTORE_404_ANSWER_BYTES = 4096;
 function restoreClassOfExit(exitCode) {
     switch (exitCode) {
         case RESTORE_STORE_BUSY_EXIT:
@@ -209,7 +214,6 @@ export class RemoteK8sExecutionEnv {
     floor;
     cwd;
     podName;
-    podIP;
     leaseRef;
     destroyed = false;
     handleSnapshotId;
@@ -280,12 +284,13 @@ export class RemoteK8sExecutionEnv {
         this.floor = this.deps.readFloor ?? createSandboxReadFloor({ lane: PROVIDER, knob: config.sandboxReadFloor, ...(config.logger !== undefined ? { logger: config.logger } : {}) });
         this.capabilities = capabilitiesFor({ remoteExec: { provider: PROVIDER, s3Snapshot: this.cfg.s3Snapshot }, sandboxReadFloor: config.sandboxReadFloor });
         const bgMax = Math.max(1, Math.min(K8S_BG_MAX_TIMEOUT_SEC, Math.floor(this.cfg.timeoutMs / 1000)));
+        const backgroundShellOn = DEPLOYMENT_SHAPE_PREDICATES.backgroundShell.holds({ remoteExec: { provider: PROVIDER } });
         this.backgroundCapabilities = {
-            supported: true,
+            supported: backgroundShellOn,
             maxConcurrent: K8S_BG_MAX_CONCURRENT,
             defaultBgTimeoutSec: Math.min(K8S_BG_DEFAULT_TIMEOUT_SEC, bgMax),
             maxBgTimeoutSec: bgMax,
-            supportsDetach: true,
+            supportsDetach: backgroundShellOn,
         };
     }
     get sandboxLeaseRef() {
@@ -453,13 +458,14 @@ export class RemoteK8sExecutionEnv {
     async releasePod() {
         const name = this.podName;
         this.podName = undefined;
-        this.podIP = undefined;
         if (name)
             await this.deletePod(name).catch(() => { });
     }
     armSpace() {
-        if (this.spaceLane.state === "open")
-            this.spaceLane.space.fence.arm(this.cfg.dataTimeoutMs);
+        if (this.spaceLane.state !== "open")
+            return;
+        this.spaceLane.space.fence.arm(this.cfg.dataTimeoutMs);
+        witnessEnv(this.cfg.runId);
     }
     async endSpace(space) {
         await this.fenceCall(() => space.fence.end(), "delete").catch(() => undefined);
@@ -581,6 +587,7 @@ export class RemoteK8sExecutionEnv {
                     this.cacheUnavailable("claim_failed", ensured.detail, lane.cache);
                     return undefined;
                 }
+                witnessEnv(this.cfg.runId);
                 return { ...lane.cache, fresh: ensured.fresh, bindDeadline: Date.now() + this.cfg.dataTimeoutMs };
             }
             default: {
@@ -798,7 +805,6 @@ export class RemoteK8sExecutionEnv {
             }
         }
         const deadline = Date.now() + this.cfg.readyTimeoutMs;
-        let podIP;
         for (;;) {
             if (signal?.aborted) {
                 await this.deletePod(name).catch(() => { });
@@ -815,10 +821,8 @@ export class RemoteK8sExecutionEnv {
             const status = got.json?.status;
             const phase = status?.phase;
             const brokerStatus = status?.containerStatuses?.find((c) => c.name === BROKER_CONTAINER_NAME);
-            if (phase === "Running" && (broker === undefined || brokerStatus?.ready === true)) {
-                podIP = status?.podIP;
+            if (phase === "Running" && (broker === undefined || brokerStatus?.ready === true))
                 break;
-            }
             if (phase === "Failed" || phase === "Succeeded") {
                 await this.deletePod(name).catch(() => { });
                 return this.refusePod(name, phase, status?.reason, status?.message);
@@ -858,14 +862,12 @@ export class RemoteK8sExecutionEnv {
             return { ok: false, error: new RemoteExecutionError("connect_failed", "env destroyed during connect") };
         }
         this.podName = name;
-        this.podIP = podIP;
         this.leaseRef = leaseName !== undefined && broker !== undefined ? { namespace: ns, podName: name, leaseSecretName: leaseName, origins: [...broker.origins] } : undefined;
         const askHome = this.homeSeat.asks;
         const mk = await this.runExec(ADAPTER_START, `mkdir -p ${shellQuote(this.cfg.mountPath)}${askHome ? ` && { ${LANE_HOME_PROBE_COMMAND} || true; }` : ""}`);
         if (!mk.ok || mk.value.exitCode !== 0) {
             await this.deletePod(name).catch(() => { });
             this.podName = undefined;
-            this.podIP = undefined;
             return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace mkdir failed: ${mk.ok ? mk.value.stderr : mk.error.message}`) };
         }
         if (askHome) {
@@ -873,7 +875,6 @@ export class RemoteK8sExecutionEnv {
             if (verdict !== undefined) {
                 await this.deletePod(name).catch(() => { });
                 this.podName = undefined;
-                this.podIP = undefined;
                 const error = new RemoteExecutionError("connect_failed", verdict.refusal);
                 return verdict.answered ? this.holdRefusal(error) : { ok: false, error };
             }
@@ -1081,9 +1082,9 @@ export class RemoteK8sExecutionEnv {
     storeFetch(url, init) {
         const bound = AbortSignal.timeout(Math.max(1, Math.min(this.cfg.rpcTimeoutMs, init.timeoutMs ?? this.cfg.rpcTimeoutMs)));
         const signal = init.signal ? AbortSignal.any([init.signal, bound]) : bound;
-        return (this.deps.fetch ?? fetch)(url, { method: init.method, ...(init.body !== undefined ? { body: init.body } : {}), signal });
+        return createEgressFetch("declare the final object-store address as MINIO_ENDPOINT", this.deps.fetch)(url, { method: init.method, ...(init.body !== undefined ? { body: init.body } : {}), signal });
     }
-    async readPointer(space, pointer, signal) {
+    async readPointer(space, pointer, signal, timeoutMs) {
         const url = this.presignSnapshot({ lane: "space", space, method: "GET", pointer });
         if (!url.ok)
             return url;
@@ -1091,50 +1092,83 @@ export class RemoteK8sExecutionEnv {
         let res;
         let body;
         try {
-            res = await this.storeFetch(url.value, { method: "GET", ...(signal ? { signal } : {}) });
+            res = await this.storeFetch(url.value, { method: "GET", ...(signal ? { signal } : {}), ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
             body = await res.text();
         }
         catch (e) {
             return spaceReadFailed(what, `: ${this.redactUrl(errorMessageOf(e), url.value)}`, e);
         }
         if (isNoSuchBucketAnswer(res.status, body))
-            return this.refuseMissingBucket(space);
+            return { ...spaceReadFailed(what, ` (${SNAPSHOT_BUCKET_MISSING})`), bucketMissing: true };
         if (res.status === 404)
             return ok(undefined);
         if (!res.ok)
             return spaceReadFailed(what, ` (HTTP ${res.status})`);
         return ok(body.trim());
     }
-    refuseMissingBucket(space) {
+    refuseMissingBucket(target, read) {
         const bucket = this.cfg.s3Snapshot.bucket;
-        this.spaceLine("workspace_snapshot_bucket_missing", space, {
+        const fields = {
             bucket,
             code: SNAPSHOT_BUCKET_MISSING,
             effect: "the workspace snapshot bucket does not exist in the object store (the server never creates buckets) — this run is refused rather than started on a workspace that could be neither restored nor saved",
-        });
-        return this.holdRefusal(spaceReadFailed("pointer", ` (${SNAPSHOT_BUCKET_MISSING}): the workspace snapshot bucket "${bucket}" (WORKSPACE_SNAPSHOT_BUCKET; default ${WORKSPACE_SNAPSHOT_BUCKET_DEFAULT}) does not exist in the object store — ` +
+        };
+        switch (target.lane) {
+            case "space":
+                this.spaceLine("workspace_snapshot_bucket_missing", target.space, fields);
+                break;
+            case "park":
+                this.snapshotLine("workspace_snapshot_bucket_missing", { ...(this.cfg.sessionId ? { sessionId: this.cfg.sessionId } : {}), ...fields });
+                break;
+            default: {
+                const unreachable = target;
+                throw new Error(`unhandled snapshot lane: ${String(unreachable.lane)}`);
+            }
+        }
+        const detail = ` (${SNAPSHOT_BUCKET_MISSING}): the workspace snapshot bucket "${bucket}" (WORKSPACE_SNAPSHOT_BUCKET; default ${WORKSPACE_SNAPSHOT_BUCKET_DEFAULT}) does not exist in the object store — ` +
             `refusing to start this run on a workspace that could be neither restored nor saved. Ways out: create the bucket, or set WORKSPACE_SNAPSHOT_BUCKET to one that exists, ` +
-            "or set WORKSPACE_PERSIST=off for a deployment that keeps no workspace between runs").error);
+            "or set WORKSPACE_PERSIST=off for a deployment that keeps no workspace between runs";
+        return this.holdRefusal(read === "pointer" ? spaceReadFailed("pointer", detail).error : new RemoteExecutionError("connect_failed", `workspace snapshot archive read failed${detail}`));
     }
     async readLatest(space, signal) {
         const r = await this.readPointer(space, "latest", signal);
-        return r.ok ? ok(r.value === undefined ? undefined : r.value) : r;
+        if (!r.ok)
+            return r.bucketMissing ? this.refuseMissingBucket({ lane: "space", space }, "pointer") : { ok: false, error: r.error };
+        return ok(r.value === undefined ? undefined : r.value);
     }
     async writePointer(space, pointer, body, timeoutMs) {
         const method = body === null ? "DELETE" : "PUT";
         const url = this.presignSnapshot({ lane: "space", space, method, pointer });
         if (!url.ok)
             return url;
-        const what = "pointer";
+        const what = `workspace space pointer ${method === "PUT" ? "write" : "delete"}`;
+        const until = timeoutMs !== undefined ? Date.now() + timeoutMs : undefined;
+        let lost;
         try {
             const res = await this.storeFetch(url.value, { method, ...(body === null ? {} : { body }), ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
-            if (!res.ok && !(method === "DELETE" && res.status === 404))
-                return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} ${method === "PUT" ? "write" : "delete"} failed (HTTP ${res.status})`) };
+            if (res.ok || (method === "DELETE" && res.status === 404))
+                return ok(undefined);
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `${what} failed (HTTP ${res.status})`) };
         }
         catch (e) {
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace space ${what} ${method === "PUT" ? "write" : "delete"} failed: ${this.redactUrl(errorMessageOf(e), url.value)}`, e instanceof Error ? e : undefined) };
+            lost = e;
         }
-        return ok(undefined);
+        const left = until === undefined ? undefined : until - Date.now();
+        const back = left !== undefined && left <= 0 ? undefined : await this.readPointer(space, pointer, undefined, left);
+        const inTime = until === undefined || Date.now() <= until;
+        if (back?.ok && inTime && back.value === (body ?? undefined))
+            return ok(undefined);
+        const readBack = back === undefined || !inTime
+            ? "not decided — the deadline passed before a read-back could answer"
+            : back.ok
+                ? back.value === undefined
+                    ? "the pointer is absent"
+                    : "the pointer names another snapshot"
+                : back.error.message;
+        return {
+            ok: false,
+            error: new RemoteExecutionError("connect_failed", `${what} failed: ${this.redactUrl(errorMessageOf(lost), url.value)} — read back: ${readBack}`, lost instanceof Error ? lost : undefined),
+        };
     }
     async awaitSpaceCommit(space, signal) {
         for (;;) {
@@ -1238,19 +1272,36 @@ export class RemoteK8sExecutionEnv {
         const url = presigned.value;
         const tmp = "/tmp/ws-snap.tgz";
         const excludes = (this.cfg.snapshotExclude ?? []).map((item) => ` --exclude=${shellQuote(`./${item}`)}`).join("");
-        const cmd = `tar czf ${tmp}${excludes} -C ${shellQuote(this.cfg.mountPath)} . && curl -fsS -X PUT -T ${tmp} ${shellQuote(url)}; ` +
+        const cmd = `tar czf ${tmp}${excludes} -C ${shellQuote(this.cfg.mountPath)} . && curl -sS -o /dev/null -w '%{http_code}' -X PUT -T ${tmp} ${shellQuote(url)}; ` +
             `rc=$?; rm -f ${tmp}; exit $rc`;
+        const until = Date.now() + boundMs;
         const r = await this.runExec(ADAPTER_START, cmd, { timeoutMs: boundMs });
-        if (!r.ok)
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot upload failed: ${this.redactUrl(r.error.message, url)}`, r.error) };
-        if (r.value.exitCode !== 0)
-            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot upload exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}`) };
+        const stored = await this.archiveStored(target, snapshotId, until - Date.now());
+        if (!stored.ok) {
+            const pod = !r.ok
+                ? `the upload exec failed: ${this.redactUrl(r.error.message, url)}`
+                : `the upload exited ${r.value.exitCode}${/^\d{3}$/.test(r.value.stdout.trim()) ? `, HTTP ${r.value.stdout.trim()}` : ""}${r.value.stderr.trim() ? `: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}` : ""}`;
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `workspace snapshot upload left no object under the snapshot id (read back: ${stored.why}); ${pod}`, r.ok ? undefined : r.error) };
+        }
         const name = this.podName;
         this.podName = undefined;
-        this.podIP = undefined;
         await this.deletePod(name).catch(() => { });
         return ok(snapshotId);
     }
+    async archiveStored(target, snapshotId, timeoutMs) {
+        if (timeoutMs <= 0)
+            return { ok: false, why: "no time left in the snapshot bound for the read-back" };
+        const url = this.presignSnapshot({ ...target, method: "HEAD", snapshotId });
+        if (!url.ok)
+            return { ok: false, why: url.error.message };
+        try {
+            const res = await this.storeFetch(url.value, { method: "HEAD", timeoutMs });
+            return res.ok ? { ok: true } : { ok: false, why: `HEAD ${res.status}` };
+        }
+        catch (e) {
+            return { ok: false, why: `HEAD failed: ${this.redactUrl(errorMessageOf(e), url.value)}` };
+        }
+    }
     resumeVM(snapshotId, options) {
         if (!this.cfg.s3Snapshot)
             return Promise.resolve(unsupported("resumeVM"));
@@ -1302,7 +1353,7 @@ export class RemoteK8sExecutionEnv {
         const url = presigned.value;
         const cmd = `code=$(curl -s -o /tmp/ws.tgz -w '%{http_code}' ${shellQuote(url)}); rc=$?; ` +
             `if [ "$rc" != "0" ]; then echo "snapshot GET got no whole answer (curl exit $rc, HTTP $code)" >&2; exit ${RESTORE_TRANSFER_EXIT}; fi; ` +
-            `if [ "$code" = "404" ]; then echo EMPTY_SNAPSHOT; exit 0; fi; ` +
+            `if [ "$code" = "404" ]; then echo ${RESTORE_404_MARKER}; head -c ${RESTORE_404_ANSWER_BYTES} /tmp/ws.tgz; exit 0; fi; ` +
             `case "$code" in ${RESTORE_STORE_BUSY_STATUSES.join("|")}) echo "snapshot GET HTTP $code" >&2; exit ${RESTORE_STORE_BUSY_EXIT};; esac; ` +
             `if [ "$code" != "200" ]; then echo "snapshot GET HTTP $code" >&2; exit ${RESTORE_DOWNLOAD_EXIT}; fi; ` +
             `tar xzf /tmp/ws.tgz -C ${shellQuote(this.cfg.mountPath)} && rm -f /tmp/ws.tgz`;
@@ -1317,7 +1368,11 @@ export class RemoteK8sExecutionEnv {
                     ? { cls: restoreClassOfExit(r.value.exitCode), detail: `workspace snapshot restore exited ${r.value.exitCode}: ${this.redactUrl(r.value.stderr, url).slice(0, 300)}` }
                     : undefined;
             if (!failure) {
-                if (r.ok && r.value.stdout.includes("EMPTY_SNAPSHOT")) {
+                if (r.ok && r.value.stdout.startsWith(RESTORE_404_MARKER)) {
+                    if (isNoSuchBucketAnswer(404, r.value.stdout.slice(RESTORE_404_MARKER.length))) {
+                        await this.releasePod();
+                        return this.refuseMissingBucket(target, "archive");
+                    }
                     await this.dropDangling(target, snapshotId);
                     return { ok: true, value: this.workspaceHandle(), origin: "minted" };
                 }
@@ -1404,7 +1459,6 @@ export class RemoteK8sExecutionEnv {
             if (alive?.status === 200 && phase === "Running")
                 return ok(this.workspaceHandle());
             this.podName = undefined;
-            this.podIP = undefined;
         }
         return this.connect();
     }
@@ -1457,7 +1511,6 @@ export class RemoteK8sExecutionEnv {
         }
         const name = this.podName;
         this.podName = undefined;
-        this.podIP = undefined;
         if (name)
             await this.deletePod(name).catch(() => { });
     }
@@ -1753,7 +1806,15 @@ export class RemoteK8sExecutionEnv {
             const cut = (c) => {
                 finish(() => ({ ok: false, error: createCutError(c, captured()) }));
             };
-            const wallTimer = wallMs !== undefined ? setTimeout(() => cut({ code: "timeout", message: `command exceeded ${wallMs}ms` }), wallMs) : undefined;
+            const adoptOnTimeout = opts?.detach?.adoptOnTimeout === true && bounds.wallMs !== undefined && wallMs === bounds.wallMs;
+            let onDetach;
+            const wallTimer = wallMs !== undefined
+                ? setTimeout(() => {
+                    if (adoptOnTimeout && onDetach?.("timeout") === true)
+                        return;
+                    cut({ code: "timeout", message: `command exceeded ${wallMs}ms` });
+                }, wallMs)
+                : undefined;
             let livenessTimer;
             const liveness = bounds.liveness;
             const armLiveness = () => {
@@ -1855,28 +1916,31 @@ export class RemoteK8sExecutionEnv {
             });
             const detach = opts?.detach;
             if (detach) {
-                const onDetach = () => {
+                const adoptRunning = (cause) => {
                     if (settled || spliceSink)
-                        return;
+                        return false;
                     const out = outTail ? outTail.result() : { text: "", droppedBytes: 0 };
                     const err = errBuf.result();
                     const shellId = detach.adopt({ sock, reroute: (sink) => { spliceSink = sink; } });
                     if (shellId === undefined) {
                         spliceSink = undefined;
-                        return;
+                        return false;
                     }
                     settled = true;
                     clearTimeout(wallTimer);
                     clearTimeout(livenessTimer);
                     opts?.signal?.removeEventListener("abort", onAbort);
                     removeDetachListener?.();
-                    resolvePromise(ok({ stdout: markTruncated(out.text, out.droppedBytes), stderr: markTruncated(err.text, err.droppedBytes), exitCode: 0, detached: { shellId } }));
+                    resolvePromise(ok({ stdout: markTruncated(out.text, out.droppedBytes), stderr: markTruncated(err.text, err.droppedBytes), exitCode: 0, detached: { shellId, ...(cause !== undefined ? { cause } : {}) } }));
+                    return true;
                 };
+                onDetach = adoptRunning;
+                const onDetachSignal = () => void adoptRunning();
                 if (detach.signal.aborted)
-                    onDetach();
+                    onDetachSignal();
                 else {
-                    detach.signal.addEventListener("abort", onDetach, { once: true });
-                    removeDetachListener = () => detach.signal.removeEventListener("abort", onDetach);
+                    detach.signal.addEventListener("abort", onDetachSignal, { once: true });
+                    removeDetachListener = () => detach.signal.removeEventListener("abort", onDetachSignal);
                 }
             }
         });
@@ -1920,6 +1984,7 @@ export class RemoteK8sExecutionEnv {
                 stderrFilter: call,
                 detach: {
                     signal: detachSignal,
+                    adoptOnTimeout: options?.autoBackgroundOnTimeout === true,
                     tee,
                     adopt: (io) => {
                         const adopted = this.bgManager.adoptSync((ctx) => {
@@ -1944,20 +2009,6 @@ export class RemoteK8sExecutionEnv {
             this.endExec(inflight);
         }
     }
-    async getHost(port) {
-        const c = await this.ensureConnected();
-        if (!c.ok || !this.podIP)
-            return undefined;
-        return `${this.podIP}:${port}`;
-    }
-    async startBackground(command) {
-        const wrapped = command.replace(/[\\"$`]/g, (c) => "\\" + c);
-        const r = await this.exec(`(nohup sh -c "${wrapped}" >>/tmp/lsp-bridge.log 2>&1 &) ; echo started`, { timeout: 30 });
-        if (!r.ok)
-            throw new RemoteExecutionError("connect_failed", `startBackground failed: ${r.error.message}`, r.error);
-        if (r.value.exitCode !== 0)
-            throw new RemoteExecutionError("connect_failed", `startBackground exited ${r.value.exitCode}: ${r.value.stderr.slice(0, 200)}`);
-    }
     async *execStream(command, options) {
         const c = await this.ensureConnected();
         if (!c.ok)