@sema-agent/server 7.135.0 → 7.137.0

package/dist/plugins/remote-env-k8s.js

@@ -6,11 +6,14 @@ import { StringDecoder } from "node:string_decoder";
 import WebSocket from "ws";
 import { buildCwdEnvCommand, createCwdGuardedCall, shellQuote, kindFromMode, numEnvOr } from "./remote-shell.js";
 import { presignS3Url, presignS3ListUrl } from "./s3-presign.js";
-import { K8S_WORKSPACE_MOUNT_DEFAULT, WORKSPACE_SNAPSHOT_TIMEOUT_MS_DEFAULT, mintSnapshotId, snapshotNameOwnedBy } from "../workspace-space.js";
+import { WORKSPACE_SNAPSHOT_BUCKET_DEFAULT } from "../object-store-buckets.js";
+import { SNAPSHOT_BUCKET_MISSING, isNoSuchBucketAnswer } from "./s3-answer.js";
+import { K8S_WORKSPACE_MOUNT_DEFAULT, SNAPSHOT_KEY_SEGMENT, WORKSPACE_SNAPSHOT_PREFIX_DEFAULT, WORKSPACE_SNAPSHOT_TIMEOUT_MS_DEFAULT, mintSnapshotId, snapshotKeyPrefixProblem, snapshotNameOwnedBy } from "../workspace-space.js";
 import { spaceCommitAlive } from "../workspace-space-commit.js";
 import { incarnationLogPrefix, redactIncarnations } from "../session-incarnation.js";
 import { WORKSPACE_CACHE_SPACE_LABEL, workspaceCacheClaimOf, workspaceCacheEnv } from "../workspace-cache.js";
 import { recordFailOpen } from "../observability/fail-open.js";
+import { PodDeleteRefused, reportPodDeleteFailed } from "./k8s-pod-delete-failure.js";
 import { buildLivenessCut, commandBoundsOf, laneLivenessOf, LIVENESS_MS_DEFAULT } from "../exec-bounds.js";
 import { SANDBOX_IPV6_DEFAULT, sandboxEgressGaps, sandboxPodSysctls } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
@@ -37,9 +40,12 @@ const unsupported = (op) => ({
 const SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount";
 const EXEC_SUBPROTOCOLS = ["v5.channel.k8s.io", "v4.channel.k8s.io"];
 const WRITE_CHUNK_BYTES = 64 * 1024;
-import { applyPodSpecPatch } from "./k8s-exec-protocol.js";
+import { applyPodSpecPatch, ADAPTER_VOLUME_NAMES, SANDBOX_POD_LABELS, SANDBOX_RUN_END_HOLD_ANNOTATION, SANDBOX_RUN_LABEL, SANDBOX_SECRET_SEAT_VERBS, isLabelValue } from "./k8s-exec-protocol.js";
 export { applyPodSpecPatch } from "./k8s-exec-protocol.js";
-const CACHE_VOLUME = "ws-cache";
+import { sandboxRunEndHoldMs } from "./k8s-pod-reaper.js";
+import { BROKER_COMMAND, BROKER_CONTAINER_NAME, BROKER_LEASE_DIR, BROKER_LEASE_KEY, BROKER_PROBE_ARG, BROKER_VOLUME_NAME } from "../broker/constants.js";
+import { buildSandboxLeaseFile, placeholderLeaseEntries, sandboxGitRouteEnv } from "../sandbox-git-lease.js";
+const CACHE_VOLUME = ADAPTER_VOLUME_NAMES.cache;
 function occupantKeyOf(occupant) {
     if (occupant === undefined)
         return { state: "unavailable", reason: "incarnation_unknown" };
@@ -166,12 +172,39 @@ export function errorMessageOf(e) {
 }
 export { exitCodeFromStatus } from "./k8s-exec-protocol.js";
 export const K8S_SANDBOX_NAMESPACE_DEFAULT = "sandboxes";
-const SANDBOX_POD_LABELS = { app: "sema-sandbox", "managed-by": "sema-server" };
+const SANDBOX_SECRET_NAME_PREFIX = "sbx-env-";
+const SANDBOX_SECRET_LABEL = "sema.io/sandbox-secret";
+const secretEnvKey = (name) => `env.${name}`;
+const SANDBOX_LEASE_NAME_PREFIX = "sbx-lease-";
+const SANDBOX_LEASE_LABEL = "sema.io/sandbox-lease";
+function brokerContainer(image) {
+    return {
+        name: BROKER_CONTAINER_NAME,
+        image,
+        command: [...BROKER_COMMAND],
+        env: [{ name: "BUN_RUNTIME_TRANSPILER_CACHE_PATH", value: "0" }],
+        securityContext: {
+            runAsUser: 0,
+            runAsGroup: 0,
+            runAsNonRoot: false,
+            privileged: false,
+            allowPrivilegeEscalation: false,
+            readOnlyRootFilesystem: true,
+            capabilities: { drop: ["ALL"] },
+            seccompProfile: { type: "RuntimeDefault" },
+            procMount: "Default",
+        },
+        volumeMounts: [{ name: BROKER_VOLUME_NAME, mountPath: BROKER_LEASE_DIR, readOnly: true }],
+        readinessProbe: { exec: { command: [...BROKER_COMMAND, BROKER_PROBE_ARG] }, periodSeconds: 2, timeoutSeconds: 5 },
+        resources: { limits: { cpu: "250m", memory: "256Mi" } },
+    };
+}
 export class RemoteK8sExecutionEnv {
     capabilities;
     cwd;
     podName;
     podIP;
+    leaseRef;
     destroyed = false;
     handleSnapshotId;
     inFlight = new Set();
@@ -183,6 +216,7 @@ export class RemoteK8sExecutionEnv {
     }
     homeSeat;
     cdPath = "unknown";
+    withheldEnv = [];
     cfg;
     deps;
     occupantIncarnation;
@@ -218,6 +252,9 @@ export class RemoteK8sExecutionEnv {
             ...(config.sessionId ? { sessionId: config.sessionId } : {}),
             ...(config.podSpecPatch ? { podSpecPatch: config.podSpecPatch } : {}),
             ...(config.podEnv ? { podEnv: config.podEnv } : {}),
+            ...(config.podSecretEnv && Object.keys(config.podSecretEnv).length > 0 ? { podSecretEnv: config.podSecretEnv } : {}),
+            ...(config.runId !== undefined && config.runId !== "" && isLabelValue(config.runId) ? { runId: config.runId } : {}),
+            ...(config.sandboxBroker !== undefined ? { sandboxBroker: config.sandboxBroker } : {}),
             ...(config.snapshotExclude && config.snapshotExclude.length > 0 ? { snapshotExclude: config.snapshotExclude } : {}),
         };
         this.cwd = this.cfg.mountPath;
@@ -241,6 +278,9 @@ export class RemoteK8sExecutionEnv {
             supportsDetach: true,
         };
     }
+    get sandboxLeaseRef() {
+        return this.leaseRef !== undefined && this.leaseRef.podName === this.podName ? this.leaseRef : undefined;
+    }
     get lifetimeMs() {
         return this.cfg.timeoutMs;
     }
@@ -291,7 +331,7 @@ export class RemoteK8sExecutionEnv {
                 headers: {
                     authorization: `Bearer ${token}`,
                     accept: "application/json",
-                    ...(payload ? { "content-type": "application/json", "content-length": Buffer.byteLength(payload) } : {}),
+                    ...(payload ? { "content-type": opts?.contentType ?? "application/json", "content-length": Buffer.byteLength(payload) } : {}),
                 },
                 ...(isHttps ? { ca: ca ? [ca] : undefined, rejectUnauthorized: !this.cfg.insecureTls } : {}),
                 timeout: opts?.timeoutMs ?? this.cfg.rpcTimeoutMs,
@@ -507,7 +547,7 @@ export class RemoteK8sExecutionEnv {
             const claim = {
                 apiVersion: "v1",
                 kind: "PersistentVolumeClaim",
-                metadata: { name: cache.claim.name, labels: { [WORKSPACE_CACHE_SPACE_LABEL]: cache.claim.label, "managed-by": "sema-server" } },
+                metadata: { name: cache.claim.name, labels: { [WORKSPACE_CACHE_SPACE_LABEL]: cache.claim.label, "managed-by": SANDBOX_POD_LABELS["managed-by"] } },
                 spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: cache.size } } },
             };
             const made = await this.request("POST", base, claim, { signal });
@@ -560,6 +600,38 @@ export class RemoteK8sExecutionEnv {
             applyPodSpecPatch(probe, { labels: this.cfg.podSpecPatch.labels });
         return sandboxEgressGaps({ nodes, policies, podLabels: probe.metadata.labels, ipv6: this.cfg.ipv6 });
     }
+    async secretSeatReading(seat) {
+        const answers = await Promise.all(SANDBOX_SECRET_SEAT_VERBS[seat].map(async (verb) => {
+            try {
+                const r = await this.request("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", {
+                    apiVersion: "authorization.k8s.io/v1",
+                    kind: "SelfSubjectAccessReview",
+                    spec: { resourceAttributes: { namespace: this.cfg.namespace, verb, resource: "secrets" } },
+                });
+                const allowed = r.json?.status?.allowed;
+                const answered = r.status >= 200 && r.status < 300;
+                return answered && typeof allowed === "boolean" ? { verb, allowed } : { verb, unasked: answered ? "answer without status.allowed" : `HTTP ${r.status}` };
+            }
+            catch (e) {
+                return { verb, unasked: errorMessageOf(e) };
+            }
+        }));
+        const denied = answers.flatMap((a) => ("allowed" in a && !a.allowed ? [a.verb] : []));
+        const unasked = answers.flatMap((a) => ("unasked" in a ? [`${a.verb}: ${a.unasked}`] : []));
+        if (denied.length > 0)
+            return { state: "denied", verbs: denied };
+        if (unasked.length > 0)
+            return { state: "unverified", detail: unasked.join("; ") };
+        return { state: "granted" };
+    }
+    async listSandboxPods() {
+        const selector = encodeURIComponent(Object.entries(SANDBOX_POD_LABELS).map(([k, v]) => `${k}=${v}`).join(","));
+        const r = await this.request("GET", `/api/v1/namespaces/${this.cfg.namespace}/pods?labelSelector=${selector}`);
+        if (r.status < 200 || r.status >= 300)
+            throw new Error(`sandbox pod listing answered HTTP ${r.status}`);
+        const items = r.json?.items;
+        return (Array.isArray(items) ? items : []).flatMap((item) => sandboxPodListingOf(item) ?? []);
+    }
     reclaimDeadlinePassed(what) {
         return `reclaim deadline (${this.cfg.dataTimeoutMs} ms, the snapshot time bound, one per reclaim call) passed — ${what}`;
     }
@@ -577,19 +649,36 @@ export class RemoteK8sExecutionEnv {
     }
     async createPod(signal, cache) {
         const ns = this.cfg.namespace;
-        const podEnv = { ...(cache ? workspaceCacheEnv(cache.mount) : {}), ...(this.cfg.podEnv ?? {}) };
+        const secretEnv = this.cfg.podSecretEnv ?? {};
+        const secretName = Object.keys(secretEnv).length > 0 ? `${SANDBOX_SECRET_NAME_PREFIX}${randomUUID()}` : undefined;
+        const broker = this.cfg.sandboxBroker;
+        const leaseName = broker !== undefined ? `${SANDBOX_LEASE_NAME_PREFIX}${randomUUID()}` : undefined;
+        const placeholder = broker !== undefined ? buildSandboxLeaseFile(broker.origins, placeholderLeaseEntries(broker.origins)) : undefined;
+        if (placeholder !== undefined && !placeholder.ok) {
+            return this.holdRefusal(new RemoteExecutionError("connect_failed", `the member-credential broker cannot be set up for this pod: ${placeholder.problem}`));
+        }
+        const layered = { ...(cache ? workspaceCacheEnv(cache.mount) : {}), ...(this.cfg.podEnv ?? {}) };
+        const envEntries = [
+            ...Object.entries({ ...layered, ...(broker !== undefined ? sandboxGitRouteEnv(broker.origins) : {}) })
+                .filter(([name]) => !Object.hasOwn(secretEnv, name))
+                .map(([name, value]) => ({ name, value: String(value) })),
+            ...(secretName !== undefined ? Object.keys(secretEnv).map((name) => ({ name, valueFrom: { secretKeyRef: { name: secretName, key: secretEnvKey(name) } } })) : []),
+        ];
+        const pkgSourceMarked = Boolean(this.cfg.podEnv?.SEMA_PKG_SOURCE || secretEnv.SEMA_PKG_SOURCE);
         const sysctls = sandboxPodSysctls(this.cfg.ipv6);
         const pod = {
             apiVersion: "v1",
             kind: "Pod",
             metadata: {
                 generateName: "sbx-",
-                labels: { ...SANDBOX_POD_LABELS },
+                labels: { ...SANDBOX_POD_LABELS, ...(this.cfg.runId !== undefined ? { [SANDBOX_RUN_LABEL]: this.cfg.runId } : {}) },
+                ...(this.cfg.runId !== undefined ? { annotations: { [SANDBOX_RUN_END_HOLD_ANNOTATION]: String(sandboxRunEndHoldMs(this.cfg.dataTimeoutMs)) } } : {}),
             },
             spec: {
                 ...(this.cfg.runtimeClass ? { runtimeClassName: this.cfg.runtimeClass } : {}),
                 ...(sysctls ? { securityContext: { sysctls: sysctls.map((x) => ({ ...x })) } } : {}),
                 restartPolicy: "Never",
+                ...(broker !== undefined ? { shareProcessNamespace: false } : {}),
                 automountServiceAccountToken: false,
                 activeDeadlineSeconds: Math.ceil(this.cfg.timeoutMs / 1000),
                 terminationGracePeriodSeconds: 1,
@@ -597,18 +686,30 @@ export class RemoteK8sExecutionEnv {
                     {
                         name: "sandbox",
                         image: this.cfg.image,
-                        command: this.cfg.podEnv?.SEMA_PKG_SOURCE
+                        command: pkgSourceMarked
                             ? ["sh", "-c", "[ -f /opt/sema/pkg-source.sh ] && sh /opt/sema/pkg-source.sh || true; sleep infinity"]
                             : ["sh", "-c", "sleep infinity"],
-                        ...(Object.keys(podEnv).length > 0 ? { env: Object.entries(podEnv).map(([name, value]) => ({ name, value: String(value) })) } : {}),
+                        ...(envEntries.length > 0 ? { env: envEntries } : {}),
                         ...(cache ? { volumeMounts: [{ name: CACHE_VOLUME, mountPath: cache.mount }] } : {}),
                         resources: { limits: { cpu: this.cfg.cpu, memory: this.cfg.memory } },
                     },
+                    ...(broker !== undefined ? [brokerContainer(broker.image)] : []),
                 ],
-                ...(cache ? { volumes: [{ name: CACHE_VOLUME, persistentVolumeClaim: { claimName: cache.claim.name } }] } : {}),
+                ...((volumes) => (volumes.length > 0 ? { volumes } : {}))([
+                    ...(cache ? [{ name: CACHE_VOLUME, persistentVolumeClaim: { claimName: cache.claim.name } }] : []),
+                    ...(leaseName !== undefined ? [{ name: BROKER_VOLUME_NAME, secret: { secretName: leaseName, items: [{ key: BROKER_LEASE_KEY, path: BROKER_LEASE_KEY }], defaultMode: 0o400 } }] : []),
+                ]),
             },
         };
-        if (this.cfg.podSpecPatch)
+        if (this.cfg.podSpecPatch && broker !== undefined) {
+            try {
+                applyPodSpecPatch(pod, this.cfg.podSpecPatch, { broker: true });
+            }
+            catch (e) {
+                return this.holdRefusal(new RemoteExecutionError("connect_failed", `${errorMessageOf(e)} — no pod is created (the member-credential broker never shares a pod with a patch that could reach it)`));
+            }
+        }
+        else if (this.cfg.podSpecPatch)
             applyPodSpecPatch(pod, this.cfg.podSpecPatch);
         let created;
         try {
@@ -626,6 +727,20 @@ export class RemoteK8sExecutionEnv {
         const name = created.json?.metadata?.name;
         if (!name)
             return { ok: false, error: new RemoteExecutionError("connect_failed", "pod create returned no name") };
+        if (secretName !== undefined) {
+            const sealed = await this.createPodSecret(secretName, name, created.json?.metadata?.uid, secretEnv, signal);
+            if (!sealed.ok) {
+                await this.discardRefusedPod(name, "env_secret_unsealed");
+                return sealed;
+            }
+        }
+        if (leaseName !== undefined && placeholder?.ok === true) {
+            const leased = await this.createLeaseSecret(leaseName, name, created.json?.metadata?.uid, placeholder.text, signal);
+            if (!leased.ok) {
+                await this.discardRefusedPod(name, "lease_secret_unsealed");
+                return leased;
+            }
+        }
         const deadline = Date.now() + this.cfg.readyTimeoutMs;
         let podIP;
         for (;;) {
@@ -643,7 +758,8 @@ export class RemoteK8sExecutionEnv {
             }
             const status = got.json?.status;
             const phase = status?.phase;
-            if (phase === "Running") {
+            const brokerStatus = status?.containerStatuses?.find((c) => c.name === BROKER_CONTAINER_NAME);
+            if (phase === "Running" && (broker === undefined || brokerStatus?.ready === true)) {
                 podIP = status?.podIP;
                 break;
             }
@@ -651,11 +767,16 @@ export class RemoteK8sExecutionEnv {
                 await this.deletePod(name).catch(() => { });
                 return this.refusePod(name, phase, status?.reason, status?.message);
             }
-            const waiting = status?.containerStatuses?.[0]?.state?.waiting;
+            const waiting = status?.containerStatuses?.find((c) => c.name !== BROKER_CONTAINER_NAME)?.state?.waiting;
             if (waiting?.reason === "ImagePullBackOff" || waiting?.reason === "ErrImagePull") {
                 await this.deletePod(name).catch(() => { });
                 return { ok: false, error: new RemoteExecutionError("connect_failed", `image pull failed for ${this.cfg.image}: ${waiting.message ?? waiting.reason}`) };
             }
+            const brokerWaiting = brokerStatus?.state?.waiting;
+            if (broker !== undefined && (brokerWaiting?.reason === "ImagePullBackOff" || brokerWaiting?.reason === "ErrImagePull")) {
+                await this.discardRefusedPod(name, "broker_image_pull_failed");
+                return { ok: false, error: new RemoteExecutionError("connect_failed", `image pull failed for the member-credential broker image ${broker.image} (K8S_SANDBOX_BROKER_IMAGE): ${brokerWaiting.message ?? brokerWaiting.reason}`) };
+            }
             if (Date.now() > deadline) {
                 if (cache) {
                     const bound = await this.cacheClaimBound(cache, signal);
@@ -669,6 +790,9 @@ export class RemoteK8sExecutionEnv {
                         : { cacheMiss: "claim_unbound", detail: `the claim ${cache.claim.name} was not bound within ${this.cfg.dataTimeoutMs}ms (no default StorageClass, or its provisioner is down)` };
                 }
                 await this.deletePod(name).catch(() => { });
+                if (phase === "Running" && broker !== undefined) {
+                    return { ok: false, error: new RemoteExecutionError("timeout", `the member-credential broker container was not ready within ${this.cfg.readyTimeoutMs}ms (it reads its lease file and answers its own readiness probe; image ${broker.image})`) };
+                }
                 return { ok: false, error: new RemoteExecutionError("timeout", `pod not Running within ${this.cfg.readyTimeoutMs}ms (phase=${phase ?? "?"})`) };
             }
             await new Promise((r) => setTimeout(r, 1000));
@@ -679,6 +803,7 @@ export class RemoteK8sExecutionEnv {
         }
         this.podName = name;
         this.podIP = podIP;
+        this.leaseRef = leaseName !== undefined && broker !== undefined ? { namespace: ns, podName: name, leaseSecretName: leaseName, origins: [...broker.origins] } : undefined;
         const askHome = this.homeSeat.asks;
         const mk = await this.runExec(`mkdir -p ${shellQuote(this.cfg.mountPath)}${askHome ? ` && { ${LANE_HOME_PROBE_COMMAND} || true; }` : ""}`);
         if (!mk.ok || mk.value.exitCode !== 0) {
@@ -701,23 +826,110 @@ export class RemoteK8sExecutionEnv {
             this.logger?.warn("workspace_cache_fresh", { space: incarnationLogPrefix(cache.spaceKey), claim: cache.claim.name, size: cache.size });
         return ok(this.workspaceHandle());
     }
+    async createPodSecret(secretName, podName, podUid, values, signal) {
+        const encoded = Object.entries(values).map(([n, v]) => [secretEnvKey(n), Buffer.from(String(v), "utf8").toString("base64")]);
+        return this.createOwnedSecret({
+            name: secretName,
+            kind: "env",
+            label: SANDBOX_SECRET_LABEL,
+            immutable: true,
+            data: Object.fromEntries(encoded),
+            hidden: [...Object.values(values).map(String), ...encoded.map(([, b64]) => b64)],
+            what: "sandbox secret",
+            effect: "the pod is deleted (deployer-named sandbox env reaches a pod only through its own Secret, never as plaintext in the pod spec)",
+        }, podName, podUid, signal);
+    }
+    async createLeaseSecret(leaseName, podName, podUid, leaseText, signal) {
+        const b64 = Buffer.from(leaseText, "utf8").toString("base64");
+        return this.createOwnedSecret({
+            name: leaseName,
+            kind: "lease",
+            label: SANDBOX_LEASE_LABEL,
+            immutable: false,
+            data: { [BROKER_LEASE_KEY]: b64 },
+            hidden: [b64],
+            what: "sandbox lease secret",
+            effect: "the pod is deleted (a pod that carries the member-credential broker never runs without the lease its broker reads)",
+        }, podName, podUid, signal);
+    }
+    async createOwnedSecret(seat, podName, podUid, signal) {
+        const { effect } = seat;
+        if (typeof podUid !== "string" || podUid === "")
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `pod create returned no uid — the ${seat.what} cannot be bound to its pod; ${effect}`) };
+        const secret = {
+            apiVersion: "v1",
+            kind: "Secret",
+            metadata: {
+                name: seat.name,
+                labels: { "managed-by": SANDBOX_POD_LABELS["managed-by"], [seat.label]: podName },
+                ownerReferences: [{ apiVersion: "v1", kind: "Pod", name: podName, uid: podUid, blockOwnerDeletion: false }],
+            },
+            type: "Opaque",
+            ...(seat.immutable ? { immutable: true } : {}),
+            data: seat.data,
+        };
+        const hidden = seat.hidden.filter((v) => v !== "");
+        const scrub = (text) => hidden.reduce((out, v) => out.split(v).join("<redacted>"), text);
+        let made;
+        try {
+            made = await this.request("POST", `/api/v1/namespaces/${this.cfg.namespace}/secrets`, secret, { signal });
+        }
+        catch (e) {
+            if (signal?.aborted)
+                return { ok: false, error: new RemoteExecutionError("aborted", "connect aborted") };
+            return { ok: false, error: new RemoteExecutionError("connect_failed", `${seat.what} create RPC failed: ${scrub(errorMessageOf(e)).slice(0, 300)}; ${effect}`) };
+        }
+        if (made.status === 200 || made.status === 201)
+            return { ok: true };
+        const detail = scrub(JSON.stringify(made.json) ?? "").slice(0, 300);
+        if (made.status === 401 || made.status === 403) {
+            return { ok: false, error: new RemoteExecutionError("auth_failed", `${seat.what} create denied (${made.status}): ${detail} — the sandbox Role needs secrets ${SANDBOX_SECRET_SEAT_VERBS[seat.kind].join(" / ")}; ${effect}`) };
+        }
+        return { ok: false, error: new RemoteExecutionError("connect_failed", `${seat.what} create failed (${made.status}): ${detail}; ${effect}`) };
+    }
+    async writeLease(ref, entries) {
+        const fail = (code, message) => {
+            this.logger?.warn("sandbox_lease_write_failed", { namespace: ref.namespace, pod: ref.podName, secretName: ref.leaseSecretName, code, detail: message });
+            return { ok: false, code, message };
+        };
+        const built = buildSandboxLeaseFile(ref.origins, entries);
+        if (!built.ok)
+            return fail("lease_invalid", `the lease was not sent — the broker would refuse it: ${built.problem}`);
+        const b64 = Buffer.from(built.text, "utf8").toString("base64");
+        const hidden = [b64, ...entries.flatMap((e) => (e.state === "ready" ? [e.secret, Buffer.from(`${e.username}:${e.secret}`, "utf8").toString("base64")] : []))].filter((v) => v !== "");
+        const scrub = (text) => hidden.reduce((out, v) => out.split(v).join("<redacted>"), text);
+        let r;
+        try {
+            r = await this.request("PATCH", `/api/v1/namespaces/${ref.namespace}/secrets/${encodeURIComponent(ref.leaseSecretName)}`, { data: { [BROKER_LEASE_KEY]: b64 } }, { contentType: "application/merge-patch+json" });
+        }
+        catch (e) {
+            return fail("write_failed", `the lease write request failed: ${scrub(errorMessageOf(e)).slice(0, 300)}`);
+        }
+        if (r.status === 200 || r.status === 201)
+            return { ok: true };
+        const detail = scrub(JSON.stringify(r.json) ?? "").slice(0, 300);
+        if (r.status === 401 || r.status === 403)
+            return fail("auth_failed", `the lease write was denied (${r.status}): ${detail} — the sandbox Role needs secrets ${SANDBOX_SECRET_SEAT_VERBS.lease.join(" / ")}`);
+        if (r.status === 404)
+            return fail("gone", `the lease Secret ${ref.leaseSecretName} is gone (its pod ${ref.podName} was deleted)`);
+        return fail("write_failed", `the lease write failed (${r.status}): ${detail}`);
+    }
     presignSnapshot(req) {
         const s3 = this.cfg.s3Snapshot;
-        const prefix = s3.keyPrefix ?? "kata";
         const [label, partition] = req.lane === "space" ? ["spaceIncarnation", req.space.incarnation] : ["sessionId", this.cfg.sessionId ?? "_"];
         const snapshotId = "snapshotId" in req ? req.snapshotId : undefined;
         const objectName = "object" in req ? req.object : undefined;
-        const safe = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
         const segments = [
             [label, partition],
             ...(snapshotId !== undefined ? [["snapshotId", snapshotId]] : []),
             ...(objectName !== undefined ? [["object", objectName]] : []),
-            ["keyPrefix", prefix],
         ];
         for (const [label, seg] of segments) {
-            if (!safe.test(seg))
+            if (!SNAPSHOT_KEY_SEGMENT.test(seg))
                 return { ok: false, error: new RemoteExecutionError("connect_failed", `unsafe snapshot key segment (${label}): refusing to build a traversing object key`) };
         }
+        if (snapshotKeyPrefixProblem(this.snapshotKeyPrefix()) !== undefined)
+            return { ok: false, error: new RemoteExecutionError("connect_failed", "unsafe snapshot key segment (keyPrefix): refusing to build a traversing object key") };
         const common = {
             endpoint: s3.endpoint,
             bucket: s3.bucket,
@@ -732,8 +944,11 @@ export class RemoteK8sExecutionEnv {
         const object = "pointer" in req ? req.pointer : "object" in req ? req.object : `${req.snapshotId}.tar.gz`;
         return { ok: true, value: presignS3Url({ ...common, key: `${base}${object}`, method: req.method }) };
     }
+    snapshotKeyPrefix() {
+        return this.cfg.s3Snapshot?.keyPrefix ?? WORKSPACE_SNAPSHOT_PREFIX_DEFAULT;
+    }
     snapshotBase(req) {
-        const prefix = this.cfg.s3Snapshot.keyPrefix ?? "kata";
+        const prefix = this.snapshotKeyPrefix();
         return req.lane === "space" ? `${prefix}/${req.space.incarnation}/space/` : `${prefix}/${this.cfg.sessionId ?? "_"}/`;
     }
     async reclaimSpaceSnapshots(spaceKey, deadline) {
@@ -825,12 +1040,25 @@ export class RemoteK8sExecutionEnv {
         catch (e) {
             return spaceReadFailed(what, `: ${this.redactUrl(errorMessageOf(e), url.value)}`, e);
         }
+        if (isNoSuchBucketAnswer(res.status, body))
+            return this.refuseMissingBucket(space);
         if (res.status === 404)
             return ok(undefined);
         if (!res.ok)
             return spaceReadFailed(what, ` (HTTP ${res.status})`);
         return ok(body.trim());
     }
+    refuseMissingBucket(space) {
+        const bucket = this.cfg.s3Snapshot.bucket;
+        this.spaceLine("workspace_snapshot_bucket_missing", space, {
+            bucket,
+            code: SNAPSHOT_BUCKET_MISSING,
+            effect: "the workspace snapshot bucket does not exist in the object store (the server never creates buckets) — this run is refused rather than started on a workspace that could be neither restored nor saved",
+        });
+        return this.holdRefusal(spaceReadFailed("pointer", ` (${SNAPSHOT_BUCKET_MISSING}): the workspace snapshot bucket "${bucket}" (WORKSPACE_SNAPSHOT_BUCKET; default ${WORKSPACE_SNAPSHOT_BUCKET_DEFAULT}) does not exist in the object store — ` +
+            `refusing to start this run on a workspace that could be neither restored nor saved. Ways out: create the bucket, or set WORKSPACE_SNAPSHOT_BUCKET to one that exists, ` +
+            "or set WORKSPACE_PERSIST=off for a deployment that keeps no workspace between runs").error);
+    }
     async readLatest(space, signal) {
         const r = await this.readPointer(space, "latest", signal);
         return r.ok ? ok(r.value === undefined ? undefined : r.value) : r;
@@ -1045,7 +1273,7 @@ export class RemoteK8sExecutionEnv {
         });
     }
     async dropDanglingLatest(space, snapshotId) {
-        const prefix = this.cfg.s3Snapshot?.keyPrefix ?? "kata";
+        const prefix = this.snapshotKeyPrefix();
         const dropped = await this.writePointer(space, "latest", null);
         this.spaceLine("workspace_latest_dangling", space, {
             key: `${prefix}/${space.incarnation}/space/${snapshotId}.tar.gz`,
@@ -1073,7 +1301,27 @@ export class RemoteK8sExecutionEnv {
         return this.connect();
     }
     async deletePod(name) {
-        await this.request("DELETE", `/api/v1/namespaces/${this.cfg.namespace}/pods/${name}?gracePeriodSeconds=0`, undefined, { timeoutMs: this.cfg.rpcTimeoutMs });
+        const r = await this.request("DELETE", `/api/v1/namespaces/${this.cfg.namespace}/pods/${encodeURIComponent(name)}?gracePeriodSeconds=0`, undefined, { timeoutMs: this.cfg.rpcTimeoutMs });
+        if (r.status === 404)
+            return "gone";
+        if (r.status < 200 || r.status >= 300)
+            throw new PodDeleteRefused(r.status);
+        return "deleted";
+    }
+    async discardRefusedPod(name, why) {
+        try {
+            await this.deletePod(name);
+        }
+        catch (e) {
+            reportPodDeleteFailed({
+                namespace: this.cfg.namespace,
+                pod: name,
+                why,
+                error: e,
+                effect: "the refused connect's pod, and the Secret it owns, stay in the sandbox namespace until the pod reaper reaches them (static data; the connect is refused either way)",
+                logger: this.logger,
+            });
+        }
     }
     async destroy() {
         await this.disposeBackgroundShells().catch(() => { });
@@ -1861,6 +2109,37 @@ export function createSpaceReclaimer(config, deps) {
 export async function checkSandboxEgressPolicy(config, deps) {
     return new RemoteK8sExecutionEnv(config, deps).egressPolicyGaps();
 }
+export async function readSandboxSecretSeat(config, seat, deps) {
+    return new RemoteK8sExecutionEnv(config, deps).secretSeatReading(seat);
+}
+export async function writeSandboxLease(config, ref, entries, deps) {
+    return new RemoteK8sExecutionEnv(config, deps).writeLease(ref, entries);
+}
+export function createSandboxPodApi(config, deps) {
+    let env;
+    const control = () => (env ??= new RemoteK8sExecutionEnv(config, deps));
+    return {
+        namespace: config.namespace ?? K8S_SANDBOX_NAMESPACE_DEFAULT,
+        list: () => control().listSandboxPods(),
+        delete: (name) => control().deletePod(name),
+    };
+}
+function sandboxPodListingOf(item) {
+    const o = item;
+    const name = o?.metadata?.name;
+    if (typeof name !== "string" || name === "")
+        return undefined;
+    const runId = o?.metadata?.labels?.[SANDBOX_RUN_LABEL];
+    const runEndHold = o?.metadata?.annotations?.[SANDBOX_RUN_END_HOLD_ANNOTATION];
+    const phase = o?.status?.phase;
+    return {
+        name,
+        ...(typeof runId === "string" && runId !== "" ? { runId } : {}),
+        ...(typeof runEndHold === "string" ? { runEndHold } : {}),
+        ...(typeof phase === "string" ? { phase } : {}),
+        terminating: typeof o?.metadata?.deletionTimestamp === "string",
+    };
+}
 export function k8sExecutionEnvFactory(config, deps) {
     return (ctx) => new RemoteK8sExecutionEnv({ ...config, sessionId: ctx.sessionId }, deps);
 }

package/dist/plugins/remote-env-local-docker.d.ts

@@ -74,6 +74,9 @@ export declare class RemoteLocalDockerExecutionEnv implements RemoteExecutionEnv
     /** core 7.49.0 [ref]:shell 子进程真拿到的 `CDPATH` 条目 —— 本车道的命令跑在容器里(镜像的 env),本类不读那份 env ⇒ `"unknown"`(core 契约:说不出就说 unknown,
      *  不许报一张没读过的空表;删除守卫因此对裸 `cd <相对名>` 之后的删除问人,而不是按「没有 CDPATH」放行)。 */
     readonly cdPath: "unknown";
+    /** core 7.56.0 [ref]:shell 子进程继承的环境里收不到的条目 —— 本车道 shell 的环境在容器里**构建**(本进程的 env 从不注入,只有引擎逐次交的 `env`
+     *  这条显式授予),没有一条是从本进程继承的 ⇒ `[]`(core 契约:「`[]` when its shell environment is built rather than inherited」)。 */
+    readonly withheldEnv: readonly [];
     private readonly cfg;
     private readonly spawnFn;
     /** Stable container NAME (also our handle id). Created on connect; targeted by every docker exec/cp/rm. */

package/dist/plugins/remote-env-local-docker.js

@@ -27,6 +27,7 @@ export class RemoteLocalDockerExecutionEnv {
         return this.homeSeat.home;
     }
     cdPath = "unknown";
+    withheldEnv = [];
     cfg;
     spawnFn;
     containerName;

package/dist/plugins/remote-env-ssh.d.ts

@@ -95,6 +95,9 @@ export declare class RemoteSshExecutionEnv implements RemoteExecutionEnv {
     /** core 7.49.0 [ref]:shell 子进程真拿到的 `CDPATH` 条目 —— 本车道的命令跑在远端登录 shell 里(目标机的 profile / env),本类不读那份 env ⇒ `"unknown"`(core 契约:说不出就说 unknown,
      *  不许报一张没读过的空表;删除守卫因此对裸 `cd <相对名>` 之后的删除问人,而不是按「没有 CDPATH」放行)。 */
     readonly cdPath: "unknown";
+    /** core 7.56.0 [ref]:shell 子进程继承的环境里收不到的条目 —— 本车道 shell 的环境在目标机上**构建**(本进程的 env 从不注入,只有引擎逐次交的 `env`
+     *  这条显式授予),没有一条是从本进程继承的 ⇒ `[]`(core 契约:「`[]` when its shell environment is built rather than inherited」)。 */
+    readonly withheldEnv: readonly [];
     private readonly cfg;
     private readonly injectedConnectOnce?;
     constructor(config: SshEnvConfig, deps?: SshEnvDeps);

package/dist/plugins/remote-env-ssh.js

@@ -32,6 +32,7 @@ export class RemoteSshExecutionEnv {
     homeChecked;
     homeRefusal;
     cdPath = "unknown";
+    withheldEnv = [];
     cfg;
     injectedConnectOnce;
     constructor(config, deps) {

package/dist/plugins/run-store-sql.js

@@ -10,10 +10,11 @@ import { RUN_HOLDER_DEATH_CODES, RUN_STALLED_ERROR, readLegacyHolderDeathCode }
 import { notifyRunTerminal, runTerminalLogLevel } from "../observability/run-terminal.js";
 import { mysqlDriver, pgDriver, dialectJsonEncoder, SQL_DB_NOW_MS, sqlDbNowMs } from "./sql-driver.js";
 import { isDupKeyError } from "./sql-errors.js";
+import { readRunIdentityCells, runIdentityCells } from "../identity-source.js";
 import { decideSpaceCommitAbandon, decideSpaceCommitClaim, fenceSpaceCommitAtTerminal, spaceCommitStateOf } from "../workspace-space-commit.js";
 function runRowCols(dialect) {
     return (`task_id, session_id, owner, status, ${jsonColumnText(dialect, "result")}, error, error_code, job_id, source, ` +
-        "instance_id, cancel_requested, objective_preview, created_at, updated_at");
+        "identity_source, identity_scope, identity_principal_inc, instance_id, cancel_requested, objective_preview, created_at, updated_at");
 }
 const readPersistedTaskResultBody = createObjectColumnReader();
 function decodeResultColumn(v, taskId) {
@@ -84,15 +85,19 @@ export class SqlRunStore {
             try {
                 await conn.query(this.q("INSERT INTO task_active (session_id, task_id, since) VALUES (?,?,?)", "INSERT INTO task_active (session_id, task_id, since) VALUES ($1,$2,$3)"), [sessionId, taskId, now]);
                 stage = "row";
-                await conn.query(this.q("INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
-                    "VALUES (?,?,?,'running',?,?,?,?,?,?,?,?)", "INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
-                    "VALUES ($1,$2,$3,'running',$4,$5,$6,$7,$8,$9,$10,$11)"), [
+                const cells = runIdentityCells(meta.identity ?? null);
+                await conn.query(this.q("INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
+                    "VALUES (?,?,?,'running',?,?,?,?,?,?,?,?,?,?,?)", "INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
+                    "VALUES ($1,$2,$3,'running',$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)"), [
                     taskId,
                     sessionId,
                     owner,
                     instanceId,
                     meta.jobId ?? null,
                     meta.source ?? null,
+                    cells.identitySource,
+                    cells.identityScope,
+                    cells.identityPrincipalInc,
                     this.db.dialect === "tidb"
                         ? meta.objectivePreview ?? null
                         : meta.objectivePreview == null
@@ -299,6 +304,7 @@ export class SqlRunStore {
             errorCode: readLegacyHolderDeathCode({ status: String(r.status), error: r.error ?? null, errorCode: r.error_code ?? null }),
             jobId: r.job_id ?? null,
             source: r.source ?? null,
+            identity: readRunIdentityCells(r.identity_source, r.identity_scope, r.identity_principal_inc),
             instanceId: r.instance_id ?? null,
             cancelRequested: Boolean(r.cancel_requested),
             objectivePreview: r.objective_preview ?? null,

package/dist/plugins/s3-answer.d.ts

@@ -0,0 +1,15 @@
+/**
+ * How this server reads an S3-compatible store's ANSWER — the one place a "the bucket itself is not there" reading is spelled.
+ *
+ * A 404 from an object store means two different things: `NoSuchKey` (the bucket is there, this object is not — a space's first run,
+ * a pointer never written) and `NoSuchBucket` (the bucket was never created — a deployment error no run can fix by retrying). Reading
+ * the second as the first turns a missing bucket into a silently empty workspace on every run. The ONLY evidence of a missing bucket is
+ * a 404 whose body carries the S3 error code `NoSuchBucket`: a bare 404, a 404 from a proxy or a wrong path, a 403 (credentials that
+ * cannot list ≠ no bucket) and a 5xx prove nothing about the bucket. Every consumer asks this one predicate: the boot probe of a
+ * configured bucket (`MinioBlobBackend.probeBucket`) and the runtime read of the workspace `latest` pointer (`remote-env-k8s.ts`).
+ */
+/** The code a run's refusal / operator line carries when the store answered "no such bucket" (the `workspace_snapshot_bucket_missing` line's `code`). */
+export declare const SNAPSHOT_BUCKET_MISSING = "snapshot_bucket_missing";
+/** Does this answer prove the bucket is missing? `status` = the HTTP status, `body` = the answer's text (an S3 error document). */
+export declare function isNoSuchBucketAnswer(status: number, body: string): boolean;
+//# sourceMappingURL=s3-answer.d.ts.map

package/dist/plugins/s3-answer.js

@@ -0,0 +1,5 @@
+export const SNAPSHOT_BUCKET_MISSING = "snapshot_bucket_missing";
+export function isNoSuchBucketAnswer(status, body) {
+    return status === 404 && /<Code>NoSuchBucket<\/Code>/.test(body);
+}
+//# sourceMappingURL=s3-answer.js.map