@inflexa-ai/harness 0.31.1 → 0.33.0

package/dist/sandbox/farm.js

@@ -0,0 +1,127 @@
+/**
+ * Farm-source resolution and the `inflexa.lock` farm contract.
+ *
+ * A farm carries exactly one metadata file, `inflexa.lock`, beside its link
+ * trees and its caches (the lib-store spec). The mount gate and the package
+ * inventory read this one file — `packages.txt`, `meta.json`, and
+ * `lock.json` are not part of the farm contract.
+ *
+ * The resolution and the gate refuse differently, on purpose:
+ * - A resolver refusal (`unavailable`, or a throw) is a `farm_unavailable`
+ *   `SandboxError` — the call refuses and no container is made.
+ * - A lock-gate failure degrades — the backend drops both store mounts,
+ *   logs a warning, and still makes the container.
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { err, ok } from "neverthrow";
+import { z } from "zod";
+/** The one metadata file of a farm. */
+export const FARM_LOCK_FILE = "inflexa.lock";
+/** One linked distribution. `requested` obeys the PEP 376 meaning: true for a direct ask, false for a transitive dependency. */
+export const FarmLockPackageSchema = z
+    .object({
+    name: z.string(),
+    version: z.string(),
+    /** The link subtree of the package (for example `python`, `cran`, `bioconductor`). */
+    track: z.string(),
+    store_dir: z.string(),
+    /** The full sha256 of the sorted store tree. */
+    hash: z.string(),
+    requested: z.boolean(),
+})
+    .passthrough();
+/**
+ * `inflexa.lock` at schema version 1. Additive fields pass through — a
+ * breaking change to the shape must move the schema number instead.
+ */
+export const FarmLockSchema = z
+    .object({
+    schema: z.literal(1),
+    arch: z.enum(["amd64", "arm64"]),
+    packages: z.array(FarmLockPackageSchema),
+    /** Per-language provenance. Each language object owns its own fields. */
+    languages: z
+        .object({
+        python: z.object({ version: z.string(), index: z.string() }).passthrough().optional(),
+        r: z
+            .object({
+            version: z.string(),
+            bioc_releases: z.array(z.string()),
+            /** The embedded pak lock of the build, verbatim. */
+            pak_lock: z.unknown().optional(),
+        })
+            .passthrough()
+            .optional(),
+    })
+        .passthrough(),
+    /** The warm replay record. Present in the catalog farm only. */
+    warm: z
+        .record(z.string(), z
+        .object({
+        script_sha256: z.string(),
+        cache_entries: z.array(z.string()),
+    })
+        .passthrough())
+        .optional(),
+    /** The top-level entries that the link merge kept-first or skipped. */
+    merge_conflicts: z
+        .array(z
+        .object({
+        entry: z.string(),
+        action: z.enum(["kept-first", "skipped"]),
+    })
+        .passthrough())
+        .optional(),
+})
+    .passthrough();
+/**
+ * Read and validate the `inflexa.lock` of a farm directory. The mount gate
+ * accepts a farm only when this returns ok — a farm whose lock is absent,
+ * unparseable, or at an unknown schema version is not usable.
+ */
+export function readFarmLock(farmPath) {
+    return readFarmLockFile(join(farmPath, FARM_LOCK_FILE));
+}
+/** Read and validate one `inflexa.lock` by its file path. The package inventory reads through this too. */
+export function readFarmLockFile(lockPath) {
+    let raw;
+    try {
+        raw = readFileSync(lockPath, "utf8");
+    }
+    catch (cause) {
+        return err({ type: "lock_unreadable", lockPath, cause });
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (cause) {
+        return err({ type: "lock_invalid", lockPath, cause });
+    }
+    const lock = FarmLockSchema.safeParse(parsed);
+    if (!lock.success) {
+        return err({ type: "lock_invalid", lockPath, cause: lock.error });
+    }
+    return ok(lock.data);
+}
+/**
+ * Resolve the farm source of one analysis. Runs before any container work.
+ * `fixed` answers at once. `per-analysis` calls the resolver of the
+ * embedder — an `unavailable` result and a throw both refuse the call with
+ * `farm_unavailable`, and the reason of the embedder rides in the error.
+ */
+export async function resolveFarmSource(source, analysisId, op) {
+    if (source.kind === "fixed")
+        return ok(source.location);
+    try {
+        const resolution = await source.resolve(analysisId);
+        if (resolution.kind === "unavailable") {
+            return err({ type: "farm_unavailable", op, analysisId, reason: resolution.reason });
+        }
+        return ok(resolution.location);
+    }
+    catch (cause) {
+        return err({ type: "farm_unavailable", op, analysisId, reason: "the farm resolver threw", cause });
+    }
+}

package/dist/sandbox/k8s-client.d.ts

@@ -6,16 +6,17 @@
  * Storage is wired via the shared session PVC: a flat read-only `volumeMount`
  * of the analysis tree at `/{resourceId}` plus a nested read-write mount of the
  * step's artifact dir, with the lib/ref stores mounted read-only at `/mnt/libs`
- * / `/mnt/refs` when their PVCs are set. Container paths and lib-store env come
- * from the shared mount plan; the PVC `subPath`s are derived from the same
- * `resolveWorkspaceRoot` seam the harness pre-creates the step tree under, so
- * both sides address one directory by construction.
+ * / `/mnt/refs` when their PVCs are set. The farm of the analysis mounts as a
+ * read-only `subPath` of the libs PVC, after the store mount. Container paths
+ * and lib-store env come from the shared mount plan; the PVC `subPath`s are
+ * derived from the same `resolveWorkspaceRoot` seam the harness pre-creates
+ * the step tree under, so both sides address one directory by construction.
  */
 import { BatchV1Api, CoreV1Api, type V1Toleration } from "@kubernetes/client-node";
 import { ResultAsync } from "neverthrow";
 import type { ResolveWorkspaceRoot } from "../workspace/paths.js";
 import { type SandboxError } from "./sandbox-error.js";
-import type { CreateSandboxMeta, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxTransport } from "./types.js";
+import type { CreateSandboxMeta, FarmSource, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxTransport, ToolchainSource } from "./types.js";
 import type { Logger } from "../lib/logger.js";
 /**
  * Coerce an arbitrary identifier into a valid K8s label value:
@@ -52,6 +53,24 @@ export interface K8sClientConfig {
     resolveWorkspaceRoot: ResolveWorkspaceRoot;
     /** PVC claim mounted read-only at `/mnt/libs` when set. */
     libStorePvc?: string;
+    /**
+     * Absolute mountpoint of `libStorePvc` on THIS process's filesystem, for
+     * a host that mounts the same volume (cortex mounts it at `/mnt/libs`).
+     * With the root set, the backend proves the `inflexa.lock` of the
+     * resolved farm before the mount — the same gate, and the same degrade,
+     * as the Docker backend. Without it, the proof duty stays whole on the
+     * farm resolver — see {@link ResolveAnalysisFarm}.
+     */
+    libStorePvcRoot?: string;
+    /**
+     * Where the farm of an analysis comes from. Required — the harness never
+     * invents a farm location. The farm is a `subPath` into `libStorePvc`,
+     * thus `FarmLocation.farmPath` is PVC-relative here. Consulted only when
+     * `libStorePvc` is set.
+     */
+    farmSource: FarmSource;
+    /** The declared toolchain owner. Absent means `"store"`, the legacy environment. */
+    toolchainSource?: ToolchainSource;
     /** PVC claim mounted read-only at `/mnt/refs` when set. */
     refStorePvc?: string;
     /** Node selector pinning sandbox pods to the dedicated agent pool. Omit for default scheduling. */
@@ -60,11 +79,6 @@ export interface K8sClientConfig {
     tolerations?: V1Toleration[];
     /** RuntimeClass for runtime isolation (e.g. `gvisor`). Omit for the default runtime. */
     runtimeClassName?: string;
-    /**
-     * The `/proc/cpuinfo` text that the cpu ConfigMap of each Job is cut from.
-     * Defaults to the file of the host of this process, read once. Injected for tests.
-     */
-    readHostCpuinfo?: () => Promise<string | undefined>;
     /** Injected for tests. */
     batchApi?: BatchV1Api;
     coreApi?: CoreV1Api;

package/dist/sandbox/k8s-client.js

@@ -6,18 +6,19 @@
  * Storage is wired via the shared session PVC: a flat read-only `volumeMount`
  * of the analysis tree at `/{resourceId}` plus a nested read-write mount of the
  * step's artifact dir, with the lib/ref stores mounted read-only at `/mnt/libs`
- * / `/mnt/refs` when their PVCs are set. Container paths and lib-store env come
- * from the shared mount plan; the PVC `subPath`s are derived from the same
- * `resolveWorkspaceRoot` seam the harness pre-creates the step tree under, so
- * both sides address one directory by construction.
+ * / `/mnt/refs` when their PVCs are set. The farm of the analysis mounts as a
+ * read-only `subPath` of the libs PVC, after the store mount. Container paths
+ * and lib-store env come from the shared mount plan; the PVC `subPath`s are
+ * derived from the same `resolveWorkspaceRoot` seam the harness pre-creates
+ * the step tree under, so both sides address one directory by construction.
  */
-import { relative as relativePath, sep } from "node:path";
+import { join, relative as relativePath, sep } from "node:path";
 import { BatchV1Api, CoreV1Api, KubeConfig, } from "@kubernetes/client-node";
 import { ResultAsync, err, ok } from "neverthrow";
 import { trySandbox } from "./sandbox-error.js";
+import { readFarmLock, resolveFarmSource } from "./farm.js";
 import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
 import { threadLimitEnv } from "./thread-env.js";
-import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
 import { createNoopLogger } from "../lib/console-logger.js";
 /** Read the originating HTTP status off any `SandboxError` variant that carries one. */
 function statusOf(e) {
@@ -26,8 +27,6 @@ function statusOf(e) {
 const SESSION_VOLUME_NAME = "session";
 const LIBS_VOLUME_NAME = "libs";
 const REFS_VOLUME_NAME = "refs";
-/** The ConfigMap volume that holds the two cpu files of the sandbox (`cpu-files.ts`). */
-const CPU_VOLUME_NAME = "cpu";
 const SANDBOX_SERVER_PORT = 8765;
 const POD_READY_TIMEOUT_MS = 5 * 60_000;
 const POD_POLL_INTERVAL_MS = 1_000;
@@ -63,19 +62,6 @@ export function sanitizeLabelValue(value) {
 function readOwnerWorkflowId(metadata) {
     return metadata?.annotations?.[OWNER_WORKFLOW_ANNOTATION] ?? null;
 }
-function deleteConfigMapIgnoreMissing(coreApi, namespace, name) {
-    return trySandbox(async () => {
-        await coreApi.deleteNamespacedConfigMap({ namespace, name });
-    }, (status, cause) => ({
-        type: "teardown_failed",
-        op: "k8s.teardown",
-        sandboxId: name,
-        status,
-        cause,
-    })).orElse((e) => 
-    // A 404 means the owner reference removed it with the Job already.
-    statusOf(e) === 404 ? ok(undefined) : err(e));
-}
 function deleteJobIgnoreMissing(batchApi, namespace, name) {
     return trySandbox(async () => {
         await batchApi.deleteNamespacedJob({
@@ -124,11 +110,30 @@ function workspaceSubPathFor(config, analysisId) {
     }
     return posix;
 }
-function buildJobSpec(meta, config, identity, cpu) {
+/**
+ * A farm location on K8s is a `subPath` into the libs PVC. A subPath must be
+ * a relative path with no `..` — an absolute or escaping value would mount a
+ * directory nothing resolved. `createSandbox` runs inside a DBOS workflow
+ * body, where a throw is the durable failure signal.
+ */
+function pvcSubPath(value, what) {
+    if (value.length === 0 || value.startsWith("/") || value.split("/").includes("..")) {
+        throw new Error(`k8s sandbox: ${what} must be a non-empty PVC-relative path without '..' (got ${JSON.stringify(value)})`);
+    }
+    return value;
+}
+function buildJobSpec(meta, config, identity, farm, libsMounted) {
     const { sandboxId } = identity;
+    // The lock gate of the package-store spec: `farmPath` is PVC-relative,
+    // thus this host can prove the lock only through `libStorePvcRoot`. The
+    // caller runs that proof and hands the verdict in `libsMounted`. Without
+    // the root, the proof duty stays whole on the farm resolver — see
+    // {@link ResolveAnalysisFarm}.
     const plan = buildMountPlan(meta, {
-        libs: !!config.libStorePvc,
+        libs: libsMounted,
         refs: !!config.refStorePvc,
+        toolchainSource: config.toolchainSource,
+        cache: libsMounted && farm?.cachePath !== undefined,
     });
     const transport = config.transport ?? "poll";
     const spec = meta.resources;
@@ -184,15 +189,36 @@ function buildJobSpec(meta, config, identity, cpu) {
         }
     }
     if (config.libStorePvc && plan.libsPath) {
+        const cacheSubPath = farm?.cachePath !== undefined && plan.cachePath ? pvcSubPath(farm.cachePath, "the farm cache") : undefined;
         volumes.push({
             name: LIBS_VOLUME_NAME,
-            persistentVolumeClaim: { claimName: config.libStorePvc, readOnly: true },
+            // Volume-level readOnly only when nothing writes through the claim:
+            // the cache mount needs write access, and the per-mount flags keep
+            // the store and the farm read-only either way.
+            persistentVolumeClaim: { claimName: config.libStorePvc, ...(cacheSubPath === undefined ? { readOnly: true } : {}) },
         });
         volumeMounts.push({
             name: LIBS_VOLUME_NAME,
             mountPath: plan.libsPath,
             readOnly: true,
         });
+        // The farm mount comes after the store mount, thus the nesting is stable.
+        if (farm && plan.farmPath) {
+            volumeMounts.push({
+                name: LIBS_VOLUME_NAME,
+                mountPath: plan.farmPath,
+                subPath: pvcSubPath(farm.farmPath, "the farm"),
+                readOnly: true,
+            });
+        }
+        if (cacheSubPath !== undefined && plan.cachePath) {
+            volumeMounts.push({
+                name: LIBS_VOLUME_NAME,
+                mountPath: plan.cachePath,
+                subPath: cacheSubPath,
+                readOnly: false,
+            });
+        }
     }
     if (config.refStorePvc && plan.refsPath) {
         volumes.push({
@@ -205,16 +231,6 @@ function buildJobSpec(meta, config, identity, cpu) {
             readOnly: true,
         });
     }
-    // The cpu files ride in a ConfigMap named after the Job (`ensureCpuConfigMap`).
-    // A `subPath` mount of one key over one kernel path is accepted by the API
-    // server, the kubelet, and gVisor. Under gVisor the sandbox sees no cgroup
-    // at all, thus these two files are the only source of the quota for
-    // `parallel::detectCores()` and `os.cpu_count()`.
-    volumes.push({ name: CPU_VOLUME_NAME, configMap: { name: sandboxId } });
-    volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPU_ONLINE_PATH, subPath: "online", readOnly: true });
-    if (cpu.hasCpuinfo) {
-        volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPUINFO_PATH, subPath: "cpuinfo", readOnly: true });
-    }
     const podSpec = {
         automountServiceAccountToken: false,
         restartPolicy: "Never",
@@ -385,9 +401,6 @@ function waitForJobGone(batchApi, namespace, name) {
  * means an (astronomically rare) name collision with a *different* step —
  * adopting its pod would HMAC-fail every callback, and deleting its Job would
  * kill a live sibling. Refuse loudly instead.
- *
- * The result is the uid of the Job that the pod belongs to, for the owner
- * reference of the cpu ConfigMap.
  */
 function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflowId, job) {
     const createFailed = (status, cause) => ({
@@ -400,7 +413,7 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
     return new ResultAsync((async () => {
         const created = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
         if (created.isOk())
-            return ok(created.value?.metadata?.uid);
+            return ok(undefined);
         if (statusOf(created.error) !== 409)
             return err(created.error);
         const existingRead = await trySandbox(() => batchApi.readNamespacedJob({ namespace, name: sandboxId }), createFailed);
@@ -428,92 +441,48 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
             const recreated = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
             if (recreated.isErr())
                 return err(recreated.error);
-            return ok(recreated.value?.metadata?.uid);
-        }
-        return ok(existingRead.value.metadata?.uid);
-    })());
-}
-/**
- * Make the cpu ConfigMap of a Job, or bring an existing one up to date.
- *
- * The ConfigMap comes after the Job, because the Job is its owner. Thus the
- * TTL of the Job and a Foreground delete both remove the ConfigMap with it.
- * The pod waits at its mount until the ConfigMap exists, and that wait is
- * shorter than the schedule of the pod.
- *
- * A `409 AlreadyExists` is the recovery re-run of the spawn step, or a
- * ConfigMap whose prior owner Job is not yet collected. Both cases get the
- * same data and the current owner through a replace.
- */
-function ensureCpuConfigMap(coreApi, namespace, sandboxId, jobUid, files) {
-    const createFailed = (status, cause) => ({
-        type: "container_create_failed",
-        op: "k8s.createNamespacedConfigMap",
-        sandboxId,
-        status,
-        cause,
-    });
-    const body = {
-        apiVersion: "v1",
-        kind: "ConfigMap",
-        metadata: {
-            name: sandboxId,
-            namespace,
-            labels: {
-                [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
-                role: "sandbox",
-                "cortex/sandbox-id": sandboxId,
-            },
-            ...(jobUid !== undefined && {
-                ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: sandboxId, uid: jobUid }],
-            }),
-        },
-        data: {
-            online: files.online,
-            ...(files.cpuinfo !== undefined && { cpuinfo: files.cpuinfo }),
-        },
-    };
-    return new ResultAsync((async () => {
-        const created = await trySandbox(() => coreApi.createNamespacedConfigMap({ namespace, body }), createFailed);
-        if (created.isOk())
             return ok(undefined);
-        if (statusOf(created.error) !== 409)
-            return err(created.error);
-        const existing = await trySandbox(() => coreApi.readNamespacedConfigMap({ namespace, name: sandboxId }), createFailed);
-        if (existing.isErr())
-            return err(existing.error);
-        const replaced = await trySandbox(() => coreApi.replaceNamespacedConfigMap({
-            namespace,
-            name: sandboxId,
-            body: { ...body, metadata: { ...body.metadata, resourceVersion: existing.value.metadata?.resourceVersion } },
-        }), createFailed);
-        if (replaced.isErr())
-            return err(replaced.error);
+        }
         return ok(undefined);
     })());
 }
 export function createK8sSandboxOps(config) {
     const { batchApi, coreApi } = config.batchApi && config.coreApi ? { batchApi: config.batchApi, coreApi: config.coreApi } : buildKubeApi();
-    const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
     return {
         createSandbox(meta, identity) {
             return new ResultAsync((async () => {
                 const { sandboxId } = identity;
-                // The same floor as `threadLimitEnv`: a fraction of a core still
-                // keeps one thread busy, and a count of zero is not representable.
-                const threads = Math.max(1, Math.floor(meta.resources.cpu));
-                const files = cpuFiles(threads, await readHostCpuinfo());
-                const job = buildJobSpec(meta, config, identity, { hasCpuinfo: files.cpuinfo !== undefined });
+                // Farm resolution comes before any cluster work: a resolver
+                // refusal and a resolver throw refuse the whole call, and no
+                // Job is made.
+                let farm;
+                if (config.libStorePvc) {
+                    const resolved = await resolveFarmSource(config.farmSource, meta.analysisId, "k8s.createSandbox");
+                    if (resolved.isErr())
+                        return err(resolved.error);
+                    farm = resolved.value;
+                }
+                // The `inflexa.lock` gate, for a host that mounts the libs
+                // volume itself. The resolver owes the proof either way,
+                // but a farm half-written between the resolution and this
+                // create still mounts without a second look. A gate failure
+                // degrades exactly as the Docker backend does: no store
+                // mount, the Job is still made, and the sandbox reports the
+                // store as unavailable.
+                let libsMounted = !!config.libStorePvc;
+                if (config.libStorePvc && config.libStorePvcRoot && farm) {
+                    const lock = readFarmLock(join(config.libStorePvcRoot, farm.farmPath));
+                    libsMounted = lock.isOk();
+                    if (lock.isErr()) {
+                        (config.logger ?? createNoopLogger())
+                            .named("k8s-client")
+                            .warn("the farm of the analysis has no usable inflexa.lock at sandbox creation — mounting no package store (sandbox degrades to available:false)", { libStorePvc: config.libStorePvc, farmPath: farm.farmPath, lockError: lock.error.type, sandboxId });
+                    }
+                }
+                const job = buildJobSpec(meta, config, identity, farm, libsMounted);
                 const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, meta.childWorkflowId, job);
                 if (adopted.isErr())
                     return err(adopted.error);
-                const cpuMap = await ensureCpuConfigMap(coreApi, config.namespace, sandboxId, adopted.value, files);
-                if (cpuMap.isErr()) {
-                    // A pod whose ConfigMap never comes stays at ContainerCreating
-                    // until the ready wait expires. Fail now, with the real cause.
-                    await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
-                    return err(cpuMap.error);
-                }
                 // A Job whose pod never schedules (quota rejection, no nodes, spot
                 // reclaim) is reaped by neither backoffLimit (an admission rejection
                 // creates no pod to count) nor ttlSecondsAfterFinished (the Job never
@@ -537,22 +506,20 @@ export function createK8sSandboxOps(config) {
                         cause,
                     }));
                     if (registered.isErr()) {
-                        await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
+                        await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
                         return err(registered.error);
                     }
                     return ok(ref);
                 }
-                await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
+                await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
                 return err(ready.error);
             })());
         },
-        // The owner reference removes the cpu ConfigMap with the Job. The explicit
-        // delete covers a ConfigMap that got no owner, and it is a no-op on 404.
         teardown(ref) {
-            return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, ref.sandboxId));
+            return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId);
         },
         teardownById(sandboxId) {
-            return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, sandboxId));
+            return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId);
         },
         listManagedSandboxes() {
             return trySandbox(() => batchApi.listNamespacedJob({
@@ -613,13 +580,9 @@ export function createK8sSandboxOps(config) {
  * matters; a delete error here is logged and swallowed so it never masks the
  * create error returned to the caller.
  */
-async function cleanupFailedJob(batchApi, coreApi, namespace, sandboxId, logger) {
+async function cleanupFailedJob(batchApi, namespace, sandboxId, logger) {
     const deleted = await deleteJobIgnoreMissing(batchApi, namespace, sandboxId);
     if (deleted.isErr()) {
         logger.named("k8s-client").error("failed to delete Job after startup failure", { sandboxId, err: deleted.error });
     }
-    const deletedMap = await deleteConfigMapIgnoreMissing(coreApi, namespace, sandboxId);
-    if (deletedMap.isErr()) {
-        logger.named("k8s-client").error("failed to delete cpu ConfigMap after startup failure", { sandboxId, err: deletedMap.error });
-    }
 }

package/dist/sandbox/mount-plan.d.ts

@@ -27,6 +27,7 @@
  * resolved root directly, K8s addresses it as a `subPath` into the session PVC
  * (see {@link buildSessionSubPaths}).
  */
+import type { ToolchainSource } from "./types.js";
 export declare const STEP_SUBDIRS: readonly ["output", "scripts", "figures", "logs", "notebooks"];
 /**
  * Container mountpoints of the two read-only stores. Exported because they are
@@ -35,6 +36,15 @@ export declare const STEP_SUBDIRS: readonly ["output", "scripts", "figures", "lo
  */
 export declare const LIBS_CONTAINER_PATH = "/mnt/libs";
 export declare const REFS_CONTAINER_PATH = "/mnt/refs";
+/** Container path of the per-analysis read-write cache, when a farm resolution carries a cache location. */
+export declare const FARM_CACHE_CONTAINER_PATH = "/mnt/libs/cache";
+/**
+ * Container path of the mounted farm, keyed on the declared toolchain. With
+ * `"image"` the farm mounts at `/mnt/libs/farm`. With `"store"` (or absent)
+ * it stays `/mnt/libs/current`, because the baked resolvers of the old
+ * images name that path.
+ */
+export declare function farmContainerPath(toolchainSource: ToolchainSource | undefined): string;
 export interface MountPlanCoords {
     analysisId: string;
     runId: string;
@@ -55,10 +65,18 @@ export interface MountPlanCoords {
     writableTail?: string;
 }
 export interface MountPlanStores {
-    /** Lib store is mounted at `/mnt/libs` and its env is emitted. */
+    /** Lib store is mounted at `/mnt/libs`, with the farm nested inside it, and its env is emitted. */
     libs: boolean;
     /** Ref store is mounted at `/mnt/refs`. */
     refs: boolean;
+    /**
+     * The declared toolchain owner. Keys the farm container path and the
+     * resolver env. Absent means `"store"`: the legacy environment,
+     * byte-identical to the one before the field existed.
+     */
+    toolchainSource?: ToolchainSource;
+    /** The farm resolution carries a cache location, mounted read-write at `/mnt/libs/cache`. */
+    cache?: boolean;
 }
 export interface MountPlan {
     /** Flat read-only mount of the whole analysis tree. */
@@ -75,6 +93,10 @@ export interface MountPlan {
     workingDir: string;
     /** Container path of the lib store, present only when `libs`. */
     libsPath?: string;
+    /** Container path of the farm of the analysis, present only when `libs`. Keyed on the toolchain source. */
+    farmPath?: string;
+    /** Container path of the read-write cache, present only when `libs` and `cache`. */
+    cachePath?: string;
     /** Container path of the ref store, present only when `refs`. */
     refsPath?: string;
     /** Pre-created subdirectories under the writable mount. Empty when read-only

package/dist/sandbox/mount-plan.js

@@ -36,6 +36,17 @@ export const STEP_SUBDIRS = ["output", "scripts", "figures", "logs", "notebooks"
  */
 export const LIBS_CONTAINER_PATH = "/mnt/libs";
 export const REFS_CONTAINER_PATH = "/mnt/refs";
+/** Container path of the per-analysis read-write cache, when a farm resolution carries a cache location. */
+export const FARM_CACHE_CONTAINER_PATH = "/mnt/libs/cache";
+/**
+ * Container path of the mounted farm, keyed on the declared toolchain. With
+ * `"image"` the farm mounts at `/mnt/libs/farm`. With `"store"` (or absent)
+ * it stays `/mnt/libs/current`, because the baked resolvers of the old
+ * images name that path.
+ */
+export function farmContainerPath(toolchainSource) {
+    return toolchainSource === "image" ? `${LIBS_CONTAINER_PATH}/farm` : `${LIBS_CONTAINER_PATH}/current`;
+}
 /**
  * The step tail beneath an analysis's workspace root, in both container and
  * PVC-relative space. The harness owns this layout (workspace-layout spec).
@@ -94,14 +105,37 @@ export function buildSessionSubPaths(coords, workspaceSubPath) {
     };
 }
 /**
- * Lib-store package-discovery env. PYTHONPATH is intentionally omitted —
- * system Python resolves via a `.pth` file in the lib store.
+ * Lib-store package-discovery env, keyed on the declared toolchain.
+ * PYTHONPATH is intentionally omitted — system Python resolves through a
+ * `.pth` file in the lib store.
+ *
+ * With `"store"` (or absent) the env is byte-identical to the legacy one:
+ * the conda `bin` under `/mnt/libs/current`, thus an old embedder keeps its
+ * exact environment. With `"image"` the image owns the toolchain: `PATH`
+ * holds `/opt/conda/bin` before the farm `python/bin` at the end, thus a
+ * farm script never shadows an image tool. The cache env points into the
+ * read-write cache mount only when it is present — the entrypoint fallback
+ * covers the caches otherwise.
  */
-function libStoreEnv() {
+function libStoreEnv(toolchainSource, cache) {
+    if (toolchainSource !== "image") {
+        return {
+            R_LIBS_SITE: "/mnt/libs/current/r/github:/mnt/libs/current/r/bioconductor:/mnt/libs/current/r/cran",
+            NODE_PATH: "/mnt/libs/current/node/node_modules",
+            PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/libs/current/conda/bin",
+        };
+    }
+    const farm = farmContainerPath(toolchainSource);
     return {
-        R_LIBS_SITE: "/mnt/libs/current/r/github:/mnt/libs/current/r/bioconductor:/mnt/libs/current/r/cran",
-        NODE_PATH: "/mnt/libs/current/node/node_modules",
-        PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/libs/current/conda/bin",
+        R_LIBS_SITE: `${farm}/r/github:${farm}/r/bioconductor:${farm}/r/cran`,
+        NODE_PATH: "/opt/node/node_modules",
+        PATH: `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/conda/bin:${farm}/python/bin`,
+        ...(cache
+            ? {
+                NUMBA_CACHE_DIR: `${FARM_CACHE_CONTAINER_PATH}/numba-cache`,
+                MPLCONFIGDIR: `${FARM_CACHE_CONTAINER_PATH}/matplotlib_config`,
+            }
+            : {}),
     };
 }
 export function buildMountPlan(coords, stores) {
@@ -112,16 +146,19 @@ export function buildMountPlan(coords, stores) {
     const readonlyTreePath = `/${analysisId}`;
     const write = sandboxWriteTail(coords);
     const writableStepPath = write === undefined ? undefined : `${readonlyTreePath}/${write.tail}`;
+    const cache = stores.libs && stores.cache === true;
     return {
         readonlyTreePath,
         writableStepPath,
         workingDir: writableStepPath ?? readonlyTreePath,
         libsPath: stores.libs ? LIBS_CONTAINER_PATH : undefined,
+        farmPath: stores.libs ? farmContainerPath(stores.toolchainSource) : undefined,
+        cachePath: cache ? FARM_CACHE_CONTAINER_PATH : undefined,
         refsPath: stores.refs ? REFS_CONTAINER_PATH : undefined,
         stepSubdirs: write?.subdirs ?? [],
         env: {
             PROVENANCE_WATCH_DIRS: readonlyTreePath,
-            ...(stores.libs ? libStoreEnv() : {}),
+            ...(stores.libs ? libStoreEnv(stores.toolchainSource, cache) : {}),
         },
     };
 }