@telorun/k8s-runner 0.13.0 → 0.15.0

package/src/k8s/backend.ts

@@ -15,9 +15,16 @@ import { relayDebugStream, SessionStartError, watchReachability } from "@telorun
 import type { BundleStore } from "../bundle-store.js";
 import type { K8sRunnerConfig } from "../config.js";
 import { clampLimits } from "../limits.js";
+import { apiFailure, apiReason, withCause } from "./api-error.js";
 import type { KubeClient } from "./client.js";
-import { ensureSessionImage } from "./image-build.js";
-import { buildSessionIngress, buildSessionService, endpointsFor } from "./ingress.js";
+import {
+  buildSessionService,
+  endpointsFor,
+  hostForPort,
+  watchRouteHealth,
+  type PublishedRoute,
+  type SessionRouter,
+} from "./routing/index.js";
 import { buildAppPod, buildSessionPod, INSPECT_PORT } from "./pod-spec.js";
 import {
   deletePod,
@@ -50,10 +57,13 @@ export interface K8sBackendDeps {
   kube: KubeClient;
   config: K8sRunnerConfig;
   bundleStore: BundleStore;
+  /** How this cluster publishes session routes, resolved once at boot. Absent →
+   *  logs-only: no base domain, or `SESSION_ROUTING_MODE=none`. */
+  router?: SessionRouter;
 }
 
 export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
-  const { kube, config, bundleStore } = deps;
+  const { kube, config, bundleStore, router } = deps;
   const ns = config.sessionNamespace;
 
   async function probe(probeConfig: ProbeConfig): Promise<AvailabilityReport> {
@@ -80,11 +90,9 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
   async function start(spec: BackendStartSpec): Promise<BackendSession> {
     // A watch session is a different pod shape and a different lifetime — it
     // outlives its runs — so it takes its own path rather than accreting
-    // branches through this one. It also never builds an image: the build path
-    // exists to put a dependency closure on disk before boot, and a watch
-    // session fetches its own and keeps it for the pod's life.
+    // branches through this one.
     if (spec.mode === "watch") {
-      return startWatchSession({ kube, config }, spec);
+      return startWatchSession({ kube, config, router }, spec);
     }
     return startRunSession(spec);
   }
@@ -104,9 +112,9 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
 
     let pod: V1Pod;
     if (spec.selfContained) {
-      // Operator-predefined app (catalog image): self-contained, no build and
-      // no bundle to deliver — the pod runs the image's own entrypoint with
-      // the env the core route already merged.
+      // Operator-predefined app (catalog image): self-contained, with no bundle
+      // to deliver — the pod runs the image's own entrypoint with the env the
+      // core route already merged.
       pod = buildAppPod({
         config,
         sessionId: spec.sessionId,
@@ -118,30 +126,9 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
         pullPolicy: spec.config.pullPolicy,
       });
     } else {
-      // Prebuild a self-contained per-app image on-cluster (controllers + module
-      // manifests baked in) and run it directly. Controller resolution never
-      // happens on the session start path, so a slow or unreachable package
-      // registry can't stall a session. Throws SessionStartError on a build
-      // failure, carrying the build pod's log tail.
-      const image = await ensureSessionImage(
-        {
-          kube,
-          build: config.build,
-          bundleStore,
-          initImage: config.initImage,
-          managedByLabel: config.managedByLabel,
-        },
-        {
-          bundle: spec.bundle,
-          entryRelativePath: app.entryRelativePath,
-          baseImage: spec.config.image || config.defaultImage,
-          pullPolicy: spec.config.pullPolicy,
-          onProgress: (message, done) => spec.onProgress("build", message, done, appName),
-        },
-      );
-
-      // The image is keyed on the dependency closure only, so deliver the
-      // per-session body to the Pod's /app at boot via a tokenized, single-use URL.
+      // Deliver the body to the Pod's /app at boot via a tokenized, single-use
+      // URL, and run it on the plain kernel image: the kernel resolves its own
+      // module closure into the pod's cache emptyDir on the way up.
       const bundleUrl = await bundleStore.stageSessionBundle(spec.sessionId, spec.bundle);
 
       pod = buildSessionPod({
@@ -152,7 +139,8 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
         env: spec.env,
         ports: app.ports,
         limits,
-        image,
+        image: spec.config.image || config.defaultImage,
+        pullPolicy: spec.config.pullPolicy,
         bundleUrl,
         inspect: spec.inspect,
       });
@@ -163,7 +151,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
       const created = await kube.core.createNamespacedPod({ namespace: ns, body: pod });
       podUid = created.metadata?.uid ?? "";
     } catch (err) {
-      throw new SessionStartError("start_failed", "create", `failed to create pod: ${msg(err)}`, msg(err));
+      throw apiFailure(err, "create", "could not create the session pod");
     }
 
     let finished = false;
@@ -177,7 +165,9 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
     let startDeadline: NodeJS.Timeout | undefined;
     let podIP: string | undefined;
     const debugAbort = new AbortController();
-    const reachAbort = new AbortController();
+    // Aborts every background watcher this session started: the per-port
+    // reachability probes and the route-health watch.
+    const watchersAbort = new AbortController();
 
     const stdin = new PassThrough();
     const stdout = new Writable({
@@ -200,7 +190,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
       clearStartDeadline();
       abortWatch();
       debugAbort.abort();
-      reachAbort.abort();
+      watchersAbort.abort();
       bundleStore.drop(spec.sessionId);
       spec.onStatus(status);
       try {
@@ -214,12 +204,12 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
     // Flip the session to `running` when the pod reaches Running. Deterministic
     // and independent of the session image's telo version — a readiness signal
     // would couple this to the in-image CLI (and a stale base image would never
-    // flip). The build/provision progress (streamed before this) covers the slow
-    // part; the brief post-Running validation runs while already `running`.
+    // flip). The provision progress (streamed before this) covers the slow part;
+    // module resolution and validation run while already `running`.
     const flipRunning = (): void => {
       if (readyFlipped || finished) return;
       readyFlipped = true;
-      spec.onStatus({ kind: "running", endpoints: endpointsFor(config, spec.sessionId, app.ports) });
+      spec.onStatus({ kind: "running", endpoints: endpointsFor(config, spec.sessionId, app.ports, Boolean(router)) });
     };
 
     let resolveRunning!: () => void;
@@ -301,7 +291,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
       } catch (err) {
         if (!runningSeen) {
           clearStartDeadline();
-          rejectRunning(new Error(`failed to watch pod: ${msg(err)}`));
+          rejectRunning(new Error(`could not watch the session pod: ${apiReason(err)}`, { cause: err }));
         } else if (!finished) {
           setTimeout(() => void armWatch(), WATCH_REARM_DELAY_MS).unref?.();
         }
@@ -324,7 +314,16 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
       abortWatch();
       bundleStore.drop(spec.sessionId);
       await deletePod(kube, ns, podName);
-      throw new SessionStartError("start_failed", "start", `pod failed to start: ${msg(err)}`, msg(err));
+      // `err` is one of OUR errors — a pod-status failure message, a
+      // disappeared pod, the start deadline, a watch that could not be armed —
+      // so its message is already client-safe and stands. What must not be lost
+      // is its `cause`: the watch path attaches the raw ApiException there, and
+      // rewrapping without forwarding it drops the very detail the summarized
+      // message exists to relocate into the log.
+      throw withCause(
+        new SessionStartError("start_failed", "start", `pod failed to start: ${msg(err)}`),
+        (err as { cause?: unknown })?.cause ?? err,
+      );
     }
 
     // Attach a PTY to the running container: stdout → onOutput, stdin ← writes.
@@ -333,12 +332,59 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
       socket = ws as unknown as ResizableSocket;
     } catch (err) {
       // Attach failure isn't fatal — status still flows; surface the degraded PTY.
-      spec.onOutput(appName, Buffer.from(`\r\n[runner] failed to attach PTY: ${msg(err)}\r\n`), "tty");
+      spec.onOutput(
+        appName,
+        Buffer.from(`\r\n[runner] failed to attach PTY: ${apiReason(err)}\r\n`),
+        "tty",
+      );
     }
 
-    if (config.sessionIngressBaseDomain && app.ports.length > 0) {
-      await createIngress(deps, spec.sessionId, podName, podUid, app.ports).catch((err) => {
-        spec.onOutput(appName, Buffer.from(`\r\n[runner] failed to create ingress: ${msg(err)}\r\n`), "tty");
+    if (router && app.ports.length > 0) {
+      let published: PublishedRoute[] = [];
+      try {
+        published = await publishRoutes(
+          deps,
+          router,
+          spec.sessionId,
+          podName,
+          podUid,
+          app.ports,
+        );
+      } catch (err) {
+        // A publish failure is reported on the ROUTE channel, not only as a
+        // terminal line: the endpoints were already advertised on the `running`
+        // status, so leaving this to a log line is the silence this whole change
+        // exists to remove.
+        const reason = `could not publish the session route: ${apiReason(err)}`;
+        for (const p of app.ports) {
+          if (p.protocol !== "tcp") continue;
+          spec.onRoute(appName, {
+            host: hostForPort(config, spec.sessionId, p.port),
+            port: p.port,
+            state: "unprogrammed",
+            reason,
+          });
+        }
+        spec.onOutput(appName, Buffer.from(`\r\n[runner] ${reason}\r\n`), "tty");
+      }
+      // Verify the routing layer actually PROGRAMMED what we published. Without
+      // this a cluster whose routing objects nothing reconciles is indistinguishable
+      // from a healthy one: reachability below dials the pod directly and stays
+      // green while every public URL 404s.
+      void watchRouteHealth({
+        router,
+        sessionId: spec.sessionId,
+        routes: published,
+        onState: (route, state, reason) =>
+          spec.onRoute(appName, { host: route.host, port: route.port, state, reason }),
+        onReadError: (err) =>
+          spec.onOutput(
+            appName,
+            Buffer.from(`\r\n[runner] could not read route status: ${apiReason(err)}\r\n`),
+            "tty",
+          ),
+        signal: watchersAbort.signal,
+        timeoutMs: config.sessionRouting.routeReadyTimeoutMs,
       });
     }
 
@@ -377,7 +423,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
     if (tcpPorts.length > 0) {
       void (async () => {
         const ip = await resolvePodIP();
-        if (reachAbort.signal.aborted) return;
+        if (watchersAbort.signal.aborted) return;
         if (!ip) {
           // Couldn't resolve the pod IP to probe — report unverified rather than
           // leaving the badge spinning forever.
@@ -388,7 +434,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
           host: ip,
           ports: tcpPorts,
           onState: (port, state) => spec.onReachability(appName, port, state),
-          signal: reachAbort.signal,
+          signal: watchersAbort.signal,
         });
       })();
     }
@@ -448,28 +494,27 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
   return { probe, start, reapOrphans };
 }
 
-async function createIngress(
+/** Create the session Service and publish its routes on whichever layer the
+ *  cluster resolved to. Returns the hosts published, for the route watch. */
+async function publishRoutes(
   deps: K8sBackendDeps,
+  router: SessionRouter,
   sessionId: string,
   podName: string,
   podUid: string,
   ports: PortMapping[],
-): Promise<void> {
+): Promise<PublishedRoute[]> {
   const { kube, config } = deps;
   const ns = config.sessionNamespace;
   const service = buildSessionService(config, sessionId, podName, podUid, ports);
   await kube.core.createNamespacedService({ namespace: ns, body: service });
-  const { ingress } = buildSessionIngress(
-    config,
+  return router.publish({
     sessionId,
-    service.metadata!.name!,
+    serviceName: service.metadata!.name!,
     podName,
     podUid,
     ports,
-  );
-  // No tcp ports → no HTTP-routable rules; the Service still exists for any udp.
-  if (!ingress.spec?.rules?.length) return;
-  await kube.networking.createNamespacedIngress({ namespace: ns, body: ingress });
+  });
 }
 
 async function clusterReachable(kube: KubeClient): Promise<boolean> {

package/src/k8s/client.ts

@@ -1,7 +1,8 @@
 import {
+  ApisApi,
   Attach,
-  BatchV1Api,
   CoreV1Api,
+  CustomObjectsApi,
   KubeConfig,
   NetworkingV1Api,
   Watch,
@@ -10,8 +11,13 @@ import {
 export interface KubeClient {
   kc: KubeConfig;
   core: CoreV1Api;
-  batch: BatchV1Api;
   networking: NetworkingV1Api;
+  /** Gateway API has no typed client in `@kubernetes/client-node`, so HTTPRoutes
+   *  are read and written as unstructured custom objects. */
+  custom: CustomObjectsApi;
+  /** API-group discovery, used to decide whether Gateway API is served at all
+   *  when the routing mode is `auto`. */
+  apis: ApisApi;
   attach: Attach;
   watch: Watch;
 }
@@ -31,8 +37,9 @@ export function createKubeClient(): KubeClient {
   return {
     kc,
     core: kc.makeApiClient(CoreV1Api),
-    batch: kc.makeApiClient(BatchV1Api),
     networking: kc.makeApiClient(NetworkingV1Api),
+    custom: kc.makeApiClient(CustomObjectsApi),
+    apis: kc.makeApiClient(ApisApi),
     attach: new Attach(kc),
     watch: new Watch(kc),
   };

package/src/k8s/pod-spec.test.ts

@@ -1,13 +1,68 @@
 import { describe, expect, it } from "vitest";
 
 import { loadK8sRunnerConfig } from "../config.js";
-import { buildAppPod } from "./pod-spec.js";
+import { buildAppPod, buildSessionPod } from "./pod-spec.js";
 
 const BASE_ENV = {
   RUNNER_SELF_URL: "http://k8s-runner.telo-runner.svc:8062",
-  RUNNER_IMAGE_REPOSITORY: "registry.telo-runner.svc:5000/telo-sessions",
 };
 
+describe("buildSessionPod", () => {
+  const config = loadK8sRunnerConfig({ ...process.env, ...BASE_ENV });
+  const pod = buildSessionPod({
+    config,
+    sessionId: "abc123",
+    podName: "telo-run-abc123",
+    entryRelativePath: "telo.yaml",
+    env: { API_TOKEN: "tok" },
+    ports: [{ port: 3000, protocol: "tcp" }],
+    limits: config.limits,
+    image: "telorun/node:0.30.1-slim",
+    pullPolicy: "always",
+    bundleUrl: "http://k8s-runner.telo-runner.svc:8062/internal/bundles/abc123?token=t",
+    inspect: false,
+  });
+  const container = pod.spec!.containers[0];
+
+  it("runs the picked kernel image under the requested pull policy", () => {
+    expect(container.image).toBe("telorun/node:0.30.1-slim");
+    expect(container.imagePullPolicy).toBe("Always");
+  });
+
+  it("resolves the module closure at boot into a writable cache", () => {
+    // No prebuilt image means no baked deps: the cache has to be a mounted
+    // emptyDir, and `telo run` has to be allowed to write it.
+    expect(container.env).toContainEqual({ name: "TELO_CACHE_DIR", value: "/telo-cache" });
+    expect(container.volumeMounts).toContainEqual({
+      name: "telo-cache",
+      mountPath: "/telo-cache",
+    });
+    expect(pod.spec!.volumes).toContainEqual({ name: "telo-cache", emptyDir: {} });
+    expect(container.command).toEqual(["telo", "run", "/app/telo.yaml"]);
+  });
+
+  it("is given headroom to resolve a closure, not just to run one", () => {
+    // The ceilings used to describe a pod that only RAN a prebuilt image (50m /
+    // 100Mi / 512Mi). It now downloads, unpacks and resolves the closure itself,
+    // into an emptyDir charged against ephemeral-storage — the old numbers were
+    // an OOMKill and an eviction for the ordinary case. Asserted against the
+    // app-session ceilings, which is the tier the equivalent watch container
+    // already ran under.
+    expect(container.resources?.limits?.memory).toBe(config.appLimits.memory);
+    expect(container.resources?.limits?.cpu).toBe(config.appLimits.cpu);
+    expect(container.resources?.limits?.["ephemeral-storage"]).toBe(
+      config.appLimits.ephemeralStorage,
+    );
+  });
+
+  it("delivers the body over the initContainer and keeps the rootfs read-only", () => {
+    const init = pod.spec!.initContainers![0];
+    expect(init.args?.[0]).toContain("/internal/bundles/abc123?token=t");
+    expect(container.securityContext).toMatchObject({ readOnlyRootFilesystem: true });
+    expect(pod.spec!.automountServiceAccountToken).toBe(false);
+  });
+});
+
 describe("buildAppPod", () => {
   const config = loadK8sRunnerConfig({ ...process.env, ...BASE_ENV });
   const pod = buildAppPod({

package/src/k8s/pod-spec.ts

@@ -17,11 +17,11 @@ export interface BuildPodArgs {
   env: Record<string, string>;
   ports: PortMapping[];
   limits: ResolvedLimits;
-  /** Prebuilt per-app image to run. Controllers + module manifests are baked
-   *  into `/telo-cache/{manifests,npm}` (read-only) by the on-cluster build;
-   *  the image is keyed only on the dependency closure, so the per-session body
-   *  is NOT baked — it's delivered to `/app` at boot via the initContainer. */
+  /** The kernel image to run (`telorun/node`) — the session's picked base image,
+   *  or the runner's default. Nothing is baked per app: the body arrives over the
+   *  initContainer and the kernel resolves its module closure at boot. */
   image: string;
+  pullPolicy: PullPolicy;
   /** Tokenized, single-use URL the body-delivery initContainer fetches the
    *  session bundle tarball from (`BundleStore.stageSessionBundle`). */
   bundleUrl: string;
@@ -38,12 +38,11 @@ export const INSPECT_PORT = 9230;
 
 const APP_DIR = "/app";
 const WORK_DIR = "/work";
-/** Baked, read-only deps (`telo install` output). Set via TELO_CACHE_DIR, NOT
- *  mounted — it lives on the image rootfs, so `telo run --no-cache-write` reads
- *  it without writing. */
+/** The kernel's module cache for this session. A writable emptyDir: the closure
+ *  is resolved at boot and lives as long as the pod, so a re-run of the same app
+ *  is a fresh download — the price of having no prebuilt image. */
 const DEPS_DIR = "/telo-cache";
-/** Writable HOME / npm scratch under a read-only rootfs. Separate from DEPS_DIR
- *  (which is now read-only baked deps, not scratch). */
+/** Writable HOME / npm scratch under a read-only rootfs. */
 const HOME_DIR = "/home/telo";
 const TMP_MOUNT = "/tmp";
 
@@ -53,8 +52,8 @@ const TMP_MOUNT = "/tmp";
  * seccomp RuntimeDefault); a sandbox RuntimeClass is layered on when configured.
  *
  * The body-delivery initContainer fetches the session bundle into the writable
- * `/app` emptyDir; the session container runs `telo run /app/<entry>
- * --no-cache-write` reading its deps from the baked, read-only `/telo-cache`.
+ * `/app` emptyDir; the session container runs `telo run /app/<entry>` on the
+ * plain kernel image and resolves its module closure into `/telo-cache`.
  * `readOnlyRootFilesystem` stays on — every write lands on a mounted emptyDir.
  */
 export function buildSessionPod(args: BuildPodArgs): V1Pod {
@@ -73,8 +72,8 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
   };
 
   const envVars = Object.entries(args.env).map(([name, value]) => ({ name, value }));
-  // Read deps from the baked, read-only `/telo-cache`; keep HOME/npm scratch on a
-  // separate writable emptyDir under the read-only root filesystem.
+  // Resolve deps into the session's own `/telo-cache` emptyDir; keep HOME/npm
+  // scratch on a separate one under the read-only root filesystem.
   envVars.push({ name: "TELO_CACHE_DIR", value: DEPS_DIR });
   envVars.push({ name: "HOME", value: HOME_DIR });
   envVars.push({ name: "npm_config_cache", value: `${HOME_DIR}/.npm` });
@@ -95,11 +94,9 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
       restartPolicy: "Never",
       activeDeadlineSeconds: limits.ttlSeconds,
       automountServiceAccountToken: false,
-      // Pull the per-app image from a private registry. The Secret must exist in
+      // Pull the kernel image from a private registry. The Secret must exist in
       // the session namespace (pull secrets are namespace-scoped).
-      ...(config.build.imagePullSecret
-        ? { imagePullSecrets: [{ name: config.build.imagePullSecret }] }
-        : {}),
+      ...(config.imagePullSecret ? { imagePullSecrets: [{ name: config.imagePullSecret }] } : {}),
       ...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
       securityContext: {
         runAsNonRoot: true,
@@ -110,8 +107,7 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
       },
       initContainers: [
         {
-          // Deliver the per-session body into the writable /app emptyDir. The
-          // image bakes only the dependency closure, so the body arrives here.
+          // Deliver the session body into the writable /app emptyDir.
           name: "body-fetch",
           image: config.initImage,
           command: ["sh", "-c"],
@@ -129,13 +125,10 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
         {
           name: "session",
           image: args.image,
-          // The per-app tag is an immutable content hash, so IfNotPresent lets
-          // the kubelet reuse a node-cached layer across runs of the same app.
-          imagePullPolicy: "IfNotPresent",
-          // Run the delivered body by absolute path; `--no-cache-write` reads
-          // the baked deps from TELO_CACHE_DIR and validates in-memory without
-          // touching the read-only cache. WORK_DIR is a writable emptyDir cwd so
-          // the workload's relative paths resolve under readOnlyRootFilesystem.
+          imagePullPolicy: pullPolicyToK8s(args.pullPolicy),
+          // Run the delivered body by absolute path. WORK_DIR is a writable
+          // emptyDir cwd so the workload's relative paths resolve under
+          // readOnlyRootFilesystem.
           workingDir: WORK_DIR,
           // 0.0.0.0 (not the CLI's loopback default) lets the runner reach the
           // debug server across the pod network; the port is never published.
@@ -143,7 +136,6 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
             "telo",
             "run",
             `${APP_DIR}/${args.entryRelativePath}`,
-            "--no-cache-write",
             ...(args.inspect ? ["--inspect", `0.0.0.0:${INSPECT_PORT}`, "--no-open"] : []),
           ],
           env: envVars,
@@ -156,6 +148,7 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
           resources,
           volumeMounts: [
             { name: "app", mountPath: APP_DIR },
+            { name: "telo-cache", mountPath: DEPS_DIR },
             { name: "work", mountPath: WORK_DIR },
             { name: "home", mountPath: HOME_DIR },
             { name: "tmp", mountPath: TMP_MOUNT },
@@ -165,6 +158,7 @@ export function buildSessionPod(args: BuildPodArgs): V1Pod {
       ],
       volumes: [
         { name: "app", emptyDir: {} },
+        { name: "telo-cache", emptyDir: {} },
         { name: "work", emptyDir: {} },
         { name: "home", emptyDir: {} },
         { name: "tmp", emptyDir: {} },
@@ -225,9 +219,7 @@ export function buildAppPod(args: BuildAppPodArgs): V1Pod {
       restartPolicy: "Never",
       activeDeadlineSeconds: limits.ttlSeconds,
       automountServiceAccountToken: false,
-      ...(config.build.imagePullSecret
-        ? { imagePullSecrets: [{ name: config.build.imagePullSecret }] }
-        : {}),
+      ...(config.imagePullSecret ? { imagePullSecrets: [{ name: config.imagePullSecret }] } : {}),
       ...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
       securityContext: {
         seccompProfile: { type: "RuntimeDefault" },
@@ -326,9 +318,8 @@ export interface BuildWatchPodArgs {
   agent?: ResolvedRunnerApp;
   limits: ResolvedLimits;
   /** Base image every app container and the workspace container run — the plain
-   *  kernel image (`telorun/node`). A watch session never builds an image: the
-   *  build path exists to put a closure on disk before boot, and a watch session
-   *  fetches its own and keeps it for the pod's life. */
+   *  kernel image (`telorun/node`). Each resolves its own module closure into
+   *  the shared workspace volume, which lives as long as the pod. */
   image: string;
   pullPolicy: PullPolicy;
   /** Name of the ConfigMap holding the workspace application's manifest. */
@@ -390,9 +381,7 @@ export function buildWatchPod(args: BuildWatchPodArgs): V1Pod {
       // conversation mid-turn.
       activeDeadlineSeconds: config.watch.maxTtlSeconds,
       automountServiceAccountToken: false,
-      ...(config.build.imagePullSecret
-        ? { imagePullSecrets: [{ name: config.build.imagePullSecret }] }
-        : {}),
+      ...(config.imagePullSecret ? { imagePullSecrets: [{ name: config.imagePullSecret }] } : {}),
       ...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
       securityContext: {
         runAsNonRoot: true,

package/src/k8s/pod-status.ts

@@ -1,6 +1,7 @@
 import type { V1ContainerState, V1ContainerStatus, V1Pod } from "@kubernetes/client-node";
 import type { RunStatus } from "@telorun/runner-core";
 
+import { statusCode } from "./api-error.js";
 import type { KubeClient } from "./client.js";
 
 /**
@@ -128,8 +129,15 @@ export async function deletePod(kube: KubeClient, ns: string, name: string): Pro
 }
 
 export function is404(err: unknown): boolean {
-  const e = err as { statusCode?: number; code?: number; response?: { statusCode?: number } };
-  return e?.statusCode === 404 || e?.code === 404 || e?.response?.statusCode === 404;
+  return statusCode(err) === 404;
+}
+
+/** A name-already-taken rejection — what makes a create idempotent by falling
+ *  through to a replace. Reads the status through `statusCode` like every other
+ *  classification here, so an `ECONNREFUSED` or a `DOMException`'s legacy number
+ *  in `code` cannot be mistaken for one. */
+export function isConflict(err: unknown): boolean {
+  return statusCode(err) === 409;
 }
 
 export function msg(err: unknown): string {

package/src/k8s/routing/create-or-replace.ts

@@ -0,0 +1,16 @@
+import { isConflict } from "../pod-status.js";
+
+/** A create that yields to an existing object of the same name. Every routing
+ *  object is republished on reload (an app's declared port set changes), so the
+ *  write has to be idempotent rather than create-once. */
+export async function createOrReplace(
+  create: () => Promise<unknown>,
+  replace: () => Promise<unknown>,
+): Promise<void> {
+  try {
+    await create();
+  } catch (err) {
+    if (!isConflict(err)) throw err;
+    await replace();
+  }
+}