@telorun/k8s-runner 0.10.2 → 0.12.0

package/src/k8s/pod-spec.ts

@@ -1,8 +1,13 @@
-import type { V1Pod } from "@kubernetes/client-node";
+import type { V1Container, V1Pod, V1Volume, V1VolumeMount } from "@kubernetes/client-node";
 
 import type { K8sRunnerConfig } from "../config.js";
 import type { ResolvedLimits } from "../limits.js";
-import type { PortMapping, PullPolicy } from "@telorun/runner-core";
+import type {
+  BackendAppSpec,
+  PortMapping,
+  PullPolicy,
+  ResolvedRunnerApp,
+} from "@telorun/runner-core";
 
 export interface BuildPodArgs {
   config: K8sRunnerConfig;
@@ -273,3 +278,318 @@ function pullPolicyToK8s(policy: PullPolicy): string {
       return "IfNotPresent";
   }
 }
+
+// ---------------------------------------------------------------------------
+// Watch sessions — one pod, one workspace, one container per running application
+// ---------------------------------------------------------------------------
+
+/** Where every container sees the shared workspace volume. */
+const WORKSPACE_DIR = "/workspace";
+/**
+ * Cache root for the WORKSPACE container only.
+ *
+ * The application containers deliberately have none: their cache is anchored by
+ * the `telo-workspace.yaml` marker seeded at the workspace root, which puts one
+ * cache under `/workspace/.telo` for every app in the session — so two apps
+ * importing the same module resolve it once between them. `TELO_CACHE_DIR`
+ * outranks the marker, so setting it per app is exactly what would undo that.
+ *
+ * The workspace container needs an explicit root because its manifest lives
+ * OUTSIDE the workspace: the kernel walks up from the entry file, which for it
+ * is `/opt/telo-workspace`, so it would never see the marker.
+ */
+const CACHE_ROOT = "/telo-cache";
+/** Where the workspace application's own manifest is mounted, read-only. It is
+ *  outside `/workspace` on purpose — the user's tree is theirs, and an
+ *  infrastructure manifest sitting in it would be diffed, listed and editable. */
+const WORKSPACE_APP_DIR = "/opt/telo-workspace";
+/** Port the workspace container serves its HTTP surface on, inside the pod. */
+export const WORKSPACE_PORT = 8099;
+
+/** Each app's kernel debug endpoint. Containers in one pod share a network
+ *  namespace, so the port has to differ per app or the second bind fails. */
+export function inspectPortFor(index: number): number {
+  return INSPECT_PORT + index;
+}
+
+export interface BuildWatchPodArgs {
+  config: K8sRunnerConfig;
+  sessionId: string;
+  podName: string;
+  /** Session-declared env. Goes on every `app-<name>` container and NOWHERE
+   *  else — `workspace` serves files and holds no secrets, and the agent gets
+   *  the operator's env instead. */
+  env: Record<string, string>;
+  apps: BackendAppSpec[];
+  /** Resolved catalog entry for the co-resident agent, when one was requested.
+   *  Its env is the operator's, LLM key included. */
+  agent?: ResolvedRunnerApp;
+  limits: ResolvedLimits;
+  /** Base image every app container and the workspace container run — the plain
+   *  kernel image (`telorun/node`). A watch session never builds an image: the
+   *  build path exists to put a closure on disk before boot, and a watch session
+   *  fetches its own and keeps it for the pod's life. */
+  image: string;
+  pullPolicy: PullPolicy;
+  /** Name of the ConfigMap holding the workspace application's manifest. */
+  workspaceAppConfigMap: string;
+}
+
+/**
+ * The co-resident watch pod.
+ *
+ * One pod, one workspace volume, and one container per running application. An
+ * edit reaches the volume two ways and only two: the agent writes files directly
+ * with its own filesystem tools, and everyone outside the pod goes through the
+ * `workspace` container's HTTP surface. Both land on one volume that N watchers
+ * observe — which is what replaces the tokenized body tarball, the per-session
+ * bundle re-fetch and the separate agent session.
+ *
+ * Three properties are load-bearing and easy to lose:
+ *
+ *  - **A shared GID.** Every container reads and writes `/workspace`, so the pod
+ *    needs an `fsGroup` they all share. Without it the agent writes files the app
+ *    cannot read, which surfaces as a manifest that "does not exist" one reload
+ *    after it was written.
+ *  - **The env split IS the credential boundary.** It used to be structural (two
+ *    pods) and is now a code invariant (containers in one pod): the operator env
+ *    goes on `agent` alone, every `app-<name>` gets the session's declared env
+ *    and nothing else, and `workspace` gets neither.
+ *  - **`CLICOLOR_FORCE` is set per app, and only under `io: "tty"`.** Forcing
+ *    colour in a mode whose whole purpose is "show me what production sees"
+ *    defeats the mode. It is set here rather than baked into the image because
+ *    the usual way to override an image default (an empty value in a pod spec)
+ *    sets the variable EMPTY, which the precedence order reads as present and
+ *    forcing — so a baked default would be undisableable by the normal means.
+ */
+export function buildWatchPod(args: BuildWatchPodArgs): V1Pod {
+  const { config, limits } = args;
+
+  const containers: V1Container[] = [
+    workspaceContainer(args),
+    ...args.apps.map((app, index) => appContainer(args, app, index)),
+  ];
+  if (args.agent) containers.push(agentContainer(args, args.agent));
+
+  return {
+    apiVersion: "v1",
+    kind: "Pod",
+    metadata: {
+      name: args.podName,
+      namespace: config.sessionNamespace,
+      labels: {
+        "app.kubernetes.io/managed-by": config.managedByLabel,
+        "telo.run/session-id": args.sessionId,
+        "telo.run/mode": "watch",
+      },
+    },
+    spec: {
+      restartPolicy: "Never",
+      // One deadline covers every container, so it takes the longer (agent)
+      // ceiling and lets idleness do the real work — an hour would kill a
+      // conversation mid-turn.
+      activeDeadlineSeconds: config.watch.maxTtlSeconds,
+      automountServiceAccountToken: false,
+      ...(config.build.imagePullSecret
+        ? { imagePullSecrets: [{ name: config.build.imagePullSecret }] }
+        : {}),
+      ...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
+      securityContext: {
+        runAsNonRoot: true,
+        runAsUser: 1000,
+        runAsGroup: 1000,
+        // The shared GID. Every container writes /workspace.
+        fsGroup: 1000,
+        seccompProfile: { type: "RuntimeDefault" },
+      },
+      containers,
+      volumes: watchVolumes(args),
+    },
+  };
+}
+
+function watchVolumes(args: BuildWatchPodArgs): V1Volume[] {
+  return [
+    { name: "workspace", emptyDir: {} },
+    // One cache volume, one subdirectory per app. Lives as long as the pod, so
+    // a module closure is downloaded once per app per session and every later
+    // reload resolves from local disk.
+    { name: "telo-cache", emptyDir: {} },
+    { name: "work", emptyDir: {} },
+    { name: "home", emptyDir: {} },
+    { name: "tmp", emptyDir: {} },
+    {
+      name: "workspace-app",
+      configMap: { name: args.workspaceAppConfigMap },
+    },
+  ];
+}
+
+/** Per-container scratch carved out of one volume by `subPath`, so N apps do not
+ *  mean 3N pod volumes. */
+function scratchMounts(owner: string): V1VolumeMount[] {
+  return [
+    { name: "work", mountPath: WORK_DIR, subPath: owner },
+    { name: "home", mountPath: HOME_DIR, subPath: owner },
+    { name: "tmp", mountPath: TMP_MOUNT, subPath: owner },
+  ];
+}
+
+function watchResources(limits: ResolvedLimits): Record<string, unknown> {
+  return {
+    limits: {
+      cpu: limits.cpu,
+      memory: limits.memory,
+      "ephemeral-storage": limits.ephemeralStorage,
+    },
+    requests: { cpu: limits.cpu, memory: limits.memory },
+  };
+}
+
+/**
+ * The workspace surface is RUNNER infrastructure, not agent functionality — it
+ * is part of the `/v1` session contract, so the runner owns it and the agent is
+ * one more writer on the volume beside the app containers. Hanging it off the
+ * catalog image would invert the dependency (the session contract resting on an
+ * application the operator configures) and would need a second implementation
+ * for the agentless case.
+ *
+ * It runs the plain kernel image over a manifest mounted from a ConfigMap, so
+ * there is no third image to build and publish.
+ */
+function workspaceContainer(args: BuildWatchPodArgs): V1Container {
+  return {
+    name: "workspace",
+    image: args.image,
+    imagePullPolicy: pullPolicyToK8s(args.pullPolicy),
+    workingDir: WORK_DIR,
+    command: ["telo", "run", `${WORKSPACE_APP_DIR}/telo.yaml`],
+    env: [
+      { name: "PORT", value: String(WORKSPACE_PORT) },
+      { name: "WORKSPACE_DIR", value: WORKSPACE_DIR },
+      { name: "TELO_CACHE_DIR", value: `${CACHE_ROOT}/workspace` },
+      { name: "HOME", value: HOME_DIR },
+      { name: "npm_config_cache", value: `${HOME_DIR}/.npm` },
+      ...(args.config.build.teloRegistryUrl
+        ? [{ name: "TELO_REGISTRY_URL", value: args.config.build.teloRegistryUrl }]
+        : []),
+    ],
+    ports: [{ containerPort: WORKSPACE_PORT, protocol: "TCP" }],
+    resources: watchResources(args.limits),
+    volumeMounts: [
+      { name: "workspace", mountPath: WORKSPACE_DIR },
+      { name: "telo-cache", mountPath: CACHE_ROOT },
+      { name: "workspace-app", mountPath: WORKSPACE_APP_DIR, readOnly: true },
+      ...scratchMounts("workspace"),
+    ],
+    securityContext: hardenedContainerSecurity(),
+  };
+}
+
+function appContainer(args: BuildWatchPodArgs, app: BackendAppSpec, index: number): V1Container {
+  const env = [
+    ...Object.entries(args.env).map(([name, value]) => ({ name, value })),
+    // Deliberately NO `TELO_CACHE_DIR`: it OUTRANKS the workspace marker, and
+    // the marker is what puts every app's cache in one place. The kernel walks
+    // up from the entry manifest to `telo-workspace.yaml` (seeded at the
+    // workspace root when the session starts) and anchors `.telo` there, so two
+    // apps importing the same module resolve it once between them.
+    { name: "HOME", value: HOME_DIR },
+    { name: "npm_config_cache", value: `${HOME_DIR}/.npm` },
+    // Carried as env, not interpolated into the shell line below: a path that
+    // reached a command string could close a quote.
+    { name: "TELO_ENTRY", value: `${WORKSPACE_DIR}/${app.entryRelativePath}` },
+    { name: "TELO_INSPECT_ADDR", value: `0.0.0.0:${inspectPortFor(index)}` },
+  ];
+  // Only under a terminal. `streams` forces nothing OFF either: with no terminal
+  // the precedence order already resolves to no colour, and an explicit
+  // NO_COLOR would sit ABOVE an app's own `color: always` and suppress a
+  // decision worth observing.
+  if (app.io === "tty") env.push({ name: "CLICOLOR_FORCE", value: "1" });
+
+  const tty = app.io === "tty";
+  return {
+    name: `app-${app.name}`,
+    image: args.image,
+    imagePullPolicy: pullPolicyToK8s(args.pullPolicy),
+    workingDir: WORK_DIR,
+    // Wait for the entry manifest before starting. The workspace arrives over
+    // the workspace container's HTTP surface once the pod is up, so an app that
+    // exec'd immediately would fail its first load on a file that is about to
+    // exist and report a generation nobody caused. The same wait covers a
+    // resumed pod and an app added to the set before its files are written.
+    //
+    // `--watch` + `--inspect` compose: one inspect endpoint serves the whole
+    // watch session and each rebuilt kernel re-attaches to it, so a reload is a
+    // stop/start pair on ONE debug connection — which is exactly what the
+    // runner's generation counting reads. 0.0.0.0 (not the CLI's loopback
+    // default) lets the runner reach it across the pod network; the port is
+    // never published via Service or Ingress.
+    command: [
+      "sh",
+      "-c",
+      'while [ ! -f "$TELO_ENTRY" ]; do sleep 0.2; done; ' +
+        'exec telo run "$TELO_ENTRY" --watch --inspect "$TELO_INSPECT_ADDR" --no-open',
+    ],
+    env,
+    stdin: true,
+    stdinOnce: false,
+    tty,
+    ...(app.ports.length > 0
+      ? {
+          ports: app.ports.map((p) => ({
+            containerPort: p.port,
+            protocol: p.protocol.toUpperCase(),
+          })),
+        }
+      : {}),
+    resources: watchResources(args.limits),
+    // No `telo-cache` mount: an app's cache lives under the workspace volume,
+    // anchored by the marker at its root, which is what makes it one cache for
+    // the whole session rather than one per app.
+    volumeMounts: [
+      { name: "workspace", mountPath: WORKSPACE_DIR },
+      ...scratchMounts(app.name),
+    ],
+    securityContext: hardenedContainerSecurity(),
+  };
+}
+
+/**
+ * At most one agent per session, never per app: the agent's unit is the
+ * workspace — it edits files, it does not own a process — and two agents over
+ * one workspace would contend on the same files and split one conversation in
+ * half.
+ *
+ * This is the ONLY container that receives the operator env, and the only one
+ * whose write-path hardening is relaxed (an operator-curated image, exactly as
+ * an app session is today).
+ */
+function agentContainer(args: BuildWatchPodArgs, agent: ResolvedRunnerApp): V1Container {
+  return {
+    name: "agent",
+    image: agent.image,
+    imagePullPolicy: pullPolicyToK8s(agent.pullPolicy),
+    env: [
+      ...Object.entries(agent.env).map(([name, value]) => ({ name, value })),
+      // The agent's own workspace IS the session's shared volume.
+      { name: "WORKSPACE_DIR", value: WORKSPACE_DIR },
+      { name: "CLICOLOR_FORCE", value: "1" },
+    ],
+    // Declared so the pod describes what it listens on; the Service selects the
+    // pod and every container shares its network namespace, so the agent needs
+    // no routing of its own beyond being in the session's port set.
+    ...(agent.port !== undefined
+      ? { ports: [{ containerPort: agent.port, protocol: "TCP" }] }
+      : {}),
+    resources: watchResources(args.limits),
+    volumeMounts: [
+      { name: "workspace", mountPath: WORKSPACE_DIR },
+      ...scratchMounts("agent"),
+    ],
+    securityContext: {
+      allowPrivilegeEscalation: false,
+      capabilities: { drop: ["ALL"] },
+    },
+  };
+}

package/src/k8s/pod-status.ts

@@ -0,0 +1,143 @@
+import type { V1ContainerState, V1ContainerStatus, V1Pod } from "@kubernetes/client-node";
+import type { RunStatus } from "@telorun/runner-core";
+
+import type { KubeClient } from "./client.js";
+
+/**
+ * Reading a Pod's status — shared by the run-session and watch-session paths.
+ * Pure functions over the watch payload plus the one delete that both perform;
+ * extracted so the two lifecycles cannot disagree about what "failed" means or
+ * which container's exit code counts.
+ */
+
+export function podStatus(obj: unknown): V1Pod["status"] | undefined {
+  return (obj as V1Pod | undefined)?.status;
+}
+
+export function podPhase(obj: unknown): string | undefined {
+  return podStatus(obj)?.phase;
+}
+
+/** A coming-up message for the studio feed while the Pod is still scheduling /
+ *  pulling / delivering the body / creating the container; undefined once running. */
+export function provisionMessage(obj: unknown): string | undefined {
+  const status = podStatus(obj);
+  if (status?.phase !== "Pending") return undefined;
+  const containers = [
+    ...(status.initContainerStatuses ?? []),
+    ...(status.containerStatuses ?? []),
+  ];
+  for (const cs of containers) {
+    const reason = cs.state?.waiting?.reason;
+    if (reason) return humanizeWaitReason(reason);
+  }
+  return "Scheduling";
+}
+
+function humanizeWaitReason(reason: string): string {
+  switch (reason) {
+    case "ContainerCreating":
+      return "Creating container";
+    case "PodInitializing":
+      return "Delivering application";
+    case "ErrImagePull":
+    case "ImagePullBackOff":
+      return "Pulling image";
+    default:
+      return reason;
+  }
+}
+
+export function terminalStatus(obj: unknown, userStopped: boolean): RunStatus {
+  if (userStopped) return { kind: "stopped" };
+  const phase = podPhase(obj);
+  if (phase === "Succeeded") return { kind: "exited", code: containerExitCode(obj) ?? 0 };
+  return { kind: "failed", message: podFailureMessage(obj) };
+}
+
+// Exit code of the main session container — used to report a clean exit.
+export function containerExitCode(obj: unknown): number | null {
+  const term = podStatus(obj)?.containerStatuses?.[0]?.state?.terminated;
+  return typeof term?.exitCode === "number" ? term.exitCode : null;
+}
+
+const MAX_FAILURE_DETAIL = 500;
+
+/**
+ * Builds an actionable failure message from a terminal Pod status. Init
+ * containers are inspected first: a failed init container leaves the main
+ * container unstarted, so reading only `containerStatuses` would fall through
+ * to the bare "pod failed". For prebuilt session pods the common failure is the
+ * main container itself (image pull, OOM, a non-zero exit).
+ */
+export function podFailureMessage(obj: unknown): string {
+  const status = podStatus(obj);
+  const fromContainer = firstContainerProblem(status);
+  if (fromContainer) return fromContainer;
+  if (status?.message) return truncateDetail(status.message);
+  if (status?.reason) return status.reason;
+  return "pod failed";
+}
+
+function firstContainerProblem(status: V1Pod["status"] | undefined): string | undefined {
+  const groups: Array<[string, V1ContainerStatus[] | undefined]> = [
+    ["init container", status?.initContainerStatuses],
+    ["container", status?.containerStatuses],
+  ];
+  for (const [label, statuses] of groups) {
+    for (const cs of statuses ?? []) {
+      const problem = containerStateProblem(cs.state) ?? containerStateProblem(cs.lastState);
+      if (problem) return `${label} "${cs.name}" ${problem}`;
+    }
+  }
+  return undefined;
+}
+
+function containerStateProblem(state: V1ContainerState | undefined): string | undefined {
+  const term = state?.terminated;
+  if (term && term.exitCode !== 0) {
+    const reason = term.reason ? `${term.reason} ` : "";
+    const detail = term.message ? `: ${truncateDetail(term.message)}` : "";
+    return `failed: ${reason}(exit code ${term.exitCode ?? "unknown"})${detail}`;
+  }
+  const waiting = state?.waiting;
+  if (waiting?.reason && isBlockingWaitReason(waiting.reason)) {
+    const detail = waiting.message ? `: ${truncateDetail(waiting.message)}` : "";
+    return `waiting: ${waiting.reason}${detail}`;
+  }
+  return undefined;
+}
+
+// Benign transient reasons the kubelet reports while a Pod is still coming up.
+function isBlockingWaitReason(reason: string): boolean {
+  return reason !== "PodInitializing" && reason !== "ContainerCreating";
+}
+
+function truncateDetail(text: string): string {
+  const trimmed = text.trim();
+  return trimmed.length > MAX_FAILURE_DETAIL ? `${trimmed.slice(0, MAX_FAILURE_DETAIL)}…` : trimmed;
+}
+
+export async function deletePod(kube: KubeClient, ns: string, name: string): Promise<void> {
+  try {
+    await kube.core.deleteNamespacedPod({ name, namespace: ns, gracePeriodSeconds: 0 });
+  } catch (err) {
+    // 404 = already gone (natural exit + GC). Anything else is a real failure.
+    if (!is404(err)) throw err;
+  }
+}
+
+export function is404(err: unknown): boolean {
+  const e = err as { statusCode?: number; code?: number; response?: { statusCode?: number } };
+  return e?.statusCode === 404 || e?.code === 404 || e?.response?.statusCode === 404;
+}
+
+export function msg(err: unknown): string {
+  if (err instanceof Error) return err.message;
+  if (typeof err === "string") return err;
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return String(err);
+  }
+}

package/src/k8s/watch-pod-spec.test.ts

@@ -0,0 +1,215 @@
+import type { V1Container } from "@kubernetes/client-node";
+import { describe, expect, it } from "vitest";
+
+import { loadK8sRunnerConfig } from "../config.js";
+import { buildSessionPod, buildWatchPod } from "./pod-spec.js";
+
+const BASE_ENV = {
+  RUNNER_SELF_URL: "http://k8s-runner.telo-runner.svc:8062",
+  RUNNER_IMAGE_REPOSITORY: "registry.telo-runner.svc:5000/telo-sessions",
+  RUNNER_WATCH_SESSIONS: "true",
+};
+
+const config = loadK8sRunnerConfig({ ...process.env, ...BASE_ENV });
+
+const OPERATOR_KEY = "OPENAI_API_KEY";
+const OPERATOR_VALUE = "sk-operator-secret";
+
+function build(overrides: Partial<Parameters<typeof buildWatchPod>[0]> = {}) {
+  return buildWatchPod({
+    config,
+    sessionId: "abc123",
+    podName: "telo-watch-abc123",
+    env: { APP_SETTING: "from-session" },
+    apps: [
+      { name: "web", entryRelativePath: "telo.yaml", ports: [{ port: 3000, protocol: "tcp" }], io: "tty" },
+      { name: "worker", entryRelativePath: "worker.yaml", ports: [], io: "streams" },
+    ],
+    agent: {
+      name: "authoring-agent",
+      image: "ghcr.io/telorun/authoring-agent:1",
+      env: { [OPERATOR_KEY]: OPERATOR_VALUE },
+      pullPolicy: "missing",
+      port: 8080,
+    },
+    limits: config.appLimits,
+    image: "telorun/node:latest-slim",
+    pullPolicy: "missing",
+    workspaceAppConfigMap: "telo-workspace-app-deadbeef",
+    ...overrides,
+  });
+}
+
+const byName = (pod: ReturnType<typeof build>, name: string): V1Container =>
+  pod.spec!.containers.find((c) => c.name === name)!;
+
+const envOf = (c: V1Container): Record<string, string> =>
+  Object.fromEntries((c.env ?? []).map((e) => [e.name, e.value ?? ""]));
+
+describe("buildWatchPod — the credential boundary", () => {
+  const pod = build();
+
+  it("puts the operator env on the agent container and nowhere else", () => {
+    // Structural before (two pods), a code invariant now (containers in one
+    // pod) — so it needs a test that names the property directly.
+    for (const container of pod.spec!.containers) {
+      const env = envOf(container);
+      if (container.name === "agent") {
+        expect(env[OPERATOR_KEY]).toBe(OPERATOR_VALUE);
+      } else {
+        expect(env).not.toHaveProperty(OPERATOR_KEY);
+      }
+    }
+  });
+
+  it("gives every app container the session env and the workspace container neither", () => {
+    expect(envOf(byName(pod, "app-web")).APP_SETTING).toBe("from-session");
+    expect(envOf(byName(pod, "app-worker")).APP_SETTING).toBe("from-session");
+    expect(envOf(byName(pod, "workspace"))).not.toHaveProperty("APP_SETTING");
+    expect(envOf(byName(pod, "workspace"))).not.toHaveProperty(OPERATOR_KEY);
+  });
+
+  it("roots the agent at the session's shared workspace", () => {
+    // The agent is one writer on the shared volume beside the app containers,
+    // not the owner of a directory inside its own container. Its manifest reads
+    // this to place every file tool and every spawned command.
+    expect(envOf(byName(pod, "agent")).WORKSPACE_DIR).toBe("/workspace");
+    expect(
+      byName(pod, "agent").volumeMounts?.find((m) => m.name === "workspace")?.mountPath,
+    ).toBe("/workspace");
+  });
+
+  it("runs at most one agent, whatever the app count", () => {
+    expect(pod.spec!.containers.filter((c) => c.name === "agent")).toHaveLength(1);
+  });
+
+  // The pod's containers share one network namespace, so declaring the port is
+  // all the agent needs to be reachable through the session's own Service —
+  // there is no routing of its own to arrange.
+  it("declares the agent's port so the session's Service can carry it", () => {
+    expect(byName(pod, "agent").ports).toEqual([{ containerPort: 8080, protocol: "TCP" }]);
+  });
+
+  it("declares no port for an agent whose catalog entry has none", () => {
+    const portless = build({
+      agent: {
+        name: "authoring-agent",
+        image: "ghcr.io/telorun/authoring-agent:1",
+        env: {},
+        pullPolicy: "missing",
+      },
+    });
+    expect(byName(portless, "agent").ports).toBeUndefined();
+  });
+
+  it("omits the agent container entirely when none was requested", () => {
+    const bare = build({ agent: undefined });
+    expect(bare.spec!.containers.map((c) => c.name)).toEqual([
+      "workspace",
+      "app-web",
+      "app-worker",
+    ]);
+  });
+});
+
+describe("buildWatchPod — the shared workspace", () => {
+  const pod = build();
+
+  it("shares one GID across every container", () => {
+    // Without this the agent writes files the app cannot read, which surfaces
+    // as a manifest that "does not exist" one reload after it was written.
+    expect(pod.spec!.securityContext?.fsGroup).toBe(1000);
+    for (const container of pod.spec!.containers) {
+      const mount = container.volumeMounts?.find((m) => m.name === "workspace");
+      expect(mount?.mountPath).toBe("/workspace");
+      expect(mount?.readOnly).toBeFalsy();
+    }
+  });
+
+  it("leaves every app's cache to the workspace marker", () => {
+    // `TELO_CACHE_DIR` OUTRANKS the marker, so setting it per app is exactly
+    // what would give each app its own cache. Without it the kernel walks up
+    // from the entry manifest to `telo-workspace.yaml` at the workspace root and
+    // anchors one `.telo` there for the whole session.
+    expect(envOf(byName(pod, "app-web"))).not.toHaveProperty("TELO_CACHE_DIR");
+    expect(envOf(byName(pod, "app-worker"))).not.toHaveProperty("TELO_CACHE_DIR");
+    for (const name of ["app-web", "app-worker"]) {
+      expect(byName(pod, name).volumeMounts?.map((m) => m.name)).not.toContain("telo-cache");
+    }
+  });
+
+  it("gives the workspace container an explicit cache root", () => {
+    // Its manifest lives OUTSIDE the workspace, so the walk-up from its entry
+    // would never reach the marker.
+    expect(envOf(byName(pod, "workspace")).TELO_CACHE_DIR).toBe("/telo-cache/workspace");
+  });
+
+  it("mounts the workspace application's manifest read-only, outside /workspace", () => {
+    const mount = byName(pod, "workspace").volumeMounts?.find((m) => m.name === "workspace-app");
+    expect(mount?.readOnly).toBe(true);
+    expect(mount?.mountPath).not.toContain("/workspace/");
+    expect(pod.spec!.volumes?.find((v) => v.name === "workspace-app")?.configMap?.name).toBe(
+      "telo-workspace-app-deadbeef",
+    );
+  });
+});
+
+describe("buildWatchPod — how each app runs", () => {
+  const pod = build();
+
+  it("keeps every app container's hardening identical to a run session's", () => {
+    const runPod = buildSessionPod({
+      config,
+      sessionId: "abc123",
+      podName: "telo-run-abc123",
+      entryRelativePath: "telo.yaml",
+      env: {},
+      ports: [],
+      limits: config.limits,
+      image: "prebuilt",
+      bundleUrl: "http://runner/bundle",
+      inspect: false,
+    });
+    const expected = runPod.spec!.containers[0].securityContext;
+    expect(byName(pod, "app-web").securityContext).toEqual(expected);
+    expect(byName(pod, "app-worker").securityContext).toEqual(expected);
+  });
+
+  it("waits for its entry manifest, then runs under --watch with a distinct inspect port", () => {
+    const web = envOf(byName(pod, "app-web"));
+    const worker = envOf(byName(pod, "app-worker"));
+    expect(web.TELO_ENTRY).toBe("/workspace/telo.yaml");
+    expect(worker.TELO_ENTRY).toBe("/workspace/worker.yaml");
+    // Containers in one pod share a network namespace, so a second bind on the
+    // same port would fail.
+    expect(web.TELO_INSPECT_ADDR).not.toBe(worker.TELO_INSPECT_ADDR);
+
+    const command = byName(pod, "app-web").command!.join(" ");
+    expect(command).toContain("while [ ! -f \"$TELO_ENTRY\" ]");
+    expect(command).toContain("--watch");
+    expect(command).toContain("--inspect");
+    // The path never reaches the shell line — a value that did could close a quote.
+    expect(command).not.toContain("/workspace/telo.yaml");
+  });
+
+  it("forces colour only under a terminal", () => {
+    expect(envOf(byName(pod, "app-web")).CLICOLOR_FORCE).toBe("1");
+    // `streams` exists to show what production sees; forcing colour there would
+    // defeat the mode by putting ANSI into what is meant to be a pipe.
+    expect(envOf(byName(pod, "app-worker"))).not.toHaveProperty("CLICOLOR_FORCE");
+    expect(byName(pod, "app-web").tty).toBe(true);
+    expect(byName(pod, "app-worker").tty).toBe(false);
+  });
+
+  it("takes the watch TTL, not the run-session one", () => {
+    expect(pod.spec!.activeDeadlineSeconds).toBe(config.watch.maxTtlSeconds);
+  });
+
+  it("labels the pod for orphan reaping and marks it as a watch session", () => {
+    expect(pod.metadata?.labels).toMatchObject({
+      "app.kubernetes.io/managed-by": "telo-k8s-runner",
+      "telo.run/session-id": "abc123",
+      "telo.run/mode": "watch",
+    });
+  });
+});