@jr2/orchestrator 0.3.0 → 0.5.1

package/src/sandbox-kubectl.ts

@@ -1,1108 +0,0 @@
-// The canonical SandboxPort (ADR-0012 / GAP(3)): drives the operator's Sandbox CRD through
-// `kubectl`, honoring the current kube context (ADR-0009: the kube target IS the kubectl
-// context; `--context` overrides). Shelling to kubectl instead of a client library keeps the
-// dependency surface at zero and the behavior identical to what a human debugging the cluster
-// would type; the `exec` process seam is injectable so the mapping logic is unit-testable
-// without a cluster. The kind e2e tier exercises the real thing.
-//
-// Reachability: the orchestrator always runs in-cluster (ADR-0019), so it dials
-// `status.endpoint` (`http://<name>.<ns>.svc:…`) directly — stable across orchestrator
-// restarts by nature, which is what ADR-0012's "same endpoint" re-attach promise rides on.
-//
-// All four operations are idempotent (SandboxPort contract): apply is create-or-update, attach
-// guards every clone/worktree, delete ignores absent.
-//
-// WHICH REPOS a Sandbox attaches arrive resolved from the `workspace()`'s Repo Slots (ADR-0051):
-// the CR names each by its cache key, the operator mounts the node's cache read-only at
-// `/repos/<key>` and gates Ready on it, and the attach clones off that mount. The one judgement
-// made here is the FENCE: a per-run url must match a `git.credentials` entry, or it is refused
-// before anything is applied.
-//
-// WHICH IMAGE a Sandbox runs is not an option here (ADR-0037/0038/0049). The request carries what
-// the `workspace()` wrapper statically declared — a `file:` docker context or a registry ref — and
-// the resolved key→ref map arrives as a mounted ConfigMap read on EVERY provision, so a `jr2 up`
-// that rebuilds an image reaches future Sandboxes without rolling this process.
-//
-// The pod's primary container is the Sandbox Image BYTE-FOR-BYTE (ADR-0037): no appended layers,
-// no rewritten Dockerfile, no jr2 knowledge inside it. jr2's runtime arrives at POD time instead —
-// an emptyDir at `/opt/jr2`, populated by an init container running the kit's Harness image — and
-// the container's COMMAND is overridden to start the Harness from that volume. The image's own
-// `USER` and `HOME` are respected (the human who execs in lands in the environment its author
-// built); only its `ENTRYPOINT`/`CMD` do not run, because a container has one command and the
-// Harness must own it — its death must be the container's death, which is what the operator's
-// Ready probe and restart semantics at `:8080` mean. A process the image WANTS running is not
-// lost: it has its own seat, the User Container (ADR-0005), composed here when the wrapper's static
-// `user` option names an image (ADR-0049).
-//
-// So this module composes the whole pod — two init steps and up to three containers:
-//
-//   initContainer runtime     the kit's Harness image → copies /opt/jr2 into the volume
-//   initContainer preflight   the USER'S image + that volume → ADR-0037's probe, the thing that
-//                             proves a registry ref, whose first appearance is this provision
-//   container     harness     the Sandbox Image, command overridden, /work + /opt/jr2 mounted
-//   container     custodian   jr2-composed, the pod's only credential holder (below)
-//   container     user        optional, the image's own entrypoint, the checkouts (/work, plus
-//                             /repos and /opt/jr2 read-only) and NOTHING else
-//
-// This is also where the CUSTODIAN is composed (ADR-0013, ADR-0059). The operator needs no change
-// to carry it: ADR-0001 made `Sidecars` generic container fragments it schedules WITHOUT
-// understanding, so the Custodian is exactly that — a container with an image, its mounts, and a
-// readiness probe the pod's Ready waits on. What this module builds is the pod's asymmetry:
-//
-//   harness container     JR2_CUSTODIAN_URL + the Stand-in of every credential  (no credential)
-//   custodian container   the Sandbox token and every held secret, as files     (the credentials)
-//
-// The Agent has code execution in the first and none in the second. The token is minted here — a
-// signed Sandbox name (see tokens.ts), so re-provisioning after a restart yields the SAME token and
-// the Secret re-applies as a no-op. What else the Custodian holds is `jr2 up`'s resolution, read
-// per provision from the `jr2-held` mount (custodian.ts builds both placements' pods alike).
-
-import { execFile } from "node:child_process";
-import { join } from "node:path";
-import {
-  matchCredential,
-  type GitCredential,
-  type HarnessEnvFromSource,
-  type HarnessEnvVar,
-  type SandboxPlacement,
-} from "./config.ts";
-import { custodianComposition, type CustodianComposition } from "./custodian.ts";
-import { readHeldManifest } from "./held-secrets.ts";
-import { readImageRefs, resolveSandboxImage, resolveUserImage, type ImageRefs } from "./images.ts";
-import { HELD_KEY, HELD_MOUNT, IMAGES_KEY, IMAGES_MOUNT, REPOS_MOUNT } from "./names.ts";
-import { repoIdentity } from "./repo-identity.ts";
-import type { RepoResources } from "./repos.ts";
-import { harnessTokenDigest, sandboxToken } from "./tokens.ts";
-import type { ProvisionedRepo, SandboxPort, WorkspaceSpec } from "./workspace.ts";
-
-/** Run one kubectl invocation to completion. `input` is piped to stdin (`apply -f -`). */
-export type KubectlExec = (args: string[], opts?: { input?: string }) => Promise<{ stdout: string; stderr: string }>;
-
-/** Where jr2's runtime lands in every container that gets it (ADR-0037). `/opt/jr2` and not `/app`
- * because a stranger's base may already use `/app`, and one layout must serve both the stock
- * Harness image and an arbitrary Sandbox Image. It is a PUBLISHED surface: `bin/` beside `lib/`
- * (node's rpath is `$ORIGIN/../lib`), `src/main.ts`, `node_modules/`. */
-export const RUNTIME_MOUNT = "/opt/jr2";
-
-/** Where the populate init container writes the runtime. NOT `/opt/jr2`: mounting the volume there
- * would shadow the very directory being copied out of the Harness image. */
-const RUNTIME_STAGE = "/mnt/jr2";
-
-/** The primary container's command (ADR-0037). Absolute, so it never depends on the image's
- * `WORKDIR`, and identical to the stock Harness image's own `CMD` — one runtime, two placements. */
-const HARNESS_COMMAND = [`${RUNTIME_MOUNT}/bin/node`, `${RUNTIME_MOUNT}/src/main.ts`];
-
-/** The program `origin`'s fetch url runs (ADR-0053), on the runtime volume beside `work-acl`. It
- * asks the node cache for a fetch and then serves the cache, so every seat that holds the
- * checkouts must hold this volume — which is why the User Container mounts it (ADR-0005). */
-const UPLOAD_PACK = `${RUNTIME_MOUNT}/bin/jr2-upload-pack`;
-
-/** ADR-0005's default work group. Convention, not config: the pod's `fsGroup` is granted to every
- * container as a supplemental group, so the Harness writes `/work` whatever the number and no
- * image's `/etc/group` needs to know it. The one override is the spec's `workGroup`. */
-const DEFAULT_WORK_GROUP = 2000;
-
-/**
- * The isolation baseline for a jr2-owned seat, spelled out HERE for the init containers because the
- * operator's hardened default covers the primary container and the sidecars only (ADR-0001/0005) —
- * init steps pass through verbatim, which is what keeps the operator agent-agnostic. Deliberately
- * not applied to the `user` container: that seat's identity is "what jr2 does not own".
- */
-const HARDENED = {
-  runAsNonRoot: true,
-  allowPrivilegeEscalation: false,
-  capabilities: { drop: ["ALL"] },
-  seccompProfile: { type: "RuntimeDefault" },
-};
-
-/**
- * ADR-0037's fallback seat, for a BUILT image that declares no `USER` (the converge's `docker
- * inspect` is what saw that; images.ts holds the record). jr2 sets `runAsUser` nowhere else — an
- * image's own `USER` decides its seat's uid (ADR-0005) — so this applies only where the image
- * chose nothing and the alternative is root, which the hardened context refuses.
- *
- * The home is a POD volume, not a directory in the image: uid 1000 on a stranger's base has no
- * home at all, and the floor needs a writable one (`git config --global` writes `$HOME/.gitconfig`
- * on every attach; a real toolchain wants `~/.npm`, `~/.cargo`, `~/.cache`). An emptyDir lands
- * group-writable under the pod's fsGroup, so uid 1000 owns it in practice without jr2 chown-ing
- * anything. `/home/jr2` and not `/home/node`: the number is jr2's choice here, so the path is too.
- */
-const FALLBACK_UID = 1000;
-const FALLBACK_HOME = "/home/jr2";
-
-/**
- * The kubelet's verdict on a container whose image resolves to root under `runAsNonRoot: true`.
- * Matched, not merely reported, because it is the ONE provision failure with no evidence anywhere
- * else: the container never starts, so it has no logs, and the pod sits in this waiting state until
- * the provision times out — which then blames the preflight for a container the preflight never got
- * to run. A BUILT image is caught earlier and cheaper (the converge's `docker inspect` recorded the
- * string; `resolveSandboxImage` in images.ts judges it into `refusedUser` before anything is
- * applied), so this is the brought ref's path: never inspected, never given the uid-1000 fallback,
- * knowable only from the cluster.
- *
- * Reason and message are BOTH required. The reason alone covers a missing Secret or ConfigMap key
- * too — a different fault with a different fix — and only the message distinguishes them.
- */
-const ROOT_IMAGE_REASON = "CreateContainerConfigError";
-const ROOT_IMAGE_MESSAGE = /runAsNonRoot/i;
-
-/** How many CR polls pass between two pod reads. See the provision loop for why it is not 1. */
-const POD_CHECK_EVERY = 5;
-
-/**
- * The kubelet's own words when a container's image resolves to root under `runAsNonRoot`, or
- * undefined for every other pod shape. A pure read of pod status: the caller supplies the parsed
- * `kubectl get pod -o json`, so the claim is testable without a cluster and the fault detection
- * cannot drift from the message the provision prints.
- *
- * Init containers are searched FIRST because they run first: the `preflight` step runs the user's
- * image before the Harness container ever exists, so that is where a root image dies. The primary
- * containers are searched too — the same image sits in the `harness` seat, and the `user` seat is
- * deliberately un-hardened (ADR-0005), so a fault there would mean something else entirely.
- *
- * Both the reason AND the message must match. `CreateContainerConfigError` is also what an absent
- * Secret key produces, and that fault has a different fix; a name with no evidence behind it is
- * worse than the timeout it replaces.
- */
-export function rootImageFault(pod: unknown): string | undefined {
-  const status = (pod as { status?: Record<string, unknown> } | null)?.status;
-  if (!status) return undefined;
-  type Waiting = { name?: string; state?: { waiting?: { reason?: string; message?: string } } };
-  const groups = [status["initContainerStatuses"], status["containerStatuses"]];
-  for (const group of groups) {
-    if (!Array.isArray(group)) continue;
-    for (const cs of group as Waiting[]) {
-      const waiting = cs?.state?.waiting;
-      if (!waiting || waiting.reason !== ROOT_IMAGE_REASON) continue;
-      if (!waiting.message || !ROOT_IMAGE_MESSAGE.test(waiting.message)) continue;
-      return `container "${cs.name ?? "?"}": ${waiting.message}`;
-    }
-  }
-  return undefined;
-}
-
-/**
- * The fix, not the symptom. The kubelet's message says what it refused; it cannot say that the
- * image is a Sandbox Image, that jr2 declined to patch a uid onto it, or where the one-line edit
- * goes — and without those three the reader has a Kubernetes error and no next step.
- *
- * It names the BROUGHT case specifically because that is the only one that reaches here: a built
- * image's `USER` was inspected at converge and judged before anything was applied (images.ts), and
- * an image declaring none gets the uid-1000 fallback. A ref is never inspected — that is the point
- * of refs (ADR-0037) — so it must declare a numeric non-root `USER` itself.
- */
-function rootImageError(name: string, fault: string): string {
-  return (
-    `Sandbox "${name}" cannot start: its image runs as ROOT, and every jr2-owned seat is hardened ` +
-    `with runAsNonRoot (ADR-0005). The kubelet refused it — ${fault}\n` +
-    `  - the fix is one line in the image: a NUMERIC non-root \`USER <uid>\` (e.g. \`USER 1000\`)\n` +
-    `  - numeric because the kubelet does not read the image's /etc/passwd, so \`USER app\` is ` +
-    `refused too — it cannot prove that name is non-root\n` +
-    `  - jr2 does not supply a uid for a brought registry ref: it is never inspected and never ` +
-    `modified, which is what "bring your own image" means (ADR-0037). Only an image jr2 BUILDS, ` +
-    `and only one that declares no USER at all, gets the uid-${FALLBACK_UID} fallback.`
-  );
-}
-
-/**
- * ADR-0037's preflight, VERBATIM: git present · `$HOME` writable · glibc new enough for jr2's node
- * (with the relocated `libstdc++`) · the vendored ripgrep, reached through the mounted `/opt/jr2`.
- *
- * The three commands are the floor, one each: `git config --global` proves git is on the system
- * PATH AND that `$HOME` is writable for the image's user; `node -e ""` proves the glibc is no
- * older than the one jr2's node was built against (this is where musl dies); `rg` UNQUALIFIED
- * proves the vendored static binary resolves THROUGH PATH, which is what the Harness's own append
- * buys at runtime.
- *
- * ONE prover, and this is it (ADR-0037/0041). A converge cannot hold an image to this floor: the
- * floor is a HARNESS-SEAT obligation, a built context may equally be destined for the User
- * Container seat — which owes no floor at all (ADR-0005) — and which seat a directory serves is
- * workflow-internal and statically unrecoverable (ADR-0031). Here the seat is known, and here is
- * also the only moment a registry ref exists at all, since jr2 never builds or inspects one.
- */
-const SANDBOX_PREFLIGHT = `git config --global safe.directory "*" && ${RUNTIME_MOUNT}/bin/node -e "" && rg --version`;
-
-/**
- * The probe as a shell line. The PATH append is mechanism, not part of the claim, and it is not
- * optional: nothing bakes `/opt/jr2/bin` into the user's image any more, so a probe that skipped it
- * would report `rg: not found` for every image on earth. APPENDED, never prepended — a toolchain
- * the image pinned wins, which is as much the property being proved as `rg`'s presence (ADR-0037).
- * The seat gets no login shell at either end, so `$PATH` is whatever the image itself set.
- */
-function preflightShell(): string {
-  return `export PATH="$PATH:${RUNTIME_MOUNT}/bin"; ${SANDBOX_PREFLIGHT}`;
-}
-
-/**
- * The preflight as an init step IN THE USER'S IMAGE with `/opt/jr2` mounted. It fails the pod
- * before the Harness starts, instead of surfacing as a tool failure mid-turn on a pod nobody is
- * watching.
- *
- * It PROVES, it does not set up: the `.gitconfig` it writes lives in this container's own
- * ephemeral filesystem, so the attach script still runs the same line in the Harness container.
- *
- * `sh -c`, not `sh -ec`: the failure is handled below, because "node did not execute" is not
- * actionable and the message has to name the fix.
- */
-const PREFLIGHT_SCRIPT = [
-  `${preflightShell()} && exit 0`,
-  `echo "jr2: this Sandbox Image does not meet the floor (ADR-0037): a glibc base no older than ` +
-    `jr2's node (musl is out entirely), git on the system PATH, a writable HOME for the image's ` +
-    `USER, and /opt/jr2 + /work + :8080 unclaimed. jr2 vendors the rest." >&2`,
-  `exit 1`,
-].join("\n");
-
-export type KubectlSandboxOptions = {
-  /** The mounted image map (ADR-0037/0038) — every ref this port can name, written by `jr2 up`.
-   * Default: the `jr2-images` ConfigMap's mount. No image option here: which image a Sandbox runs
-   * is the `workspace()` wrapper's static `image` option (ADR-0049) — carried on the Machine, read
-   * off it at invoke time, handed to `provision()` as a string — and resolved against this map at
-   * provision. The per-run spec never names one (ADR-0051). */
-  imagesPath?: string;
-  /** The held-secret manifest `jr2 up` resolved (ADR-0059) — read per provision like the image map,
-   * so a held-secret edit reaches future Sandboxes without rolling this process. Default: the
-   * `jr2-held` ConfigMap's mount. */
-  heldPath?: string;
-  /** Extra env for the HARNESS container (`harness.env`) — merged ahead of the
-   * mechanism-owned vars, which win on collision. */
-  env?: HarnessEnvVar[];
-  /** Whole-Secret/ConfigMap env for the Harness container (`harness.envFrom`) — how a real
-   * Harness gets its model API key without the value ever touching jr2 config. */
-  envFrom?: HarnessEnvFromSource[];
-  /** Where a Sandbox may land (ADR-0052): the Instance's `sandbox.nodeSelector` and
-   * `sandbox.tolerations`, written on the CR verbatim and copied onto the pod by the operator, which
-   * merges nothing with them. Absent → wherever an ordinary pod lands. */
-  placement?: SandboxPlacement;
-  /** The instance ships a private-CA bundle (ADR-0020): the HARNESS container trusts it, and the
-   * Custodian verifies a bound host with it (custodian.ts). */
-  caBundle?: boolean;
-  /** The key Sandbox tokens are signed with — from the instance Secret (ADR-0013/0019). */
-  signingKey?: Buffer;
-  /** Kube namespace for Sandbox CRs. Default `default`. */
-  namespace?: string;
-  /** kubectl `--context` override. Default: the current context (ADR-0009). */
-  context?: string;
-  /** In-pod root for the pod-local clones + worktrees (ADR-0004 layout). Default `/work`. */
-  workRoot?: string;
-  /** CR `spec.idleTimeout` — the operator's abandoned-Sandbox GC backstop (ADR-0001). Default `30m`. */
-  idleTimeout?: string;
-  /** How often a workspace's lease actor renews (ADR-0001/0021): the cadence at which
-   * `jr2.dev/keepalive` is re-stamped AND continuity is read back. Must be ≪ idleTimeout, since
-   * a lapsed lease is what lets the operator reap. Default 5m. */
-  leaseIntervalMs?: number;
-  /** Await-Ready budget for the POD: from the CR apply until the operator reports the pod Ready.
-   * Default 120s, polled every second. A pod that never comes up (an image that misses ADR-0037's
-   * floor) is what this bounds; a pod that is up and waiting on its Repos is `repoTimeoutMs`'s. */
-  readyTimeoutMs?: number;
-  /**
-   * Await-Ready budget for the REPOS (ADR-0051): once the operator holds a Sandbox whose pod is
-   * Ready on a Repo reason — the node's cache agent is cloning a cold node, or fetching before the
-   * attach — the wait is measured against this, from the same CR apply. Sized for a clone, not a
-   * pod: the agent's clone budget is 20m, and this must exceed it so a clone that runs out of
-   * time fails BY NAME (`RepoCloneFailed`, git's words) instead of as this port's timeout; and it
-   * must stay inside `idleTimeout` (30m), because the lease starts after provision, so the operator
-   * reaps a Sandbox that waits longer than that. Default 25m.
-   */
-  repoTimeoutMs?: number;
-  pollMs?: number;
-  /**
-   * The Repo-resource port (ADR-0051, repos.ts): every Repo a provision names must exist as a
-   * `Repo` resource before the CR names it, or the operator reports it missing and the Sandbox
-   * never reaches Ready. So `provision` ensures each one here — a per-run url's resource is
-   * created at first attach, a bound one is found as the boot stated it — and REFUSES to run
-   * without the port: a Sandbox whose Repos nobody creates parks on `RepoMissing` for the whole
-   * Ready budget. The other operations (attach, renew, destroy) need no port, so it is optional
-   * at construction.
-   */
-  repos?: RepoResources;
-  /**
-   * The instance's `git.credentials` (ADR-0051) — THE FENCE. A per-run url is run input, a
-   * ticket field, and otherwise a way to spend the cluster's credential against any host: one
-   * whose identity matches no entry is refused here, before a Secret or a CR exists, naming the
-   * list. A bound url is code the instance typechecked and deployed, admitted without a match.
-   * Default: no entries, so every per-run url is refused.
-   */
-  credentials?: readonly GitCredential[];
-  /** Process seam, injectable for tests. Defaults shell to the `kubectl` on PATH. */
-  exec?: KubectlExec;
-};
-
-export function kubectlSandbox(opts: KubectlSandboxOptions = {}): SandboxPort {
-  const ns = opts.namespace ?? "default";
-  const imagesPath = opts.imagesPath ?? join(IMAGES_MOUNT, IMAGES_KEY);
-  const heldPath = opts.heldPath ?? join(HELD_MOUNT, HELD_KEY);
-  const workRoot = opts.workRoot ?? "/work";
-  const readyTimeoutMs = opts.readyTimeoutMs ?? 120_000;
-  const repoTimeoutMs = opts.repoTimeoutMs ?? 25 * 60_000;
-  const pollMs = opts.pollMs ?? 1_000;
-  const exec = opts.exec ?? defaultKubectlExec;
-  const credentials = opts.credentials ?? [];
-
-  const leaseIntervalMs = opts.leaseIntervalMs ?? 5 * 60_000;
-  const base = ["--namespace", ns, ...(opts.context ? ["--context", opts.context] : [])];
-
-  /** The Sandbox's token Secret — mounted into the Custodian container, and nothing else in the pod. */
-  const secretName = (name: string) => `${name}-token`;
-
-  /**
-   * The User Container (ADR-0005): the opt-in third seat, composed only when the wrapper's static
-   * `user` option names an image (ADR-0049). The ZERO-CONTRACT seat — jr2 injects nothing, probes
-   * nothing, overrides nothing. So: no `command` (its own entrypoint runs, untouched), no `env`,
-   * no `envFrom`, no CA bundle, no ports, no resources. Every key jr2 forwarded would be a crack in
-   * "jr2 puts nothing in it", and widening the one authoring string to an object stays compatible
-   * if a concrete need ever argues its own way in. (Git's dubious-ownership guard is the line's
-   * cost, accepted with
-   * eyes open — ADR-0005: safe.directory is honored only from files this seat's image owns, so an
-   * image whose sessions run git carries its own line.)
-   *
-   * `/work` read-write plus the checkouts' two read-only halves are the single exception, and they
-   * are not an injection but the point: this seat and the Harness mount ONE worktree, so the human
-   * and the Agent see identical files — which is also why ADR-0005's cross-uid pair (the pod's
-   * `fsGroup`, the attach's default ACL) exists at all. The caches ride along because they are
-   * half of the same files: the worktrees are `--shared` clones whose alternates resolve objects
-   * from `/repos/<key>` (ADR-0004/0051), so a seat with `/work` alone holds checkouts whose every
-   * borrowed object is missing ("unable to normalize alternate object path"). `/opt/jr2` is the
-   * other half (ADR-0053): `origin`'s fetch url is a program on that volume, so a seat without it
-   * holds checkouts whose `git fetch` dies — and with it the human gets the same fetch as the
-   * Agent, with no credential of their own. Nothing else follows it in: `ext::` names the program
-   * by absolute path, so this seat still gets no env, no command, and no probe. The repo volumes
-   * are the operator's — it defines `repo-<key>` for every key the CR names — so this seat mounts
-   * them by name. The Custodian is deliberately not given any of the three: it reads no worktree, and it is the
-   * container holding the pod's credentials, so it gets the narrowest mount set that works.
-   * It also carries no `securityContext`, which the operator reads as the exemption — root is
-   * ALLOWED here, because hardening a seat whose identity is "what jr2 does not own" is an opinion,
-   * and the standard managed-access shape (a root sshd that setuids sessions down) must run
-   * unmodified.
-   */
-  const userSidecar = async (refs: ImageRefs, image: string, keys: string[]) => ({
-    name: "user",
-    image: await resolveUserImage(refs, image),
-    volumeMounts: [
-      { name: "work", mountPath: workRoot },
-      { name: "runtime", mountPath: RUNTIME_MOUNT, readOnly: true },
-      ...keys.map((key) => ({ name: repoVolumeName(key), mountPath: repoMountPath(key), readOnly: true })),
-    ],
-  });
-
-  /** The pod's sidecar list (ADR-0001: opaque fragments the operator schedules verbatim). The
-   * Custodian is ALWAYS here: a Sandbox without one is a pod that comes up Ready and then parks its
-   * Machine forever on a Menu it cannot read (ADR-0013). The User Container joins it only when the
-   * spec named one. */
-  const sidecarsFor = async (custodian: CustodianComposition, refs: ImageRefs, keys: string[], user?: string) => [
-    custodian.custodianContainer,
-    ...(user !== undefined ? [await userSidecar(refs, user, keys)] : []),
-  ];
-
-  // The Harness container's env: the instance's passthrough (`harness.env` — e.g. model
-  // config) first, then the mechanism-owned vars (the Custodian's address and every Stand-in, the
-  // proxy and the trust bundles — custodian.ts), which win on collision. Note the asymmetry stands
-  // (ADR-0013): user env/envFrom land on the HARNESS container only — never on the Custodian, whose
-  // mounts are composed here and carry the pod's credentials.
-  //
-  // The gate rides LAST (ADR-0058): the digest of the bearer the Orchestrator derives for THIS
-  // Sandbox. A digest because the Agent reads this env; last because a `harness.env` entry of the
-  // same name must not be able to choose the Harness's credential.
-  const harnessEnv = (name: string, custodian: CustodianComposition): HarnessEnvVar[] => {
-    if (!opts.signingKey) throw new Error("kubectlSandbox: a Harness needs a signingKey to check its bearer");
-    return [
-      ...(opts.env ?? []),
-      ...custodian.harnessEnv,
-      { name: "JR2_HARNESS_TOKEN_SHA256", value: harnessTokenDigest(opts.signingKey, name) },
-    ];
-  };
-
-  /**
-   * The two init steps, in order (ADR-0037). They are plain container fragments the operator
-   * schedules without understanding, exactly like sidecars — the operator stays agent-agnostic
-   * (ADR-0001), so "how a Sandbox gets its runtime" is composed here, not reconciled there.
-   *
-   * 1. `runtime` — the kit's Harness image, copying its `/opt/jr2` into the shared emptyDir. This
-   *    is what makes the runtime's version ride the VOLUME rather than the image: a kit edit moves
-   *    the harness image's own tag and re-images future pods without touching a single Sandbox
-   *    Image tag, which is the only way a registry-ref image could ever follow a kit update.
-   * 2. `preflight` — the USER'S image with that volume mounted, running the probe. Ordered second
-   *    because it needs what the first one wrote.
-   *
-   * Both carry jr2's hardened context explicitly, and `preflight` runs the probe in the SAME seat
-   * the Harness will get — the image's own user, or ADR-0037's fallback — because a probe that
-   * proved a different uid's `$HOME` proved nothing.
-   */
-  const initContainersFor = (refs: ImageRefs, seat: Seat) => [
-    {
-      name: "runtime",
-      image: refs.harness,
-      // The copy's rules live beside the tree they copy (`deploy/harness/init-copy`), not in a
-      // string here: `/opt/jr2` is a published surface whose SHAPE is load-bearing — node's rpath
-      // is `$ORIGIN/../lib`, so `bin/` and `lib/` must land as siblings — and the script proves
-      // its own result by running the copied node before the pod moves on.
-      command: [`${RUNTIME_MOUNT}/bin/init-copy`, RUNTIME_STAGE],
-      volumeMounts: [{ name: "runtime", mountPath: RUNTIME_STAGE }],
-      securityContext: HARDENED,
-    },
-    {
-      name: "preflight",
-      image: seat.image,
-      command: ["/bin/sh", "-c", PREFLIGHT_SCRIPT],
-      ...(seat.env.length ? { env: seat.env } : {}),
-      volumeMounts: [{ name: "runtime", mountPath: RUNTIME_MOUNT, readOnly: true }, ...seat.homeMount],
-      securityContext: seat.securityContext,
-    },
-  ];
-
-  /**
-   * The primary container's seat: which image runs, as whom, and with what home. One value, built
-   * once per provision and shared by the Harness container and the preflight, so the probe cannot
-   * drift from the thing it proves.
-   */
-  type Seat = {
-    image: string;
-    env: HarnessEnvVar[];
-    homeVolume: Array<{ name: string; emptyDir: Record<string, never> }>;
-    homeMount: Array<{ name: string; mountPath: string }>;
-    securityContext: typeof HARDENED & { runAsUser?: number };
-  };
-
-  const seatFor = async (refs: ImageRefs, name?: string): Promise<Seat> => {
-    // ONE resolution, so the ref and the two seat facts can never come off different legs of
-    // ADR-0037's chain (images.ts). It is async because a `file:` context is keyed by its content
-    // digest, which is a directory walk — the price of the host and the pod agreeing about an
-    // image without a path table (ADR-0049).
-    const { ref: image, fallbackSeat, refusedUser } = await resolveSandboxImage(refs, name);
-    // Fail HERE, before a Secret or a CR exists, on a `USER` the kubelet will refuse (images.ts).
-    // The alternative is the worst shape a failure has: the preflight container never starts, so
-    // it has no logs, and the whole 120s Ready budget burns before anything is said. The converge
-    // already inspected the image, so this is knowable at zero cost — and the message names the
-    // edit, because the fix is one line of the caller's own Dockerfile.
-    if (refusedUser !== undefined) {
-      throw new Error(
-        `the Sandbox Image "${name ?? "default"}" declares \`USER ${refusedUser}\`, which cannot run a jr2 seat: ` +
-          "every jr2-owned container is `runAsNonRoot` with no `runAsUser`, so the kubelet needs a NUMERIC " +
-          "non-zero uid it can check without reading the image (ADR-0005). Change the Dockerfile's last " +
-          "`USER` to that uid (e.g. `USER 1000`, or drop the line entirely and jr2 supplies uid 1000 with a " +
-          "writable HOME — ADR-0037), then re-run `jr2 up`.",
-      );
-    }
-    // The common case, and the one ADR-0037 is written around: the image chose its `USER` and its
-    // `HOME`, and jr2 touches neither — the human who execs in lands in the environment the
-    // image's author built, dotfiles included.
-    if (!fallbackSeat) {
-      return { image, env: [], homeVolume: [], homeMount: [], securityContext: HARDENED };
-    }
-    return {
-      image,
-      env: [{ name: "HOME", value: FALLBACK_HOME }],
-      homeVolume: [{ name: "home", emptyDir: {} }],
-      homeMount: [{ name: "home", mountPath: FALLBACK_HOME }],
-      securityContext: { ...HARDENED, runAsUser: FALLBACK_UID },
-    };
-  };
-
-  const crFor = async (
-    req: { name: string; runId: string; workflow: string; image?: string; user?: string; workGroup?: number },
-    refs: ImageRefs,
-    repos: FencedRepo[],
-    custodian: CustodianComposition,
-  ) => {
-    const seat = await seatFor(refs, req.image);
-    const sidecars = await sidecarsFor(
-      custodian,
-      refs,
-      repos.map((r) => r.key),
-      req.user,
-    );
-    return {
-      apiVersion: "core.jr2.dev/v1alpha1",
-      kind: "Sandbox",
-      metadata: {
-        name: req.name,
-        namespace: ns,
-        // The run↔workspace link `jr2 ls` groups by (ADR-0009/0012) — readable without the host.
-        labels: { "jr2.dev/run": req.runId, "jr2.dev/workflow": req.workflow },
-      },
-      spec: {
-        // The Sandbox Image, unmodified (ADR-0037) — the user's tools, its own USER and HOME, and
-        // jr2's runtime arriving beside it on a volume. This container is both the Harness and the
-        // human's `exec` shell.
-        image: seat.image,
-        // The one thing jr2 takes from the image: its command. A container has exactly one, and it
-        // must be the Harness's — a pod whose main process is the user's entrypoint keeps
-        // "Running" through a Harness death, which makes the operator's Ready probe a lie.
-        command: HARNESS_COMMAND,
-        // Hardened, and the ONLY place jr2 ever names a uid: ADR-0037's fallback for an image that
-        // declared none. Stating the whole context here rather than leaving it to the operator's
-        // default is what makes that possible — the operator hardens only what says nothing.
-        securityContext: seat.securityContext,
-        idleTimeout: opts.idleTimeout ?? "30m",
-        // Placement (ADR-0052): the Instance's word on which nodes are Sandbox nodes, raw pod-spec
-        // shapes the operator copies verbatim. Absent keys are absent here too — the CR says
-        // nothing, and the pod lands wherever an ordinary pod lands. The operator's own soft
-        // affinity toward nodes holding this Sandbox's caches sits beside these untouched: a
-        // preference never conflicts with a requirement.
-        ...(opts.placement?.nodeSelector ? { nodeSelector: opts.placement.nodeSelector } : {}),
-        ...(opts.placement?.tolerations?.length ? { tolerations: opts.placement.tolerations } : {}),
-        // The work group (ADR-0005), the ownership half of cross-uid sharing on `/work`.
-        // Kubernetes grants it as a supplemental group to every container, and puts a setgid
-        // group on the volume root that propagates down; the WRITABILITY half is the default ACL
-        // the attach stamps on each repo root (attachScript below), without which fsGroup gives
-        // group-READ, which is the trap. Both are inert when the uids match.
-        fsGroup: req.workGroup ?? DEFAULT_WORK_GROUP,
-        // Ordered, and before any container starts: populate `/opt/jr2`, then prove the image on it.
-        initContainers: initContainersFor(refs, seat),
-        // Never empty: the Custodian's address is unconditional. The seat's own vars (the fallback
-        // `HOME`) come FIRST, so the instance's `harness.env` can still override them the way it
-        // overrides anything the image set.
-        env: [...seat.env, ...harnessEnv(req.name, custodian)],
-        ...(opts.envFrom?.length ? { envFrom: opts.envFrom } : {}),
-        // What the AGENT gets: an address on its own loopback and a Stand-in for every credential,
-        // and no credential anywhere. The Custodian beside it holds them (ADR-0013, ADR-0059).
-        sidecars,
-        // The Repos this Sandbox attaches, by cache key (ADR-0051). The operator does the rest: a
-        // `repo-<key>` volume per entry — the node's cache, hostPath, read-only in the primary
-        // container at `/repos/<key>` — a soft affinity toward nodes already holding them, and
-        // `Ready` only once every one is present on the pod's node and fetched since this CR
-        // asked. Read-only is load-bearing twice (ADR-0004): no write contention, and nothing in
-        // a Sandbox can `gc` the object store its `--shared` clones borrow from.
-        repos: repos.map(({ key, url }) => ({ key, url })),
-        volumes: [
-          // The worktree root is a POD volume, not a directory baked into the image. Two reasons,
-          // both load-bearing: every jr2-owned seat runs as an unprivileged uid, which cannot mkdir
-          // under `/` — so an image-owned `/work` would make every attach fail — and `/work` is
-          // the one thing all three containers share (ADR-0005), so human and Agent see identical
-          // files. An emptyDir lands group-writable under the pod's fsGroup, so it is writable
-          // whatever uid the Sandbox Image runs as: `/work` unclaimed is the only image contract.
-          { name: "work", emptyDir: {} },
-          // jr2's runtime (ADR-0037). An emptyDir, so it lives and dies with the pod and carries
-          // the version the pod STARTED with — a live Sandbox keeps its runtime across a kit
-          // update, the same create-if-absent stance ADR-0038 takes for images.
-          { name: "runtime", emptyDir: {} },
-          // Only for ADR-0037's fallback seat: uid 1000 on a stranger's base has no home at all.
-          ...seat.homeVolume,
-          // The Custodian's (its values, leaves and config), and the trust ConfigMap (ADR-0020).
-          ...custodian.volumes,
-        ],
-        // CR-level volumeMounts land on the HARNESS container only (the operator's contract): the
-        // trust bundles, and none of the Custodian's volumes. The Repo caches are not listed: the
-        // operator mounts each `repo-<key>` into this container itself.
-        volumeMounts: [
-          { name: "work", mountPath: workRoot },
-          // Read-only: nothing writes under `/opt/jr2` at runtime, and the Agent has code execution
-          // in this container — leaving its own runtime writable would let a turn edit it.
-          { name: "runtime", mountPath: RUNTIME_MOUNT, readOnly: true },
-          ...seat.homeMount,
-          ...custodian.harnessMounts,
-        ],
-      },
-    };
-  };
-
-  type SandboxStatus = {
-    phase?: string;
-    endpoint?: string;
-    podUID?: string;
-    uid?: string;
-    conditions?: Condition[];
-  };
-
-  /** Parse a Sandbox CR off any kubectl call that printed one (`get -o json`, and the lease's
-   * `annotate -o json` — which returns the object AFTER the patch, status included). */
-  const readSandbox = (stdout: string): SandboxStatus => {
-    const parsed = JSON.parse(stdout) as {
-      metadata?: { uid?: string };
-      status?: { phase?: string; endpoint?: string; podUID?: string; conditions?: Condition[] };
-    };
-    return { ...(parsed.status ?? {}), uid: parsed.metadata?.uid };
-  };
-
-  const conditionOf = (status: SandboxStatus | undefined, type: string): Condition | undefined =>
-    status?.conditions?.find((c) => c.type === type);
-
-  const getSandbox = async (name: string): Promise<SandboxStatus | undefined> => {
-    try {
-      const { stdout } = await exec(["get", "sandbox", name, ...base, "-o", "json"]);
-      return readSandbox(stdout);
-    } catch (err) {
-      if (isNotFound(err)) return undefined;
-      throw err;
-    }
-  };
-
-  /**
-   * Ask the POD whether it is stuck on a fault the Sandbox's phase cannot express (see
-   * {@link rootImageFault}). The CR is the port's normal window on a provision; this is the one
-   * question it cannot answer, because the operator reports "not Ready yet" for a pod that will
-   * never be Ready and one that simply has not started.
-   *
-   * Absent or unreadable answers undefined: the pod trails the CR by a moment at every provision,
-   * and a missing pod is a normal early poll, never evidence of a fault. This may only ever CONVERT
-   * a failure that was already going to happen into a named one.
-   */
-  const podFault = async (name: string): Promise<string | undefined> => {
-    try {
-      const { stdout } = await exec(["get", "pod", name, ...base, "-o", "json"]);
-      return rootImageFault(JSON.parse(stdout));
-    } catch {
-      return undefined;
-    }
-  };
-
-  /**
-   * Mint this Sandbox's token into a Secret, BEFORE the CR exists — the operator creates the pod
-   * the moment it sees the CR, and a pod whose volume names an absent Secret never starts.
-   * Idempotent by construction: the token is the Sandbox's name, signed (tokens.ts), so a
-   * re-provision after an orchestrator restart re-applies the SAME value, and the Custodian that has
-   * been holding it all along stays valid.
-   */
-  const applyTokenSecret = async (name: string): Promise<void> => {
-    if (!opts.signingKey) throw new Error("kubectlSandbox: a Custodian needs a signingKey to mint its Sandbox token");
-    const secret = {
-      apiVersion: "v1",
-      kind: "Secret",
-      metadata: { name: secretName(name), namespace: ns, labels: { "jr2.dev/sandbox": name } },
-      type: "Opaque",
-      stringData: { JR2_SANDBOX_TOKEN: sandboxToken(opts.signingKey, name) },
-    };
-    await exec(["apply", ...base, "-f", "-"], { input: JSON.stringify(secret) });
-  };
-
-  /**
-   * Make the Secret a child of the Sandbox CR, so Kubernetes reaps it whenever the CR goes — including
-   * the paths no jr2 code observes (the operator's idle-timeout GC, a `kubectl delete sandbox` by hand).
-   * Needs the CR's uid, so it can only happen after the apply; a failure here leaks a Secret, never a
-   * pod, so it is not worth failing the provision over.
-   */
-  const ownSecret = async (name: string, uid: string | undefined): Promise<void> => {
-    if (!uid) return;
-    const ownerRef = [
-      { apiVersion: "core.jr2.dev/v1alpha1", kind: "Sandbox", name, uid, controller: true, blockOwnerDeletion: false },
-    ];
-    await exec([
-      "patch",
-      "secret",
-      secretName(name),
-      ...base,
-      "--type",
-      "merge",
-      "-p",
-      JSON.stringify({ metadata: { ownerReferences: ownerRef } }),
-    ]).catch(() => {});
-  };
-
-  /**
-   * Every Repo the provision names, resolved to its identity and key — and FENCED (ADR-0051). A
-   * per-run url is the run's input; one no `git.credentials` entry admits is refused here, before
-   * anything is read or applied, naming the list. A bound url is admitted without a match: it is
-   * code the instance typechecked and deployed. Two slots spelling one repository collapse to one
-   * CR entry (first spelling wins) — one cache, however many slots borrow from it — and the
-   * resource is BOUND when any of those slots is the Machine's: a per-run slot alone leaves it on
-   * `jr2 gc`'s clock.
-   */
-  const fencedRepos = (name: string, repos: ProvisionedRepo[]): FencedRepo[] => {
-    const byKey = new Map<string, FencedRepo>();
-    for (const repo of repos) {
-      const { identity, key } = repoIdentity(repo.url);
-      if (repo.perRun && !matchCredential(identity, credentials)) {
-        const entries = credentials.map((c) => c.match).join(", ") || "none";
-        throw new Error(
-          `Sandbox "${name}" refuses the per-run repo ${repo.url} for slot "${repo.slot}": no git.credentials ` +
-            `entry matches "${identity}" (entries: ${entries}). A per-run url can spend the cluster's credential ` +
-            "against any host, so jr2.config.ts must admit it by prefix (ADR-0051).",
-        );
-      }
-      const seen = byKey.get(key);
-      if (seen === undefined) byKey.set(key, { key, url: repo.url, identity, bound: !repo.perRun });
-      else seen.bound ||= !repo.perRun;
-    }
-    return [...byKey.values()];
-  };
-
-  /**
-   * What the operator will hold this Sandbox's Ready on, and what an attach reads afterwards
-   * (ADR-0051). `Ready` with reason `RepoCloneFailed` is terminal for the provision — the cache
-   * agent could not clone onto the node the pod landed on, and the reason names it — so the loop
-   * fails on it by name rather than burning the budget. `ReposFresh=False` is the other verdict:
-   * the caches are there but a fetch since this CR asked failed, so the attach proceeds STALE and
-   * says so. Remembered per name until the Sandbox is destroyed, because the attach is a separate
-   * call. The operator takes that verdict once per pod and keeps it, so a provision re-run on
-   * snapshot restore reads the same one — unless the pod was replaced, which is the lease's news.
-   */
-  const staleByName = new Map<string, string>();
-
-  return {
-    async provision(req) {
-      // The fence first: a refused url costs nothing — no map read, no Secret, no CR.
-      const repos = fencedRepos(req.name, req.repos);
-      if (opts.repos === undefined) {
-        throw new Error(
-          `Sandbox "${req.name}" cannot be provisioned: this port has no Repo-resource port (ADR-0051). The ` +
-            "operator holds a Sandbox's Ready until every Repo it names exists as a resource, and creating " +
-            "them is this provision's job — build the port with `repos: kubectlRepos(...)`.",
-        );
-      }
-      // Read PER PROVISION, and next (ADR-0038). Not hoisted into `kubectlSandbox()`: a boot-time
-      // read would freeze the map for the process lifetime, which is precisely the Deployment-env
-      // behavior the ConfigMap mount was chosen over — the point of the mount is that a `jr2 up`
-      // reaches future Sandboxes without rolling the Orchestrator. Reading before the Secret apply
-      // also means an unknown image name costs nothing: no Secret, no CR, nothing to clean up.
-      const refs = await readImageRefs(imagesPath);
-      // The same stance for what the Custodian holds (ADR-0059): `jr2 up` resolved it host-side,
-      // where `.env` exists, and this process reads the result — never its own config, whose
-      // `.env` values are absent in-cluster.
-      const manifest = await readHeldManifest(heldPath);
-      const custodian = custodianComposition(manifest, {
-        image: refs.custodian,
-        token: { secret: secretName(req.name), key: "JR2_SANDBOX_TOKEN" },
-        sandbox: req.name,
-        caBundle: opts.caBundle === true,
-      });
-      const cr = await crFor(req, refs, repos, custodian);
-
-      // The Repo resources, BEFORE the CR names them (ADR-0051): a bound one already exists from
-      // the boot and only its eviction clock moves — this run's spelling never rewrites the spec
-      // the boot stated; a per-run one is created here, at its first attach, and every later
-      // attach anywhere finds it and restates its credential, so the `git.credentials` fix
-      // `repoCloneError` names reaches the cache at the next run. After the image resolution, so
-      // a refused image still costs nothing; before the token Secret, so no Secret is minted for
-      // a Sandbox whose Repo could not be recorded.
-      for (const repo of repos) await opts.repos.ensure(repo);
-
-      await applyTokenSecret(req.name); // before the CR: the pod's Custodian mounts it at start
-      await exec(["apply", ...base, "-f", "-"], { input: JSON.stringify(cr) });
-
-      const applied = Date.now();
-      // Two budgets from one instant (see the options): the pod's until the operator has seen the
-      // pod Ready, the Repos' from the first poll that finds the Sandbox held on a Repo reason —
-      // which the operator reports only once the pod IS Ready, so the preflight has already passed
-      // and what remains is a clone or a fetch on the node. Sticky: a pod that came up once is not
-      // a pod that will never come up, whatever it does afterwards.
-      let held = false;
-      let owned = false;
-      let polls = 0;
-      for (;;) {
-        const status = await getSandbox(req.name);
-        if (!owned && status?.uid) ((owned = true), await ownSecret(req.name, status.uid));
-        // The pod is read at a COARSER cadence than the CR, and only while not Ready. The fault it
-        // looks for is terminal — the kubelet never retries out of it — so learning about it a few
-        // seconds late costs nothing, while reading the pod on every poll would double this port's
-        // API traffic for every healthy provision in the cluster.
-        if (status?.phase !== "Ready" && polls++ % POD_CHECK_EVERY === 0) {
-          const fault = await podFault(req.name);
-          if (fault) throw new Error(rootImageError(req.name, fault));
-        }
-        // Terminal for THIS provision: the cache agent tried to clone onto the pod's node and git
-        // refused. The operator's message carries the key, the node, and git's own words; the
-        // agent keeps retrying on its own, so `jr2 status` will show the same error until it is fixed.
-        const ready = conditionOf(status, "Ready");
-        if (status?.phase !== "Ready" && ready?.reason === REPO_CLONE_FAILED) {
-          throw new Error(repoCloneError(req.name, ready.message ?? ready.reason));
-        }
-        if (ready?.reason !== undefined && REPO_HELD.has(ready.reason)) held = true;
-        if (status?.phase === "Ready") {
-          // Only `phase: Ready` means serving — status.endpoint appears earlier (ADR-0001).
-          if (!status.endpoint) throw new Error(`Sandbox "${req.name}" is Ready but reports no endpoint`);
-          // Freshness degrades, absence does not (ADR-0051): Ready with `ReposFresh=False` is a
-          // Sandbox whose caches exist but could not be fetched since it asked. Remembered for the
-          // attach, which is where a slot can be named; forgotten when the caches are fresh.
-          const fresh = conditionOf(status, "ReposFresh");
-          if (fresh?.status === "False" && fresh.message) staleByName.set(req.name, fresh.message);
-          else staleByName.delete(req.name);
-          // The identity the lease will hold this workspace to (ADR-0021). Ready means the pod
-          // is up, so the operator has published it; an operator too old to do so leaves it
-          // undefined and the lease falls back to presence.
-          return { endpoint: status.endpoint, identity: status.podUID };
-        }
-        if (Date.now() >= applied + (held ? repoTimeoutMs : readyTimeoutMs)) {
-          // A Sandbox the operator held on its Repos ran out the Repo budget: the pod is up and the
-          // preflight passed, so the hint about the image would be a lie. What is true is the
-          // operator's own verdict — which Repo, on which node — and that the agent is still at it.
-          if (held) throw new Error(repoWaitError(req.name, repoTimeoutMs, ready));
-          // One last look before falling back to the hint: the fault may have appeared inside the
-          // final interval, and a named error beats a timeout in every case where both are true.
-          const fault = await podFault(req.name);
-          if (fault) throw new Error(rootImageError(req.name, fault));
-          // Otherwise name where to look, because the next most likely cause is an image that
-          // misses ADR-0037's floor, and that failure is an INIT container's — invisible in the
-          // phase alone. A musl or git-less base dies INSIDE the preflight, on jr2's own message.
-          // The operator's own verdict rides along when it has one: a Repo still pending on the
-          // node (a slow clone) reads very differently from a preflight death.
-          throw new Error(
-            `Sandbox "${req.name}" never reached Ready (last phase: ${status?.phase ?? "absent"}) — if its ` +
-              "Sandbox Image is new, check the preflight: `kubectl logs " +
-              req.name +
-              " -c preflight` (ADR-0037's floor: glibc, git, a writable HOME, a numeric non-root USER)." +
-              (ready?.message
-                ? `\n  the operator's Ready condition says: ${ready.reason ?? "?"}: ${ready.message}`
-                : ""),
-          );
-        }
-        await sleep(pollMs);
-      }
-    },
-
-    async attach(req) {
-      const { script, repos, review } = attachScript(req.spec, req.repos, { reposMount: REPOS_MOUNT, workRoot });
-      // `-c harness` is unchanged and still correct after ADR-0037: the primary container runs the
-      // Sandbox Image, so `git` here is the git the user chose. Never `-c user` — that seat is
-      // zero-contract, may hold no git at all, and jr2 commands nothing in it (ADR-0005).
-      await exec(["exec", `pod/${req.name}`, ...base, "-c", "harness", "--", "sh", "-ec", script]);
-      const stale = staleSlots(req.repos, staleByName.get(req.name));
-      return { repos, ...(review ? { review } : {}), ...(stale ? { stale } : {}) };
-    },
-
-    leaseIntervalMs,
-
-    async renew(name) {
-      // ONE call, both directions: `annotate --overwrite -o json` writes the stamp and prints the
-      // object as it stands afterwards, status included. So asserting liveness and learning
-      // whether the workspace survived cost exactly one API round trip (ADR-0021).
-      try {
-        const { stdout } = await exec([
-          "annotate",
-          "sandbox",
-          name,
-          ...base,
-          `jr2.dev/keepalive=${new Date().toISOString()}`,
-          "--overwrite",
-          "-o",
-          "json",
-        ]);
-        return { present: true, identity: readSandbox(stdout).podUID };
-      } catch (err) {
-        if (isNotFound(err)) return { present: false };
-        throw err; // anything else is UNKNOWN — the caller must not read it as loss
-      }
-    },
-
-    async destroy(name) {
-      staleByName.delete(name);
-      // The Secret is an owned child of the CR, so deleting the CR reaps it — this is belt and
-      // braces for the case where the ownerRef patch didn't land.
-      await exec(["delete", "sandbox", name, ...base, "--ignore-not-found"]);
-      await exec(["delete", "secret", secretName(name), ...base, "--ignore-not-found"]).catch(() => {});
-    },
-  };
-}
-
-/** The pod volume the operator defines for one Repo's node cache, and where it lands in the
- * primary container (ADR-0051). Two halves of one contract with the operator, spelled here so the
- * User Container's mounts and the attach's clone source agree with it by construction. */
-export const repoVolumeName = (key: string): string => `repo-${key}`;
-export const repoMountPath = (key: string): string => `${REPOS_MOUNT}/${key}`;
-
-/** One Repo as the provision names it: the CR entry, plus what its resource records — the
- * identity, and whether a Machine's slot (not only the run's) binds it. */
-type FencedRepo = { key: string; url: string; identity: string; bound: boolean };
-
-/** One entry of a Sandbox CR's `status.conditions`, as the operator writes it. */
-type Condition = { type: string; status: string; reason?: string; message?: string };
-
-/** The operator's Ready reason when the cache agent could not clone onto the pod's node
- * (sandbox_controller.go) — the one Ready verdict a provision cannot wait out. */
-const REPO_CLONE_FAILED = "RepoCloneFailed";
-
-/** The operator's Ready reasons that hold a Sandbox whose POD is Ready on its Repos
- * (sandbox_controller.go, `reposReadiness`): the resource not yet seen, or the node's cache
- * agent still cloning or fetching. The wait against them is the Repo budget, not the pod's. */
-const REPO_HELD: ReadonlySet<string> = new Set(["RepoMissing", "RepoPending"]);
-
-/**
- * The Repo budget ran out with the operator still holding the Sandbox. The pod is up, so the
- * preflight is not the question; the verdict names the Repo and the node, and the port adds
- * what the operator cannot say: the agent is still working, `jr2 status` shows it per node, and
- * the budget is the port's, not the clone's.
- */
-function repoWaitError(name: string, budgetMs: number, ready: Condition | undefined): string {
-  const verdict = ready?.message ? `${ready.reason ?? "?"}: ${ready.message}` : "no Ready condition reported";
-  return (
-    `Sandbox "${name}" waited ${Math.round(budgetMs / 60_000)}m for its Repos and the operator still holds it — ` +
-    `${verdict} (ADR-0051). The node's cache agent clones a cold node once and fetches before every attach; ` +
-    "`jr2 status` reports each Repo per node. Start the run again once the cache is present, or raise " +
-    "the port's `repoTimeoutMs` for a repository whose clone outlasts it."
-  );
-}
-
-/**
- * The fix beside the symptom. The operator's message carries the Repo, the node, and git's own
- * words; what it cannot say is that the cache agent keeps retrying, that `jr2 status` reports the
- * same line per node, or where a credential is configured (ADR-0047/0051).
- */
-function repoCloneError(name: string, verdict: string): string {
-  return (
-    `Sandbox "${name}" cannot start: ${verdict} (ADR-0051). The node's cache agent keeps retrying on its own — ` +
-    "fix the url or its git.credentials entry (an ssh url needs its deploy key registered with the host, " +
-    "ADR-0047), then start the run again; `jr2 status` reports the same error per node until it clears."
-  );
-}
-
-/**
- * The `ReposFresh=False` message, keyed back to SLOTS for the attach's `stale`. The operator
- * writes one clause per stale Repo — `Repo "<key>" on node <n> is stale: <error>` — joined by
- * `; `; each slot whose key a clause names gets that clause. A message that names no key at all
- * lands on every slot: a verdict with no address is still a verdict.
- */
-function staleSlots(
-  repos: Array<{ slot: string; url: string }>,
-  message: string | undefined,
-): Record<string, string> | undefined {
-  if (!message) return undefined;
-  const byKey = new Map<string, string>();
-  for (const clause of message.split("; ")) {
-    const key = /^Repo "([^"]+)"/.exec(clause)?.[1];
-    if (key !== undefined) byKey.set(key, clause);
-  }
-  const stale: Record<string, string> = {};
-  for (const repo of repos) {
-    const clause = byKey.size === 0 ? message : byKey.get(repoIdentity(repo.url).key);
-    if (clause !== undefined) stale[repo.slot] = clause;
-  }
-  return Object.keys(stale).length ? stale : undefined;
-}
-
-/**
- * The post-Ready attach step as one idempotent in-pod script (ADR-0004, ADR-0051): per Repo Slot
- * in declaration order, a pod-local `git clone --shared` borrowing objects from the node's
- * read-only cache at `/repos/<key>` — `<slot>/default/`, a checkout of the Repo's default branch
- * — then the branch worktree as a sibling (gwtmux layout: `<slot>/default/` + `<slot>/<branch>/`). With a `reviewSha`, also the detached review worktree
- * (ADR-0028) — another sibling. `repos` keeps the slots' declaration order. Exported for the port's
- * tests; the workflow never sees it.
- */
-export function attachScript(
-  spec: WorkspaceSpec,
-  repos: Array<{ slot: string; url: string; ref?: string }>,
-  paths: { reposMount: string; workRoot: string },
-): { script: string; repos: Record<string, string>; review?: Record<string, string> } {
-  const worktrees: Record<string, string> = {};
-  const review: Record<string, string> = {};
-  // The cache is written by the node's cache agent and read here as the Harness's unprivileged
-  // uid (ADR-0001/0004/0051), so git's dubious-ownership guard would refuse the clone source.
-  // safe.directory is only honored from global/system config (never `-c`), and inside the pod
-  // every path is jr2-owned — trusting them all is the honest scope.
-  // The attach runs via exec, not as a child of the Harness process, so it does NOT inherit the
-  // Harness's `umask 002` — without its own, the repo roots it mkdirs land 755 and the work group
-  // could never create a file at a tree's top. INSIDE the trees the umask stops mattering: the
-  // default ACL stamped below governs everything created beneath a repo root (ADR-0005).
-  const lines: string[] = [`umask 002`, `git config --global safe.directory '*'`];
-  const branchDir = spec.branch.replace(/\//g, "-");
-  for (const repo of repos) {
-    const slotDir = `${paths.workRoot}/${repo.slot}`;
-    const dflt = `${slotDir}/default`;
-    const worktree = `${slotDir}/${branchDir}`;
-    const { identity, key } = repoIdentity(repo.url);
-    const cache = `${paths.reposMount}/${key}`;
-    worktrees[repo.slot] = worktree;
-    lines.push(
-      `mkdir -p ${sq(slotDir)}`,
-      // BEFORE the clone fills it: a default ACL is inherited at creation, never retrofitted, so
-      // the stamp must exist while the tree is still empty. From here down, both seats' files land
-      // group-writable with zero umask lines in any image (ADR-0005); on a filesystem without
-      // POSIX ACLs the helper warns and exits 0, degrading to the umask sharing above.
-      `/opt/jr2/bin/work-acl ${sq(slotDir)}`,
-      `[ -d ${sq(`${dflt}/.git`)} ] || git clone --shared ${sq(cache)} ${sq(dflt)}`,
-      // No ref → the Repo's own default branch: this clone's `origin/HEAD` tracks the cache's
-      // HEAD, which the cache agent's clone pointed at the remote's default (ADR-0004).
-      `[ -d ${sq(worktree)} ] || git -C ${sq(dflt)} worktree add ${sq(worktree)} -b ${sq(spec.branch)} ${repo.ref === undefined ? sq("origin/HEAD") : baseOf(dflt, repo.ref)}`,
-      // Fetch/push split (ADR-0005). The FETCH url is a command, not a path (ADR-0053): git's
-      // built-in `ext::` transport runs the program on the runtime volume, which asks the node
-      // cache to fetch the remote, waits for the landing, then serves the cache — so every fetch
-      // inside the pod is a fetch of the remote's now, and a stale attach is stale only until the
-      // next fetch anyone in the pod runs. Git substitutes `%S` with the service it wants
-      // (`git-upload-pack`), and splits the rest on spaces with no quoting of its own, so the
-      // url's arguments carry none: the Repo's IDENTITY, never the cache key — that is the name a
-      // human reads in `git remote -v`, and a key is a derived directory name (ADR-0004). The
-      // program discovers the cache from the checkout's alternates, so the url says nothing about
-      // where the objects are. `git push` goes to the REAL remote — the Binding's own spelling, so
-      // a Machine that bound over ssh pushes over ssh even when the cache was cloned over https
-      // (ADR-0051). Push still succeeds only with a caller-supplied credential (a forwarded agent
-      // in the User Container); the pod itself holds none. `--` keeps the url an operand, never an
-      // option.
-      `git -C ${sq(dflt)} remote set-url origin -- ${sq(fetchUrl(identity))}`,
-      `git -C ${sq(dflt)} remote set-url --push origin -- ${sq(repo.url)}`,
-      // `ext` is on git's own "known scary" list, so its built-in default is `never` and the url
-      // above would die with `fatal: transport 'ext' not allowed` before the program ever ran.
-      // `user` is the policy ADR-0053 argues for, said out loud: a fetch A PERSON OR THE AGENT
-      // runs is allowed, and a recursive one git makes for itself (a submodule url, anything with
-      // `GIT_PROTOCOL_FROM_USER=0`) is still refused — so a repository cannot smuggle a program
-      // into this pod through a url jr2 did not write. Repo-level, on the pod-local clone: the
-      // linked worktrees share this config, so the branch worktree and the review worktree inherit
-      // it with no env and no `--global`.
-      `git -C ${sq(dflt)} config protocol.ext.allow user`,
-    );
-    if (spec.reviewSha) {
-      // The reviewer's seat (ADR-0028): a DETACHED HEAD at the sha under review, so a rogue write
-      // cannot move the branch and a rogue commit evaporates with the checkout. Forced checkout
-      // AND clean on every attach: a previous round's rogue edits (tracked) and leftovers
-      // (untracked) must not survive into this round — the review worktree's contents are the
-      // sha under review, period.
-      const reviewDir = `${worktree}-review`;
-      review[repo.slot] = reviewDir;
-      lines.push(
-        `[ -d ${sq(reviewDir)} ] || git -C ${sq(dflt)} worktree add --detach ${sq(reviewDir)} ${sq(spec.reviewSha)}`,
-        `git -C ${sq(reviewDir)} checkout --detach -f ${sq(spec.reviewSha)}`,
-        `git -C ${sq(reviewDir)} clean -fd`,
-      );
-    }
-  }
-  if (repos.length === 0)
-    throw new Error("the attach names no Repo Slot — nothing to attach (a workspace() declares at least one)");
-  return {
-    script: lines.join("\n"),
-    repos: worktrees,
-    ...(spec.reviewSha ? { review } : {}),
-  };
-}
-
-/**
- * `origin`'s fetch url for one Repo (ADR-0053): the `ext::` transport, the program's absolute path
- * on the runtime volume, the service git asks for, and the Repo's identity. No address: the
- * program asks the Custodian at its one loopback address, which every Harness pod has (ADR-0059).
- */
-function fetchUrl(identity: string): string {
-  return `ext::${UPLOAD_PACK} %S ${extArg(identity)}`;
-}
-
-/**
- * One argument of an `ext::` url, in git's own escaping. Git splits the url on spaces and reads
- * `%` as a placeholder introducer — `%S` is the service it substitutes — so it DIES on a `%` it
- * does not recognize (`fatal: Bad remote-ext placeholder '%2'`) and silently splits an argument
- * that holds a space. An identity carries both: a forge path may be percent-encoded
- * (`dev.azure.com/org/My%20Project/_git/repo`) and an scp-style url may hold a literal space. Git
- * spells those two `%%` and `% `, and the program receives the identity back exactly as written —
- * which it must, because the Orchestrator derives the cache key from that same string. The
- * placeholder jr2 writes itself (`%S`) is not escaped: it is git's, not an argument's.
- */
-function extArg(value: string): string {
-  return value.replace(/%/g, "%%").replace(/ /g, "% ");
-}
-
-/**
- * The commit-ish a Binding's `ref` names inside the pod-local clone, as a shell expression: the
- * remote-tracking branch `refs/remotes/origin/<ref>` when the clone has one, else `<ref>` as
- * written (a tag, a sha). A fresh clone holds ONE local branch — the default — so a bare
- * branch name is never a local ref here, and git's "worktree add" DWIM would then create the BASE
- * branch tracking `origin/<ref>` and discard `-b`: the Agent would commit on, and push to, the base
- * it was meant to branch FROM. Naming the remote-tracking ref outright leaves nothing to guess.
- */
-function baseOf(dflt: string, ref: string): string {
-  const remote = sq(`refs/remotes/origin/${ref}`);
-  return `"$(git -C ${sq(dflt)} rev-parse --verify -q ${remote} >/dev/null && printf %s ${remote} || printf %s ${sq(ref)})"`;
-}
-
-/** POSIX single-quote an argument for the in-pod `sh -ec` script. */
-function sq(s: string): string {
-  return `'${s.replace(/'/g, `'\\''`)}'`;
-}
-
-function isNotFound(err: unknown): boolean {
-  return err instanceof Error && /NotFound|not found/i.test(err.message);
-}
-
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-/** The `kubectl` on PATH, as every kubectl-driven port shells to it (this one and repos.ts). */
-export const defaultKubectlExec: KubectlExec = (args, opts) =>
-  new Promise((resolve, reject) => {
-    const child = execFile("kubectl", args, { maxBuffer: 8 * 1024 * 1024 }, (err, stdout, stderr) => {
-      if (err) reject(new Error(`kubectl ${args[0]} failed: ${stderr || err.message}`));
-      else resolve({ stdout, stderr });
-    });
-    if (opts?.input !== undefined) child.stdin?.end(opts.input);
-  });