@jr2/cli 0.3.0 → 0.5.1

package/src/kube.ts

@@ -5,6 +5,7 @@
 // one, shelling out to the same `kubectl` a human debugging the cluster would use.
 
 import { execFile, spawn } from "node:child_process";
+import { setTimeout as sleep } from "node:timers/promises";
 import { promisify } from "node:util";
 
 const exec = promisify(execFile);
@@ -31,7 +32,7 @@ function unreachable(e: unknown): string {
 
 // The fixed in-namespace object names live with the orchestrator (its entrypoint consumes them
 // too); re-exported here for the CLI's own modules.
-export { INSTANCE_SECRET, ORCHESTRATOR_PORT, ORCHESTRATOR_SERVICE } from "@jr2/orchestrator";
+export { INSTANCE_SECRET, ORCHESTRATOR_PORT, ORCHESTRATOR_SERVICE } from "@jr2/orchestrator/internal";
 
 export type KubePort = {
   /** The current kubectl context, or undefined when there is none configured. */
@@ -52,8 +53,8 @@ export type KubePort = {
 const ctxArgs = (context?: string): string[] => (context ? ["--context", context] : []);
 
 /** The loosely-typed kube object shape the converge logic inspects: labels carry the instance's
- * identity and the instance image's content hash; annotations carry the converged image map, which
- * is too long for a label value (ADR-0038). */
+ * identity; annotations carry the converged image map, which is too long for a label value
+ * (ADR-0038). */
 export type KubeObject = {
   metadata: {
     name: string;
@@ -101,8 +102,23 @@ export type KubeAdmin = {
   /** `kubectl apply -f -` of a multi-doc YAML or JSON manifest string. `serverSide` is for an
    * object too large for client-side apply's `last-applied-configuration` annotation (256 KiB) —
    * the trust bundles, which carry every root certificate (ADR-0059); jr2 is its only writer, so it
-   * takes the fields it owns (`--force-conflicts`). */
-  apply(opts: { manifest: string; context?: string; serverSide?: boolean }): Promise<void>;
+   * takes the fields it owns (`--force-conflicts`).
+   *
+   * A Secret is ALWAYS applied server-side, whatever `serverSide` says (`splitSecrets`): a
+   * client-side apply copies the whole object, its values too, into that annotation, and a value
+   * there is in plaintext for every reader of the object's metadata. On a Secret that an older
+   * `jr2 up` applied client-side, the first server-side apply migrates: kubectl (1.26 and later)
+   * moves the client-side manager's fields to `jr2`, so that apply removes the annotation and each
+   * key that it does not name. A migration that fails is only a warning on kubectl's stderr. So a
+   * fallback removes the annotation from each Secret after the apply; when the fallback fails, it is
+   * one line to `warn` (default: stderr), and the apply goes on. The values go on stdin, never in
+   * argv. */
+  apply(opts: {
+    manifest: string;
+    context?: string;
+    serverSide?: boolean;
+    warn?: (line: string) => void;
+  }): Promise<void>;
   /** `kubectl label --overwrite`. */
   label(opts: {
     kind: string;
@@ -138,24 +154,235 @@ export type KubeAdmin = {
     previous?: boolean;
     context?: string;
   }): Promise<string>;
-  /** Run a one-shot node script IN the cluster (`kubectl run --rm`) and return its output — the
-   * provider-preflight seam (ADR-0019: reachability must be probed from where pods live).
-   * `caPem` makes the probe trust a private CA the same way the Harness does (ADR-0020):
-   * `NODE_EXTRA_CA_CERTS`, which must be process env at node start — it cannot be set from
-   * inside the script. */
-  runOneShot(opts: {
+  /** The API server's `gitVersion` (`kubectl get --raw /version`), e.g. `v1.35.0-gke.1200` — what
+   * ADR-0037's Kubernetes floor is judged against. THROWS when the server does not answer, like
+   * `listJson`: a floor check that passed because it could not look is no check at all. */
+  serverVersion(opts: { context?: string }): Promise<string>;
+  /** `kubectl create --dry-run=server -o json -f -`: the object as admission would store it,
+   * stored nowhere. THROWS on a refusal, with kubectl's reason. */
+  dryRunCreate(opts: { manifest: string; context?: string }): Promise<{ spec?: { resources?: unknown } }>;
+  /** Wait until one pod has run to its end, and read its logs — the provider preflight's seam
+   * (ADR-0019, ADR-0059). Its pod has a Custodian beside the check container, which `kubectl run` cannot
+   * make, so `jr2 up` applies it and waits here. Two bounds: `startSeconds` until the check
+   * container runs (the image pulls and the Custodian's start), then `runSeconds` for the check. It
+   * stops early on a pod that cannot start (`podProgress`). Never throws. */
+  awaitPodEnd(opts: {
     namespace: string;
     name: string;
-    script: string;
-    caPem?: string;
+    container: string;
+    startSeconds: number;
+    runSeconds: number;
     context?: string;
-  }): Promise<string>;
+    /** Called with each new reason the pod gives for not running yet (`Unschedulable: …`). */
+    onPending?: (line: string) => void;
+    /** Called with the check container's log on each poll while it runs. */
+    onLog?: (log: string) => void;
+  }): Promise<PodEnd>;
+};
+
+/** How one pod ended (`awaitPodEnd`). `phase` is the pod's own, or `stuck` (it cannot start:
+ * `stuck` says why) or `timeout` (`started` says whether the check ran). `output` is the check
+ * container's whole log; `custodian` the Custodian's, its previous run's first when it restarted.
+ * `reasons` is each container's waiting or ended reason as the kubelet says it. */
+export type PodEnd = {
+  phase: "Succeeded" | "Failed" | "stuck" | "timeout";
+  started: boolean;
+  stuck?: string;
+  output: string;
+  custodian: string;
+  reasons: string[];
+};
+
+/** A container's status as the kubelet reports it: the part `podProgress` reads. */
+type CheckContainerStatus = {
+  name?: string;
+  restartCount?: number;
+  state?: Record<string, { reason?: string; message?: string }>;
+  /** Why the run before the current one ended: where an OOMKilled sidecar says so. */
+  lastState?: { terminated?: { reason?: string; exitCode?: number } };
+};
+
+/** The pod as `podProgress` reads it. */
+export type CheckPodObject = {
+  status?: {
+    phase?: string;
+    conditions?: Array<{ type?: string; status?: string; reason?: string; message?: string }>;
+    initContainerStatuses?: CheckContainerStatus[];
+    containerStatuses?: CheckContainerStatus[];
+  };
 };
 
+/** Why a pod that has not ended is not running yet, for a person who waits on it: the scheduler's
+ * reason, else the first container's waiting reason, else its phase. Undefined when it gives none,
+ * and for a pod that runs with no container waiting. */
+export function podPending(pod: CheckPodObject | undefined): string | undefined {
+  const unscheduled = pod?.status?.conditions?.find((c) => c.type === "PodScheduled" && c.status === "False");
+  if (unscheduled)
+    return `${unscheduled.reason ?? "not scheduled"}${unscheduled.message ? `: ${unscheduled.message}` : ""}`;
+  const statuses = [...(pod?.status?.initContainerStatuses ?? []), ...(pod?.status?.containerStatuses ?? [])];
+  for (const c of statuses) {
+    const waiting = c.state?.waiting;
+    if (waiting?.reason) return `${c.name ?? "?"}: ${waiting.reason}`;
+  }
+  // A pod that runs, with no container waiting, is not waiting to start: its check is on its way.
+  return pod?.status?.phase === "Running" ? undefined : pod?.status?.phase;
+}
+
+/** A volume a pod cannot mount because a Secret, a ConfigMap or a key it names is missing: the
+ * kubelet keeps the pod in ContainerCreating and says so in a FailedMount event alone — `secret "x"
+ * not found`, `references non-existent secret key: y`, `configmap references non-existent config
+ * key`. The first such event can be a race with the objects that the same apply wrote, so it counts
+ * only when the kubelet says it again: its `count` or `lastTimestamp` went up since an earlier
+ * poll. `seen` keeps each message's last value from poll to poll. */
+export function failedMount(
+  events: ReadonlyArray<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>,
+  seen: Map<string, string>,
+): string | undefined {
+  let found: string | undefined;
+  for (const e of events) {
+    if (e.reason !== "FailedMount" || !/not found|non-existent|couldn't find key|does not exist/i.test(e.message ?? ""))
+      continue;
+    const message = e.message ?? "";
+    const mark = `${e.count ?? 1}|${e.lastTimestamp ?? ""}`;
+    const before = seen.get(message);
+    if (before !== undefined && before !== mark) found ??= `FailedMount: ${message}`;
+    seen.set(message, mark);
+  }
+  return found;
+}
+
+/** The waiting reasons from which a pod does not come back by itself (ADR-0019): an image no node
+ * can pull, a Secret or ConfigMap key it names that is missing, a container that crashes at start. */
+export const TERMINAL_WAITING = new Set(["InvalidImageName", "CreateContainerConfigError", "CrashLoopBackOff"]);
+
+/** A pull that ends with one of these comes back by itself (a slow registry, a timeout, a rate
+ * limit); a pull whose message says that the image or the right to it is not there does not. */
+const PULL_WAITING = new Set(["ErrImagePull", "ImagePullBackOff"]);
+const PULL_FINAL = /not found|manifest unknown|unauthorized|denied/i;
+
+/**
+ * Where a check pod is (`awaitPodEnd`): ended (`phase`), unable to start (`stuck`, the reason), or
+ * still on its way, with whether `container` runs yet. A restart of `sidecar` is `stuck`: the
+ * Custodian restarts only when it refused to start, and the next start refuses alike. Pure, for its
+ * test.
+ */
+export function podProgress(
+  pod: CheckPodObject | undefined,
+  container: string,
+  sidecar: string,
+): { phase?: "Succeeded" | "Failed"; stuck?: string; running: boolean; reasons: string[] } {
+  const statuses = [...(pod?.status?.initContainerStatuses ?? []), ...(pod?.status?.containerStatuses ?? [])];
+  const reasons = statuses.flatMap((c) => [
+    ...Object.entries(c.state ?? {}).flatMap(([state, s]) =>
+      s.reason || s.message ? [`${c.name ?? "?"}: ${state} ${s.reason ?? ""} ${s.message ?? ""}`.trim()] : [],
+    ),
+    ...(c.lastState?.terminated?.reason ? [`${c.name ?? "?"}: last ended ${c.lastState.terminated.reason}`] : []),
+  ]);
+  const running = statuses.some((c) => c.name === container && c.state?.running !== undefined);
+  const phase = pod?.status?.phase;
+  if (phase === "Succeeded" || phase === "Failed") return { phase, running, reasons };
+  for (const c of statuses) {
+    const waiting = c.state?.waiting?.reason;
+    const pullFinal =
+      waiting !== undefined && PULL_WAITING.has(waiting) && PULL_FINAL.test(c.state?.waiting?.message ?? "");
+    if (waiting && (TERMINAL_WAITING.has(waiting) || pullFinal)) {
+      const last = c.lastState?.terminated?.reason;
+      return { stuck: `${c.name ?? "?"}: ${waiting}${last ? ` (last ended ${last})` : ""}`, running, reasons };
+    }
+    if (c.name === sidecar && (c.restartCount ?? 0) > 0) {
+      const last = c.lastState?.terminated?.reason;
+      return { stuck: `${sidecar}: restarted${last ? ` (last ended ${last})` : ""}`, running, reasons };
+    }
+  }
+  return { running, reasons };
+}
+
+/** What `waitPodEnd` reads the pod and the clock through: kubectl and the wall clock in
+ * `kubectlAdmin`, a script and a fake clock in its test. */
+export type PodEndPorts = {
+  pod(): Promise<CheckPodObject | undefined>;
+  /** The pod's events: where a FailedMount is told. */
+  events(): Promise<Array<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>>;
+  /** Each new reason the pod gives for not running yet. */
+  pending?(line: string): void;
+  /** The container's log so far, on each poll while it runs: its results as they come. */
+  log?(output: string): void;
+  logs(container: string, opts: { tailLines: number; previous?: boolean }): Promise<string>;
+  sleep(ms: number): Promise<unknown>;
+  now(): number;
+};
+
+/**
+ * The wait of `awaitPodEnd`: poll the pod every 2 s until it ends (`podProgress`), cannot start
+ * (a terminal waiting reason, a Custodian restart, a `FailedMount` event), or runs out of time —
+ * `startSeconds` until `container` runs, then `runSeconds` from then on. While it waits to start,
+ * each new reason it gives goes to `pending`. Then it reads the logs: the check container's whole log, for
+ * every model's line, read again when a pod that ended answers nothing; and the Custodian's last
+ * 50 lines, its run before first when it restarted.
+ */
+export async function waitPodEnd(
+  ports: PodEndPorts,
+  opts: { container: string; startSeconds: number; runSeconds: number },
+): Promise<PodEnd> {
+  const startBy = ports.now() + opts.startSeconds * 1000;
+  let runBy: number | undefined;
+  let seen: ReturnType<typeof podProgress>;
+  let end: PodEnd["phase"];
+  let restarted = false;
+  let said: string | undefined;
+  const mounts = new Map<string, string>();
+  for (;;) {
+    const pod = await ports.pod();
+    seen = podProgress(pod, opts.container, "custodian");
+    restarted ||= seen.stuck?.startsWith("custodian: restarted") === true;
+    if (seen.running && runBy === undefined) runBy = ports.now() + opts.runSeconds * 1000;
+    if (seen.phase) {
+      end = seen.phase;
+      break;
+    }
+    if (!seen.stuck && !seen.running) {
+      const mount = failedMount(await ports.events(), mounts);
+      if (mount) seen = { ...seen, stuck: mount };
+    }
+    if (seen.stuck) {
+      end = "stuck";
+      break;
+    }
+    if (ports.now() >= (runBy ?? startBy)) {
+      end = "timeout";
+      break;
+    }
+    if (seen.running && ports.log) ports.log(await ports.logs(opts.container, { tailLines: -1 }));
+    const pending = seen.running ? undefined : podPending(pod);
+    if (pending && pending !== said) ports.pending?.(pending);
+    said = pending;
+    await ports.sleep(2000);
+  }
+  // The whole log: one line per model. A check that ended always wrote one, so an empty read is a
+  // read that failed (kubectl answers nothing then): read again before calling it empty.
+  let output = await ports.logs(opts.container, { tailLines: -1 });
+  for (let n = 0; n < 3 && !output.trim() && (end === "Succeeded" || end === "Failed"); n++) {
+    await ports.sleep(1000);
+    output = await ports.logs(opts.container, { tailLines: -1 });
+  }
+  const custodian = [
+    ...(restarted ? [await ports.logs("custodian", { tailLines: 50, previous: true })] : []),
+    await ports.logs("custodian", { tailLines: 50 }),
+  ].join("\n");
+  return {
+    phase: end,
+    started: runBy !== undefined,
+    ...(seen.stuck ? { stuck: seen.stuck } : {}),
+    output,
+    custodian,
+    reasons: seen.reasons,
+  };
+}
+
 const nsArgs = (namespace?: string): string[] => (namespace ? ["--namespace", namespace] : []);
 
-/** The two workload kinds `jr2 up` rolls out and waits on. */
-export type RolloutKind = "deployment" | "daemonset";
+/** The workload kinds `jr2 up` rolls out and waits on. */
+export type RolloutKind = "deployment" | "daemonset" | "statefulset";
 
 /** One rollout to wait for: the object, by kind and name, in its namespace. */
 export type RolloutRequest = {
@@ -186,6 +413,28 @@ export function rolloutStatusArgs({
   ];
 }
 
+/** ADR-0037's Kubernetes floor: every Sandbox's `/opt/jr2` is an `image` volume of the harness
+ * image, and image volumes are on by default from 1.35 (GA in 1.36). There is no fallback below it
+ * — a second pod shape kept alive for clusters no Instance runs on — so `jr2 up` refuses there. */
+export const KUBE_FLOOR = { major: 1, minor: 35 } as const;
+
+/**
+ * Why this server version cannot host an Instance, or undefined when it can. Pure, so the floor is
+ * checkable without a cluster. Judged on the leading `v<major>.<minor>` alone: distributions append
+ * their own suffixes (`-gke.1200`, `+k3s1`, `-eks-…`), and none of them moves the feature gate. A
+ * version with no such prefix is refused BY NAME rather than passed — a floor that waves through
+ * what it cannot read is the timeout ADR-0046 exists to replace.
+ */
+export function kubeFloorRefusal(gitVersion: string): string | undefined {
+  const floor = `${KUBE_FLOOR.major}.${KUBE_FLOOR.minor}`;
+  const why = `jr2 needs Kubernetes ${floor} or newer (every Sandbox mounts jr2's runtime as an image volume, ADR-0037)`;
+  const m = /^v?(\d+)\.(\d+)/.exec(gitVersion);
+  if (!m) return `reports version "${gitVersion}", which names no major.minor; ${why}`;
+  const [major, minor] = [Number(m[1]), Number(m[2])];
+  if (major > KUBE_FLOOR.major || (major === KUBE_FLOOR.major && minor >= KUBE_FLOOR.minor)) return undefined;
+  return `runs Kubernetes ${gitVersion}; ${why}`;
+}
+
 /**
  * Did this read fail because the cluster has no such RESOURCE TYPE (`kubectl get sandboxes… ` on a
  * cluster with no jr2 CRD)? kubectl exits 1 with `the server doesn't have a resource type "…"`, and
@@ -241,9 +490,30 @@ export const kubectlAdmin: KubeAdmin = {
     return (JSON.parse(stdout) as { items?: never[] }).items ?? [];
   },
 
-  async apply({ manifest, context, serverSide }) {
-    const mode = serverSide ? ["--server-side", "--force-conflicts", "--field-manager=jr2"] : [];
-    await execStdin(["kubectl", ...ctxArgs(context), "apply", ...mode, "-f", "-"], manifest);
+  async apply({ manifest, context, serverSide, warn = (line) => process.stderr.write(`${line}\n`) }) {
+    const { secrets, identities, rest } = splitSecrets(manifest);
+    if (secrets !== undefined) {
+      await execStdin(["kubectl", ...ctxArgs(context), "apply", ...SERVER_SIDE, "-f", "-"], secrets);
+      // The fallback for a migration that failed. Removing an annotation that is not there is no
+      // fault, so this changes nothing once it is gone.
+      try {
+        await execStdin(
+          ["kubectl", ...ctxArgs(context), "annotate", "--overwrite", "-f", "-", `${LAST_APPLIED}-`],
+          identities!,
+        );
+      } catch (err) {
+        const names = (JSON.parse(identities!) as { items: Array<{ metadata: { name: string } }> }).items
+          .map((o) => o.metadata.name)
+          .join(", ");
+        warn(
+          `warning: the ${LAST_APPLIED} annotation was not removed from Secret ${names} ` +
+            `(${(err as Error).message}) — the next \`jr2 up\` tries again`,
+        );
+      }
+    }
+    if (rest !== undefined) {
+      await execStdin(["kubectl", ...ctxArgs(context), "apply", ...(serverSide ? SERVER_SIDE : []), "-f", "-"], rest);
+    }
   },
 
   async label({ kind, name, namespace, labels, context }) {
@@ -288,58 +558,53 @@ export const kubectlAdmin: KubeAdmin = {
     }
   },
 
-  async runOneShot({ namespace, name, script, caPem, context }) {
-    // With a CA bundle: write the PEM to a file BEFORE node starts, because NODE_EXTRA_CA_CERTS
-    // is only honored as process-start env. The PEM rides an env var base64'd (multiline values
-    // and `--env` don't mix); `"$1"` keeps the script out of shell parsing entirely (execFile
-    // passes argv verbatim, no host shell either).
-    const command = caPem
-      ? ["sh", "-ec", 'echo "$JR2_CA_B64" | base64 -d > /tmp/jr2-ca.crt && exec node -e "$1"', "sh", script]
-      : ["node", "-e", script];
+  async serverVersion({ context }) {
+    // `get --raw /version`, not `kubectl version`: that one reports the CLIENT too; this asks the
+    // server alone, and answers in the server's own JSON. Bounded like every first contact: it is
+    // the first thing `jr2 up` asks, so a dead context fails here, fast, with kubectl's own reason.
     try {
-      const { stdout } = await exec(
-        "kubectl",
-        [
-          ...ctxArgs(context),
-          ...nsArgs(namespace),
-          "run",
-          name,
-          "--rm",
-          "--attach",
-          "--restart=Never",
-          "--quiet",
-          "--image=node:24-slim",
-          ...(caPem
-            ? [`--env=JR2_CA_B64=${Buffer.from(caPem).toString("base64")}`, "--env=NODE_EXTRA_CA_CERTS=/tmp/jr2-ca.crt"]
-            : []),
-          "--command",
-          "--",
-          ...command,
-        ],
-        { timeout: 180_000 },
-      );
-      return stdout;
-    } catch (err) {
-      throw oneShotFailure(err);
+      const { stdout } = await exec("kubectl", [...ctxArgs(context), "get", "--raw", "/version", ...reachArgs], {
+        timeout: REACH_BUDGET_MS,
+      });
+      return (JSON.parse(stdout) as { gitVersion?: string }).gitVersion ?? "";
+    } catch (e) {
+      throw new Error(unreachable(e));
     }
   },
-};
 
-/**
- * A one-shot probe's failure, made readable. `kubectl run --attach` streams the POD's own output
- * on ITS stdout, and kubectl writes only its verdict ("pod … terminated (Error)") to stderr — so
- * the one line that says WHY (a stack, a DNS error, an HTTP status) is exactly what execFile's
- * error message drops, since that message carries stderr alone. The provider preflight (ADR-0019)
- * exists to be the cheapest diagnosis available; without the pod's own words it can only report
- * that something in the cluster exited non-zero, and the caller is back to reproducing the probe
- * by hand. So the attached output leads, and the kubectl verdict follows it.
- */
-export function oneShotFailure(err: unknown): Error {
-  const attached = typeof (err as { stdout?: unknown })?.stdout === "string" ? (err as { stdout: string }).stdout : "";
-  const base = err instanceof Error ? err.message : String(err);
-  const trimmed = attached.trim();
-  return trimmed ? new Error(`${trimmed}\n${base}`) : err instanceof Error ? err : new Error(base);
-}
+  async dryRunCreate({ manifest, context }) {
+    const out = await execStdinOut(
+      ["kubectl", ...ctxArgs(context), "create", "--dry-run=server", "-o", "json", "-f", "-"],
+      manifest,
+    );
+    return JSON.parse(out) as { spec?: { resources?: unknown } };
+  },
+
+  async awaitPodEnd({ namespace, name, container, startSeconds, runSeconds, context, onPending, onLog }) {
+    const ctx = context ? { context } : {};
+    return waitPodEnd(
+      {
+        pod: () => kubectlAdmin.getJson<CheckPodObject>({ kind: "pod", name, namespace, ...ctx }),
+        // A read that fails is no event: the wait goes on, and its bounds still hold.
+        events: () =>
+          kubectlAdmin
+            .listJson<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>({
+              kind: "event",
+              fieldSelector: `involvedObject.name=${name}`,
+              namespace,
+              ...ctx,
+            })
+            .catch(() => []),
+        ...(onPending ? { pending: onPending } : {}),
+        ...(onLog ? { log: onLog } : {}),
+        logs: (c, opts) => kubectlAdmin.logs({ namespace, pod: name, container: c, ...opts, ...ctx }),
+        sleep,
+        now: Date.now,
+      },
+      { container, startSeconds, runSeconds },
+    );
+  },
+};
 
 /** What a rollout wait needs to know to go looking: the workload that did not come up (a Deployment
  * unless `kind` says otherwise), and the label selector its pods carry (each converge layer already
@@ -350,6 +615,8 @@ export type RolloutTarget = {
   namespace: string;
   selector: string;
   context?: string;
+  /** Default 180s (see `rolloutStatusArgs`). */
+  timeoutSeconds?: number;
 };
 
 /** One container's state, as the kubelet tells it. `reason`/`message` are its words verbatim — the
@@ -425,8 +692,7 @@ type EventObject = {
  * A rollout's failure, made readable (ADR-0046). `kubectl rollout status` reports its verdict and
  * nothing else — `error: timed out waiting for the condition` — so the fact that says WHY (an
  * `exec format error`, a pull the kubelet gave up on, a Secret that is not there) lives in the
- * pods, which the caller then goes and reads by hand. This is `oneShotFailure()` one layer up: the
- * evidence leads, kubectl's verdict follows, and `jr2 up`'s three rollout waits share the one pair
+ * pods, which the caller then goes and reads by hand. So the evidence leads, kubectl's verdict follows, and `jr2 up`'s three rollout waits share the one pair
  * of eyes.
  *
  * Every read here is best-effort by construction. It runs only after a failure has already
@@ -474,7 +740,12 @@ async function gatherRolloutEvidence(kube: KubeAdmin, target: RolloutTarget): Pr
   const live = pods.filter((p) => !p.metadata.deletionTimestamp);
   if (pods.length > 0 && live.length === 0) notes.push(`every pod matching ${selector} is terminating`);
   if (pods.length === 0 && notes.length === 0) {
-    const maker = target.kind === "daemonset" ? "the DaemonSet" : "the ReplicaSet";
+    const maker =
+      target.kind === "daemonset"
+        ? "the DaemonSet"
+        : target.kind === "statefulset"
+          ? "the StatefulSet"
+          : "the ReplicaSet";
     notes.push(`no pod matches ${selector} — ${maker} made none (check quota, node taints, and the selector)`);
   }
 
@@ -716,6 +987,60 @@ function resolveRef(ref?: string): string {
   return `docker.io/${ref.includes("/") ? ref : `library/${ref}`}`;
 }
 
+/** A server-side apply as jr2's own field manager. jr2 is the only writer of what it applies, so it
+ * takes the fields it names (`--force-conflicts`) from an older client-side apply. */
+const SERVER_SIDE = ["--server-side", "--force-conflicts", "--field-manager=jr2"];
+
+/** The annotation in which a client-side apply keeps a copy of the whole object it applied. */
+export const LAST_APPLIED = "kubectl.kubernetes.io/last-applied-configuration";
+
+/**
+ * A manifest, split for `apply`: its Secrets as one List to apply server-side, the names of those
+ * Secrets alone (for the annotation's removal), and the other objects as they were given. A
+ * Secret's `stringData` goes into `data` (base64): a server-side apply owns the fields it names,
+ * and `stringData` is write-only, so a key that a later apply leaves out would stay in `data`. A
+ * manifest that is not JSON (the operator's YAML, which holds no Secret) is all `rest`. Pure, for
+ * its test.
+ */
+export function splitSecrets(manifest: string): { secrets?: string; identities?: string; rest?: string } {
+  let parsed: { kind?: string; items?: Array<Record<string, unknown>> } & Record<string, unknown>;
+  try {
+    parsed = JSON.parse(manifest);
+  } catch {
+    return { rest: manifest };
+  }
+  const items = parsed.kind === "List" ? (parsed.items ?? []) : [parsed];
+  const secrets = items.filter((o) => o.kind === "Secret");
+  if (!secrets.length) return { rest: manifest };
+  const others = items.filter((o) => o.kind !== "Secret");
+  const list = (objects: unknown[]) => JSON.stringify({ apiVersion: "v1", kind: "List", items: objects });
+  return {
+    secrets: list(secrets.map(serverSideSecret)),
+    identities: list(
+      secrets.map((o) => {
+        const m = o.metadata as { name: string; namespace?: string };
+        return {
+          apiVersion: "v1",
+          kind: "Secret",
+          metadata: { name: m.name, ...(m.namespace ? { namespace: m.namespace } : {}) },
+        };
+      }),
+    ),
+    ...(others.length ? { rest: list(others) } : {}),
+  };
+}
+
+/** A Secret with its `stringData` in `data`, as Kubernetes stores it: a key in both takes the
+ * `stringData` value, as the API server merges them. */
+function serverSideSecret(o: Record<string, unknown>): Record<string, unknown> {
+  const { stringData, ...secret } = o as { stringData?: Record<string, string> } & Record<string, unknown>;
+  if (stringData === undefined) return secret;
+  const encoded = Object.fromEntries(
+    Object.entries(stringData).map(([k, v]) => [k, Buffer.from(v, "utf8").toString("base64")]),
+  );
+  return { ...secret, data: { ...((secret.data as Record<string, string> | undefined) ?? {}), ...encoded } };
+}
+
 /** Run a command feeding `stdin`, surfacing stderr in the thrown error (kubectl's messages are the
  * useful part of an apply failure). */
 function execStdin(cmd: string[], stdin: string): Promise<void> {
@@ -732,6 +1057,50 @@ function execStdin(cmd: string[], stdin: string): Promise<void> {
   });
 }
 
+/** `execStdin`, keeping stdout. */
+function execStdinOut(cmd: string[], stdin: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd[0]!, cmd.slice(1));
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (d: Buffer) => (out += d.toString()));
+    child.stderr.on("data", (d: Buffer) => (err += d.toString()));
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0 ? resolve(out) : reject(new Error(`${cmd.join(" ")} exited (${code}): ${err.trim()}`)),
+    );
+    child.stdin.write(stdin);
+    child.stdin.end();
+  });
+}
+
+/**
+ * The pod ADR-0060's check dry-runs: one that states pod-level resources (KEP-2837). A server
+ * whose PodLevelResources gate is off drops `spec.resources` at admission without a word, and
+ * every Sandbox then runs Burstable — a User Container with no split unbounded, against "jr2
+ * reserves the whole Size". On by default from 1.34, below KUBE_FLOOR, so only an owner who turned
+ * it off trips this. Shaped for the `restricted` Pod Security level, so a namespace that enforces
+ * it does not refuse the probe for another reason. Never scheduled, never pulled.
+ */
+export function podResourcesProbe(namespace: string): object {
+  return {
+    apiVersion: "v1",
+    kind: "Pod",
+    metadata: { name: "jr2-pod-resources-probe", namespace },
+    spec: {
+      resources: { requests: { cpu: "10m", memory: "16Mi" }, limits: { cpu: "10m", memory: "16Mi" } },
+      securityContext: { runAsNonRoot: true, runAsUser: 65532, seccompProfile: { type: "RuntimeDefault" } },
+      containers: [
+        {
+          name: "probe",
+          image: "registry.k8s.io/pause:3.10",
+          securityContext: { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] } },
+        },
+      ],
+    },
+  };
+}
+
 /** The real kube port, over `kubectl` subprocesses. */
 export const kubectlKube: KubePort = {
   async currentContext() {

package/src/nodes.ts

@@ -7,13 +7,19 @@
 // at the moment of the read. The set moves (a pool autoscales, a node is cordoned), which is why an
 // empty set is reported and never refused.
 
-import type { SandboxPlacement, Toleration } from "@jr2/orchestrator";
+import {
+  cpuMillis,
+  memoryBytes,
+  type ResolvedSize,
+  type SandboxPlacement,
+  type Toleration,
+} from "@jr2/orchestrator/internal";
 
 /** A cluster node, read for the facts that decide placement and platform (ADR-0045/0052). */
 export type NodeObject = {
   metadata: { name: string; labels?: Record<string, string> };
   spec?: { unschedulable?: boolean; taints?: readonly Taint[] };
-  status?: { nodeInfo?: { architecture?: string } };
+  status?: { nodeInfo?: { architecture?: string }; allocatable?: Record<string, string> };
 };
 
 export type Taint = { key: string; value?: string; effect: string };
@@ -72,3 +78,28 @@ export function tolerates(t: Toleration, taint: Taint): boolean {
   if (t.operator === "Exists") return true;
   return (t.value ?? "") === (taint.value ?? "");
 }
+
+/**
+ * Whether any of `nodes` could hold `size` (ADR-0060): its allocatable memory and cpu both at least
+ * the whole Size — what the scheduler compares a Guaranteed pod against on an empty node. `undefined`
+ * when no node reports allocatable, which is an unknown, never a no. A WARNING at `jr2 up`, never a
+ * refusal: the node set moves (ADR-0052), and a pool may scale up a bigger node tomorrow.
+ */
+export function anyNodeHolds(nodes: readonly NodeObject[], size: ResolvedSize): boolean | undefined {
+  let known = false;
+  for (const node of nodes) {
+    const alloc = node.status?.allocatable;
+    if (alloc?.memory === undefined || alloc.cpu === undefined) continue;
+    let memory: number;
+    let cpu: number;
+    try {
+      memory = memoryBytes(alloc.memory);
+      cpu = cpuMillis(alloc.cpu);
+    } catch {
+      continue;
+    }
+    known = true;
+    if (memory >= size.memory.bytes && cpu >= size.cpu.millis) return true;
+  }
+  return known ? false : undefined;
+}